From f4113033acea2a97932cea400f14e2555708f76f Mon Sep 17 00:00:00 2001 From: Zebreus Date: Tue, 27 Oct 2026 20:36:52 +0100 Subject: [PATCH] Start the server again when the cleanup job's start was dropped The task that runs the server ignores a start while it thinks it is still running, so the cleanup job could stop the server and then fail to start it again: the site had no shell for forty minutes. The start is now tried three times, with the processes of the task killed in between. The job now only counts the established connections when it decides whether anybody is connected - the server leaves the processes of finished sessions behind, and they kept it from ever restarting the server. Those processes are killed with the server, because they hold the runtime and the protected programs open. The pacman calls are bounded, so a database lock cannot keep the server down, and they run before the server is stopped. The exit code of a program started this way is only available when its handle is taken while it runs, and the pattern for the files to overwrite has to be quoted, or the runtime expands it. Co-Authored-By: Claude Code --- server/README.md | 8 +++- server/scripts/install_restore_task.ps1 | 63 ++++++++++++++++++------- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/server/README.md b/server/README.md index 8bbd580..0870675 100644 --- a/server/README.md +++ b/server/README.md @@ -50,8 +50,8 @@ The runtime itself is not among the protected copies. A program and the shell th runtime DLL, or the program does not run at all - a second copy of it in one process is refused. The login path therefore uses the one in `/usr/bin`, next to the user's own programs, and the user may replace and upgrade it. What is protected instead is that it is the one the package says it is: pacman checks the files of `msys2-runtime`, -and the cleanup job puts the package back if it was changed - before it starts the server, which runs as SYSTEM and -would otherwise run code the user of the website wrote. +and the cleanup job puts the package back if it was changed - while the server is stopped, before it starts it +again, because the server runs as SYSTEM and would otherwise run code the user of the website wrote. `/etc/passwd` (the login shell and home directory) and `/etc/ssh` (the host keys and config) are read-only to the user as well; everything else in the installation is theirs. @@ -71,6 +71,10 @@ it comes back by itself if it died, it takes over a server that was left behind, were just restored. A session that is open keeps the port, so a server that is started now could not bind it, and the job never disturbs a session. +The task that runs the server ignores a start while it thinks it is still running, so a start that is dropped +leaves the site without a shell. The cleanup job therefore tries three times, killing the processes of the task +in between, before it gives up. + The deployment does not restart the server. It runs in a session, and that session keeps the port, so a server that starts during a deployment cannot bind it. The new configuration and the restored programs therefore take effect at the next run of the cleanup job, which happens within half an hour. diff --git a/server/scripts/install_restore_task.ps1 b/server/scripts/install_restore_task.ps1 index a5531da..afe3883 100644 --- a/server/scripts/install_restore_task.ps1 +++ b/server/scripts/install_restore_task.ps1 @@ -160,6 +160,9 @@ function Stop-Sshd { Get-CimInstance Win32_Process -Filter "Name='cmd.exe'" | Where-Object { $_.CommandLine -match 'cmd\.exe\s+/c\s+"?C:\\ProgramData\\kik-restore\\run-sshd\.bat' } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } Get-Process -Name sshd -ErrorAction SilentlyContinue | Stop-Process -Force + # the server leaves the processes of the sessions it has served behind, and they hold the runtime and the + # protected programs open, so those cannot be replaced while they are there + Get-Process -Name 'sshd-session' -ErrorAction SilentlyContinue | Stop-Process -Force $waited = 0 while ((Get-ScheduledTask -TaskName 'KiK sshd').State -ne 'Ready' -and $waited -lt 30) { Start-Sleep 1; $waited++ } $waited = 0 @@ -167,30 +170,58 @@ function Stop-Sshd { } # The server is started again and waited for: a deployment or a crash must never leave the site without a shell. +# The task ignores a new instance while it thinks one is running, so when the port stays closed its processes are +# killed and it is started again. function Start-Sshd { - Start-ScheduledTask -TaskName 'KiK sshd' - $waited = 0 - while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } - return [bool](Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) + for ($attempt = 1; $attempt -le 3; $attempt++) { + Start-ScheduledTask -TaskName 'KiK sshd' + $waited = 0 + while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } + if (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) { return $true } + Write-Log "the server did not come up (attempt $attempt)" + Stop-Sshd + } + return $false +} + +# Runs a program and gives up after a while: pacman can wait for the lock of its database, and the server must not +# stay down because of it. The exit code is returned, or -1 when it did not finish in time. +function Invoke-Bounded($program, $arguments, $seconds) { + $process = Start-Process -FilePath $program -ArgumentList $arguments -NoNewWindow -PassThru ` + -RedirectStandardOutput (Join-Path $env:TEMP 'kik-pacman.out') -RedirectStandardError (Join-Path $env:TEMP 'kik-pacman.err') + # the handle is taken while the process runs, otherwise its exit code is not available afterwards + $null = $process.Handle + if (-not $process.WaitForExit($seconds * 1000)) { + Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue + return -1 + } + return $process.ExitCode } $netError = $null $connections = @(Get-NetTCPConnection -LocalPort 22 -ErrorAction SilentlyContinue -ErrorVariable netError) -$sessions = @(Get-Process -Name 'sshd-session' -ErrorAction SilentlyContinue) -# A session that is open keeps the port and the runtime, so the server is only restarted when there is none: it -# comes back by itself if it died, it takes over a server that was left behind, and it picks up the programs that -# were just restored. A query that fails is not a reason to disturb a session. -$busy = @($connections | Where-Object { $_.State -eq 'Established' }).Count -gt 0 -or $sessions.Count -gt 0 +# An open session keeps the port, so the server is only restarted when there is none: it comes back by itself if it +# died, it takes over a server that was left behind, and it picks up the programs that were just restored. A query +# that fails is not a reason to disturb a session. The processes of the sessions are left behind by the server when a +# client goes away, so they say nothing about whether anybody is connected. +$busy = @($connections | Where-Object { $_.State -eq 'Established' }).Count -gt 0 $restart = -not $netError -and -not $busy +# The runtime of the installation belongs to the user's programs and is not protected: a program and the shell that +# starts it have to use the same one, or the program does not run at all. What has to be protected is that it is the +# one the package says it is, because the server runs as SYSTEM. The check takes a while, so it runs while the +# server is up - the server must not wait for it. +$runtimeBroken = $false +if ($restart) { + $runtimeBroken = (Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-Qkk', 'msys2-runtime') 120) -ne 0 +} + if ($restart) { Stop-Sshd - # The runtime of the installation belongs to the user's programs and is not protected: a program and the shell - # that starts it have to use the same one, or the program does not run at all. What has to be protected is that - # it is the one the package says it is, because the server runs as SYSTEM. Remove-Item "$kik\msys-2.0.dll" -Force -ErrorAction SilentlyContinue - & "$msys\usr\bin\pacman.exe" -Qkk msys2-runtime 2>&1 | Out-Null - if ($LASTEXITCODE -ne 0) { - & "$msys\usr\bin\pacman.exe" -S --noconfirm --overwrite '*' msys2-runtime 2>&1 | Out-Null + if ($runtimeBroken) { + # the package is put back while the server is stopped: the file cannot be replaced while it is mapped. + # The pattern is quoted, or the runtime expands it to the files of the working directory. + Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-S', '--noconfirm', '--overwrite', "'*'", 'msys2-runtime') 300 | Out-Null Write-Log 'restored the runtime of the installation' } } @@ -465,7 +496,7 @@ $restoreTriggers = @( New-ScheduledTaskTrigger -Once -At (Get-Date) -RepetitionInterval (New-TimeSpan -Minutes 30) -RepetitionDuration (New-TimeSpan -Days 3650) New-ScheduledTaskTrigger -AtStartup ) -$restoreSettings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10) -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 5) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries +$restoreSettings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 30) -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 5) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries Register-ScheduledTask -TaskName $restoreTask -Action $restoreAction -Trigger $restoreTriggers -Principal $atSystem -Settings $restoreSettings -Force | Out-Null # The server is started and waited for: it must listen on port 22, or the site has no shell.