diff --git a/server/README.md b/server/README.md index 8bbd580..0870675 100644 --- a/server/README.md +++ b/server/README.md @@ -50,8 +50,8 @@ The runtime itself is not among the protected copies. A program and the shell th runtime DLL, or the program does not run at all - a second copy of it in one process is refused. The login path therefore uses the one in `/usr/bin`, next to the user's own programs, and the user may replace and upgrade it. What is protected instead is that it is the one the package says it is: pacman checks the files of `msys2-runtime`, -and the cleanup job puts the package back if it was changed - before it starts the server, which runs as SYSTEM and -would otherwise run code the user of the website wrote. +and the cleanup job puts the package back if it was changed - while the server is stopped, before it starts it +again, because the server runs as SYSTEM and would otherwise run code the user of the website wrote. `/etc/passwd` (the login shell and home directory) and `/etc/ssh` (the host keys and config) are read-only to the user as well; everything else in the installation is theirs. @@ -71,6 +71,10 @@ it comes back by itself if it died, it takes over a server that was left behind, were just restored. A session that is open keeps the port, so a server that is started now could not bind it, and the job never disturbs a session. +The task that runs the server ignores a start while it thinks it is still running, so a start that is dropped +leaves the site without a shell. The cleanup job therefore tries three times, killing the processes of the task +in between, before it gives up. + The deployment does not restart the server. It runs in a session, and that session keeps the port, so a server that starts during a deployment cannot bind it. The new configuration and the restored programs therefore take effect at the next run of the cleanup job, which happens within half an hour. diff --git a/server/scripts/install_restore_task.ps1 b/server/scripts/install_restore_task.ps1 index a5531da..afe3883 100644 --- a/server/scripts/install_restore_task.ps1 +++ b/server/scripts/install_restore_task.ps1 @@ -160,6 +160,9 @@ function Stop-Sshd { Get-CimInstance Win32_Process -Filter "Name='cmd.exe'" | Where-Object { $_.CommandLine -match 'cmd\.exe\s+/c\s+"?C:\\ProgramData\\kik-restore\\run-sshd\.bat' } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } Get-Process -Name sshd -ErrorAction SilentlyContinue | Stop-Process -Force + # the server leaves the processes of the sessions it has served behind, and they hold the runtime and the + # protected programs open, so those cannot be replaced while they are there + Get-Process -Name 'sshd-session' -ErrorAction SilentlyContinue | Stop-Process -Force $waited = 0 while ((Get-ScheduledTask -TaskName 'KiK sshd').State -ne 'Ready' -and $waited -lt 30) { Start-Sleep 1; $waited++ } $waited = 0 @@ -167,30 +170,58 @@ function Stop-Sshd { } # The server is started again and waited for: a deployment or a crash must never leave the site without a shell. +# The task ignores a new instance while it thinks one is running, so when the port stays closed its processes are +# killed and it is started again. function Start-Sshd { - Start-ScheduledTask -TaskName 'KiK sshd' - $waited = 0 - while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } - return [bool](Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) + for ($attempt = 1; $attempt -le 3; $attempt++) { + Start-ScheduledTask -TaskName 'KiK sshd' + $waited = 0 + while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } + if (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) { return $true } + Write-Log "the server did not come up (attempt $attempt)" + Stop-Sshd + } + return $false +} + +# Runs a program and gives up after a while: pacman can wait for the lock of its database, and the server must not +# stay down because of it. The exit code is returned, or -1 when it did not finish in time. +function Invoke-Bounded($program, $arguments, $seconds) { + $process = Start-Process -FilePath $program -ArgumentList $arguments -NoNewWindow -PassThru ` + -RedirectStandardOutput (Join-Path $env:TEMP 'kik-pacman.out') -RedirectStandardError (Join-Path $env:TEMP 'kik-pacman.err') + # the handle is taken while the process runs, otherwise its exit code is not available afterwards + $null = $process.Handle + if (-not $process.WaitForExit($seconds * 1000)) { + Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue + return -1 + } + return $process.ExitCode } $netError = $null $connections = @(Get-NetTCPConnection -LocalPort 22 -ErrorAction SilentlyContinue -ErrorVariable netError) -$sessions = @(Get-Process -Name 'sshd-session' -ErrorAction SilentlyContinue) -# A session that is open keeps the port and the runtime, so the server is only restarted when there is none: it -# comes back by itself if it died, it takes over a server that was left behind, and it picks up the programs that -# were just restored. A query that fails is not a reason to disturb a session. -$busy = @($connections | Where-Object { $_.State -eq 'Established' }).Count -gt 0 -or $sessions.Count -gt 0 +# An open session keeps the port, so the server is only restarted when there is none: it comes back by itself if it +# died, it takes over a server that was left behind, and it picks up the programs that were just restored. A query +# that fails is not a reason to disturb a session. The processes of the sessions are left behind by the server when a +# client goes away, so they say nothing about whether anybody is connected. +$busy = @($connections | Where-Object { $_.State -eq 'Established' }).Count -gt 0 $restart = -not $netError -and -not $busy +# The runtime of the installation belongs to the user's programs and is not protected: a program and the shell that +# starts it have to use the same one, or the program does not run at all. What has to be protected is that it is the +# one the package says it is, because the server runs as SYSTEM. The check takes a while, so it runs while the +# server is up - the server must not wait for it. +$runtimeBroken = $false +if ($restart) { + $runtimeBroken = (Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-Qkk', 'msys2-runtime') 120) -ne 0 +} + if ($restart) { Stop-Sshd - # The runtime of the installation belongs to the user's programs and is not protected: a program and the shell - # that starts it have to use the same one, or the program does not run at all. What has to be protected is that - # it is the one the package says it is, because the server runs as SYSTEM. Remove-Item "$kik\msys-2.0.dll" -Force -ErrorAction SilentlyContinue - & "$msys\usr\bin\pacman.exe" -Qkk msys2-runtime 2>&1 | Out-Null - if ($LASTEXITCODE -ne 0) { - & "$msys\usr\bin\pacman.exe" -S --noconfirm --overwrite '*' msys2-runtime 2>&1 | Out-Null + if ($runtimeBroken) { + # the package is put back while the server is stopped: the file cannot be replaced while it is mapped. + # The pattern is quoted, or the runtime expands it to the files of the working directory. + Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-S', '--noconfirm', '--overwrite', "'*'", 'msys2-runtime') 300 | Out-Null Write-Log 'restored the runtime of the installation' } } @@ -465,7 +496,7 @@ $restoreTriggers = @( New-ScheduledTaskTrigger -Once -At (Get-Date) -RepetitionInterval (New-TimeSpan -Minutes 30) -RepetitionDuration (New-TimeSpan -Days 3650) New-ScheduledTaskTrigger -AtStartup ) -$restoreSettings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10) -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 5) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries +$restoreSettings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 30) -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 5) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries Register-ScheduledTask -TaskName $restoreTask -Action $restoreAction -Trigger $restoreTriggers -Principal $atSystem -Settings $restoreSettings -Force | Out-Null # The server is started and waited for: it must listen on port 22, or the site has no shell.