From 727f8f6e9ccf8ac37483ef531bf6bbafe8586cf7 Mon Sep 17 00:00:00 2001 From: Zebreus Date: Fri, 6 Nov 2026 00:17:08 +0100 Subject: [PATCH] Restart the shell server only when it has to, and record failed starts Every outage since 28 Oct began at the cleanup job's routine restart of a working server: the start failed and kept failing until a reboot. The job now leaves a working server alone and restarts it only when it does not listen, is older than its configuration, or holds open a program or the runtime that has to be repaired. A failed start left no trace anywhere, so windows now records the history of the scheduled tasks and the start and end of every process, and the job writes the processes and the state of the server's task to failed-starts.log when the server does not come up. The security log grows to a gigabyte, because the scanners of the remote desktop fill a hundred megabytes in a week. Co-Authored-By: Claude Opus 5.5 (1M context) --- server/README.md | 16 ++- server/scripts/install_restore_task.ps1 | 130 +++++++++++++++--------- 2 files changed, 92 insertions(+), 54 deletions(-) diff --git a/server/README.md b/server/README.md index 3e5405d..10b4a89 100644 --- a/server/README.md +++ b/server/README.md @@ -69,10 +69,11 @@ Three scheduled tasks run as SYSTEM, all at startup, and the user cannot stop an the server only starts it again when it failed, never when it was stopped, so a cleanup job that is killed while it has the server stopped would otherwise take the site down with it. -The cleanup job also restarts the server when nobody is connected, and when the server does not listen at all: -it comes back by itself if it died, it takes over a server that was left behind, and it picks up the programs that -were just restored. A session that is open keeps the port, so a server that is started now could not bind it, and -the job never disturbs a session. +The cleanup job only restarts the server when it has to: when it does not listen at all, when it is older than its +configuration, or when the protected copy or the runtime has to be repaired - those cannot be replaced while the +server holds them open. A working server is left alone, because every start can fail, and failed starts have kept +the site down for hours. The job also never restarts the server while somebody is connected: a session that is open +keeps the port, so a server that is started then could not bind it. The server is only down for the short part of the job that needs it down: the protected copy cannot be replaced while the server holds it open. Everything else - the home directory, the four texts, the permissions of the @@ -90,7 +91,12 @@ at the next run of the cleanup job, which happens within half an hour. The server logs to `C:\ProgramData\kik-restore\sshd.log`. The log is appended to, not overwritten, because it is the only trace of what the server did: a server that is restarted would otherwise take the evidence of an outage with it. It is rotated when it grows past a hundred megabytes, which takes years. The event logs of windows are -made that big as well, so that they keep the history of the box itself for just as long. +made bigger as well, the security log to a gigabyte: the scanners of the remote desktop alone fill a hundred +megabytes of it in a week. + +A start of the server that fails leaves nothing in that log. So windows records the history of the scheduled tasks +and the start and end of every process, and when the server does not come up, the cleanup job writes the processes +and the state of the server's task to `C:\ProgramData\kik-restore\failed-starts.log`. Only keys may log in over SSH. The box stands in the open internet and both of its accounts have a password. diff --git a/server/scripts/install_restore_task.ps1 b/server/scripts/install_restore_task.ps1 index 1b5873b..18230dc 100644 --- a/server/scripts/install_restore_task.ps1 +++ b/server/scripts/install_restore_task.ps1 @@ -150,9 +150,9 @@ function New-Home($path) { Get-ChildItem -LiteralPath "$msys\etc\skel" -Force | Copy-Item -Destination $path -Recurse -Force } -# The server is stopped and started again when nobody is connected: it picks up the programs that are restored -# just now, and it comes back by itself if it died. It has to be stopped before the copies are made, because a -# running server holds its programs open and they cannot be replaced. +# The server is only stopped when it has to be: when it does not run, when it is older than its configuration, or +# when it holds open what has to be repaired - its programs in opt/kik or the runtime. A working server is left alone, +# because every start can fail, and a failed start has kept the site down for hours. # # Stopping the task does not take the server with it, so its processes are killed by hand and the task is waited # for: it ignores a new instance while one is running, so a start would be dropped. @@ -170,6 +170,19 @@ function Stop-Sshd { while ((Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } } +# What the box looks like when the server did not come up. Nothing else records a failed start, and the reboot that +# ends the outage takes the evidence with it. +function Save-Snapshot($attempt) { + $task = Get-ScheduledTask -TaskName 'KiK sshd' + $info = $task | Get-ScheduledTaskInfo + $lines = @("== $(Get-Date -Format s) attempt ${attempt}: the task is $($task.State), last run $($info.LastRunTime), result 0x$('{0:X}' -f $info.LastTaskResult)") + $lines += Get-NetTCPConnection -LocalPort 22 -ErrorAction SilentlyContinue | ForEach-Object { "port 22: $($_.LocalAddress) $($_.State) pid $($_.OwningProcess)" } + $lines += Get-CimInstance Win32_Process | Sort-Object CreationDate | ForEach-Object { + '{0:MM-dd HH:mm:ss} {1} pid {2} parent {3} session {4}: {5}' -f $_.CreationDate, $_.Name, $_.ProcessId, $_.ParentProcessId, $_.SessionId, $_.CommandLine + } + $lines | Out-File -FilePath (Join-Path $PSScriptRoot 'failed-starts.log') -Append +} + # The server is started again and waited for: a deployment or a crash must never leave the site without a shell. # The task ignores a new instance while it thinks one is running, so when the port stays closed its processes are # killed and it is started again. @@ -180,6 +193,7 @@ function Start-Sshd { while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } if (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) { return $true } Write-Log "the server did not come up (attempt $attempt)" + Save-Snapshot $attempt Stop-Sshd } return $false @@ -201,18 +215,17 @@ function Invoke-Bounded($program, $arguments, $seconds) { $netError = $null $connections = @(Get-NetTCPConnection -LocalPort 22 -ErrorAction SilentlyContinue -ErrorVariable netError) -# An open session keeps the port, so the server is only restarted when there is none: it comes back by itself if it -# died, it takes over a server that was left behind, and it picks up the programs that were just restored. A query -# that fails is not a reason to disturb a session. The processes of the sessions are left behind by the server when a -# client goes away, so they say nothing about whether anybody is connected. +# An open session keeps the port, so the server is only stopped when there is none. A query that fails is not a +# reason to disturb a session. The processes of the sessions are left behind by the server when a client goes away, +# so they say nothing about whether anybody is connected. $busy = @($connections | Where-Object { $_.State -eq 'Established' }).Count -gt 0 -$restart = -not $netError -and -not $busy +$mayStop = -not $netError -and -not $busy # The runtime of the installation belongs to the user's programs and is not protected: a program and the shell that # starts it have to use the same one, or the program does not run at all. What has to be protected is that it is the # one the package says it is, because the server runs as SYSTEM. The check takes a while, so it runs while the # server is up - the server must not wait for it. $runtimeBroken = $false -if ($restart) { +if ($mayStop) { $runtimeBroken = (Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-Qkk', 'msys2-runtime') 120) -ne 0 } @@ -266,6 +279,23 @@ foreach ($copy in Get-ChildItem -LiteralPath $copies -Recurse -File -Force) { # permissions because they are set without inheritance. if ($sid) { icacls $msys /grant "*${sid}:(OI)(CI)F" | Out-Null } +# The protected login path: the copy the sshd and the login shell run from. The user cannot write it, so even after +# they replace or remove their own copies in /usr/bin the server and the journey keep working. The trusted originals +# are in bin\; anything missing from opt/kik or different from its original is put back. Replacing them is the only +# work that needs the server to be stopped, because a running server holds its programs open. +$stale = @(if (Test-Path -LiteralPath $binDir) { + Get-ChildItem -LiteralPath $binDir -File -Force | Where-Object { + $target = Join-Path $kik $_.Name + $_.Name -ne 'msys-2.0.dll' -and (-not (Test-Path -LiteralPath $target) -or (Get-Item -LiteralPath $target -Force).Length -ne $_.Length) + } +}) +# A deployment writes the configuration but cannot restart the server, so a server that is older than its +# configuration is restarted here. +$listener = Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue | Select-Object -First 1 +$outdated = $listener -and @(Get-Item -LiteralPath "$msys\etc\ssh\sshd_config", (Join-Path $PSScriptRoot 'run-sshd.bat') | + Where-Object { $_.LastWriteTime -gt (Get-Process -Id $listener.OwningProcess).StartTime }).Count -gt 0 +$restart = $mayStop -and (-not $listener -or $outdated -or $runtimeBroken -or $stale.Count -gt 0) + if ($restart) { Stop-Sshd Remove-Item "$kik\msys-2.0.dll" -Force -ErrorAction SilentlyContinue @@ -275,47 +305,43 @@ if ($restart) { Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-S', '--noconfirm', '--overwrite', "'*'", 'msys2-runtime') 300 | Out-Null Write-Log 'restored the runtime of the installation' } - - try { - # The protected login path: the copy the sshd and the login shell run from. The user cannot write it, so even - # after they replace or remove their own copies in /usr/bin the server and the journey keep working. The trusted - # originals are in bin\; anything missing from opt/kik is put back. This is the only work that needs the server - # to be stopped, because a running server holds its programs open. - if (Test-Path -LiteralPath $binDir) { - if (-not (Test-Path -LiteralPath $kik)) { New-Item -ItemType Directory -Path $kik | Out-Null } - foreach ($file in Get-ChildItem -LiteralPath $binDir -File -Force | Where-Object { $_.Name -ne 'msys-2.0.dll' }) { - $target = Join-Path $kik $file.Name - try { - if (-not (Test-Path -LiteralPath $target) -or (Get-Item -LiteralPath $target -Force).Length -ne $file.Length) { - Copy-Item -LiteralPath $file.FullName -Destination $target -Force - Write-Log "restored opt/kik/$($file.Name)" - } - } catch { - # a running server holds its programs open; they are replaced in one of the next runs - Write-Log "failed opt/kik/$($file.Name) : $_" - } - } - New-Item -ItemType Directory -Force -Path "$kik\dev\shm", "$kik\dev\mqueue" | Out-Null - Set-LockedAcl $kik $true 'ReadAndExecute' - # /dev is scratch: the user may write it so POSIX shared memory works, the rest of opt/kik stays read-only - if ($sid) { icacls "$kik\dev" /grant "*${sid}:(OI)(CI)M" | Out-Null } + if (-not (Test-Path -LiteralPath $kik)) { New-Item -ItemType Directory -Path $kik | Out-Null } + foreach ($file in $stale) { + try { + Copy-Item -LiteralPath $file.FullName -Destination (Join-Path $kik $file.Name) -Force + Write-Log "restored opt/kik/$($file.Name)" + } catch { + # a process the server left behind may still hold it open; it is replaced in one of the next runs + Write-Log "failed opt/kik/$($file.Name) : $_" } + } +} - # The account database and the ssh keys and config the SYSTEM server reads: the user may read passwd (programs - # map ids through it) but change none of it, and has no business in /etc/ssh at all. - $rootShell = '/opt/kik/zsh.exe' - $pw = "$msys\etc\passwd" - $lines = Get-Content $pw - if ($lines | Where-Object { $_ -match '^root:' -and $_ -notmatch ':/opt/kik/zsh\.exe$' }) { - ($lines | ForEach-Object { if ($_ -match '^root:') { $_ -replace ':[^:]*$', ":$rootShell" } else { $_ } }) | Set-Content $pw -Encoding ascii - Write-Log 'set the root login shell to the protected zsh' - } - Set-LockedAcl $pw $false 'Read' - if (Test-Path "$msys\etc\ssh") { Set-LockedAcl "$msys\etc\ssh" $true 'None' } - } catch { - Write-Log "the protected login path is not fully in place: $_" +# The permissions do not need the server to be stopped, so they are put back on every run. +try { + if (Test-Path -LiteralPath $binDir) { + New-Item -ItemType Directory -Force -Path "$kik\dev\shm", "$kik\dev\mqueue" | Out-Null + Set-LockedAcl $kik $true 'ReadAndExecute' + # /dev is scratch: the user may write it so POSIX shared memory works, the rest of opt/kik stays read-only + if ($sid) { icacls "$kik\dev" /grant "*${sid}:(OI)(CI)M" | Out-Null } } + # The account database and the ssh keys and config the SYSTEM server reads: the user may read passwd (programs + # map ids through it) but change none of it, and has no business in /etc/ssh at all. + $rootShell = '/opt/kik/zsh.exe' + $pw = "$msys\etc\passwd" + $lines = Get-Content $pw + if ($lines | Where-Object { $_ -match '^root:' -and $_ -notmatch ':/opt/kik/zsh\.exe$' }) { + ($lines | ForEach-Object { if ($_ -match '^root:') { $_ -replace ':[^:]*$', ":$rootShell" } else { $_ } }) | Set-Content $pw -Encoding ascii + Write-Log 'set the root login shell to the protected zsh' + } + Set-LockedAcl $pw $false 'Read' + if (Test-Path "$msys\etc\ssh") { Set-LockedAcl "$msys\etc\ssh" $true 'None' } +} catch { + Write-Log "the protected login path is not fully in place: $_" +} + +if ($restart) { if (Start-Sshd) { Write-Log 'restarted the sshd' } else { Write-Log 'the sshd does not listen on port 22' } } '@ @@ -507,11 +533,17 @@ Set-Content -LiteralPath $sshdBat -Encoding ascii -Value @( 'C:\msys2\opt\kik\sshd.exe -D -e >> C:\ProgramData\kik-restore\sshd.log 2>&1' ) -# The event logs of windows are made bigger as well: they are the only record of what the box itself did, and -# twenty megabytes are not much with all the connections. -foreach ($eventLog in 'System', 'Application', 'Security') { +# The event logs of windows are made bigger as well: they are the only record of what the box itself did. The +# security log gets the most, because the scanners of the remote desktop alone fill a hundred megabytes in a week. +foreach ($eventLog in 'System', 'Application') { wevtutil sl $eventLog /ms:104857600 | Out-Null } +wevtutil sl Security /ms:1073741824 | Out-Null + +# A start of the server that fails leaves no trace in its own log, so windows records what the task of the server +# launched and how each process ended: the history of the scheduled tasks, and the start and end of every process. +wevtutil sl Microsoft-Windows-TaskScheduler/Operational /e:true /ms:104857600 | Out-Null +auditpol /set '/subcategory:{0CCE922B-69AE-11D9-BED3-505054503030},{0CCE922C-69AE-11D9-BED3-505054503030}' /success:enable | Out-Null # The installation's own cygrunsrv sshd service would fight for port 22; the scheduled task owns the sshd instead. Set-Service $service -StartupType Disabled -ErrorAction SilentlyContinue