diff --git a/server/README.md b/server/README.md index 3e5405d..10b4a89 100644 --- a/server/README.md +++ b/server/README.md @@ -69,10 +69,11 @@ Three scheduled tasks run as SYSTEM, all at startup, and the user cannot stop an the server only starts it again when it failed, never when it was stopped, so a cleanup job that is killed while it has the server stopped would otherwise take the site down with it. -The cleanup job also restarts the server when nobody is connected, and when the server does not listen at all: -it comes back by itself if it died, it takes over a server that was left behind, and it picks up the programs that -were just restored. A session that is open keeps the port, so a server that is started now could not bind it, and -the job never disturbs a session. +The cleanup job only restarts the server when it has to: when it does not listen at all, when it is older than its +configuration, or when the protected copy or the runtime has to be repaired - those cannot be replaced while the +server holds them open. A working server is left alone, because every start can fail, and failed starts have kept +the site down for hours. The job also never restarts the server while somebody is connected: a session that is open +keeps the port, so a server that is started then could not bind it. The server is only down for the short part of the job that needs it down: the protected copy cannot be replaced while the server holds it open. Everything else - the home directory, the four texts, the permissions of the @@ -90,7 +91,12 @@ at the next run of the cleanup job, which happens within half an hour. The server logs to `C:\ProgramData\kik-restore\sshd.log`. The log is appended to, not overwritten, because it is the only trace of what the server did: a server that is restarted would otherwise take the evidence of an outage with it. It is rotated when it grows past a hundred megabytes, which takes years. The event logs of windows are -made that big as well, so that they keep the history of the box itself for just as long. +made bigger as well, the security log to a gigabyte: the scanners of the remote desktop alone fill a hundred +megabytes of it in a week. + +A start of the server that fails leaves nothing in that log. So windows records the history of the scheduled tasks +and the start and end of every process, and when the server does not come up, the cleanup job writes the processes +and the state of the server's task to `C:\ProgramData\kik-restore\failed-starts.log`. Only keys may log in over SSH. The box stands in the open internet and both of its accounts have a password. diff --git a/server/scripts/install_restore_task.ps1 b/server/scripts/install_restore_task.ps1 index 1b5873b..18230dc 100644 --- a/server/scripts/install_restore_task.ps1 +++ b/server/scripts/install_restore_task.ps1 @@ -150,9 +150,9 @@ function New-Home($path) { Get-ChildItem -LiteralPath "$msys\etc\skel" -Force | Copy-Item -Destination $path -Recurse -Force } -# The server is stopped and started again when nobody is connected: it picks up the programs that are restored -# just now, and it comes back by itself if it died. It has to be stopped before the copies are made, because a -# running server holds its programs open and they cannot be replaced. +# The server is only stopped when it has to be: when it does not run, when it is older than its configuration, or +# when it holds open what has to be repaired - its programs in opt/kik or the runtime. A working server is left alone, +# because every start can fail, and a failed start has kept the site down for hours. # # Stopping the task does not take the server with it, so its processes are killed by hand and the task is waited # for: it ignores a new instance while one is running, so a start would be dropped. @@ -170,6 +170,19 @@ function Stop-Sshd { while ((Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } } +# What the box looks like when the server did not come up. Nothing else records a failed start, and the reboot that +# ends the outage takes the evidence with it. +function Save-Snapshot($attempt) { + $task = Get-ScheduledTask -TaskName 'KiK sshd' + $info = $task | Get-ScheduledTaskInfo + $lines = @("== $(Get-Date -Format s) attempt ${attempt}: the task is $($task.State), last run $($info.LastRunTime), result 0x$('{0:X}' -f $info.LastTaskResult)") + $lines += Get-NetTCPConnection -LocalPort 22 -ErrorAction SilentlyContinue | ForEach-Object { "port 22: $($_.LocalAddress) $($_.State) pid $($_.OwningProcess)" } + $lines += Get-CimInstance Win32_Process | Sort-Object CreationDate | ForEach-Object { + '{0:MM-dd HH:mm:ss} {1} pid {2} parent {3} session {4}: {5}' -f $_.CreationDate, $_.Name, $_.ProcessId, $_.ParentProcessId, $_.SessionId, $_.CommandLine + } + $lines | Out-File -FilePath (Join-Path $PSScriptRoot 'failed-starts.log') -Append +} + # The server is started again and waited for: a deployment or a crash must never leave the site without a shell. # The task ignores a new instance while it thinks one is running, so when the port stays closed its processes are # killed and it is started again. @@ -180,6 +193,7 @@ function Start-Sshd { while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } if (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) { return $true } Write-Log "the server did not come up (attempt $attempt)" + Save-Snapshot $attempt Stop-Sshd } return $false @@ -201,18 +215,17 @@ function Invoke-Bounded($program, $arguments, $seconds) { $netError = $null $connections = @(Get-NetTCPConnection -LocalPort 22 -ErrorAction SilentlyContinue -ErrorVariable netError) -# An open session keeps the port, so the server is only restarted when there is none: it comes back by itself if it -# died, it takes over a server that was left behind, and it picks up the programs that were just restored. A query -# that fails is not a reason to disturb a session. The processes of the sessions are left behind by the server when a -# client goes away, so they say nothing about whether anybody is connected. +# An open session keeps the port, so the server is only stopped when there is none. A query that fails is not a +# reason to disturb a session. The processes of the sessions are left behind by the server when a client goes away, +# so they say nothing about whether anybody is connected. $busy = @($connections | Where-Object { $_.State -eq 'Established' }).Count -gt 0 -$restart = -not $netError -and -not $busy +$mayStop = -not $netError -and -not $busy # The runtime of the installation belongs to the user's programs and is not protected: a program and the shell that # starts it have to use the same one, or the program does not run at all. What has to be protected is that it is the # one the package says it is, because the server runs as SYSTEM. The check takes a while, so it runs while the # server is up - the server must not wait for it. $runtimeBroken = $false -if ($restart) { +if ($mayStop) { $runtimeBroken = (Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-Qkk', 'msys2-runtime') 120) -ne 0 } @@ -266,6 +279,23 @@ foreach ($copy in Get-ChildItem -LiteralPath $copies -Recurse -File -Force) { # permissions because they are set without inheritance. if ($sid) { icacls $msys /grant "*${sid}:(OI)(CI)F" | Out-Null } +# The protected login path: the copy the sshd and the login shell run from. The user cannot write it, so even after +# they replace or remove their own copies in /usr/bin the server and the journey keep working. The trusted originals +# are in bin\; anything missing from opt/kik or different from its original is put back. Replacing them is the only +# work that needs the server to be stopped, because a running server holds its programs open. +$stale = @(if (Test-Path -LiteralPath $binDir) { + Get-ChildItem -LiteralPath $binDir -File -Force | Where-Object { + $target = Join-Path $kik $_.Name + $_.Name -ne 'msys-2.0.dll' -and (-not (Test-Path -LiteralPath $target) -or (Get-Item -LiteralPath $target -Force).Length -ne $_.Length) + } +}) +# A deployment writes the configuration but cannot restart the server, so a server that is older than its +# configuration is restarted here. +$listener = Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue | Select-Object -First 1 +$outdated = $listener -and @(Get-Item -LiteralPath "$msys\etc\ssh\sshd_config", (Join-Path $PSScriptRoot 'run-sshd.bat') | + Where-Object { $_.LastWriteTime -gt (Get-Process -Id $listener.OwningProcess).StartTime }).Count -gt 0 +$restart = $mayStop -and (-not $listener -or $outdated -or $runtimeBroken -or $stale.Count -gt 0) + if ($restart) { Stop-Sshd Remove-Item "$kik\msys-2.0.dll" -Force -ErrorAction SilentlyContinue @@ -275,47 +305,43 @@ if ($restart) { Invoke-Bounded "$msys\usr\bin\pacman.exe" @('-S', '--noconfirm', '--overwrite', "'*'", 'msys2-runtime') 300 | Out-Null Write-Log 'restored the runtime of the installation' } - - try { - # The protected login path: the copy the sshd and the login shell run from. The user cannot write it, so even - # after they replace or remove their own copies in /usr/bin the server and the journey keep working. The trusted - # originals are in bin\; anything missing from opt/kik is put back. This is the only work that needs the server - # to be stopped, because a running server holds its programs open. - if (Test-Path -LiteralPath $binDir) { - if (-not (Test-Path -LiteralPath $kik)) { New-Item -ItemType Directory -Path $kik | Out-Null } - foreach ($file in Get-ChildItem -LiteralPath $binDir -File -Force | Where-Object { $_.Name -ne 'msys-2.0.dll' }) { - $target = Join-Path $kik $file.Name - try { - if (-not (Test-Path -LiteralPath $target) -or (Get-Item -LiteralPath $target -Force).Length -ne $file.Length) { - Copy-Item -LiteralPath $file.FullName -Destination $target -Force - Write-Log "restored opt/kik/$($file.Name)" - } - } catch { - # a running server holds its programs open; they are replaced in one of the next runs - Write-Log "failed opt/kik/$($file.Name) : $_" - } - } - New-Item -ItemType Directory -Force -Path "$kik\dev\shm", "$kik\dev\mqueue" | Out-Null - Set-LockedAcl $kik $true 'ReadAndExecute' - # /dev is scratch: the user may write it so POSIX shared memory works, the rest of opt/kik stays read-only - if ($sid) { icacls "$kik\dev" /grant "*${sid}:(OI)(CI)M" | Out-Null } + if (-not (Test-Path -LiteralPath $kik)) { New-Item -ItemType Directory -Path $kik | Out-Null } + foreach ($file in $stale) { + try { + Copy-Item -LiteralPath $file.FullName -Destination (Join-Path $kik $file.Name) -Force + Write-Log "restored opt/kik/$($file.Name)" + } catch { + # a process the server left behind may still hold it open; it is replaced in one of the next runs + Write-Log "failed opt/kik/$($file.Name) : $_" } + } +} - # The account database and the ssh keys and config the SYSTEM server reads: the user may read passwd (programs - # map ids through it) but change none of it, and has no business in /etc/ssh at all. - $rootShell = '/opt/kik/zsh.exe' - $pw = "$msys\etc\passwd" - $lines = Get-Content $pw - if ($lines | Where-Object { $_ -match '^root:' -and $_ -notmatch ':/opt/kik/zsh\.exe$' }) { - ($lines | ForEach-Object { if ($_ -match '^root:') { $_ -replace ':[^:]*$', ":$rootShell" } else { $_ } }) | Set-Content $pw -Encoding ascii - Write-Log 'set the root login shell to the protected zsh' - } - Set-LockedAcl $pw $false 'Read' - if (Test-Path "$msys\etc\ssh") { Set-LockedAcl "$msys\etc\ssh" $true 'None' } - } catch { - Write-Log "the protected login path is not fully in place: $_" +# The permissions do not need the server to be stopped, so they are put back on every run. +try { + if (Test-Path -LiteralPath $binDir) { + New-Item -ItemType Directory -Force -Path "$kik\dev\shm", "$kik\dev\mqueue" | Out-Null + Set-LockedAcl $kik $true 'ReadAndExecute' + # /dev is scratch: the user may write it so POSIX shared memory works, the rest of opt/kik stays read-only + if ($sid) { icacls "$kik\dev" /grant "*${sid}:(OI)(CI)M" | Out-Null } } + # The account database and the ssh keys and config the SYSTEM server reads: the user may read passwd (programs + # map ids through it) but change none of it, and has no business in /etc/ssh at all. + $rootShell = '/opt/kik/zsh.exe' + $pw = "$msys\etc\passwd" + $lines = Get-Content $pw + if ($lines | Where-Object { $_ -match '^root:' -and $_ -notmatch ':/opt/kik/zsh\.exe$' }) { + ($lines | ForEach-Object { if ($_ -match '^root:') { $_ -replace ':[^:]*$', ":$rootShell" } else { $_ } }) | Set-Content $pw -Encoding ascii + Write-Log 'set the root login shell to the protected zsh' + } + Set-LockedAcl $pw $false 'Read' + if (Test-Path "$msys\etc\ssh") { Set-LockedAcl "$msys\etc\ssh" $true 'None' } +} catch { + Write-Log "the protected login path is not fully in place: $_" +} + +if ($restart) { if (Start-Sshd) { Write-Log 'restarted the sshd' } else { Write-Log 'the sshd does not listen on port 22' } } '@ @@ -507,11 +533,17 @@ Set-Content -LiteralPath $sshdBat -Encoding ascii -Value @( 'C:\msys2\opt\kik\sshd.exe -D -e >> C:\ProgramData\kik-restore\sshd.log 2>&1' ) -# The event logs of windows are made bigger as well: they are the only record of what the box itself did, and -# twenty megabytes are not much with all the connections. -foreach ($eventLog in 'System', 'Application', 'Security') { +# The event logs of windows are made bigger as well: they are the only record of what the box itself did. The +# security log gets the most, because the scanners of the remote desktop alone fill a hundred megabytes in a week. +foreach ($eventLog in 'System', 'Application') { wevtutil sl $eventLog /ms:104857600 | Out-Null } +wevtutil sl Security /ms:1073741824 | Out-Null + +# A start of the server that fails leaves no trace in its own log, so windows records what the task of the server +# launched and how each process ended: the history of the scheduled tasks, and the start and end of every process. +wevtutil sl Microsoft-Windows-TaskScheduler/Operational /e:true /ms:104857600 | Out-Null +auditpol /set '/subcategory:{0CCE922B-69AE-11D9-BED3-505054503030},{0CCE922C-69AE-11D9-BED3-505054503030}' /success:enable | Out-Null # The installation's own cygrunsrv sshd service would fight for port 22; the scheduled task owns the sshd instead. Set-Service $service -StartupType Disabled -ErrorAction SilentlyContinue