diff --git a/server/README.md b/server/README.md index 822ea4e..8bbd580 100644 --- a/server/README.md +++ b/server/README.md @@ -46,6 +46,13 @@ The copy has to be the *same* MSYS2 installation, not a second one: `C:\msys2\op runtime. A separate installation does not work - two runtimes in one process tree corrupt each other's file descriptors and pacman cannot even lock its database. +The runtime itself is not among the protected copies. A program and the shell that starts it have to use the same +runtime DLL, or the program does not run at all - a second copy of it in one process is refused. The login path +therefore uses the one in `/usr/bin`, next to the user's own programs, and the user may replace and upgrade it. +What is protected instead is that it is the one the package says it is: pacman checks the files of `msys2-runtime`, +and the cleanup job puts the package back if it was changed - before it starts the server, which runs as SYSTEM and +would otherwise run code the user of the website wrote. + `/etc/passwd` (the login shell and home directory) and `/etc/ssh` (the host keys and config) are read-only to the user as well; everything else in the installation is theirs. @@ -59,6 +66,21 @@ Two scheduled tasks run as SYSTEM, both at startup, and the user cannot stop eit re-applies the permissions, and puts the home directory, the shell configuration, the website's key and the four texts back. Its second job is to best-effort revert other mockery and bring a fresh server to the expected state. +The cleanup job also restarts the server when nobody is connected, and when the server does not listen at all: +it comes back by itself if it died, it takes over a server that was left behind, and it picks up the programs that +were just restored. A session that is open keeps the port, so a server that is started now could not bind it, and +the job never disturbs a session. + +The deployment does not restart the server. It runs in a session, and that session keeps the port, so a server that +starts during a deployment cannot bind it. The new configuration and the restored programs therefore take effect +at the next run of the cleanup job, which happens within half an hour. + +The server logs to `C:\ProgramData\kik-restore\sshd.log`. The log is appended to, not overwritten, because it is +the only trace of what the server did: a server that is restarted would otherwise take the evidence of an outage +with it. It is rotated when it grows past a megabyte. + +Only keys may log in over SSH. The box stands in the open internet and both of its accounts have a password. + The files that have to stay belong to the administrators, so the user cannot delete them, and they cannot change the shell configuration or the key the website logs in with either. The shell reads its configuration from `~/.zsh`, so the startup files the user could write in the home directory are ignored. diff --git a/server/scripts/install_restore_task.ps1 b/server/scripts/install_restore_task.ps1 index 2547df8..a5531da 100644 --- a/server/scripts/install_restore_task.ps1 +++ b/server/scripts/install_restore_task.ps1 @@ -37,6 +37,11 @@ $service = 'msys2-sshd' $loginBinaries = "$msys\usr\bin\sshd.exe", "$msys\usr\lib\ssh\sshd-session.exe", "$msys\usr\lib\ssh\sshd-auth.exe", "$msys\usr\bin\zsh.exe", "$msys\usr\bin\cat.exe" +# The runtime is not among them: a program and the shell that starts it have to use the same runtime, or the +# program does not run at all. The login path therefore uses the one in /usr/bin, and the user may replace it - +# pacman checks that it is the one the package says it is, and the cleanup job puts it back if it is not. +$runtimeFiles = '/usr/bin/msys-2.0.dll' + # The copies the task restores from and the script it runs both live in $installDir, which only SYSTEM and the # administrators may read or write. $restoreScript = @' @@ -144,17 +149,68 @@ function New-Home($path) { Get-ChildItem -LiteralPath "$msys\etc\skel" -Force | Copy-Item -Destination $path -Recurse -Force } +# The server is stopped and started again when nobody is connected: it picks up the programs that are restored +# just now, and it comes back by itself if it died. It has to be stopped before the copies are made, because a +# running server holds its programs open and they cannot be replaced. +# +# Stopping the task does not take the server with it, so its processes are killed by hand and the task is waited +# for: it ignores a new instance while one is running, so a start would be dropped. +function Stop-Sshd { + Stop-ScheduledTask -TaskName 'KiK sshd' -ErrorAction SilentlyContinue + Get-CimInstance Win32_Process -Filter "Name='cmd.exe'" | Where-Object { $_.CommandLine -match 'cmd\.exe\s+/c\s+"?C:\\ProgramData\\kik-restore\\run-sshd\.bat' } | + ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } + Get-Process -Name sshd -ErrorAction SilentlyContinue | Stop-Process -Force + $waited = 0 + while ((Get-ScheduledTask -TaskName 'KiK sshd').State -ne 'Ready' -and $waited -lt 30) { Start-Sleep 1; $waited++ } + $waited = 0 + while ((Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } +} + +# The server is started again and waited for: a deployment or a crash must never leave the site without a shell. +function Start-Sshd { + Start-ScheduledTask -TaskName 'KiK sshd' + $waited = 0 + while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } + return [bool](Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) +} + +$netError = $null +$connections = @(Get-NetTCPConnection -LocalPort 22 -ErrorAction SilentlyContinue -ErrorVariable netError) +$sessions = @(Get-Process -Name 'sshd-session' -ErrorAction SilentlyContinue) +# A session that is open keeps the port and the runtime, so the server is only restarted when there is none: it +# comes back by itself if it died, it takes over a server that was left behind, and it picks up the programs that +# were just restored. A query that fails is not a reason to disturb a session. +$busy = @($connections | Where-Object { $_.State -eq 'Established' }).Count -gt 0 -or $sessions.Count -gt 0 +$restart = -not $netError -and -not $busy +if ($restart) { + Stop-Sshd + # The runtime of the installation belongs to the user's programs and is not protected: a program and the shell + # that starts it have to use the same one, or the program does not run at all. What has to be protected is that + # it is the one the package says it is, because the server runs as SYSTEM. + Remove-Item "$kik\msys-2.0.dll" -Force -ErrorAction SilentlyContinue + & "$msys\usr\bin\pacman.exe" -Qkk msys2-runtime 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + & "$msys\usr\bin\pacman.exe" -S --noconfirm --overwrite '*' msys2-runtime 2>&1 | Out-Null + Write-Log 'restored the runtime of the installation' + } +} + try { # The protected login path: the copy the sshd and the login shell run from. The user cannot write it, so even # after they replace or remove their own copies in /usr/bin the server and the journey keep working. The trusted # originals are in bin\; anything missing from opt/kik is put back. if (Test-Path -LiteralPath $binDir) { if (-not (Test-Path -LiteralPath $kik)) { New-Item -ItemType Directory -Path $kik | Out-Null } - foreach ($file in Get-ChildItem -LiteralPath $binDir -File -Force) { + foreach ($file in Get-ChildItem -LiteralPath $binDir -File -Force | Where-Object { $_.Name -ne 'msys-2.0.dll' }) { $target = Join-Path $kik $file.Name - if (-not (Test-Path -LiteralPath $target) -or (Get-Item -LiteralPath $target -Force).Length -ne $file.Length) { - Copy-Item -LiteralPath $file.FullName -Destination $target -Force - Write-Log "restored opt/kik/$($file.Name)" + try { + if (-not (Test-Path -LiteralPath $target) -or (Get-Item -LiteralPath $target -Force).Length -ne $file.Length) { + Copy-Item -LiteralPath $file.FullName -Destination $target -Force + Write-Log "restored opt/kik/$($file.Name)" + } + } catch { + # a running server holds its programs open; they are replaced in one of the next runs + Write-Log "failed opt/kik/$($file.Name) : $_" } } New-Item -ItemType Directory -Force -Path "$kik\dev\shm", "$kik\dev\mqueue" | Out-Null @@ -225,6 +281,10 @@ foreach ($copy in Get-ChildItem -LiteralPath $copies -Recurse -File -Force) { Write-Log "failed $($copy.Name) : $_" } } + +if ($restart) { + if (Start-Sshd) { Write-Log 'restarted the sshd' } else { Write-Log 'the sshd does not listen on port 22' } +} '@ if (-not ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { @@ -270,7 +330,7 @@ function Get-Libraries($program) { foreach ($line in & "$msys\usr\bin\ldd.exe" $posix 2>$null) { foreach ($m in [regex]::Matches($line, '/usr/[^\s]+\.dll')) { $paths += $m.Value } } - $paths | Select-Object -Unique | ForEach-Object { $msys + ($_ -replace '/', '\') } + $paths | Select-Object -Unique | Where-Object { $_ -notin $runtimeFiles } | ForEach-Object { $msys + ($_ -replace '/', '\') } } if ($Uninstall) { @@ -338,12 +398,38 @@ foreach ($b in $loginBinaries) { } foreach ($f in $files.Keys) { Copy-Item -LiteralPath $f -Destination $binDir -Force } +# Remote Desktop is the way in for the administrator, and the port is open to the internet. It needs network level +# authentication, so that a scanner cannot get a session before it has proven that it knows the password. +$rdpTcp = 'HKLM:\SYSTEM\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' +$nla = (Get-ItemProperty -Path $rdpTcp -Name UserAuthentication).UserAuthentication +Set-ItemProperty -Path $rdpTcp -Name UserAuthentication -Value 1 -Type DWord +if ($nla -ne 1) { + # the setting only counts when the service starts again, which drops the sessions that are open + Restart-Service TermService -Force +} + +# Only the administrator may log in over Remote Desktop. The user of the website is in no group that may log in +# over it, and it must never be an administrator - that is taken away if it somehow got there. +$rootAccount = Get-LocalUser -Name 'root' -ErrorAction SilentlyContinue +if ($rootAccount) { + foreach ($group in 'Administrators', 'Remote Desktop Users') { + $member = Get-LocalGroupMember $group -ErrorAction SilentlyContinue | Where-Object { $_.SID -eq $rootAccount.SID } + if ($member) { + Remove-LocalGroupMember $group $member -Confirm:$false + Write-Host "The root user was in '$group' and has been removed from it." + } + } +} + # The sshd reads its configuration and finds its helpers in opt/kik; nothing here depends on the user's /usr. $sshdConfig = "$msys\etc\ssh\sshd_config" Set-Content -LiteralPath $sshdConfig -Encoding ascii -Value @( '# The website logs in as root, the deployers as Administrator. Both come in over the websocket bridge.', 'Port 22', 'AllowUsers root Administrator', + '# Only keys may log in: the box is in the open internet and the accounts have passwords.', + 'PasswordAuthentication no', + 'KbdInteractiveAuthentication no', '# Files are protected by Windows ACLs; under MSYS2''s noacl mounts the POSIX modes sshd checks are fake.', 'StrictModes no', 'Subsystem sftp internal-sftp', @@ -354,10 +440,14 @@ Set-Content -LiteralPath $sshdConfig -Encoding ascii -Value @( # The sshd runs as SYSTEM from the protected copy, with opt/kik first on the path so its own DLLs and the session's # programs (zsh, cat) are the protected ones. The log stays in the install directory, out of the user's reach. +# The log is appended to, not overwritten: it is the only trace of what the server did, and a server that is +# restarted would otherwise take the evidence of the outage with it. Set-Content -LiteralPath $sshdBat -Encoding ascii -Value @( '@echo off', 'set PATH=C:\msys2\opt\kik;C:\msys2\usr\bin;%PATH%', - 'C:\msys2\opt\kik\sshd.exe -D -e > C:\ProgramData\kik-restore\sshd.log 2>&1' + 'rem Keep the previous log if it grew too big; the current one is appended to.', + 'if exist C:\ProgramData\kik-restore\sshd.log for %%f in (C:\ProgramData\kik-restore\sshd.log) do if %%~zf GEQ 1048576 move /y C:\ProgramData\kik-restore\sshd.log C:\ProgramData\kik-restore\sshd.log.1', + 'C:\msys2\opt\kik\sshd.exe -D -e >> C:\ProgramData\kik-restore\sshd.log 2>&1' ) # The installation's own cygrunsrv sshd service would fight for port 22; the scheduled task owns the sshd instead. @@ -378,13 +468,32 @@ $restoreTriggers = @( $restoreSettings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10) -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 5) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries Register-ScheduledTask -TaskName $restoreTask -Action $restoreAction -Trigger $restoreTriggers -Principal $atSystem -Settings $restoreSettings -Force | Out-Null +# The server is started and waited for: it must listen on port 22, or the site has no shell. +function Start-Sshd { + Start-ScheduledTask -TaskName $sshdTask + $waited = 0 + while (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) -and $waited -lt 30) { Start-Sleep 1; $waited++ } + return [bool](Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue) +} + # The restore task builds opt/kik and sets every permission; run it before starting the sshd that depends on it. Start-ScheduledTask -TaskName $restoreTask Start-Sleep -Seconds 6 $waited = 0 while ((Get-ScheduledTask -TaskName $restoreTask).State -eq 'Running' -and $waited -lt 60) { Start-Sleep 2; $waited += 2 } -Start-ScheduledTask -TaskName $sshdTask -Start-Sleep -Seconds 4 + +# The server is not restarted here. It cannot be: the deployment runs in a session, and the port is still in use by +# that session, so a server that starts now cannot bind it. The cleanup job restarts the server when nobody is +# connected, and the new configuration and the restored programs take effect then. A server that is not running at +# all is started here, because then there is no session that could hold the port. +if (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue)) { + if (-not (Start-Sshd)) { + throw "The sshd does not listen on port 22; the task '$sshdTask' says: $((Get-ScheduledTask -TaskName $sshdTask).State), $((Get-ScheduledTaskInfo -TaskName $sshdTask).LastTaskResult)" + } +} +if (-not (Get-NetTCPConnection -LocalPort 22 -State Listen -ErrorAction SilentlyContinue)) { + throw "The sshd does not listen on port 22." +} Write-Host "The sshd runs as SYSTEM from C:\msys2\opt\kik (task '$sshdTask'); '$restoreTask' keeps it and the home directory in shape." Get-Content (Join-Path $installDir 'restore.log') -Tail 10 -ErrorAction SilentlyContinue