Watch
0
0
Fork
You've already forked hyperhive
0

sign ssh certificates on demand from ssh itself, warn before setup, say what to do when a host key changes

This commit is contained in:
atlas 2026-11-16 19:42:44 +01:00 • committed by mara
commit 3d7541c0af
5 changed files with 521 additions and 67 deletions

View file

@ -3,18 +3,26 @@
#
# On every agent that has a store address (the same switch as ./bao.nix: with no
# store there is nothing to sign with). The agent has no standing SSH credential:
# `${unitName}.service` generates an ed25519 key in its own `/run` directory
# (tmpfs, never on disk), logs in to the swarm secret store with the identity
# ./bao.nix keeps current, and has the public half signed by the store's SSH
# engine (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through
# issues a user certificate with this agent's name as its only principal, valid
# for an hour; a timer signs again well before that runs out.
# the helper below generates an ed25519 key in `/run/${unitName}` (tmpfs, never
# on disk), logs in to the swarm secret store with the identity ./bao.nix keeps
# current, and has the public half signed by the store's SSH engine
# (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through issues a
# user certificate with this agent's name as its only principal, valid for an
# hour.
#
# Whether the store signs is the operator's per-agent switch in the swarm UI
# (the `ssh` permission, `swarm-controller`'s `agent_ssh`). With it off the store
# refuses, the unit logs one line and ends successfully, and the next tick asks
# again; with it on the agent has a certificate within one tick. The unit
# needs no change to be switched.
# (the `ssh` permission, `swarm-controller`'s `agent_ssh`). Every signing makes a
# fresh login, so a token never outlives the policies the role had when it was
# issued by more than the helper's own run. Two things run the helper:
#
# - `${unitName}.service`, from a timer, renews well before the certificate
# runs out. With the switch off the store refuses: the unit logs one line, ends
# successfully and asks again at the next tick.
# - `ssh` itself, through a `Match exec` line in the system config: when the
# certificate is missing or close to its end, the helper signs before the
# connection starts. That is what makes the first `ssh` after the operator
# switches SSH on work at once. It is silent and its exit status is whether a
# usable certificate exists; nothing in the config depends on the answer.
#
# A target machine trusts the CA, not this key: ../target-modules/agent-ssh.nix.
{
@ -38,28 +46,57 @@ let
unitName = "hive-agent-ssh-cert";
runtimeDir = unitName;
keyFile = "/run/${runtimeDir}/key";
dir = "/run/${runtimeDir}";
keyFile = "${dir}/key";
pubFile = "${keyFile}.pub";
# `ssh` reads `<IdentityFile>-cert.pub` by itself, but the config names it so
# that the file a Host block uses is visible in the config.
certFile = "${keyFile}-cert.pub";
stagingFile = "/run/${runtimeDir}/cert.new";
errFile = "/run/${runtimeDir}/bao.err";
stagingFile = "${dir}/cert.new";
errFile = "${dir}/bao.err";
# The store's CA, copied out of the unit's credentials so that the helper can
# reach the store from an `ssh` run, which has no credentials directory.
serverCaCopy = "${dir}/server-ca.pem";
# Written when the store refuses, so that a loop of `ssh` calls does not log in
# to the store each time the switch is off.
refusedFile = "${dir}/refused";
# In the state dir, which is bound from the host, so a pin survives a restart.
knownHostsFile = "/agents/${agentName}/state/ssh_known_hosts";
signPath = "${ca.mountPath}/sign/${ca.roleName}";
# A certificate with less than this left is signed again by `ssh`. The timer
# fires every `ca.renewEveryMinutes`, so a healthy agent never gets here.
ensureMarginSec = 600;
# How long a refusal is remembered. The delay the operator sees after
# switching on is at most this, then one login and one signing.
refusedForSec = 10;
storeRetry = import ../host-modules/lib/store-retry.nix { };
binPath = lib.makeBinPath [
pkgs.openbao
pkgs.openssh
pkgs.coreutils
pkgs.gnused
pkgs.util-linux
];
# The helper, and the unit's script: one text. `HIVE_SSH_CERT_ENSURE=1` is the
# `ssh` entry; without it this is the timer's. The tools go after the caller's
# `PATH`, so a caller that has its own wins.
helper = pkgs.writeShellScript unitName config.systemd.services.${unitName}.script;
in
{
config = lib.mkIf (baoCfg.addr != null) {
# Every host: the certificate, and the pins in the state dir, which outlives
# the container root. `accept-new` records a host's key at first contact and
# refuses a changed one after that.
# refuses a changed one after that. The `Match` line has no settings: ssh
# runs its command while it reads the config, before it loads any identity,
# and the answer is not used.
programs.ssh.extraConfig = ''
Match exec "HIVE_SSH_CERT_ENSURE=1 ${helper} </dev/null"
Host *
IdentityFile ${keyFile}
CertificateFile ${certFile}
@ -77,11 +114,6 @@ in
"hive-agent-bao-identity.service"
];
wantedBy = [ "multi-user.target" ];
path = [
pkgs.openbao
pkgs.openssh
pkgs.coreutils
];
# ../host-modules/lib/store-retry.nix.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
@ -98,19 +130,77 @@ in
UMask = "0377";
LoadCredential = [ serverCaCredential ];
};
environment = {
BAO_ADDR = toString baoCfg.addr;
BAO_CLIENT_CERT = baoCfg.clientCertFile;
BAO_CLIENT_KEY = baoCfg.clientKeyFile;
};
script = ''
set -euo pipefail
# `UMask=0377` in the unit makes every file this script creates `0400`,
# so each one is removed before the command that creates it. `ssh` runs
# the same text, which therefore sets it too.
umask 0377
export PATH="$PATH:${binPath}"
export BAO_ADDR=${lib.escapeShellArg (toString baoCfg.addr)}
export BAO_CLIENT_CERT=${lib.escapeShellArg baoCfg.clientCertFile}
export BAO_CLIENT_KEY=${lib.escapeShellArg baoCfg.clientKeyFile}
dir=${lib.escapeShellArg dir}
key=${lib.escapeShellArg keyFile}
pub=${lib.escapeShellArg pubFile}
cert=${lib.escapeShellArg certFile}
staged=${lib.escapeShellArg stagingFile}
err=${lib.escapeShellArg errFile}
server_ca=${lib.escapeShellArg serverCaCopy}
refused=${lib.escapeShellArg refusedFile}
ensure=
if [ -n "''${HIVE_SSH_CERT_ENSURE-}" ]; then ensure=1; fi
# The `ssh` entry is silent and says yes or no by its exit status; the
# unit's output is its log.
if [ -n "$ensure" ]; then exec >/dev/null 2>&1; fi
# `ssh` can run before the unit's first start has made the directory.
if [ ! -d "$dir" ]; then mkdir -m 0700 "$dir"; fi
# One run at a time, the unit's or an `ssh`'s: two would race on the key
# and the staged file.
exec {lock}<"$dir"
if ! flock -w 20 "$lock"; then
echo "another run holds $dir and did not finish in 20 seconds." >&2
exit 1
fi
# Epoch second the certificate ends, 0 when there is none.
cert_end() {
local end
end=$(ssh-keygen -L -f "$cert" 2>/dev/null | sed -n 's/^ *Valid: from .* to //p') || end=
if [ -n "$end" ]; then date -d "$end" +%s 2>/dev/null || echo 0; else echo 0; fi
}
if [ -n "$ensure" ]; then
now=$(date +%s)
# Enough left: nothing to do, and nothing asked of the store.
if [ "$(cert_end)" -gt $((now + ${toString ensureMarginSec})) ]; then exit 0; fi
# The store refused a moment ago: the switch is off.
if [ -e "$refused" ] && [ $((now - $(stat -c %Y "$refused"))) -lt ${toString refusedForSec} ]; then
exit 1
fi
fi
# The unit has the credentials directory and keeps the store's CA where
# an `ssh` run can find it.
if [ -n "''${CREDENTIALS_DIRECTORY-}" ] && [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
rm -f "$server_ca"
cp "$CREDENTIALS_DIRECTORY/${serverCaCredential}" "$server_ca"
fi
if [ -s "$server_ca" ]; then export BAO_CACERT="$server_ca"; fi
trap 'rm -f "$err" "$staged"' EXIT
# No identity delivered: ./bao.nix's check reports that; saying it
# twice adds nothing.
for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
if [ ! -s "$f" ]; then
echo "this agent has no store identity, so it cannot get an SSH certificate." >&2
if [ -n "$ensure" ]; then exit 1; fi
exit 0
fi
if [ ! -r "$f" ]; then
@ -119,19 +209,6 @@ in
fi
done
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
fi
# `UMask=0377` makes every file this script creates `0400`, so each one
# is removed before the command that creates it.
key=${lib.escapeShellArg keyFile}
pub=${lib.escapeShellArg pubFile}
cert=${lib.escapeShellArg certFile}
staged=${lib.escapeShellArg stagingFile}
err=${lib.escapeShellArg errFile}
trap 'rm -f "$err" "$staged"' EXIT
# The key lives as long as /run does, which is as long as this
# container: a restart is a new key. Only a pair with both halves is
# kept.
@ -141,6 +218,8 @@ in
chmod 0400 "$key"
fi
# A fresh login every time: a token carries the policies the role had
# when it was issued, so only a new one sees the operator's switch.
rm -f "$err"
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "the swarm secret store at $BAO_ADDR refused or could not answer this agent's certificate login:" >&2
@ -157,7 +236,10 @@ in
# so no failed unit. Anything else (a sealed or unreachable store) is
# one, and retried.
if [[ "$(<"$err")" == *"Code: 403"* ]]; then
rm -f "$refused"
: >"$refused"
echo "SSH certificates are not enabled for this agent (the swarm store refused to sign); asking again next tick."
if [ -n "$ensure" ]; then exit 1; fi
exit 0
fi
echo "the swarm secret store would not sign this agent's SSH key at ${signPath}:" >&2
@ -173,6 +255,7 @@ in
# One rename: ssh never reads a half-written certificate.
mv -f "$staged" "$cert"
rm -f "$refused"
echo "signed this agent's SSH key as ${agentName}, valid for ${toString ca.certTtlMinutes} minutes."
'';
};

View file

@ -236,6 +236,12 @@ in
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
# Runs ./agent-modules/ssh.nix's helper, the timer's entry and the one `ssh`
# runs, against a stub `bao` and a real `ssh-keygen`.
script-test-agent-ssh-cert = import ./script-tests/agent-ssh-cert.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
# Runs ./agent-modules/backend-env.nix's fetch-and-seed unit against a stub
# `bao` that keeps its leaves in files.
script-test-agent-backend-env = import ./script-tests/agent-backend-env.nix {

View file

@ -562,7 +562,7 @@ let
&& !(lib.hasInfix "valid_principals" s)
&& !(lib.hasInfix ".ssh" s)
&& lib.elem "hive-agent-bao-identity.service" u.after
&& u.environment.BAO_CLIENT_CERT == agentForgeBao.services.hyperhive.agent.bao.clientCertFile;
&& lib.hasInfix "export BAO_CLIENT_CERT=${agentForgeBao.services.hyperhive.agent.bao.clientCertFile}" s;
}
{
# The certificate lives an hour and the timer fires every third of
@ -599,17 +599,35 @@ let
&& !(lib.hasInfix "hive-agent-ssh-cert" agentForgeNoBao.environment.etc."ssh/ssh_config".text);
}
{
# A refused signing is the switch being off, not a fault: one line and a
# clean exit. Any other failure still fails the unit so it retries.
name = "the ssh certificate unit ends cleanly on a 403 and fails on any other signing error";
# `ssh` runs the helper before it loads an identity, so the first `ssh`
# after the operator's switch has a certificate. The line comes before the
# `Host *` block it serves, runs the `ssh` entry, and is on every agent
# with a store and no other.
name = "the ssh config signs on demand: a Match exec line runs the helper's ssh entry ahead of the Host block";
ok =
let
conf = agentForgeBao.environment.etc."ssh/ssh_config".text;
# What comes before the first `Host *`.
before = lib.head (lib.splitString "Host *" conf);
in
lib.hasInfix "Match exec \"HIVE_SSH_CERT_ENSURE=1 /nix/store/" before
&& lib.hasInfix "</dev/null\"" conf
&& !(lib.hasInfix "Match exec" agentForgeNoBao.environment.etc."ssh/ssh_config".text);
}
{
# The helper is the unit's script and the one `ssh` runs: the ssh entry
# is silent, signs from a fresh login, and remembers a refusal.
name = "the ssh helper is silent in the ssh entry, logs in afresh to sign, and keeps the store's CA for the ssh run";
ok =
let
s = (sshUnit agentForgeBao).script;
at = lib.splitString "Code: 403" s;
in
lib.length at == 2
&& lib.hasInfix "exit 0" (lib.head (lib.splitString "exit 1" (lib.elemAt at 1)))
&& lib.hasInfix "exit 1" (lib.elemAt at 1);
lib.hasInfix "HIVE_SSH_CERT_ENSURE" s
&& lib.hasInfix "exec >/dev/null 2>&1" s
&& lib.hasInfix "bao login -method=cert -token-only" s
&& lib.hasInfix "export BAO_TOKEN" s
&& lib.hasInfix "server-ca.pem" s
&& lib.hasInfix "Code: 403" s;
}
];
in

View file

@ -0,0 +1,92 @@
# `checks.script-test-agent-ssh-cert` — runs ../agent-modules/ssh.nix's helper,
# the text of the unit `hive-agent-ssh-cert`, against a stub `bao` and a real
# `ssh-keygen`, in both entries: the timer's and the `ssh` one. The cases are in
# ./agent-ssh-cert.sh.
#
# The one edit to the text is its `/run/` prefix, moved under the build
# directory because the sandbox has no writable `/run`. The helper adds its own
# tools after the caller's `PATH`, so the stub in front wins.
{
pkgs,
lib,
self,
nixosSystem,
}:
let
inherit
(import ../module-eval/lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
agentWith
;
machine = agentWith { services.hyperhive.agent.bao.addr = "https://bao.t.local:8200"; };
# `login` prints a token; `write` signs `public_key=@<file>` with the test CA
# for `FAKE_SIGN_VALID` (an `ssh-keygen -V` interval), or answers 403 when
# `FAKE_SIGN_MODE` is `refuse`, or fails otherwise when it is `fail`. A call
# without the token the last login printed fails, so a script that signs
# without a login of its own cannot pass.
fakeBao = pkgs.writeShellScriptBin "bao" ''
umask 022
echo "$1" >> "$FAKE_BAO_LOG"
case "$1" in
login)
printf 'tok-%s' "$(wc -l < "$FAKE_BAO_LOG")"
;;
write)
if [ "''${BAO_TOKEN-}" != "tok-$(( $(wc -l < "$FAKE_BAO_LOG") - 1 ))" ]; then
echo "fake bao: write without the token of the login just before it" >&2
exit 97
fi
case "$FAKE_SIGN_MODE" in
refuse)
printf 'Error writing data: Code: 403. Errors:\n* permission denied\n' >&2
exit 2
;;
fail)
printf 'Error writing data: Code: 503. Errors:\n* sealed\n' >&2
exit 2
;;
esac
pub=
for a in "$@"; do
case "$a" in public_key=@*) pub=''${a#public_key=@} ;; esac
done
tmp=$(mktemp -d)
cp "$pub" "$tmp/k.pub"
${pkgs.openssh}/bin/ssh-keygen -q -s "$FAKE_CA_KEY" -I fake -n a1 -V "$FAKE_SIGN_VALID" "$tmp/k.pub"
cat "$tmp/k-cert.pub"
;;
*)
echo "fake bao: unexpected call: $*" >&2
exit 98
;;
esac
'';
script = pkgs.writeText "agent-ssh-cert-script" machine.systemd.services.hive-agent-ssh-cert.script;
in
pkgs.runCommand "hyperhive-script-test-agent-ssh-cert"
{
nativeBuildInputs = [
pkgs.openssh
pkgs.coreutils
pkgs.util-linux
];
SSH_CERT_SCRIPT = script;
FAKE_BAO_BIN = "${fakeBao}/bin";
SSH_CERT_PATH = lib.makeBinPath [
pkgs.coreutils
pkgs.bash
];
}
''
${pkgs.bash}/bin/bash ${./agent-ssh-cert.sh}
touch "$out"
''

View file

@ -0,0 +1,181 @@
# Cases for ./agent-ssh-cert.nix. Each case gets a fresh runtime directory and
# runs the helper in one entry (the timer's, or `ssh`'s with
# HIVE_SSH_CERT_ENSURE=1) with a clean environment, and asserts on the exit
# code, the output, the files left behind and the `bao` calls made.
set -uo pipefail
failures=0
count=0
case=
fail() {
echo "FAILED: $case: $*" >&2
failures=$((failures + 1))
}
top=$(mktemp -d)
ssh-keygen -q -t ed25519 -N "" -f "$top/ca"
# setup <description>
setup() {
case="$1"
dir=$(mktemp -d)
rt=$dir/run/hive-agent-ssh-cert
mkdir -p "$dir/run/hive-agent-bao-identity/current" "$dir/creds"
mkdir -m 0700 "$rt"
printf cert >"$dir/run/hive-agent-bao-identity/current/cert"
printf key >"$dir/run/hive-agent-bao-identity/current/key"
printf ca >"$dir/creds/hive-agent-bao-server-ca"
log=$dir/bao.log
: >"$log"
sed -e "s|/run/|$dir/run/|g" "$SSH_CERT_SCRIPT" >"$dir/script"
chmod +x "$dir/script"
if grep -q ' /run/\|"/run/\|=/run/' "$dir/script"; then fail "the script still names /run after the rewrite"; fi
sign_mode=ok
valid=+1h
creds=$dir/creds
}
# go [ensure]
go() {
count=$((count + 1))
local ensure=
if [ "${1-}" = ensure ]; then ensure=1; fi
(
cd "$dir" || exit 99
exec env -i \
PATH="$FAKE_BAO_BIN:$SSH_CERT_PATH" \
HOME="$dir" \
CREDENTIALS_DIRECTORY="$creds" \
HIVE_SSH_CERT_ENSURE="$ensure" \
FAKE_BAO_LOG="$log" \
FAKE_CA_KEY="$top/ca" \
FAKE_SIGN_MODE="$sign_mode" \
FAKE_SIGN_VALID="$valid" \
"$dir/script"
) >"$dir/out" 2>"$dir/err"
rc=$?
}
expect_rc() {
if [ "$rc" != "$1" ]; then fail "exit $rc, expected $1; stderr: $(<"$dir/err")"; fi
}
expect_silent() {
if [ -s "$dir/out" ] || [ -s "$dir/err" ]; then fail "output where none was wanted: $(<"$dir/out") $(<"$dir/err")"; fi
}
expect_calls() {
local want
want=$(printf '%s\n' "$@")
if [ "$(<"$log")" != "${want%$'\n'}" ]; then fail "bao calls were '$(<"$log" tr '\n' ' ')', expected '$*'"; fi
}
expect_cert() {
if ! ssh-keygen -L -f "$rt/key-cert.pub" >/dev/null 2>&1; then fail "no readable certificate at $rt/key-cert.pub"; fi
}
# The seconds the certificate at $rt has left; 0 when there is none.
left() {
local end
end=$(ssh-keygen -L -f "$rt/key-cert.pub" 2>/dev/null | sed -n 's/^ *Valid: from .* to //p')
if [ -z "$end" ]; then echo 0; else echo $(($(date -d "$end" +%s) - $(date +%s))); fi
}
# A certificate for the key the helper will keep, valid for `$1`, put in place
# as a first run would have left it.
seed_cert() {
mkdir -p "$rt"
rm -f "$rt/key" "$rt/key.pub" "$rt/key-cert.pub"
(umask 0377 && ssh-keygen -q -t ed25519 -N "" -f "$rt/key" && chmod 0400 "$rt/key")
ssh-keygen -q -s "$top/ca" -I seed -n a1 -V "$1" "$rt/key.pub"
}
setup "ssh finds no certificate: it logs in, signs, and is silent and true"
go ensure
expect_rc 0
expect_silent
expect_calls login write
expect_cert
if [ "$(left)" -lt 3000 ]; then fail "the new certificate has only $(left)s left"; fi
setup "ssh finds a certificate with time left: it asks the store nothing"
seed_cert +1h
go ensure
expect_rc 0
expect_silent
expect_calls
setup "ssh finds a certificate close to its end: it signs again with a fresh login"
seed_cert +5m
before=$(left)
go ensure
expect_rc 0
expect_silent
expect_calls login write
if [ "$(left)" -le "$before" ]; then fail "the certificate was not replaced ($before then $(left))"; fi
setup "ssh finds an expired certificate: it signs again"
seed_cert -2h:-1h
go ensure
expect_rc 0
expect_calls login write
if [ "$(left)" -lt 3000 ]; then fail "an expired certificate was kept ($(left)s)"; fi
setup "the store refuses (switch off): ssh gets false, quietly, and no certificate"
sign_mode=refuse
go ensure
expect_rc 1
expect_silent
expect_calls login write
if [ -e "$rt/key-cert.pub" ]; then fail "a certificate appeared although the store refused"; fi
setup "right after a refusal ssh does not ask again, and does once the memory has aged"
sign_mode=refuse
go ensure
go ensure
expect_rc 1
expect_calls login write
touch -d '1 minute ago' "$rt/refused"
sign_mode=ok
go ensure
expect_rc 0
expect_calls login write login write
expect_cert
setup "the timer's entry signs even with time left, and a refusal is one line and a clean exit"
seed_cert +1h
go
expect_rc 0
expect_calls login write
sign_mode=refuse
go
expect_rc 0
if ! grep -q 'not enabled for this agent' "$dir/out"; then fail "no line saying signing is off: $(<"$dir/out")"; fi
if [ -s "$dir/err" ]; then fail "stderr on a refusal: $(<"$dir/err")"; fi
setup "any other signing failure fails both entries, and ssh stays quiet"
sign_mode=fail
go
expect_rc 1
go ensure
expect_rc 1
expect_silent
setup "a store identity that is missing is true for the timer and false for ssh"
rm "$dir/run/hive-agent-bao-identity/current/cert"
go
expect_rc 0
go ensure
expect_rc 1
expect_calls
setup "the unit keeps the store's CA where an ssh run finds it, and the key stays unreadable to others"
go
expect_rc 0
if [ "$(<"$rt/server-ca.pem")" != ca ]; then fail "the store's CA was not copied"; fi
if [ "$(stat -c %a "$rt/key")" != 400 ]; then fail "the key is mode $(stat -c %a "$rt/key")"; fi
if [ "$(stat -c %a "$rt/key-cert.pub")" != 400 ]; then fail "the certificate is mode $(stat -c %a "$rt/key-cert.pub")"; fi
echo "$count runs, $failures failures"
if [ "$failures" -ne 0 ]; then exit 1; fi