# Outbound SSH for this agent: a certificate for a key it generates itself, # signed by the swarm's SSH CA, and a client config that offers it. # # On every agent that has a store address (the same switch as ./bao.nix: with no # store there is nothing to sign with). The agent has no standing SSH credential: # the helper below generates an ed25519 key in `/run/${unitName}` (tmpfs, never # on disk), logs in to the swarm secret store with the identity ./bao.nix keeps # current, and has the public half signed by the store's SSH engine # (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through issues a # user certificate with this agent's name as its only principal, valid for an # hour. # # Whether the store signs is the operator's per-agent switch in the swarm UI # (the `ssh` permission, `swarm-controller`'s `agent_ssh`). Every signing makes a # fresh login, so a token never outlives the policies the role had when it was # issued by more than the helper's own run. Two things run the helper: # # - `${unitName}.service`, from a timer, renews well before the certificate # runs out. With the switch off the store refuses: the unit logs one line, ends # successfully and asks again at the next tick. # - `ssh` itself, through a `Match exec` line in the system config: when the # certificate is missing or close to its end, the helper signs before the # connection starts. That is what makes the first `ssh` after the operator # switches SSH on work at once. It is silent and its exit status is whether a # usable certificate exists; nothing in the config depends on the answer. # # A target machine trusts the CA, not this key: ../target-modules/agent-ssh.nix. { pkgs, lib, config, ... }: let baoCfg = config.services.hyperhive.agent.bao; # The unix user is named for the agent, so this is also the principal the # role puts on the certificate (the identity entity swarm-controller names # after the agent). agentName = config.services.hyperhive.agent.user.name; ca = import ../host-modules/lib/ssh-agent-ca.nix { }; # The server CA id ./bao.nix loads. serverCaCredential = "hive-agent-bao-server-ca"; unitName = "hive-agent-ssh-cert"; runtimeDir = unitName; dir = "/run/${runtimeDir}"; keyFile = "${dir}/key"; pubFile = "${keyFile}.pub"; # `ssh` reads `-cert.pub` by itself, but the config names it so # that the file a Host block uses is visible in the config. certFile = "${keyFile}-cert.pub"; stagingFile = "${dir}/cert.new"; errFile = "${dir}/bao.err"; # The store's CA, copied out of the unit's credentials so that the helper can # reach the store from an `ssh` run, which has no credentials directory. serverCaCopy = "${dir}/server-ca.pem"; # Written when the store refuses, so that a loop of `ssh` calls does not log in # to the store each time the switch is off. refusedFile = "${dir}/refused"; # In the state dir, which is bound from the host, so a pin survives a restart. knownHostsFile = "/agents/${agentName}/state/ssh_known_hosts"; signPath = "${ca.mountPath}/sign/${ca.roleName}"; # A certificate with less than this left is signed again by `ssh`. The timer # fires every `ca.renewEveryMinutes`, so a healthy agent never gets here. ensureMarginSec = 600; # How long a refusal is remembered. The delay the operator sees after # switching on is at most this, then one login and one signing. refusedForSec = 10; storeRetry = import ../host-modules/lib/store-retry.nix { }; binPath = lib.makeBinPath [ pkgs.openbao pkgs.openssh pkgs.coreutils pkgs.gnused pkgs.util-linux ]; # The helper, and the unit's script: one text. `HIVE_SSH_CERT_ENSURE=1` is the # `ssh` entry; without it this is the timer's. The tools go after the caller's # `PATH`, so a caller that has its own wins. helper = pkgs.writeShellScript unitName config.systemd.services.${unitName}.script; in { config = lib.mkIf (baoCfg.addr != null) { # Every host: the certificate, and the pins in the state dir, which outlives # the container root. `accept-new` records a host's key at first contact and # refuses a changed one after that. The `Match` line has no settings: ssh # runs its command while it reads the config, before it loads any identity, # and the answer is not used. programs.ssh.extraConfig = '' Match exec "HIVE_SSH_CERT_ENSURE=1 ${helper} /dev/null 2>&1; fi # `ssh` can run before the unit's first start has made the directory. if [ ! -d "$dir" ]; then mkdir -m 0700 "$dir"; fi # One run at a time, the unit's or an `ssh`'s: two would race on the key # and the staged file. exec {lock}<"$dir" if ! flock -w 20 "$lock"; then echo "another run holds $dir and did not finish in 20 seconds." >&2 exit 1 fi # Epoch second the certificate ends, 0 when there is none. cert_end() { local end end=$(ssh-keygen -L -f "$cert" 2>/dev/null | sed -n 's/^ *Valid: from .* to //p') || end= if [ -n "$end" ]; then date -d "$end" +%s 2>/dev/null || echo 0; else echo 0; fi } if [ -n "$ensure" ]; then now=$(date +%s) # Enough left: nothing to do, and nothing asked of the store. if [ "$(cert_end)" -gt $((now + ${toString ensureMarginSec})) ]; then exit 0; fi # The store refused a moment ago: the switch is off. if [ -e "$refused" ] && [ $((now - $(stat -c %Y "$refused"))) -lt ${toString refusedForSec} ]; then exit 1 fi fi # The unit has the credentials directory and keeps the store's CA where # an `ssh` run can find it. if [ -n "''${CREDENTIALS_DIRECTORY-}" ] && [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then rm -f "$server_ca" cp "$CREDENTIALS_DIRECTORY/${serverCaCredential}" "$server_ca" fi if [ -s "$server_ca" ]; then export BAO_CACERT="$server_ca"; fi trap 'rm -f "$err" "$staged"' EXIT # No identity delivered: ./bao.nix's check reports that; saying it # twice adds nothing. for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do if [ ! -s "$f" ]; then echo "this agent has no store identity, so it cannot get an SSH certificate." >&2 if [ -n "$ensure" ]; then exit 1; fi exit 0 fi if [ ! -r "$f" ]; then echo "cannot read $f, so this agent cannot present its store identity." >&2 exit 1 fi done # The key lives as long as /run does, which is as long as this # container: a restart is a new key. Only a pair with both halves is # kept. if [ ! -s "$key" ] || [ ! -s "$pub" ]; then rm -f "$key" "$pub" "$cert" ssh-keygen -q -t ed25519 -N "" -C ${lib.escapeShellArg "hive-agent-${agentName}"} -f "$key" chmod 0400 "$key" fi # A fresh login every time: a token carries the policies the role had # when it was issued, so only a new one sees the operator's switch. rm -f "$err" if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "the swarm secret store at $BAO_ADDR refused or could not answer this agent's certificate login:" >&2 if [ -s "$err" ]; then cat "$err" >&2; fi exit 1 fi export BAO_TOKEN # No principal is requested: the role's default is this agent's own # name, and it refuses any other. rm -f "$err" "$staged" if ! bao write -field=signed_key ${lib.escapeShellArg signPath} public_key=@"$pub" >"$staged" 2>"$err"; then # A 403 is the operator's switch being off for this agent: not a fault, # so no failed unit. Anything else (a sealed or unreachable store) is # one, and retried. if [[ "$(<"$err")" == *"Code: 403"* ]]; then rm -f "$refused" : >"$refused" echo "SSH certificates are not enabled for this agent (the swarm store refused to sign); asking again next tick." if [ -n "$ensure" ]; then exit 1; fi exit 0 fi echo "the swarm secret store would not sign this agent's SSH key at ${signPath}:" >&2 if [ -s "$err" ]; then cat "$err" >&2; fi exit 1 fi # A reply that is not a certificate must not replace the one in use. if ! ssh-keygen -L -f "$staged" >/dev/null 2>&1; then echo "what the store returned at ${signPath} is not an SSH certificate; keeping the current one." >&2 exit 1 fi # One rename: ssh never reads a half-written certificate. mv -f "$staged" "$cert" rm -f "$refused" echo "signed this agent's SSH key as ${agentName}, valid for ${toString ca.certTtlMinutes} minutes." ''; }; # Every third of the certificate's lifetime, so a store that is down for # a run or two does not outlast it. systemd.timers.${unitName} = { description = "sign this agent's SSH key again before its certificate expires"; wantedBy = [ "timers.target" ]; timerConfig = { OnUnitInactiveSec = "${toString ca.renewEveryMinutes}min"; RandomizedDelaySec = "1min"; }; }; }; }