274 lines
11 KiB
Nix
274 lines
11 KiB
Nix
# Outbound SSH for this agent: a certificate for a key it generates itself,
|
|
# signed by the swarm's SSH CA, and a client config that offers it.
|
|
#
|
|
# On every agent that has a store address (the same switch as ./bao.nix: with no
|
|
# store there is nothing to sign with). The agent has no standing SSH credential:
|
|
# the helper below generates an ed25519 key in `/run/${unitName}` (tmpfs, never
|
|
# on disk), logs in to the swarm secret store with the identity ./bao.nix keeps
|
|
# current, and has the public half signed by the store's SSH engine
|
|
# (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through issues a
|
|
# user certificate with this agent's name as its only principal, valid for an
|
|
# hour.
|
|
#
|
|
# Whether the store signs is the operator's per-agent switch in the swarm UI
|
|
# (the `ssh` permission, `swarm-controller`'s `agent_ssh`). Every signing makes a
|
|
# fresh login, so a token never outlives the policies the role had when it was
|
|
# issued by more than the helper's own run. Two things run the helper:
|
|
#
|
|
# - `${unitName}.service`, from a timer, renews well before the certificate
|
|
# runs out. With the switch off the store refuses: the unit logs one line, ends
|
|
# successfully and asks again at the next tick.
|
|
# - `ssh` itself, through a `Match exec` line in the system config: when the
|
|
# certificate is missing or close to its end, the helper signs before the
|
|
# connection starts. That is what makes the first `ssh` after the operator
|
|
# switches SSH on work at once. It is silent and its exit status is whether a
|
|
# usable certificate exists; nothing in the config depends on the answer.
|
|
#
|
|
# A target machine trusts the CA, not this key: ../target-modules/agent-ssh.nix.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
baoCfg = config.services.hyperhive.agent.bao;
|
|
|
|
# The unix user is named for the agent, so this is also the principal the
|
|
# role puts on the certificate (the identity entity swarm-controller names
|
|
# after the agent).
|
|
agentName = config.services.hyperhive.agent.user.name;
|
|
|
|
ca = import ../host-modules/lib/ssh-agent-ca.nix { };
|
|
|
|
# The server CA id ./bao.nix loads.
|
|
serverCaCredential = "hive-agent-bao-server-ca";
|
|
|
|
unitName = "hive-agent-ssh-cert";
|
|
runtimeDir = unitName;
|
|
dir = "/run/${runtimeDir}";
|
|
keyFile = "${dir}/key";
|
|
pubFile = "${keyFile}.pub";
|
|
# `ssh` reads `<IdentityFile>-cert.pub` by itself, but the config names it so
|
|
# that the file a Host block uses is visible in the config.
|
|
certFile = "${keyFile}-cert.pub";
|
|
stagingFile = "${dir}/cert.new";
|
|
errFile = "${dir}/bao.err";
|
|
# The store's CA, copied out of the unit's credentials so that the helper can
|
|
# reach the store from an `ssh` run, which has no credentials directory.
|
|
serverCaCopy = "${dir}/server-ca.pem";
|
|
# Written when the store refuses, so that a loop of `ssh` calls does not log in
|
|
# to the store each time the switch is off.
|
|
refusedFile = "${dir}/refused";
|
|
|
|
# In the state dir, which is bound from the host, so a pin survives a restart.
|
|
knownHostsFile = "/agents/${agentName}/state/ssh_known_hosts";
|
|
|
|
signPath = "${ca.mountPath}/sign/${ca.roleName}";
|
|
|
|
# A certificate with less than this left is signed again by `ssh`. The timer
|
|
# fires every `ca.renewEveryMinutes`, so a healthy agent never gets here.
|
|
ensureMarginSec = 600;
|
|
# How long a refusal is remembered. The delay the operator sees after
|
|
# switching on is at most this, then one login and one signing.
|
|
refusedForSec = 10;
|
|
|
|
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
|
|
|
binPath = lib.makeBinPath [
|
|
pkgs.openbao
|
|
pkgs.openssh
|
|
pkgs.coreutils
|
|
pkgs.gnused
|
|
pkgs.util-linux
|
|
];
|
|
|
|
# The helper, and the unit's script: one text. `HIVE_SSH_CERT_ENSURE=1` is the
|
|
# `ssh` entry; without it this is the timer's. The tools go after the caller's
|
|
# `PATH`, so a caller that has its own wins.
|
|
helper = pkgs.writeShellScript unitName config.systemd.services.${unitName}.script;
|
|
in
|
|
{
|
|
config = lib.mkIf (baoCfg.addr != null) {
|
|
# Every host: the certificate, and the pins in the state dir, which outlives
|
|
# the container root. `accept-new` records a host's key at first contact and
|
|
# refuses a changed one after that. The `Match` line has no settings: ssh
|
|
# runs its command while it reads the config, before it loads any identity,
|
|
# and the answer is not used.
|
|
programs.ssh.extraConfig = ''
|
|
Match exec "HIVE_SSH_CERT_ENSURE=1 ${helper} </dev/null"
|
|
Host *
|
|
IdentityFile ${keyFile}
|
|
CertificateFile ${certFile}
|
|
IdentitiesOnly yes
|
|
StrictHostKeyChecking accept-new
|
|
UserKnownHostsFile ${knownHostsFile}
|
|
'';
|
|
|
|
systemd.services.${unitName} = {
|
|
description = "sign this agent's SSH key with the swarm's SSH CA";
|
|
after = [
|
|
"network.target"
|
|
# Ordering only: that unit reports a store this agent cannot reach as
|
|
# itself, and should get to say so first.
|
|
"hive-agent-bao-identity.service"
|
|
];
|
|
wantedBy = [ "multi-user.target" ];
|
|
# ../host-modules/lib/store-retry.nix.
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
Type = "oneshot";
|
|
# Not `RemainAfterExit`: the timer has to be able to start this unit
|
|
# again. `RuntimeDirectoryPreserve` keeps the key between runs.
|
|
RemainAfterExit = false;
|
|
TimeoutStartSec = 30;
|
|
User = agentName;
|
|
Group = agentName;
|
|
RuntimeDirectory = runtimeDir;
|
|
RuntimeDirectoryMode = "0700";
|
|
RuntimeDirectoryPreserve = "yes";
|
|
UMask = "0377";
|
|
LoadCredential = [ serverCaCredential ];
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
# `UMask=0377` in the unit makes every file this script creates `0400`,
|
|
# so each one is removed before the command that creates it. `ssh` runs
|
|
# the same text, which therefore sets it too.
|
|
umask 0377
|
|
export PATH="$PATH:${binPath}"
|
|
export BAO_ADDR=${lib.escapeShellArg (toString baoCfg.addr)}
|
|
export BAO_CLIENT_CERT=${lib.escapeShellArg baoCfg.clientCertFile}
|
|
export BAO_CLIENT_KEY=${lib.escapeShellArg baoCfg.clientKeyFile}
|
|
|
|
dir=${lib.escapeShellArg dir}
|
|
key=${lib.escapeShellArg keyFile}
|
|
pub=${lib.escapeShellArg pubFile}
|
|
cert=${lib.escapeShellArg certFile}
|
|
staged=${lib.escapeShellArg stagingFile}
|
|
err=${lib.escapeShellArg errFile}
|
|
server_ca=${lib.escapeShellArg serverCaCopy}
|
|
refused=${lib.escapeShellArg refusedFile}
|
|
|
|
ensure=
|
|
if [ -n "''${HIVE_SSH_CERT_ENSURE-}" ]; then ensure=1; fi
|
|
|
|
# The `ssh` entry is silent and says yes or no by its exit status; the
|
|
# unit's output is its log.
|
|
if [ -n "$ensure" ]; then exec >/dev/null 2>&1; fi
|
|
|
|
# `ssh` can run before the unit's first start has made the directory.
|
|
if [ ! -d "$dir" ]; then mkdir -m 0700 "$dir"; fi
|
|
|
|
# One run at a time, the unit's or an `ssh`'s: two would race on the key
|
|
# and the staged file.
|
|
exec {lock}<"$dir"
|
|
if ! flock -w 20 "$lock"; then
|
|
echo "another run holds $dir and did not finish in 20 seconds." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Epoch second the certificate ends, 0 when there is none.
|
|
cert_end() {
|
|
local end
|
|
end=$(ssh-keygen -L -f "$cert" 2>/dev/null | sed -n 's/^ *Valid: from .* to //p') || end=
|
|
if [ -n "$end" ]; then date -d "$end" +%s 2>/dev/null || echo 0; else echo 0; fi
|
|
}
|
|
|
|
if [ -n "$ensure" ]; then
|
|
now=$(date +%s)
|
|
# Enough left: nothing to do, and nothing asked of the store.
|
|
if [ "$(cert_end)" -gt $((now + ${toString ensureMarginSec})) ]; then exit 0; fi
|
|
# The store refused a moment ago: the switch is off.
|
|
if [ -e "$refused" ] && [ $((now - $(stat -c %Y "$refused"))) -lt ${toString refusedForSec} ]; then
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# The unit has the credentials directory and keeps the store's CA where
|
|
# an `ssh` run can find it.
|
|
if [ -n "''${CREDENTIALS_DIRECTORY-}" ] && [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
|
rm -f "$server_ca"
|
|
cp "$CREDENTIALS_DIRECTORY/${serverCaCredential}" "$server_ca"
|
|
fi
|
|
if [ -s "$server_ca" ]; then export BAO_CACERT="$server_ca"; fi
|
|
|
|
trap 'rm -f "$err" "$staged"' EXIT
|
|
|
|
# No identity delivered: ./bao.nix's check reports that; saying it
|
|
# twice adds nothing.
|
|
for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
|
|
if [ ! -s "$f" ]; then
|
|
echo "this agent has no store identity, so it cannot get an SSH certificate." >&2
|
|
if [ -n "$ensure" ]; then exit 1; fi
|
|
exit 0
|
|
fi
|
|
if [ ! -r "$f" ]; then
|
|
echo "cannot read $f, so this agent cannot present its store identity." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# The key lives as long as /run does, which is as long as this
|
|
# container: a restart is a new key. Only a pair with both halves is
|
|
# kept.
|
|
if [ ! -s "$key" ] || [ ! -s "$pub" ]; then
|
|
rm -f "$key" "$pub" "$cert"
|
|
ssh-keygen -q -t ed25519 -N "" -C ${lib.escapeShellArg "hive-agent-${agentName}"} -f "$key"
|
|
chmod 0400 "$key"
|
|
fi
|
|
|
|
# A fresh login every time: a token carries the policies the role had
|
|
# when it was issued, so only a new one sees the operator's switch.
|
|
rm -f "$err"
|
|
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
|
echo "the swarm secret store at $BAO_ADDR refused or could not answer this agent's certificate login:" >&2
|
|
if [ -s "$err" ]; then cat "$err" >&2; fi
|
|
exit 1
|
|
fi
|
|
export BAO_TOKEN
|
|
|
|
# No principal is requested: the role's default is this agent's own
|
|
# name, and it refuses any other.
|
|
rm -f "$err" "$staged"
|
|
if ! bao write -field=signed_key ${lib.escapeShellArg signPath} public_key=@"$pub" >"$staged" 2>"$err"; then
|
|
# A 403 is the operator's switch being off for this agent: not a fault,
|
|
# so no failed unit. Anything else (a sealed or unreachable store) is
|
|
# one, and retried.
|
|
if [[ "$(<"$err")" == *"Code: 403"* ]]; then
|
|
rm -f "$refused"
|
|
: >"$refused"
|
|
echo "SSH certificates are not enabled for this agent (the swarm store refused to sign); asking again next tick."
|
|
if [ -n "$ensure" ]; then exit 1; fi
|
|
exit 0
|
|
fi
|
|
echo "the swarm secret store would not sign this agent's SSH key at ${signPath}:" >&2
|
|
if [ -s "$err" ]; then cat "$err" >&2; fi
|
|
exit 1
|
|
fi
|
|
|
|
# A reply that is not a certificate must not replace the one in use.
|
|
if ! ssh-keygen -L -f "$staged" >/dev/null 2>&1; then
|
|
echo "what the store returned at ${signPath} is not an SSH certificate; keeping the current one." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# One rename: ssh never reads a half-written certificate.
|
|
mv -f "$staged" "$cert"
|
|
rm -f "$refused"
|
|
echo "signed this agent's SSH key as ${agentName}, valid for ${toString ca.certTtlMinutes} minutes."
|
|
'';
|
|
};
|
|
|
|
# Every third of the certificate's lifetime, so a store that is down for
|
|
# a run or two does not outlast it.
|
|
systemd.timers.${unitName} = {
|
|
description = "sign this agent's SSH key again before its certificate expires";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnUnitInactiveSec = "${toString ca.renewEveryMinutes}min";
|
|
RandomizedDelaySec = "1min";
|
|
};
|
|
};
|
|
};
|
|
}
|