# This agent's own identity at the swarm secret store: the check that proves # it works, and the live copy every store reader in the container presents. # # `swarm-controller` mints the leaf at agent creation and publishes it; this # agent's hive collects it under the hive's own certificate and hands it in as # systemd credentials (`hive_c0re::lifecycle::agent_identity`). That copy is # fixed for the life of the container, and the controller re-issues the leaf # at half its validity, so this unit seeds a live copy from it and, on a timer, # reads the current leaf back from the store under the one it holds. Because # that login needs the held leaf to still be valid, a container whose leaf # expired while every pull failed recovers only at its next start. # # ⚠️ The identity arrives as credentials and NOT as a bind mount, and the mode # is why: the host file is `0600` to the hive daemon and this unit runs as the # unprivileged agent user. nspawn's `--load-credential` is read by the # container manager as root and re-exposed under this unit's own `User=`; a # bind would deliver a private key this user cannot open. Same answer, same # reason, as ./queue.nix's credential pair. # # This is the courier's other end: the hive-side collector has no purpose # without it, so the two ship together and neither is reachable alone. # # 🩸 This unit fails LOUDLY where the hive-side readers degrade quietly, and # that is the opposite default on purpose. A missing OIDC client secret means # a hive whose publisher has yet to run; a missing or refused certificate # means an agent that believes it can reach the store and cannot, which every # later pull would report as its own unrelated failure. The one place that # knows the real cause is the login itself, so this is where it is said. { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.agent.bao; # This container's agent name. The same string the hive published the # identity under, because the agent's unix user is named for the agent — # see ./user.nix. agentName = config.services.hyperhive.agent.user.name; # The three ids `hive_c0re::lifecycle::agent_identity` forwards under. # Neither side can discover the other's spelling, so a rename is a rename # there too. certCredential = "hive-agent-bao-cert"; keyCredential = "hive-agent-bao-key"; serverCaCredential = "hive-agent-bao-server-ca"; unitName = "hive-agent-bao-identity"; runtimeDir = unitName; # A symlink to the directory that holds the live pair. Each pair is written # into a fresh directory and goes live with one rename of this link, so the # link always names one whole pair, old or new. liveLink = "/run/${runtimeDir}/current"; certFile = "${liveLink}/cert"; keyFile = "${liveLink}/key"; # The store's whole answer, key included, in the unit's own `0700` directory. rawFile = "/run/${runtimeDir}/bao-mtls.json"; errFile = "/run/${runtimeDir}/bao.err"; # Where the identity lives in the store, spelled from the same pieces the # publisher uses. `swarm_secret_client::mtls::identity_path` builds # `swarm/agents//bao-mtls` and `path::MOUNT` is `secret`; this # literal is the nix half of that one agreement, exactly as # ../host-modules/glue-queue-agent-credential.nix spells its own. identityPath = "secret/swarm/agents/${agentName}/bao-mtls"; # The address is the whole switch — no separate `enable`, the same shape # ./queue.nix and ./logs.nix gate themselves with. A hive that has a store # forwards its address and every agent on it gets the check; a hive that has # none forwards nothing and no agent does. An `enable` beside it would be a # knob whose only correct setting is whatever the address already says, and # its default would decide whether the hive-side courier delivers into a # container that reads what it is given or into one that never looks. configured = cfg.addr != null; storeRetry = import ../host-modules/lib/store-retry.nix { }; in { options.services.hyperhive.agent.bao = { addr = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "https://bao.example.com:8200"; description = '' Where the swarm secret store listens, as this container reaches it. Set by the generated meta flake from the host's own `BAO_ADDR`, which is the address this hive already uses, and setting it is what generates `${unitName}.service`: at boot that unit logs in with the certificate its hive delivered and reads this agent's own path back, failing if either step does not succeed. `systemctl status ${unitName}` inside the container is then the answer to "can this agent reach the store as itself", which nothing else in the tree reports. `null` means the hive was given no store. No unit is generated then, because an agent whose swarm never minted an identity has nothing to log in with, and a failed unit at every boot would say that in the loudest possible way about a deployment that never asked for it. ''; }; clientCertFile = lib.mkOption { type = lib.types.str; readOnly = true; default = certFile; description = '' This agent's current store certificate, kept by `${unitName}.service`. Read-only: it is where that unit writes, not a knob. Every unit that logs in to the store as this agent points `BAO_CLIENT_CERT` here. ''; }; clientKeyFile = lib.mkOption { type = lib.types.str; readOnly = true; default = keyFile; description = '' The private key for `clientCertFile`, `0400` to the agent user. 🩸 A PATH and never a value. ''; }; }; config = lib.mkIf configured { systemd.services.${unitName} = { description = "keep this agent's swarm secret store identity current, and prove it works"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = [ pkgs.openbao pkgs.coreutils pkgs.diffutils pkgs.jq ]; # ../host-modules/lib/store-retry.nix. From an agent the store is reached # through the gateway's stream passthrough, so either being down fails # the login below. inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { Type = "oneshot"; # Not `RemainAfterExit`, so the timer below can start it again. # `RuntimeDirectoryPreserve` keeps the live pair between runs. RemainAfterExit = false; TimeoutStartSec = 30; User = agentName; Group = agentName; RuntimeDirectory = runtimeDir; RuntimeDirectoryMode = "0700"; RuntimeDirectoryPreserve = "yes"; UMask = "0377"; # Bare ids, no paths: the terse `LoadCredential=` form that inherits a # credential the service *manager* received, which is what the # container manager passed in. ./queue.nix states the same shape. LoadCredential = [ certCredential keyCredential serverCaCredential ]; }; environment = { BAO_ADDR = cfg.addr; BAO_CLIENT_CERT = certFile; BAO_CLIENT_KEY = keyFile; }; script = '' set -euo pipefail # Absent credentials are the first thing checked, because every later # message would blame the store for a delivery that never happened. # 🩸 The files are named and never read here: what went wrong is a # property of the path, and the bytes at it are a private key. missing= for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then missing="$missing $id" fi done if [ -n "$missing" ]; then echo "this agent has no store identity: the hive delivered no$missing." >&2 echo "swarm-controller publishes it at ${identityPath} when the agent is created, and the hive collects it from there." >&2 exit 1 fi # Only when one was delivered. Absent means the container verifies the # store's listener against its own trust store, which is what a # deployment with a real CA wants; pointing BAO_CACERT at a file that # is not there would fail the handshake and name the wrong cause. if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}" fi # `UMask=0377` makes every file this script creates `0400`, so each one # is removed before the command that creates it. err=${lib.escapeShellArg errFile} raw=${lib.escapeShellArg rawFile} trap 'rm -f "$err" "$raw"' EXIT live=${lib.escapeShellArg liveLink} dir=''${live%/*} # A fresh directory for one pair. `chmod`, because the umask cuts # mktemp's `0700` down to a `0400` no file can be created in. new_pair() { local d d=$(mktemp -d -p "$dir" pair.XXXXXX) chmod 0700 "$d" echo "$d" } # The only step that changes the live pair: one rename of the link. A # reader whose two opens straddle it gets one file of each pair for # that one login; the next login reads a whole pair. go_live() { rm -f "$live.new" ln -s "''${1##*/}" "$live.new" mv -Tf "$live.new" "$live" } # Every pair directory but the live one is left over: the pair the # last swap replaced, kept until now for a reader that resolved the # link before the swap, or a pair a stopped run staged. current=$(readlink "$live" || true) for d in "$dir"/pair.*; do if [ -d "$d" ] && [ "''${d##*/}" != "$current" ]; then chmod 0700 "$d" rm -rf "$d" fi done rm -f "$live.new" # The delivered pair is the start of every container's life: /run is # empty at boot and kept across this unit's runs. if [ ! -s "$BAO_CLIENT_CERT" ] || [ ! -s "$BAO_CLIENT_KEY" ]; then seed=$(new_pair) cp "$CREDENTIALS_DIRECTORY/${keyCredential}" "$seed/key" cp "$CREDENTIALS_DIRECTORY/${certCredential}" "$seed/cert" go_live "$seed" fi # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*` # variables above only decide which certificate the TLS handshake # presents; without a token `bao` asks its token helper instead, and # that is a `sh` this unit's `path` does not carry. `-token-only` # answers on stdout and skips the helper on both sides. # # No `name=`: a cert role pins both its authority and the common name # it accepts, so this agent's leaf matches its own role and no other. # Naming the role here would be a third copy of a string # `swarm_secret_client::policy::agent_object_name` already owns. rm -f "$err" if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 1 fi export BAO_TOKEN # A token is not yet an answer: the login proves the certificate, this # proves the policy attached to it, and the answer is the current # leaf. One read, so the certificate and key come from one version. # # ⚠️ Written by redirect, never printed: the object holds a private # key. rm -f "$err" "$raw" if ! bao kv get -format=json ${lib.escapeShellArg identityPath} >"$raw" 2>"$err"; then echo "this agent logged in to the swarm secret store but cannot read ${identityPath}, so its policy does not cover its own path." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 1 fi # `-j`: the bytes as the store holds them, which is what the hive # delivered, so an unchanged leaf compares equal to the seeded copy. staged=$(new_pair) if ! jq -ej '.data.data.cert | strings' "$raw" >"$staged/cert" \ || ! jq -ej '.data.data.key | strings' "$raw" >"$staged/key"; then rm -rf "$staged" echo "${identityPath} holds no certificate and key; this agent keeps the one it has." >&2 exit 1 fi if cmp -s "$staged/cert" "$BAO_CLIENT_CERT" && cmp -s "$staged/key" "$BAO_CLIENT_KEY"; then rm -rf "$staged" echo "authenticated to the swarm secret store at $BAO_ADDR as ${agentName}; its certificate at ${identityPath} is unchanged." else # This unit's next login uses whatever goes live, so a pair the # store refuses must never replace the one it accepts. rm -f "$err" if ! BAO_CLIENT_CERT="$staged/cert" BAO_CLIENT_KEY="$staged/key" \ bao login -method=cert -token-only >/dev/null 2>"$err"; then rm -rf "$staged" echo "the swarm secret store refused the certificate at ${identityPath}; this agent keeps the one it has." >&2 if [ -s "$err" ]; then cat "$err" >&2; fi exit 1 fi go_live "$staged" echo "replaced this agent's store certificate with the one at ${identityPath}." fi ''; }; # The controller re-issues a leaf at half its validity, so ten minutes is # small against the time the old one stays valid. systemd.timers.${unitName} = { description = "re-read this agent's store certificate from the secret store"; wantedBy = [ "timers.target" ]; timerConfig = { OnUnitInactiveSec = "10min"; RandomizedDelaySec = "1min"; }; }; }; }