# Glue: this hive's agent queue credential comes out of the secret store. # # The store's second reader, and deliberately the same shape as its first # (./glue-matrix-bao-token.nix): a cert login that fails LOUDLY because every # state it fails on is one a retry fixes, then a read that degrades QUIETLY # because no retry turns "no value there" into a value. # # ⚠️ Two files, not one, and that is the consumer's shape rather than a # preference. `swarm_queue_client::QueueConfig::from_env` takes the secret as # a PATH (`_OIDC_CLIENT_SECRET_FILE`) and the client id as a VALUE # (`_OIDC_CLIENT_ID`), so splitting them here is what lets the next # slice hand both to an agent without parsing anything. # # ⚠️ An ABSENT file means this hive's agents do not connect to the queue, and # that is correct rather than degraded. The publisher runs on the authelia # host and authelia mints on its first boot, so "nothing at that path yet" is # the ordinary early state of a swarm. Nothing here writes a local stand-in: # unlike a matrix registration token there is no such thing as a locally valid # OIDC client secret, so a placeholder would turn a hive that cannot connect # into one that is refused, which reaches the agent as a timeout. # # 📌 This runs wherever a client identity is configured, NOT only where the # store is — the rule ./glue-matrix-bao-token.nix states in full. There is no # "this hive runs agents" condition to gate it on as well: agent containers # are created at runtime by hive-c0re, so every hyperhive host is a host that # may run one. { pkgs, lib, config, ... }: let hyperhiveCfg = config.services.hyperhive; deployCfg = hyperhiveCfg.deploy; baoCfg = hyperhiveCfg.swarm.bao; baoDeploy = deployCfg.bao; # What decides whether this unit exists at all. A reader is defined by holding # a certificate the store accepts, and that is true on the store's own host # and on a hive three networks away for exactly the same reason. haveClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null; credentialDir = toString deployCfg.hive-controller.queue.agentCredentialDir; secretFile = "${credentialDir}/secret"; clientIdFile = "${credentialDir}/client_id"; # Where the credential lives in the store, spelled from the same pieces the # writer uses. Whoever writes it and whoever reads it must agree, and the # agreement belongs in one visible place. # # ⚠️ The `hives/` segment is not decoration — it is what the reader's # own grant covers, so a path outside it is a 403 rather than a miss, however # correct it looks. `swarm-secret-client`'s `queue::agent_client_path` builds # the same string from the same pieces; this literal is the nix half of that # one agreement. # # `hiveName` has no fallback here for the reason ./glue-bao-tls.nix gives at # its own use of it: it is asserted set for every hyperhive host. credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent"; in { options.services.hyperhive.deploy.hive-controller.queue = { agentCredentialDir = lib.mkOption { type = lib.types.path; default = "/var/lib/hyperhive/queue-agent"; description = '' Host directory holding this hive's agent queue credential: `secret` (the OIDC client secret, `0600`) and `client_id` (the client that secret authenticates, `0644` — it is sent to the token endpoint on every connection and is not itself a secret). Both files are written by `swarm-bao-queue-agent.service`, which also creates this directory. Named here because nothing else creates it: the parent `/var/lib/hyperhive` belongs to `hive-c0re.service`'s `StateDirectory=`, which re-applies its own mode on every start, and a second declaration of *that* path is the two-mechanisms-one-path trap ./hive-gateway/default.nix records — so this directory is the unit's to make and the parent stays c0re's. Absent files mean this hive has no queue credential yet, which is what a swarm looks like before the publisher on the authelia host has run. ''; }; }; config = lib.mkIf (hyperhiveCfg.enable && haveClientIdentity) { # Same rule as the unit's own gate: this reader exists on any host holding # a client identity, which is not every host that runs the store, so the # store's module cannot name it. services.hyperhive.swarm.otel.journaldUnits = [ "swarm-bao-queue-agent" ]; systemd.services.swarm-bao-queue-agent = { description = "fetch this hive's agent queue credential from the swarm secret store"; # Every one of these names a unit that exists only where the store runs. # `Requires=` on an absent unit fails the job outright, so the ordering is # conditional even though the read is not: off-host there is nothing local # to wait for, and the timeout below is what bounds the attempt instead. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" ]; wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; # Ordered `Before=` nothing, because the consumer does not exist yet. # Agent containers are created at runtime, so no static unit name can be # named here anyway; the slice that bind-mounts these files in adds the # edge through hive-c0re's `container@h-.service` drop-in. wantedBy = [ "multi-user.target" ]; path = [ baoDeploy.package pkgs.coreutils ]; # Sized for the race this loses, not for an unseal: `swarm-bao` comes up # seconds before this unit asks, and the cert-auth role it logs in # against is written seconds after, so a few short attempts cover it. # An hours-long window would be a bet on a store that is sealed, and the # degrade below is already correct for that. # # `StartLimit*` are `[Unit]` settings, so they go here and not in # `serviceConfig` — systemd ignores them under `[Service]`. The window # has to exceed `RestartSec × burst`. startLimitBurst = 4; startLimitIntervalSec = 300; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; # What actually bounds the reads below. Stated here rather than left to # systemd's default, so the number a boot waits on is in the file that # waits. TimeoutStartSec = 30; Restart = "on-failure"; RestartSec = 15; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; BAO_CLIENT_CERT = baoDeploy.clientCertFile; BAO_CLIENT_KEY = baoDeploy.clientKeyFile; } # Absent means the system trust store, which is what a deployment with a # real CA wants and what a self-signed one must not be left with. // lib.optionalAttrs (baoDeploy.serverCaFile != null) { BAO_CACERT = baoDeploy.serverCaFile; }; script = '' set -euo pipefail # `bao`'s own message is the only thing separating a missing value from # a refused identity from an unreachable host. This unit's degraded # mode is correct for all three, so it reports which one rather than # asserting all three in a sentence of ours. err="$(mktemp)" trap 'rm -f "$err"' EXIT # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*` # variables above only decide which certificate the TLS handshake # presents; without a token `bao` asks its token helper instead, and # that is a `sh` this unit's `path` does not carry. `-token-only` # answers on stdout and skips the helper on both sides. # # Fails LOUDLY, unlike the reads below: the three states a login # failure covers — store not up, sealed, role not written yet — are all # things a retry fixes, and `Restart=on-failure` above is what retries. if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "could not log in to swarm-bao with this host's certificate; leaving the queue credential in ${credentialDir} as it is." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 1 fi export BAO_TOKEN # Two reads of one object rather than one `-format=json` parsed with # `jq`: no sibling unit carries `jq` on its `path`, and the pair cannot # actually disagree — the client id is derived from this hive's name, so # a rotation landing between these two calls changes the secret and # rewrites the same id. if ! secret="$(bao kv get -field=value ${lib.escapeShellArg credentialPath} 2>"$err")"; then echo "swarm-bao did not return ${credentialPath}; this hive's agents have no queue credential yet." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 0 fi if ! client_id="$(bao kv get -field=client_id ${lib.escapeShellArg credentialPath} 2>"$err")"; then echo "${credentialPath} holds no client_id; the secret alone is not a usable credential, so nothing is written." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 0 fi # Both or neither, for the reason `QueueConfig::from_env` refuses a # half-set environment: a client that finds one of the two comes up # "fine" and never connects. if [ -z "$secret" ] || [ -z "$client_id" ]; then echo "swarm-bao returned an empty field of ${credentialPath}; leaving the files as they are." >&2 exit 0 fi install -d -m 0755 ${lib.escapeShellArg credentialDir} umask 077 printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile} chmod 0600 ${lib.escapeShellArg secretFile} # `0644` on purpose: an OIDC client id is presented to the token # endpoint on every connection and is public by construction. printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile} chmod 0644 ${lib.escapeShellArg clientIdFile} ''; }; }; }