agent: fetch this agent's own swarm-queue credential from the store

Every agent on a hive authenticates to the swarm queue with the same
hive-scoped OIDC client, so at the auth callout one agent is
indistinguishable from its co-hived neighbours. The commit before this
one mints a secret per agent at swarm level into
secret/swarm/agents/<agent>/queue; nothing read it.

Read it here, and read it from the container itself. A hive courier in
the path would be the hive vouching for which agent this is, which is
the property a per-agent credential exists to remove -- so the agent
logs in to the store with the certificate hive-agent-bao-identity
already proves it can log in with, and reads its own path. The store
certificate is for reaching the store and nothing else: what the new
unit writes to /run is the secret it read back, and nothing hands a
BAO_CLIENT_* path to anything queue-shaped.

The read needs no policy change. render_agent grants read on
secret/data/swarm/agents/<agent>/*, which covers this path and the
bao-mtls one beside it alike -- which is also why this unit degrades
where the identity check fails. A refusal this unit sees and that check
did not cannot be a policy that drifted; it is an object not yet minted,
the ordinary state of every agent created before its swarm knew to mint
one.

The harness resolves the path and reports which credential this agent
can present. It does not yet present it: the auth-callout responder
still verifies only the hive-scoped token, and an agent offering a
credential nothing on the other end reads back would simply be refused.
Teaching swarm-nats-auth to read the same path is the next slice.

hive-agent/src/swarm_queue.rs

@@ -13,8 +13,21 @@
 //! has already spawned threads, and this one has. So the rule is restated
 //! here over the inputs this consumer actually has, and [`decide`] is the
 //! single place it lives.
+//!
+//! Beside all four sits a fifth coordinate, resolved by
+//! [`decide_agent_secret`] and not part of their group. The four are the
+//! *hive's* — one OIDC client shared by every container on it — so at the
+//! queue's auth callout they say which hive is connecting and never which
+//! agent. The fifth is this agent's own, minted per agent at swarm level and
+//! fetched by the container itself (`nix/agent-modules/queue-identity.nix`).
+//!
+//! It is reported here but not yet *presented*: the queue's auth-callout
+//! responder (`swarm-nats-auth`) validates only the hive-scoped token, and an
+//! agent offering a credential nothing on the other end reads back would be
+//! refused. Until that responder learns the same path, the connect path below
+//! is unchanged and this is the fetching half.
 
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
 
 use tokio::sync::OnceCell;
@@ -49,6 +62,13 @@ struct QueueEnv {
     /// `hive_c0re::meta` embeds at build time — so it is here for a
     /// deployment that needs a different one, not for ours.
     ca_file: Option<String>,
+    /// Where `nix/agent-modules/queue-identity.nix` fetched this agent's own
+    /// per-agent secret to. Outside the all-or-none group above because it is
+    /// governed by a different switch entirely — that unit is generated by
+    /// the agent having a *store* address, not by its hive having queue
+    /// coordinates — so an agent can legally have this and none of the four,
+    /// or the four and not this.
+    agent_secret_file: Option<String>,
 }
 
 impl QueueEnv {
@@ -60,6 +80,7 @@ impl QueueEnv {
             client_id_file: var("OIDC_CLIENT_ID_FILE"),
             client_secret_file: var("OIDC_CLIENT_SECRET_FILE"),
             ca_file: var("OIDC_CA_FILE"),
+            agent_secret_file: var("QUEUE_AGENT_SECRET_FILE"),
         }
     }
 }
@@ -96,6 +117,36 @@ fn read_client_id(path: &Path) -> Option<String> {
     }
 }
 
+/// Decide whether this agent has its own per-agent queue secret, given the
+/// path the fetch unit was told to write it to.
+///
+/// Three states collapse to two answers. No variable means no store address
+/// for this container, so no fetch unit was generated at all. A variable
+/// naming a file that is missing or empty means the unit ran and found
+/// nothing minted — the ordinary state of an agent created before its swarm
+/// knew to mint one, which that unit reports and survives. Only a non-empty
+/// file is a credential.
+///
+/// The file is not read. Its *contents* are the secret and belong nowhere but
+/// the moment of use; what a caller needs from here is whether there is one
+/// and where, which `metadata` answers without opening it.
+fn decide_agent_secret(path: Option<&str>) -> Option<PathBuf> {
+    let path = PathBuf::from(path?);
+    match std::fs::metadata(&path) {
+        Ok(m) if m.len() > 0 => Some(path),
+        Ok(_) => None,
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
+        Err(e) => {
+            tracing::warn!(
+                path = %path.display(),
+                error = %e,
+                "checking for this agent's own queue credential failed"
+            );
+            None
+        }
+    }
+}
+
 /// Decide what this agent's queue configuration is, given the environment and
 /// whatever the client-id file held.
 ///
@@ -163,6 +214,28 @@ pub fn init() {
         }
     };
     let _ = CONFIG.set(resolved);
+
+    // Independent of everything above: this agent may hold its own secret on
+    // a hive with no queue coordinates, or hold the coordinates and no secret
+    // of its own yet. Reported either way, because "which credential is this
+    // agent able to present" is a question only this process can answer, and
+    // it is the one the next slice's rollout will be asked repeatedly.
+    //
+    // The answer is only logged here. Presenting it needs the queue's
+    // auth-callout responder to verify it, which is the next slice — see this
+    // module's header.
+    if let Some(path) = decide_agent_secret(env.agent_secret_file.as_deref()) {
+        // The path, never the bytes: the file holds the secret itself.
+        tracing::info!(
+            path = %path.display(),
+            "this agent has its own swarm queue credential"
+        );
+    } else {
+        tracing::info!(
+            "no per-agent swarm queue credential; this agent is known to the queue \
+             by its hive's shared client"
+        );
+    }
 }
 
 /// What [`init`] resolved, or `None` when this agent has no queue.
@@ -213,7 +286,7 @@ async fn connect_once() -> Option<async_nats::Client> {
 
 #[cfg(test)]
 mod tests {
-    use super::{QueueEnv, Resolution, decide, read_client_id};
+    use super::{QueueEnv, Resolution, decide, decide_agent_secret, read_client_id};
 
     fn env(parts: [Option<&str>; 4]) -> QueueEnv {
         let [nats_url, token_endpoint, client_id_file, client_secret_file] = parts;
@@ -223,6 +296,7 @@ mod tests {
             client_id_file: client_id_file.map(str::to_owned),
             client_secret_file: client_secret_file.map(str::to_owned),
             ca_file: None,
+            agent_secret_file: None,
         }
     }
 
@@ -313,4 +387,52 @@ mod tests {
             );
         }
     }
+
+    /// The whole point of the fetch: a non-empty file is this agent's own
+    /// credential, and the answer is the path rather than what is in it.
+    #[test]
+    fn a_fetched_secret_resolves_to_its_path() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = dir.path().join("secret");
+        std::fs::write(&path, "s3cr3t").expect("write");
+        assert_eq!(
+            decide_agent_secret(path.to_str()).as_deref(),
+            Some(path.as_path())
+        );
+    }
+
+    /// The rollout state, and the one this must not confuse with a
+    /// credential: the fetch unit ran, found nothing minted for this agent,
+    /// and left no file. Treating that as a secret would have the harness
+    /// present zero bytes to the queue.
+    #[test]
+    fn a_missing_or_empty_fetched_secret_is_no_credential() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = dir.path().join("secret");
+        assert_eq!(decide_agent_secret(path.to_str()), None);
+        std::fs::write(&path, "").expect("write");
+        assert_eq!(decide_agent_secret(path.to_str()), None);
+    }
+
+    /// No variable at all: this container was given no store address, so no
+    /// fetch unit exists to have written anything.
+    #[test]
+    fn no_fetch_path_is_no_credential() {
+        assert_eq!(decide_agent_secret(None), None);
+    }
+
+    /// The two credentials are resolved by separate switches, and this is the
+    /// asymmetry that makes keeping them apart worth it: an agent whose hive
+    /// has no queue can still hold its own swarm-minted secret, because that
+    /// one is minted with no hive in the chain.
+    #[test]
+    fn the_per_agent_secret_is_independent_of_the_hive_coordinates() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = dir.path().join("secret");
+        std::fs::write(&path, "s3cr3t").expect("write");
+
+        let e = env([None, None, None, None]);
+        assert!(matches!(decide(&e, None), Resolution::Absent(_)));
+        assert!(decide_agent_secret(path.to_str()).is_some());
+    }
 }

nix/agent-modules/default.nix

@@ -55,6 +55,7 @@ in
     ./otel.nix
     ./packages.nix
     ./queue.nix
+    ./queue-identity.nix
     ./renamed-options.nix
     ./user.nix
     ./screen.nix

nix/agent-modules/queue-identity.nix

@@ -0,0 +1,203 @@
+# This agent's own credential for the swarm queue, fetched from the store by
+# the agent itself.
+#
+# The credential beside it in ./queue.nix is keyed per **hive**: one OIDC
+# client minted at deploy time and handed to every agent container on the
+# hive, so at the queue's auth callout one agent is indistinguishable from its
+# co-hived neighbours. This one is keyed per **agent** — `swarm-controller`
+# mints it at agent creation into `swarm/agents/<agent>/queue`
+# (`swarm_secret_client::queue::agent_queue_path`), at swarm level, with no
+# hive anywhere in the chain.
+#
+# That is also why this unit *fetches* rather than being handed a credential:
+# a hive courier in the path would be the hive vouching for which agent this
+# is, which is the property the per-agent credential exists to remove. The
+# agent authenticates to the store as itself, with the certificate ./bao.nix
+# already proves it can log in with, and reads its own path.
+#
+# ⛔ The store certificate is for reaching the store and nothing else. It is
+# never presented to the queue: what goes to the queue is the secret read
+# back from this path.
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.services.hyperhive.agent.bao;
+
+  # This container's agent name — the same string `swarm-controller` minted
+  # the credential under, because the agent's unix user is named for the
+  # agent (see ./user.nix).
+  agentName = config.services.hyperhive.agent.user.name;
+
+  # The same three ids ./bao.nix loads. Both units present the same
+  # certificate because there is one store identity per agent; the ids are
+  # `hive_c0re::lifecycle::agent_identity`'s and a rename is a rename there
+  # too.
+  certCredential = "hive-agent-bao-cert";
+  keyCredential = "hive-agent-bao-key";
+  serverCaCredential = "hive-agent-bao-server-ca";
+
+  unitName = "hive-agent-queue-credential";
+
+  # The nix half of `swarm_secret_client::queue::agent_queue_path` plus
+  # `path::MOUNT`, spelled exactly as ./bao.nix spells its own sibling path.
+  queuePath = "secret/swarm/agents/${agentName}/queue";
+
+  # `RuntimeDirectory=` under the unit's own `User=`, so the file is owned by
+  # the agent and readable by the harness without a mode change. /run and not
+  # the state dir on purpose: a secret fetched at boot has no business
+  # surviving one.
+  runtimeDir = "${unitName}";
+  secretFile = "/run/${runtimeDir}/secret";
+
+  # The store's address is the whole switch, exactly as in ./bao.nix — and
+  # deliberately *not* the queue coordinates in ./queue.nix. Those are the
+  # hive's, and gating a swarm-minted per-agent credential on a hive-level
+  # option would put the hive back in a path whose entire purpose is not
+  # having one.
+  configured = cfg.addr != null;
+in
+{
+  options.services.hyperhive.agent.queue.agentSecretFile = lib.mkOption {
+    type = lib.types.str;
+    readOnly = true;
+    default = secretFile;
+    description = ''
+      Path this agent's own swarm-queue secret is fetched to, for a consumer
+      outside the harness unit. Read-only for the same reason
+      {option}`services.hyperhive.agent.queue.clientSecretFile` is: it is a
+      fact about where the fetch writes, not a knob.
+
+      🩸 A PATH and never a value. The file is `0400` to the agent user and
+      is read at the moment it is needed; nothing in this tree puts its
+      contents in an environment variable, where `/proc/<pid>/environ` would
+      publish them to every process in the container.
+
+      The file exists only once the swarm has minted a credential for this
+      agent. An agent created before its swarm did so has none, and
+      `${unitName}.service` says so in the journal rather than failing —
+      see that unit.
+    '';
+  };
+
+  config = lib.mkIf configured {
+    systemd.services.${unitName} = {
+      description = "fetch this agent's own swarm-queue credential from the secret store";
+      after = [
+        "network.target"
+        # Ordering only, not a requirement: that unit is the one that reports
+        # a store this agent cannot reach as itself, and it should get to say
+        # so before this one reports a path it could not read.
+        "hive-agent-bao-identity.service"
+      ];
+      before = [ "hive-agent.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [
+        pkgs.openbao
+        pkgs.coreutils
+      ];
+      # Same sizing and the same `[Unit]`-not-`[Service]` placement as
+      # ./bao.nix's check: a few short attempts cover a store that comes up
+      # alongside this container, and a longer window only delays the report.
+      startLimitBurst = 4;
+      startLimitIntervalSec = 300;
+      serviceConfig = {
+        Type = "oneshot";
+        # Keeps the unit active, which is what keeps `RuntimeDirectory=`
+        # from being removed out from under the harness.
+        RemainAfterExit = true;
+        TimeoutStartSec = 30;
+        Restart = "on-failure";
+        RestartSec = 15;
+        User = agentName;
+        Group = agentName;
+        RuntimeDirectory = runtimeDir;
+        RuntimeDirectoryMode = "0700";
+        UMask = "0377";
+        # Bare ids, no paths: the terse `LoadCredential=` form that inherits
+        # a credential the service *manager* received, which is what the
+        # container manager passed in. ./bao.nix and ./queue.nix state the
+        # same shape.
+        LoadCredential = [
+          certCredential
+          keyCredential
+          serverCaCredential
+        ];
+      };
+      environment = {
+        BAO_ADDR = cfg.addr;
+        # `%d` is `$CREDENTIALS_DIRECTORY`, per-unit and owned by `User=`.
+        BAO_CLIENT_CERT = "%d/${certCredential}";
+        BAO_CLIENT_KEY = "%d/${keyCredential}";
+      };
+      script = ''
+        set -euo pipefail
+
+        # No identity delivered at all. ./bao.nix's check reports this as the
+        # failure it is; there is nothing for this unit to add, and failing
+        # here too would be the same cause stated twice.
+        for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do
+          if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then
+            echo "this agent has no store identity, so it cannot fetch its own queue credential." >&2
+            exit 0
+          fi
+        done
+
+        # Only when one was delivered — absent means verify the store's
+        # listener against the container's own trust store, which is what a
+        # deployment with a real CA wants. ./bao.nix says the same.
+        if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
+          export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
+        fi
+
+        err="$(mktemp)"
+        trap 'rm -f "$err"' EXIT
+
+        # Cert auth is a login, not a transport setting: the `BAO_CLIENT_*`
+        # variables above only pick the certificate the handshake presents.
+        # `-token-only` answers on stdout and skips the token helper, which
+        # is a `sh` this unit's `path` does not carry.
+        if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
+          echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2
+          if [ -s "$err" ]; then cat "$err" >&2; fi
+          exit 1
+        fi
+        export BAO_TOKEN
+
+        # 🩸 Degrades where ./bao.nix's check fails, and the reason is that
+        # the two reads are governed by the *same* policy stanza:
+        # `swarm_secret_client::policy::render_agent` grants read on
+        # `secret/data/swarm/agents/<agent>/*`, which covers this path and
+        # the `bao-mtls` one beside it alike. So a refusal this unit sees and
+        # that check did not cannot be a policy that drifted — it is an
+        # object that has not been minted, which is the ordinary state of
+        # every agent created before its swarm knew to mint one. A unit that
+        # failed at every boot over that would be loud about a deployment
+        # doing nothing wrong.
+        #
+        # ⚠️ Written by redirect into the runtime directory, never echoed:
+        # the field is the secret itself.
+        if ! bao kv get -field=value ${lib.escapeShellArg queuePath} > ${lib.escapeShellArg secretFile} 2>"$err"; then
+          rm -f ${lib.escapeShellArg secretFile}
+          echo "no per-agent queue credential at ${queuePath} yet; this agent falls back to its hive's shared one." >&2
+          if [ -s "$err" ]; then cat "$err" >&2; fi
+          exit 0
+        fi
+
+        echo "fetched this agent's own queue credential from ${queuePath}."
+      '';
+    };
+
+    # The harness reads a path and never a value, the same shape ./queue.nix
+    # hands it the hive-scoped secret in. Not `%d` here: this credential is
+    # not a systemd credential at all — it is a file this container fetched
+    # for itself, which is the whole point.
+    systemd.services.hive-agent = {
+      after = [ "${unitName}.service" ];
+      environment.HIVE_AGENT_QUEUE_AGENT_SECRET_FILE = secretFile;
+    };
+  };
+}

nix/module-eval/agent-queue-bao.nix

@@ -41,6 +41,7 @@ let
   agentNoBao = agentWith { };
 
   agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity;
+  agentQueueCredential = machine: machine.systemd.services.hive-agent-queue-credential;
   cases = [
     {
       # Both ids or neither: the secret authenticates nobody without the id it
@@ -164,6 +165,93 @@ let
       name = "an agent told no store address runs no identity check";
       ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity);
     }
+    {
+      # The per-agent credential's fetch rides on the same store identity the
+      # check above proves, because there is one identity per agent. A fetch
+      # unit loading different ids would be a second certificate nothing
+      # mints.
+      name = "the queue-credential fetch presents the agent's own store identity";
+      ok =
+        let
+          u = agentQueueCredential agentBao;
+        in
+        builtins.elem "hive-agent-bao-cert" u.serviceConfig.LoadCredential
+        && u.environment.BAO_CLIENT_CERT == "%d/hive-agent-bao-cert"
+        && u.environment.BAO_CLIENT_KEY == "%d/hive-agent-bao-key";
+    }
+    {
+      # Same 403-not-a-miss reason as every other reader here: the path
+      # `swarm_secret_client::queue::agent_queue_path` builds is the one this
+      # agent's own policy stanza covers. Built from the agent's own name,
+      # because the name is what makes it this agent's credential and not a
+      # neighbour's — which is the entire point of minting one per agent.
+      name = "the queue-credential fetch reads the agent's own per-agent path";
+      ok =
+        let
+          m = agentBao;
+          name = m.services.hyperhive.agent.user.name;
+        in
+        lib.hasInfix "secret/swarm/agents/${name}/queue" (agentQueueCredential m).script;
+    }
+    {
+      # ⛔ The store certificate authenticates against the store and nothing
+      # else. It must never reach the queue, so nothing in this unit may hand
+      # a `BAO_CLIENT_*` path to anything queue-shaped.
+      name = "the queue-credential fetch never points the queue at the store certificate";
+      ok =
+        let
+          u = agentQueueCredential agentBao;
+          harness = agentHarness agentBao;
+        in
+        !(lib.hasInfix "nats" u.script)
+        && !(lib.any (lib.hasPrefix "BAO_") (builtins.attrNames harness.environment));
+    }
+    {
+      # A secret fetched at boot has no business surviving one, and the
+      # directory has to be the unit's own so the file is owned by the agent
+      # rather than needing a mode change. `RemainAfterExit` is what keeps
+      # systemd from removing it out from under the harness.
+      name = "the fetched credential lands in a runtime directory the unit keeps alive";
+      ok =
+        let
+          u = agentQueueCredential agentBao;
+          c = u.serviceConfig;
+        in
+        c.RuntimeDirectory == "hive-agent-queue-credential"
+        && c.RemainAfterExit
+        && lib.hasInfix "/run/hive-agent-queue-credential/secret" u.script;
+    }
+    {
+      # 🩸 Degrades where the identity check fails, and the reason is that
+      # both reads are governed by one policy stanza: a refusal this unit
+      # sees and that check did not is an object not yet minted, not a policy
+      # that drifted. An agent created before its swarm minted one is a
+      # deployment doing nothing wrong.
+      name = "a per-agent credential that was never minted does not fail the unit";
+      ok = lib.hasInfix "exit 0" (agentQueueCredential agentBao).script;
+    }
+    {
+      # The harness reads a path and never a value — the same discipline
+      # ../agent-modules/queue.nix keeps for the hive-scoped secret. Not
+      # `%d`: this one is not a systemd credential, it is a file the
+      # container fetched for itself.
+      name = "the harness is handed the fetched credential as a path";
+      ok =
+        let
+          m = agentBao;
+        in
+        (agentHarness m).environment.HIVE_AGENT_QUEUE_AGENT_SECRET_FILE
+        == m.services.hyperhive.agent.queue.agentSecretFile;
+    }
+    {
+      # The absence arm. An agent whose swarm gave it no store has nothing to
+      # log in with, so there is nothing to fetch with either — and the
+      # harness is then told no path rather than one that never fills.
+      name = "an agent told no store address fetches no queue credential";
+      ok =
+        !(agentNoBao.systemd.services ? hive-agent-queue-credential)
+        && !((agentHarness agentNoBao).environment ? HIVE_AGENT_QUEUE_AGENT_SECRET_FILE);
+    }
   ];
 in
 runGroup "agent-queue-bao" cases