swarm-nats-auth: verify an agent's own token against the store

An `auth_token` spelled `swarm-agent.<agent>.<secret>` is no longer sent
to introspection. The responder reads `swarm/agents/<agent>/queue` with
an identity of its own, checks that the stored object names the same
agent, compares the secret in constant time, and grants the subjects
`--agent-token-publish-subject` lists with `{agent}` expanded. Every
other outcome denies: a malformed token, no store identity, nothing
stored, a failed or slow lookup, a different secret. A token without
the prefix takes the OIDC path unchanged.

The journal's `auth request` line names such a caller `agent:<agent>`;
the hive-shared credential keeps `hive-<h>-agent`.

The new principal: a `swarm-nats-auth` cert-auth role and policy with
read on `secret/data/swarm/agents/+/queue` alone, a leaf signed by the
store's PKI glue, and `glue-nats-auth-bao-identity.nix` pairing the two.
The copy unit delivers the identity into the queue's container, and an
absent leaf is delivered empty so the responder still starts and only
agent tokens are refused.

The policy and role are written by `swarm-bao-nats-auth-policy`, logged in
as the bao granter: both names fall under its `swarm-*` globs, so the
deploy writes them with no operator step. module-eval counts it among the
granting units, so every generic granting-unit case covers it.

The secret compare uses `subtle`, already in the lock file through the
TLS stack; no workspace crate offered one directly.

nix/host-modules/default.nix

@@ -32,6 +32,7 @@
     ./glue-grafana-oidc-client.nix
     ./glue-matrix-bao-token.nix
     ./glue-matrix-ctl-bao-identity.nix
+    ./glue-nats-auth-bao-identity.nix
     ./glue-nats-bao-identity.nix
     ./glue-queue-agent-credential.nix
     ./glue-secret-publisher-bao-identity.nix

nix/host-modules/glue-bao-tls.nix

@@ -260,6 +260,12 @@ in
         # leaf logs in with. Stays on this host; see its default above.
         [ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \
           ${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth
+
+        # The queue's auth-callout responder, which reads agent queue
+        # credentials. Minted here for the matrix-ctl leaf's reason: the queue
+        # is a swarm singleton, so elsewhere this is the file an operator copies.
+        [ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \
+          ${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth
       '';
     };
   };

nix/host-modules/glue-nats-auth-bao-identity.nix

@@ -0,0 +1,37 @@
+# Glue: point the queue's auth-callout responder at the bao leaf minted for it.
+#
+# ONE PAIRING PER FILE — the swarm-nats-auth principal ← bao, and nothing else.
+# Deleting this leaves a responder with no store identity unless the operator
+# names one: every agent token is then denied, and OIDC clients are unaffected.
+#
+# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the
+# leaf. What belongs here is the pairing: which paths the responder presents.
+#
+# ⚠️ Gated on the leaf existing, not on the store being enabled, for the reason
+# ./glue-nats-bao-identity.nix states: the hive hosting the queue need not be
+# the hive hosting the store.
+#
+# Everything is `mkDefault`. An operator naming their own paths wins.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  hyperhiveCfg = config.services.hyperhive;
+  deployCfg = hyperhiveCfg.deploy;
+  baoDeploy = deployCfg.bao;
+
+  # Where ./glue-bao-tls.nix puts the leaves, derived from the reader's own path
+  # rather than repeating that file's directory literal.
+  haveMintedPki = baoDeploy.clientCertFile != null;
+  pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null;
+in
+{
+  config = lib.mkIf (hyperhiveCfg.enable && deployCfg.nats.enable && haveMintedPki) {
+    services.hyperhive.deploy.nats = {
+      authBaoClientCertFile = lib.mkDefault "${pkiDir}/nats-auth.pem";
+      authBaoClientKeyFile = lib.mkDefault "${pkiDir}/nats-auth-key.pem";
+    };
+  };
+}

nix/host-modules/swarm-bao.nix

@@ -754,6 +754,18 @@ let
     }
   ];
 
+  # The queue's auth-callout responder, which checks the secret an agent
+  # presents against the one stored for it. One leaf under every agent: `+` is
+  # one path segment, where `*` globs only at the end and would reach every
+  # other credential an agent holds.
+  natsAuthReaders = [
+    {
+      name = "swarm-nats-auth";
+      cn = baoDeploy.natsAuthCommonName;
+      policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue";
+    }
+  ];
+
   # The role name IS the policy name, as for the three service principals
   # above: the role attaches the policy by spelling it identically, and one
   # string for both objects removes the way they drift apart.
@@ -1438,6 +1450,24 @@ in
       '';
     };
 
+    natsAuthCommonName = lib.mkOption {
+      type = lib.types.str;
+      default = "swarm-nats-auth";
+      description = ''
+        Subject the store's `swarm-nats-auth` cert-auth role accepts: the
+        identity the queue's auth-callout responder presents to read agent
+        queue credentials, `swarm/agents/<agent>/queue` and nothing else under
+        an agent.
+
+        Its own principal rather than
+        {option}`services.hyperhive.deploy.bao.natsCommonName`: that one
+        issues the queue's TLS leaf from a host unit, this one reads secrets
+        from inside the queue's container.
+
+        ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
+      '';
+    };
+
     matrixCtlHiveName = lib.mkOption {
       type = lib.types.str;
       default = toString hyperhiveCfg.hiveName;
@@ -2021,6 +2051,7 @@ in
         "swarm-bao-services-issuer-policy"
         "swarm-bao-nats-tls-policy"
         "swarm-bao-agent-pki"
+        "swarm-bao-nats-auth-policy"
       ];
 
       # 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is
@@ -2723,6 +2754,8 @@ in
 
       systemd.services.swarm-bao-forwarder-oidc-policy = readerPolicyUnit "write the store forwarder's OIDC-secret-reader bao policy and cert-auth role" forwarderOidcReaders;
 
+      systemd.services.swarm-bao-nats-auth-policy = readerPolicyUnit "write the queue responder's agent-credential-reader bao policy and cert-auth role" natsAuthReaders;
+
       # A FOURTH sibling, same shape and same reasons as the two above. This
       # one is what turns `swarm-services-issuer` from a declaration into a
       # grant: a bao policy reaches nothing until a login role hands it to a

nix/host-modules/swarm-nats.nix

@@ -219,6 +219,25 @@ let
   tlsKeyCredential = "tls-key";
   tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
 
+  # The responder's own store identity, for reading agent queue credentials.
+  # Without it the responder denies every agent token; agents then fall back to
+  # their hive's OIDC client.
+  authStoreActive =
+    deployCfg.nats.authBaoClientCertFile != null && deployCfg.nats.authBaoClientKeyFile != null;
+  # The role ./swarm-bao.nix writes on the store's `cert` mount, by the same name.
+  authCertRole = "swarm-nats-auth";
+  # Store identity files the copy unit delivers, as credential id → host source.
+  authStoreFiles = lib.optionalAttrs authStoreActive (
+    {
+      "bao-client.pem" = deployCfg.nats.authBaoClientCertFile;
+      "bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile;
+    }
+    // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
+      "bao-ca.pem" = baoDeploy.serverCaFile;
+    }
+  );
+  authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
+
   # Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats`
   # grants it (./swarm-bao.nix), so the daily timer below has two weeks of
   # retries before it lapses.
@@ -498,6 +517,34 @@ in
         A path, never a value.
       '';
     };
+
+    authBaoClientCertFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "/var/lib/swarm-bao-pki/nats-auth.pem";
+      description = ''
+        Client certificate the auth-callout responder presents to the swarm's
+        secret store to read agent queue credentials. Its subject must be
+        {option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
+
+        No default. ./glue-nats-auth-bao-identity.nix points it at the leaf
+        ./glue-bao-tls.nix mints, where this host mints one. Unset, or set to
+        a file that does not exist, the responder denies every agent token and
+        admits OIDC clients as before.
+
+        A path, never a value.
+      '';
+    };
+
+    authBaoClientKeyFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "/var/lib/swarm-bao-pki/nats-auth-key.pem";
+      description = ''
+        Private key for {option}`services.hyperhive.deploy.nats.authBaoClientCertFile`.
+        A path, never a value.
+      '';
+    };
   };
 
   config = lib.mkIf deployCfg.nats.enable {
@@ -905,6 +952,11 @@ in
                 # an append-only row stream, a header is one current value
                 # republished on change.
                 "--agent-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{hive}.>"}"
+                # What an agent that proved its own credential may publish to:
+                # the same two streams, keyed on the agent alone.
+                "--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{agent}"}"
+                "--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{agent}"}"
+                "--store-cert-role ${lib.escapeShellArg authCertRole}"
               ];
               # Every credential arrives by `LoadCredential` and is named
               # on the command line only as a **path** — `argv` is
@@ -914,12 +966,25 @@ in
                 "callout-user.seed:${inContainer "callout-user.seed"}"
                 "issuer.seed:${inContainer "issuer.seed"}"
                 "oidc-client.secret:${inContainer "oidc-client.secret"}"
-              ];
+              ]
+              ++ lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles;
               DynamicUser = true;
               Restart = "on-failure";
               RestartSec = "5s";
               SyslogIdentifier = "swarm-nats-auth";
             };
+            # The store the responder reads agent credentials from. Absent
+            # without an identity, which the responder reads as "no store".
+            environment = lib.optionalAttrs authStoreActive (
+              {
+                BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
+                BAO_CLIENT_CERT = authCredential "bao-client.pem";
+                BAO_CLIENT_KEY = authCredential "bao-client-key.pem";
+              }
+              // lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") {
+                BAO_CACERT = authCredential "bao-ca.pem";
+              }
+            );
           };
 
           # The server binary, so an operator with a shell in here can
@@ -957,7 +1022,10 @@ in
         # being up says nothing about whether its in-container secrets unit
         # has finished. The wait in the script is what actually closes it;
         # this only stops us spinning for the full timeout on every boot.
-        ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service";
+        ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"
+        # Where the store's PKI is minted on this host, the responder's leaf is
+        # one of its files. Ordering only: elsewhere the unit does not exist.
+        ++ lib.optional authStoreActive "swarm-bao-pki.service";
       requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service";
       serviceConfig = {
         Type = "oneshot";
@@ -1006,7 +1074,22 @@ in
 
         install -m 0400 "$secret" \
           ${lib.escapeShellArg (hostPath "oidc-client.secret")}
-      '';
+      ''
+      # The store identity is optional to the responder, so an absent source
+      # is delivered as an empty file rather than failing this unit: a missing
+      # `LoadCredential` source stops the responder starting, and a responder
+      # that does not start denies every client. An empty identity fails only
+      # the store login, which denies agent tokens.
+      + lib.concatStrings (
+        lib.mapAttrsToList (id: source: ''
+          if [ -s ${lib.escapeShellArg source} ]; then
+            install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)}
+          else
+            echo "${source} is absent; the queue responder denies agent tokens until it exists" >&2
+            install -m 0400 /dev/null ${lib.escapeShellArg (hostPath id)}
+          fi
+        '') authStoreFiles
+      );
     };
 
     # ⚠️ Minted on the HOST, not in the container, because the responder is

nix/host-modules/swarm.nix

@@ -53,6 +53,7 @@ let
     deployCfg.bao.servicesIssuerCommonName
     deployCfg.bao.natsCommonName
     deployCfg.bao.granterCommonName
+    deployCfg.bao.natsAuthCommonName
   ]
   # The two per-hive readers' subjects, spelled out per hive rather than as the
   # prefix. The prefix alone would reserve the wrong string: the role for hive