swarm-nats-auth: verify an agent's own token against the store

An `auth_token` spelled `swarm-agent.<agent>.<secret>` is no longer sent
to introspection. The responder reads `swarm/agents/<agent>/queue` with
an identity of its own, checks that the stored object names the same
agent, compares the secret in constant time, and grants the subjects
`--agent-token-publish-subject` lists with `{agent}` expanded. Every
other outcome denies: a malformed token, no store identity, nothing
stored, a failed or slow lookup, a different secret. A token without
the prefix takes the OIDC path unchanged.

The journal's `auth request` line names such a caller `agent:<agent>`;
the hive-shared credential keeps `hive-<h>-agent`.

The new principal: a `swarm-nats-auth` cert-auth role and policy with
read on `secret/data/swarm/agents/+/queue` alone, a leaf signed by the
store's PKI glue, and `glue-nats-auth-bao-identity.nix` pairing the two.
The copy unit delivers the identity into the queue's container, and an
absent leaf is delivered empty so the responder still starts and only
agent tokens are refused.

The policy and role are written by `swarm-bao-nats-auth-policy`, logged in
as the bao granter: both names fall under its `swarm-*` globs, so the
deploy writes them with no operator step. module-eval counts it among the
granting units, so every generic granting-unit case covers it.

The secret compare uses `subtle`, already in the lock file through the
TLS stack; no workspace crate offered one directly.

Cargo.lock

@@ -4828,7 +4828,9 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2 0.11.0",
+ "subtle",
  "swarm-queue-client",
+ "swarm-secret-client",
  "tokio",
  "tracing",
  "tracing-subscriber",

Cargo.toml

@@ -230,6 +230,9 @@ matrix-sdk = { version = "0.18", default-features = false, features = [
 futures-util = "0.3"
 hmac = "0.13"
 sha2 = "0.11"
+# Constant-time comparison of a presented secret against the stored one
+# (`swarm-nats-auth::agent_token`). Already in the tree through the TLS stack.
+subtle = "2.6"
 # The NATS protocol client, for the swarm queue's auth-callout responder.
 # `default-features = false` because the default set is broad - jetstream, kv,
 # object-store, websockets, service - and a callout responder speaks none of

nix/host-modules/default.nix

@@ -32,6 +32,7 @@
     ./glue-grafana-oidc-client.nix
     ./glue-matrix-bao-token.nix
     ./glue-matrix-ctl-bao-identity.nix
+    ./glue-nats-auth-bao-identity.nix
     ./glue-nats-bao-identity.nix
     ./glue-queue-agent-credential.nix
     ./glue-secret-publisher-bao-identity.nix

nix/host-modules/glue-bao-tls.nix

@@ -260,6 +260,12 @@ in
         # leaf logs in with. Stays on this host; see its default above.
         [ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \
           ${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth
+
+        # The queue's auth-callout responder, which reads agent queue
+        # credentials. Minted here for the matrix-ctl leaf's reason: the queue
+        # is a swarm singleton, so elsewhere this is the file an operator copies.
+        [ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \
+          ${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth
       '';
     };
   };

nix/host-modules/glue-nats-auth-bao-identity.nix

@@ -0,0 +1,37 @@
+# Glue: point the queue's auth-callout responder at the bao leaf minted for it.
+#
+# ONE PAIRING PER FILE — the swarm-nats-auth principal ← bao, and nothing else.
+# Deleting this leaves a responder with no store identity unless the operator
+# names one: every agent token is then denied, and OIDC clients are unaffected.
+#
+# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the
+# leaf. What belongs here is the pairing: which paths the responder presents.
+#
+# ⚠️ Gated on the leaf existing, not on the store being enabled, for the reason
+# ./glue-nats-bao-identity.nix states: the hive hosting the queue need not be
+# the hive hosting the store.
+#
+# Everything is `mkDefault`. An operator naming their own paths wins.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  hyperhiveCfg = config.services.hyperhive;
+  deployCfg = hyperhiveCfg.deploy;
+  baoDeploy = deployCfg.bao;
+
+  # Where ./glue-bao-tls.nix puts the leaves, derived from the reader's own path
+  # rather than repeating that file's directory literal.
+  haveMintedPki = baoDeploy.clientCertFile != null;
+  pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null;
+in
+{
+  config = lib.mkIf (hyperhiveCfg.enable && deployCfg.nats.enable && haveMintedPki) {
+    services.hyperhive.deploy.nats = {
+      authBaoClientCertFile = lib.mkDefault "${pkiDir}/nats-auth.pem";
+      authBaoClientKeyFile = lib.mkDefault "${pkiDir}/nats-auth-key.pem";
+    };
+  };
+}

nix/host-modules/swarm-bao.nix

@@ -754,6 +754,18 @@ let
     }
   ];
 
+  # The queue's auth-callout responder, which checks the secret an agent
+  # presents against the one stored for it. One leaf under every agent: `+` is
+  # one path segment, where `*` globs only at the end and would reach every
+  # other credential an agent holds.
+  natsAuthReaders = [
+    {
+      name = "swarm-nats-auth";
+      cn = baoDeploy.natsAuthCommonName;
+      policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue";
+    }
+  ];
+
   # The role name IS the policy name, as for the three service principals
   # above: the role attaches the policy by spelling it identically, and one
   # string for both objects removes the way they drift apart.
@@ -1438,6 +1450,24 @@ in
       '';
     };
 
+    natsAuthCommonName = lib.mkOption {
+      type = lib.types.str;
+      default = "swarm-nats-auth";
+      description = ''
+        Subject the store's `swarm-nats-auth` cert-auth role accepts: the
+        identity the queue's auth-callout responder presents to read agent
+        queue credentials, `swarm/agents/<agent>/queue` and nothing else under
+        an agent.
+
+        Its own principal rather than
+        {option}`services.hyperhive.deploy.bao.natsCommonName`: that one
+        issues the queue's TLS leaf from a host unit, this one reads secrets
+        from inside the queue's container.
+
+        ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
+      '';
+    };
+
     matrixCtlHiveName = lib.mkOption {
       type = lib.types.str;
       default = toString hyperhiveCfg.hiveName;
@@ -2021,6 +2051,7 @@ in
         "swarm-bao-services-issuer-policy"
         "swarm-bao-nats-tls-policy"
         "swarm-bao-agent-pki"
+        "swarm-bao-nats-auth-policy"
       ];
 
       # 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is
@@ -2723,6 +2754,8 @@ in
 
       systemd.services.swarm-bao-forwarder-oidc-policy = readerPolicyUnit "write the store forwarder's OIDC-secret-reader bao policy and cert-auth role" forwarderOidcReaders;
 
+      systemd.services.swarm-bao-nats-auth-policy = readerPolicyUnit "write the queue responder's agent-credential-reader bao policy and cert-auth role" natsAuthReaders;
+
       # A FOURTH sibling, same shape and same reasons as the two above. This
       # one is what turns `swarm-services-issuer` from a declaration into a
       # grant: a bao policy reaches nothing until a login role hands it to a

nix/host-modules/swarm-nats.nix

@@ -219,6 +219,25 @@ let
   tlsKeyCredential = "tls-key";
   tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
 
+  # The responder's own store identity, for reading agent queue credentials.
+  # Without it the responder denies every agent token; agents then fall back to
+  # their hive's OIDC client.
+  authStoreActive =
+    deployCfg.nats.authBaoClientCertFile != null && deployCfg.nats.authBaoClientKeyFile != null;
+  # The role ./swarm-bao.nix writes on the store's `cert` mount, by the same name.
+  authCertRole = "swarm-nats-auth";
+  # Store identity files the copy unit delivers, as credential id → host source.
+  authStoreFiles = lib.optionalAttrs authStoreActive (
+    {
+      "bao-client.pem" = deployCfg.nats.authBaoClientCertFile;
+      "bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile;
+    }
+    // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
+      "bao-ca.pem" = baoDeploy.serverCaFile;
+    }
+  );
+  authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
+
   # Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats`
   # grants it (./swarm-bao.nix), so the daily timer below has two weeks of
   # retries before it lapses.
@@ -498,6 +517,34 @@ in
         A path, never a value.
       '';
     };
+
+    authBaoClientCertFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "/var/lib/swarm-bao-pki/nats-auth.pem";
+      description = ''
+        Client certificate the auth-callout responder presents to the swarm's
+        secret store to read agent queue credentials. Its subject must be
+        {option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
+
+        No default. ./glue-nats-auth-bao-identity.nix points it at the leaf
+        ./glue-bao-tls.nix mints, where this host mints one. Unset, or set to
+        a file that does not exist, the responder denies every agent token and
+        admits OIDC clients as before.
+
+        A path, never a value.
+      '';
+    };
+
+    authBaoClientKeyFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "/var/lib/swarm-bao-pki/nats-auth-key.pem";
+      description = ''
+        Private key for {option}`services.hyperhive.deploy.nats.authBaoClientCertFile`.
+        A path, never a value.
+      '';
+    };
   };
 
   config = lib.mkIf deployCfg.nats.enable {
@@ -905,6 +952,11 @@ in
                 # an append-only row stream, a header is one current value
                 # republished on change.
                 "--agent-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{hive}.>"}"
+                # What an agent that proved its own credential may publish to:
+                # the same two streams, keyed on the agent alone.
+                "--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{agent}"}"
+                "--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{agent}"}"
+                "--store-cert-role ${lib.escapeShellArg authCertRole}"
               ];
               # Every credential arrives by `LoadCredential` and is named
               # on the command line only as a **path** — `argv` is
@@ -914,12 +966,25 @@ in
                 "callout-user.seed:${inContainer "callout-user.seed"}"
                 "issuer.seed:${inContainer "issuer.seed"}"
                 "oidc-client.secret:${inContainer "oidc-client.secret"}"
-              ];
+              ]
+              ++ lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles;
               DynamicUser = true;
               Restart = "on-failure";
               RestartSec = "5s";
               SyslogIdentifier = "swarm-nats-auth";
             };
+            # The store the responder reads agent credentials from. Absent
+            # without an identity, which the responder reads as "no store".
+            environment = lib.optionalAttrs authStoreActive (
+              {
+                BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
+                BAO_CLIENT_CERT = authCredential "bao-client.pem";
+                BAO_CLIENT_KEY = authCredential "bao-client-key.pem";
+              }
+              // lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") {
+                BAO_CACERT = authCredential "bao-ca.pem";
+              }
+            );
           };
 
           # The server binary, so an operator with a shell in here can
@@ -957,7 +1022,10 @@ in
         # being up says nothing about whether its in-container secrets unit
         # has finished. The wait in the script is what actually closes it;
         # this only stops us spinning for the full timeout on every boot.
-        ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service";
+        ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"
+        # Where the store's PKI is minted on this host, the responder's leaf is
+        # one of its files. Ordering only: elsewhere the unit does not exist.
+        ++ lib.optional authStoreActive "swarm-bao-pki.service";
       requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service";
       serviceConfig = {
         Type = "oneshot";
@@ -1006,7 +1074,22 @@ in
 
         install -m 0400 "$secret" \
           ${lib.escapeShellArg (hostPath "oidc-client.secret")}
-      '';
+      ''
+      # The store identity is optional to the responder, so an absent source
+      # is delivered as an empty file rather than failing this unit: a missing
+      # `LoadCredential` source stops the responder starting, and a responder
+      # that does not start denies every client. An empty identity fails only
+      # the store login, which denies agent tokens.
+      + lib.concatStrings (
+        lib.mapAttrsToList (id: source: ''
+          if [ -s ${lib.escapeShellArg source} ]; then
+            install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)}
+          else
+            echo "${source} is absent; the queue responder denies agent tokens until it exists" >&2
+            install -m 0400 /dev/null ${lib.escapeShellArg (hostPath id)}
+          fi
+        '') authStoreFiles
+      );
     };
 
     # ⚠️ Minted on the HOST, not in the container, because the responder is

nix/host-modules/swarm.nix

@@ -53,6 +53,7 @@ let
     deployCfg.bao.servicesIssuerCommonName
     deployCfg.bao.natsCommonName
     deployCfg.bao.granterCommonName
+    deployCfg.bao.natsAuthCommonName
   ]
   # The two per-hive readers' subjects, spelled out per hive rather than as the
   # prefix. The prefix alone would reserve the wrong string: the role for hive

nix/module-eval/bao-grants.nix

@@ -151,7 +151,7 @@ let
     _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
   ) baoGrantWithConsumers.systemd.services;
 
-  # The eleven units that write a `swarm-*` grant, by name, for the discovery
+  # The twelve units that write a `swarm-*` grant, by name, for the discovery
   # control below.
   grantingUnitNames = [
     "swarm-bao-controller-policy"
@@ -165,6 +165,7 @@ let
     "swarm-bao-services-issuer-policy"
     "swarm-bao-nats-tls-policy"
     "swarm-bao-agent-pki"
+    "swarm-bao-nats-auth-policy"
   ];
 
   # Comment lines dropped first: both the HCL and the scripts explain
@@ -561,6 +562,33 @@ let
         && !(lib.hasInfix "swarm-grafana" s)
         && !(lib.hasInfix "sys/policies/acl" s);
     }
+    {
+      # `+` is one path segment, so this reaches `swarm/agents/<agent>/queue`
+      # and no other credential an agent holds; `swarm/agents/*` would reach
+      # all of them.
+      name = "the queue responder's grant is every agent's queue credential and nothing else";
+      ok =
+        let
+          s = baoGrantHere.systemd.services.swarm-bao-nats-auth-policy.script;
+        in
+        lib.hasInfix "path \"secret/data/swarm/agents/+/queue\" {" s
+        && lib.hasInfix "capabilities = [\"read\"]" s
+        && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1
+        && !(lib.hasInfix "secret/data/swarm/agents/*" s)
+        && !(lib.hasInfix "secret/data/swarm/hives" s)
+        && !(lib.hasInfix "secret/data/swarm/services" s)
+        && lib.hasInfix "auth/cert/certs/swarm-nats-auth" s
+        && lib.hasInfix "allowed_common_names=swarm-nats-auth" s
+        && lib.hasInfix "token_policies=swarm-nats-auth" s;
+    }
+    {
+      name = "the PKI unit signs the queue responder's leaf under its own subject";
+      ok =
+        let
+          s = baoGrantHere.systemd.services.swarm-bao-pki.script;
+        in
+        lib.hasInfix "/nats-auth.pem ]" s && lib.hasInfix "swarm-nats-auth \"\" clientAuth" s;
+    }
     {
       # 🩸 The half that makes the policies above bind: a policy grants only
       # through a token that carries it, and a token is minted by a cert-auth
@@ -748,24 +776,24 @@ let
     }
     {
       # A store host without the granter's pair writes its grants some other
-      # way, so none of the eleven units may exist. Without this arm
+      # way, so none of the twelve units may exist. Without this arm
       # `lib.mkIf haveGranter` could be dropped from any of them and every other
       # case here would still pass.
-      name = "without the granter's pair none of the eleven granting units render";
+      name = "without the granter's pair none of the twelve granting units render";
       ok =
         let
           s = baoGranterOptOut.systemd.services;
         in
         lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
-        # The control: the same store with the pair renders all eleven.
+        # The control: the same store with the pair renders all twelve.
         && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
     }
     {
-      # 🩸 What replaced the silent skip. With no bootstrap token the eleven still
+      # 🩸 What replaced the silent skip. With no bootstrap token the twelve still
       # render, and a refused granter fails them with the step that fixes it.
       # A store host that never named a token is told to name one, since the
       # unit that sets the granter up renders only where it has.
-      name = "a store host without a bootstrap token renders the eleven, each failing loudly with the one-time step";
+      name = "a store host without a bootstrap token renders the twelve, each failing loudly with the one-time step";
       ok =
         let
           s = baoGranterNoToken.systemd.services;
@@ -1129,8 +1157,8 @@ let
     }
     {
       # What makes the case above mean something: discovery by the granter's
-      # certificate reaches all eleven units, and each yields calls.
+      # certificate reaches all twelve units, and each yields calls.
-      name = "the granter-policy check sees all eleven granting units, and parses calls from each";
+      name = "the granter-policy check sees all twelve granting units, and parses calls from each";
       ok =
         lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
         && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)

nix/module-eval/nats-authelia.nix

@@ -60,6 +60,28 @@ let
     swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
   };
 
+  # The queue with a store identity for its responder, placed by hand: the
+  # queue host that is not the store's.
+  natsWithStoreIdentity = hive {
+    deploy.nats.enable = true;
+    deploy.nats.autoGenerateCallout = false;
+    deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+    deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+    deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
+    deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
+    deploy.nats.authPackage = pkgs.emptyDirectory;
+    deploy.nats.authBaoClientCertFile = "/etc/pki/nats-auth.pem";
+    deploy.nats.authBaoClientKeyFile = "/etc/pki/nats-auth-key.pem";
+  };
+
+  # The queue on the store's own host, where the PKI glue mints every leaf.
+  natsOnStoreHost = hive {
+    deploy.bao.enable = true;
+    deploy.nats.enable = true;
+  };
+
+  responderOf = h: h.containers.swarm-nats.config.systemd.services.swarm-nats-auth;
+
   # A hive running NOTHING of the swarm's own services — no IdP here, no
   # `swarm.authelia.url` set by hand. The whole point of the fixture is what it
   # does *not* say: it is the shape whose IdP address used to be null, and
@@ -117,6 +139,73 @@ let
         in
         lib.hasInfix "--agent-publish-subject '$$SWARM.agent-state.{hive}.>'" exec;
     }
+    {
+      # The per-agent grant: the same two streams keyed on the agent alone,
+      # with the same doubled dollar, and the role the store writes for it.
+      name = "the responder grants a verified agent its own hive-free subjects";
+      ok =
+        let
+          exec = (responderOf natsOldPath).serviceConfig.ExecStart;
+        in
+        lib.hasInfix "--agent-token-publish-subject '$$SWARM.term.{agent}'" exec
+        && lib.hasInfix "--agent-token-publish-subject '$$SWARM.agent-state.{agent}'" exec
+        && lib.hasInfix "--store-cert-role swarm-nats-auth" exec;
+    }
+    {
+      # Each credential by `LoadCredential`, and the environment naming where
+      # the unit sees it: a DynamicUser cannot read the copies directly.
+      name = "a responder with a store identity loads it and is pointed at the store";
+      ok =
+        let
+          u = responderOf natsWithStoreIdentity;
+          creds = u.serviceConfig.LoadCredential;
+        in
+        lib.elem "bao-client.pem:/var/lib/swarm-nats-auth/bao-client.pem" creds
+        && lib.elem "bao-client-key.pem:/var/lib/swarm-nats-auth/bao-client-key.pem" creds
+        && u.environment.BAO_CLIENT_CERT == "/run/credentials/swarm-nats-auth.service/bao-client.pem"
+        && u.environment.BAO_CLIENT_KEY == "/run/credentials/swarm-nats-auth.service/bao-client-key.pem"
+        && lib.hasPrefix "https://" u.environment.BAO_ADDR;
+    }
+    {
+      # An absent leaf must not stop the responder, which would deny every
+      # client: it is delivered empty, and the three credentials the responder
+      # cannot run without are delivered too.
+      name = "the copy unit delivers the store identity, and an absent one as an empty file";
+      ok =
+        let
+          s = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script;
+        in
+        lib.hasInfix "install -m 0400 /etc/pki/nats-auth.pem " s
+        && lib.hasInfix "install -m 0400 /etc/pki/nats-auth-key.pem " s
+        && lib.hasInfix "install -m 0400 /dev/null " s
+        && lib.hasInfix "oidc-client.secret" s;
+    }
+    {
+      # The control: no identity, no store wiring, and the responder's
+      # credentials are exactly the three it cannot run without.
+      name = "a responder with no store identity is not pointed at a store";
+      ok =
+        let
+          u = responderOf natsOldPath;
+        in
+        !(u.environment ? BAO_ADDR)
+        && !(u.environment ? BAO_CLIENT_CERT)
+        && lib.length u.serviceConfig.LoadCredential == 3;
+    }
+    {
+      # On the store's host the glue pairs the responder with a leaf of its
+      # own, never the queue's TLS-issuing one or the hive's.
+      name = "on the store's host the responder presents its own leaf";
+      ok =
+        let
+          n = natsOnStoreHost.services.hyperhive.deploy.nats;
+          b = natsOnStoreHost.services.hyperhive.deploy.bao;
+        in
+        n.authBaoClientCertFile == "/var/lib/swarm-bao-pki/nats-auth.pem"
+        && n.authBaoClientKeyFile == "/var/lib/swarm-bao-pki/nats-auth-key.pem"
+        && n.authBaoClientCertFile != n.baoClientCertFile
+        && n.authBaoClientCertFile != b.clientCertFile;
+    }
     {
       # Not a rename test. `hostClientSecretDir` is `readOnly`, so the fixture
       # cannot define it; what can break is a reader left pointing at the

swarm-nats-auth/Cargo.toml

@@ -22,12 +22,17 @@ serde.workspace = true
 serde_json.workspace = true
 # The jti digest: base32hex(sha256(claims)) over every JWT this crate signs.
 sha2.workspace = true
+# Comparing an agent's presented secret with the stored one.
+subtle.workspace = true
 # For `status::BUCKET` and `notices::STREAM` - the subjects a hive may
 # publish to are derived from these names, and every end that touches them
 # must agree on the same one. Deliberately WITHOUT the `kv` feature: this
 # crate derives subject strings, it never opens the bucket. `notices`
 # is name-only too (no `jetstream`/`kv` surface), same reason.
 swarm-queue-client = { workspace = true, features = ["notices"] }
+# The agent-token spelling the agent also uses, and the read of the stored
+# credential it is checked against.
+swarm-secret-client.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true

swarm-nats-auth/src/agent_token.rs

@@ -0,0 +1,382 @@
+//! Verifying an agent's own credential, presented as `auth_token`.
+//!
+//! The spelling is `swarm_queue_client::agent_token`'s, shared with the agent
+//! that presents it: a prefix, the agent's name, and its secret. The name is only a
+//! claim. It becomes an identity when the secret equals the one stored at
+//! `swarm/agents/<agent>/queue`, and nothing in this module grants on the name
+//! alone: every path that does not reach that comparison, and every lookup
+//! that fails, is a denial.
+//!
+//! A token without the prefix is not this module's: it goes to introspection
+//! unchanged.
+
+use std::future::Future;
+
+use anyhow::Context;
+use subtle::ConstantTimeEq;
+use swarm_queue_client::agent_token::{AgentToken, Malformed, parse_agent_token};
+use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings};
+use swarm_secret_client::queue::{self, AgentCredential};
+
+use crate::policy::{Permissions, Policy};
+
+/// What a presented `auth_token` is.
+pub enum Presented<'a> {
+    /// An agent's own credential, to be checked against the store.
+    Agent(AgentToken<'a>),
+    /// Carries the agent-token prefix but is not well-formed. Denied without a
+    /// lookup, and never handed to introspection: it holds a secret meant for
+    /// the store, not for the `IdP`.
+    Malformed(Malformed),
+    /// Anything else, which is an OIDC access token for introspection.
+    Bearer(&'a str),
+}
+
+/// Sort `token` onto the agent path or the introspection path.
+pub fn classify(token: &str) -> Presented<'_> {
+    match parse_agent_token(token) {
+        Some(Ok(agent)) => Presented::Agent(agent),
+        Some(Err(e)) => Presented::Malformed(e),
+        None => Presented::Bearer(token),
+    }
+}
+
+/// Where the stored credential for an agent comes from.
+pub trait CredentialSource {
+    /// The object at `agent`'s queue path, `None` when nothing is stored
+    /// there, or an error when the store could not answer.
+    fn lookup(
+        &self,
+        agent: &str,
+    ) -> impl Future<Output = anyhow::Result<Option<AgentCredential>>> + Send;
+}
+
+/// The swarm secret store, reached with this responder's own certificate.
+pub struct Store {
+    settings: Settings,
+    cert_role: String,
+}
+
+impl Store {
+    /// The store named by the `BAO_*` environment, or `None` when that is
+    /// unset: a responder with no store identity denies every agent token and
+    /// serves the OIDC path unchanged.
+    pub fn from_env(cert_role: String) -> Option<Self> {
+        match Settings::from_env() {
+            Ok(settings) => Some(Self {
+                settings,
+                cert_role,
+            }),
+            Err(e) => {
+                tracing::info!(reason = %e, "no secret store configured; agent tokens are denied");
+                None
+            }
+        }
+    }
+}
+
+impl CredentialSource for Store {
+    /// Logs in per lookup. An agent connects once per boot and per reconnect,
+    /// so a login each time costs little, and it leaves no token to expire
+    /// inside a long-lived process.
+    async fn lookup(&self, agent: &str) -> anyhow::Result<Option<AgentCredential>> {
+        let path = queue::agent_queue_path(agent)?;
+        let store = swarm_secret_client::SecretStore::connect(
+            &self.settings,
+            &self.cert_role,
+            DEFAULT_CERT_MOUNT,
+        )
+        .await
+        .context("logging in to the secret store")?;
+        store
+            .read_optional(&path)
+            .await
+            .with_context(|| format!("reading {path}"))
+    }
+}
+
+/// Whether `token`'s secret is the one stored for the agent it names.
+///
+/// The lookup is bounded by introspection's budget, under the server's
+/// `authorization.timeout`. The callout loop answers one request at a time, so
+/// a store that does not answer holds every request behind it, OIDC ones
+/// included, for at most that long, and this then denies.
+async fn verify(source: &impl CredentialSource, token: &AgentToken<'_>) -> bool {
+    let agent = token.agent;
+    let stored = match tokio::time::timeout(
+        crate::introspect::INTROSPECTION_TIMEOUT,
+        source.lookup(agent),
+    )
+    .await
+    {
+        Ok(Ok(Some(stored))) => stored,
+        Ok(Ok(None)) => {
+            tracing::warn!(
+                agent,
+                "no queue credential is stored for this agent; denying"
+            );
+            return false;
+        }
+        Ok(Err(e)) => {
+            tracing::warn!(
+                agent,
+                error = format!("{e:#}"),
+                "credential lookup failed; denying"
+            );
+            return false;
+        }
+        Err(_) => {
+            tracing::warn!(agent, "credential lookup timed out; denying");
+            return false;
+        }
+    };
+    if stored.agent != agent {
+        tracing::warn!(
+            agent,
+            stored_agent = %stored.agent,
+            "the stored credential names a different agent than its path; denying"
+        );
+        return false;
+    }
+    // Constant time, so the time to refuse says nothing about how much of the
+    // secret was right. A length mismatch returns early; the length is not
+    // secret, every minted secret has the same one.
+    stored
+        .value
+        .as_bytes()
+        .ct_eq(token.secret.as_bytes())
+        .into()
+}
+
+/// The grant for an agent token, or `None` for a denial.
+///
+/// `source` is `None` when this responder has no store identity, which denies.
+pub async fn authorize<S: CredentialSource>(
+    policy: &Policy,
+    source: Option<&S>,
+    token: &AgentToken<'_>,
+) -> Option<Permissions> {
+    let Some(source) = source else {
+        tracing::warn!(
+            agent = token.agent,
+            "agent token presented, but this responder has no secret store; denying"
+        );
+        return None;
+    };
+    if !verify(source, token).await {
+        return None;
+    }
+    let permissions = policy.agent_token_permissions(token.agent);
+    if permissions.is_none() {
+        tracing::warn!(
+            agent = token.agent,
+            "verified agent token, but no --agent-token-publish-subject is configured; denying"
+        );
+    }
+    permissions
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A store holding at most one credential, or failing outright.
+    enum Fake {
+        /// `credential` is stored at `path_agent`'s queue path.
+        Holds {
+            path_agent: &'static str,
+            credential: AgentCredential,
+        },
+        Fails,
+        /// A store that accepts the request and never answers.
+        Hangs,
+    }
+
+    impl CredentialSource for Fake {
+        async fn lookup(&self, agent: &str) -> anyhow::Result<Option<AgentCredential>> {
+            match self {
+                Self::Holds {
+                    path_agent,
+                    credential,
+                } => Ok((agent == *path_agent).then(|| credential.clone())),
+                Self::Fails => anyhow::bail!("store unreachable"),
+                Self::Hangs => std::future::pending().await,
+            }
+        }
+    }
+
+    fn stored(path_agent: &'static str, named: &str) -> Fake {
+        Fake::Holds {
+            path_agent,
+            credential: AgentCredential {
+                value: "Ab9_-zSECRET".to_owned(),
+                agent: named.to_owned(),
+            },
+        }
+    }
+
+    fn atlas() -> Fake {
+        stored("atlas", "atlas")
+    }
+
+    fn policy() -> Policy {
+        Policy::new(
+            "hive-".to_owned(),
+            "-agent".to_owned(),
+            "hive-status".to_owned(),
+            vec!["swarm-controller".to_owned()],
+            vec![],
+            vec!["$SWARM.term.{hive}.>".to_owned()],
+        )
+        .expect("valid")
+        .with_agent_token_subjects(vec![
+            "$SWARM.term.{agent}".to_owned(),
+            "$SWARM.agent-state.{agent}".to_owned(),
+        ])
+        .expect("valid")
+    }
+
+    async fn grant(source: Option<&Fake>, token: &str) -> Option<Permissions> {
+        let Presented::Agent(token) = classify(token) else {
+            panic!("{token:?} must classify as an agent token");
+        };
+        authorize(&policy(), source, &token).await
+    }
+
+    #[tokio::test]
+    async fn a_valid_agent_token_is_granted_exactly_its_own_subjects() {
+        let g = grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECRET")
+            .await
+            .expect("granted");
+        assert_eq!(
+            g.publish,
+            vec![
+                "$SWARM.term.atlas".to_owned(),
+                "$SWARM.agent-state.atlas".to_owned(),
+            ]
+        );
+    }
+
+    #[tokio::test]
+    async fn a_wrong_secret_is_denied() {
+        assert!(
+            grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECREX")
+                .await
+                .is_none()
+        );
+        assert!(
+            grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECRE")
+                .await
+                .is_none()
+        );
+    }
+
+    /// The secret check is what stops one agent claiming another's name: the
+    /// right secret under someone else's name finds that agent's credential,
+    /// or none.
+    #[tokio::test]
+    async fn another_agents_name_with_this_agents_secret_is_denied() {
+        assert!(
+            grant(Some(&atlas()), "swarm-agent.argus.Ab9_-zSECRET")
+                .await
+                .is_none()
+        );
+    }
+
+    #[tokio::test]
+    async fn an_agent_with_nothing_stored_is_denied() {
+        assert!(
+            grant(
+                Some(&stored("argus", "argus")),
+                "swarm-agent.atlas.Ab9_-zSECRET"
+            )
+            .await
+            .is_none()
+        );
+    }
+
+    #[tokio::test]
+    async fn a_failed_lookup_is_denied() {
+        assert!(
+            grant(Some(&Fake::Fails), "swarm-agent.atlas.Ab9_-zSECRET")
+                .await
+                .is_none()
+        );
+    }
+
+    /// The callout loop answers one request at a time, so a store that never
+    /// answers must cost at most the bound, and then deny.
+    #[tokio::test]
+    async fn a_store_that_never_answers_is_denied_within_the_bound() {
+        let bound = crate::introspect::INTROSPECTION_TIMEOUT;
+        let started = std::time::Instant::now();
+        assert!(
+            grant(Some(&Fake::Hangs), "swarm-agent.atlas.Ab9_-zSECRET")
+                .await
+                .is_none()
+        );
+        let took = started.elapsed();
+        assert!(took >= bound, "denied before the bound: {took:?}");
+        assert!(
+            took < bound + std::time::Duration::from_millis(250),
+            "denied well after the bound: {took:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn no_store_is_denied() {
+        assert!(
+            grant(None, "swarm-agent.atlas.Ab9_-zSECRET")
+                .await
+                .is_none()
+        );
+    }
+
+    #[tokio::test]
+    async fn a_stored_object_naming_another_agent_is_denied() {
+        assert!(
+            grant(
+                Some(&stored("argus", "atlas")),
+                "swarm-agent.argus.Ab9_-zSECRET"
+            )
+            .await
+            .is_none()
+        );
+    }
+
+    #[tokio::test]
+    async fn a_verified_agent_with_no_subjects_configured_is_denied() {
+        let bare = Policy::new(
+            "hive-".to_owned(),
+            "-agent".to_owned(),
+            "hive-status".to_owned(),
+            vec![],
+            vec![],
+            vec![],
+        )
+        .expect("valid");
+        let Presented::Agent(token) = classify("swarm-agent.atlas.Ab9_-zSECRET") else {
+            panic!("an agent token");
+        };
+        assert!(authorize(&bare, Some(&atlas()), &token).await.is_none());
+    }
+
+    /// Everything without the prefix reaches introspection as it arrived.
+    #[test]
+    fn a_token_without_the_prefix_goes_to_introspection_unchanged() {
+        for token in ["authelia_at_abc.def", "atlas.Ab9_-zSECRET"] {
+            let Presented::Bearer(t) = classify(token) else {
+                panic!("{token:?} is not an agent token");
+            };
+            assert_eq!(t, token);
+        }
+    }
+
+    #[test]
+    fn a_malformed_agent_token_goes_nowhere() {
+        assert!(matches!(
+            classify("swarm-agent.atlas"),
+            Presented::Malformed(_)
+        ));
+    }
+}

swarm-nats-auth/src/main.rs

@@ -8,7 +8,8 @@
 //! It connects as the one callout-exempt user (by nkey, never by name — the
 //! server refuses to start if that entry carries a username), subscribes to
 //! `$SYS.REQ.USER.AUTH`, validates the presented bearer token against
-//! authelia's introspection endpoint, and answers with a NATS user JWT signed
+//! authelia's introspection endpoint (or, for an agent's own token, against the
+//! secret store: see `agent_token`), and answers with a NATS user JWT signed
 //! by the account key. A rejection is answered explicitly: silence is
 //! indistinguishable from the responder being down, and the queue is the
 //! swarm's control path.
@@ -27,6 +28,7 @@ use anyhow::Context;
 use clap::Parser;
 use futures_util::StreamExt;
 
+mod agent_token;
 mod introspect;
 mod policy;
 mod request;
@@ -89,7 +91,7 @@ struct Args {
     /// config change plus a reload.
     ///
     /// So this identity says which hive an agent belongs to and never which
-    /// agent: two agents on one hive are indistinguishable to this responder.
+    /// agent: two agents on one hive are indistinguishable under it.
     ///
     /// A suffix on the hive's id rather than a prefix of its own, because
     /// `agent-<name>` reads as *the agent called `<name>`* — the one thing
@@ -127,6 +129,18 @@ struct Args {
     /// is refused, which is loud rather than silently over-broad.
     #[arg(long = "agent-publish-subject")]
     agent_publish_subjects: Vec<String>,
+
+    /// Subjects an agent that presented its own credential may publish to,
+    /// with `{agent}` standing for its name. Repeatable, empty by default,
+    /// which denies every agent token.
+    #[arg(long = "agent-token-publish-subject")]
+    agent_token_publish_subjects: Vec<String>,
+
+    /// Role on the secret store's `cert` auth mount this responder logs in
+    /// as to read agent credentials. The store is found through the `BAO_*`
+    /// environment; with none, every agent token is denied.
+    #[arg(long, default_value = "swarm-nats-auth")]
+    store_cert_role: String,
 }
 
 /// Read a secret file and strip surrounding whitespace.
@@ -177,7 +191,9 @@ async fn main() -> anyhow::Result<()> {
         args.reader_clients.clone(),
         args.hive_publish_subjects.clone(),
         args.agent_publish_subjects.clone(),
-    )?;
+    )?
+    .with_agent_token_subjects(args.agent_token_publish_subjects.clone())?;
+    let store = agent_token::Store::from_env(args.store_cert_role.clone());
     let http = reqwest::Client::new();
     let issuer = nkeys::KeyPair::from_seed(&read_secret(&args.issuer_seed_file)?)
         .context("parse the account signing seed")?;
@@ -210,44 +226,33 @@ async fn main() -> anyhow::Result<()> {
         };
         // No token is a denial, not an error: an anonymous connect is a
         // normal thing for a client to attempt and an abnormal thing to
-        // grant. Introspection is only reached once something was presented.
+        // grant. Neither the store nor introspection is reached until
-        //
+        // something was presented.
-        // The caller is an identity or nothing — see `introspect`'s module
+        let (caller, permissions) = match req
-        // docs. There is no "admitted, identity unknown" branch to write here
+            .connect_opts
-        // because there is no such value to receive.
+            .auth_token
-        let caller = match &req.connect_opts.auth_token {
+            .as_deref()
-            Some(token) => introspect::identify_caller(
+            .map(agent_token::classify)
-                &http,
+        {
-                &args.introspection_url,
+            // The caller is the name the token claims, logged whether or not
-                &args.client_id,
+            // the secret proved it; `granted` says which.
-                &client_secret,
+            Some(agent_token::Presented::Agent(token)) => (
-                token,
+                Some(format!("agent:{}", token.agent)),
-            )
+                agent_token::authorize(&policy, store.as_ref(), &token).await,
-            .await
+            ),
-            // An introspection that could not be *made* is a denial too. The
+            Some(agent_token::Presented::Malformed(e)) => {
-            // failure modes of an HTTP call are exactly the conditions under
+                tracing::warn!(error = %e, "malformed agent token; denying");
-            // which an attacker would most like this to fall open.
+                (None, None)
-            .unwrap_or_else(|e| {
-                tracing::warn!(error = ?e, "introspection failed; denying");
-                None
-            }),
-            None => None,
-        };
-        // Admission said who; the policy says what. A caller the `IdP`
-        // vouches for but no rule matches is denied — see `policy`'s module
-        // docs for why that is deny and not "connect with nothing".
-        let permissions = caller.as_deref().and_then(|id| policy.permissions(id));
-        if let (Some(id), None) = (caller.as_deref(), permissions.as_ref()) {
-            // Loud, and the one case an operator has to be able to find: a
-            // valid credential refused by our own policy. The alternative is
-            // a client that authenticates fine and mysteriously cannot work.
-            tracing::warn!(
-                caller = %id,
-                "authenticated client matches no policy rule; denying"
-            );
             }
-        // The client id is an identifier, not a credential, and it is the
+            Some(agent_token::Presented::Bearer(token)) => {
-        // only thing tying a connection in this log to a hive.
+                introspected(&policy, &http, &args, &client_secret, token).await
+            }
+            None => (None, None),
+        };
+        // The caller is an identifier, not a credential: a client id, or
+        // `agent:<name>` for an agent token. One line per auth request, a connect
+        // or a reconnect, so `hive-<h>-agent` lines count requests made on a
+        // hive's shared credential, not agents.
         tracing::info!(
             user_nkey = %req.user_nkey,
             server_id = %req.server_id.id,
@@ -281,6 +286,50 @@ async fn main() -> anyhow::Result<()> {
     anyhow::bail!("subscription to {AUTH_SUBJECT} ended")
 }
 
+/// The OIDC path: who the `IdP` says presented `token`, and what the policy
+/// grants that client. `None` permissions is a denial.
+///
+/// The caller is an identity or nothing — see `introspect`'s module docs.
+/// There is no "admitted, identity unknown" branch to write here because
+/// there is no such value to receive.
+async fn introspected(
+    policy: &policy::Policy,
+    http: &reqwest::Client,
+    args: &Args,
+    client_secret: &str,
+    token: &str,
+) -> (Option<String>, Option<policy::Permissions>) {
+    let caller = introspect::identify_caller(
+        http,
+        &args.introspection_url,
+        &args.client_id,
+        client_secret,
+        token,
+    )
+    .await
+    // An introspection that could not be *made* is a denial too. The
+    // failure modes of an HTTP call are exactly the conditions under
+    // which an attacker would most like this to fall open.
+    .unwrap_or_else(|e| {
+        tracing::warn!(error = ?e, "introspection failed; denying");
+        None
+    });
+    // Admission said who; the policy says what. A caller the `IdP`
+    // vouches for but no rule matches is denied — see `policy`'s module
+    // docs for why that is deny and not "connect with nothing".
+    let permissions = caller.as_deref().and_then(|id| policy.permissions(id));
+    if let (Some(id), None) = (caller.as_deref(), permissions.as_ref()) {
+        // Loud, and the one case an operator has to be able to find: a
+        // valid credential refused by our own policy. The alternative is
+        // a client that authenticates fine and mysteriously cannot work.
+        tracing::warn!(
+            caller = %id,
+            "authenticated client matches no policy rule; denying"
+        );
+    }
+    (caller, permissions)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -306,9 +355,16 @@ mod tests {
             "hive-",
             "--agent-client-suffix",
             "-agent",
+            "--agent-token-publish-subject",
+            "$SWARM.term.{agent}",
+            "--agent-token-publish-subject",
+            "$SWARM.agent-state.{agent}",
+            "--store-cert-role",
+            "swarm-nats-auth",
         ])
         .expect("the unit's own argument vector must parse");
         assert_eq!(args.agent_client_suffix, "-agent");
+        assert_eq!(args.agent_token_publish_subjects.len(), 2);
     }
 
     /// The control for the case above: an ordinary value parses through the

swarm-nats-auth/src/policy.rs

@@ -45,12 +45,16 @@ pub struct Policy {
     readers: Vec<String>,
     extra_hive_subjects: Vec<String>,
     extra_agent_subjects: Vec<String>,
+    agent_token_subjects: Vec<String>,
 }
 
 /// Placeholder replaced with the hive's own name in `extra_hive_subjects` and
 /// `extra_agent_subjects`.
 const HIVE_PLACEHOLDER: &str = "{hive}";
 
+/// Placeholder replaced with the agent's own name in `agent_token_subjects`.
+const AGENT_PLACEHOLDER: &str = "{agent}";
+
 impl Policy {
     /// `hive_prefix` is the client-id prefix that marks a hive and
     /// `agent_suffix` what a hive's agent containers carry **on top of** it —
@@ -130,9 +134,43 @@ impl Policy {
             readers,
             extra_hive_subjects,
             extra_agent_subjects,
+            agent_token_subjects: Vec::new(),
         })
     }
 
+    /// Set the subjects an agent that proved its own credential may publish
+    /// to, with `{agent}` standing for its name.
+    ///
+    /// # Errors
+    ///
+    /// A template with no `{agent}` in it is refused: it would be one subject
+    /// shared by every agent in the swarm rather than the agent's own.
+    pub fn with_agent_token_subjects(mut self, subjects: Vec<String>) -> anyhow::Result<Self> {
+        if let Some(bad) = subjects.iter().find(|s| !s.contains(AGENT_PLACEHOLDER)) {
+            anyhow::bail!(
+                "--agent-token-publish-subject {bad:?} contains no {AGENT_PLACEHOLDER}: every \
+                 agent in the swarm would be granted that exact subject"
+            );
+        }
+        self.agent_token_subjects = subjects;
+        Ok(self)
+    }
+
+    /// The permissions for an agent whose own credential was verified, or
+    /// `None` when none are configured.
+    ///
+    /// Keyed on the agent alone: its identity is not tied to a hive. `agent`
+    /// must already be a single `[A-Za-z0-9_-]` segment, which the token parse
+    /// guarantees, so it cannot widen a subject with `.`, `*` or `>`.
+    pub fn agent_token_permissions(&self, agent: &str) -> Option<Permissions> {
+        let publish: Vec<String> = self
+            .agent_token_subjects
+            .iter()
+            .map(|s| s.replace(AGENT_PLACEHOLDER, agent))
+            .collect();
+        (!publish.is_empty()).then_some(Permissions { publish })
+    }
+
     /// The permissions for `client_id`, or `None` when no rule matches.
     ///
     /// `None` is a denial. It is not "grant nothing and let them connect":
@@ -1175,4 +1213,35 @@ mod tests {
             "an empty hive name must never be expanded into a subject: {g:?}"
         );
     }
+
+    #[test]
+    fn an_agent_token_grant_is_the_agents_own_subjects_and_nothing_else() {
+        let p = policy_with_agent_subject()
+            .with_agent_token_subjects(vec![
+                "$SWARM.term.{agent}".to_owned(),
+                "$SWARM.agent-state.{agent}".to_owned(),
+            ])
+            .expect("per-agent templates are valid");
+        let g = p.agent_token_permissions("atlas").expect("configured");
+        assert_eq!(
+            g.publish,
+            vec![
+                "$SWARM.term.atlas".to_owned(),
+                "$SWARM.agent-state.atlas".to_owned(),
+            ]
+        );
+    }
+
+    #[test]
+    fn an_agent_token_subject_without_the_placeholder_is_refused() {
+        let err = policy()
+            .with_agent_token_subjects(vec!["$SWARM.term.all".to_owned()])
+            .expect_err("a subject shared by every agent is not the agent's own");
+        assert!(format!("{err}").contains("$SWARM.term.all"), "{err}");
+    }
+
+    #[test]
+    fn with_no_agent_token_subject_configured_an_agent_token_is_refused() {
+        assert!(policy().agent_token_permissions("atlas").is_none());
+    }
 }