diff --git a/Cargo.lock b/Cargo.lock index 11a6dd49..41b34768 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4828,7 +4828,9 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", + "subtle", "swarm-queue-client", + "swarm-secret-client", "tokio", "tracing", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index 22eb6dda..7368ead1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -230,6 +230,9 @@ matrix-sdk = { version = "0.18", default-features = false, features = [ futures-util = "0.3" hmac = "0.13" sha2 = "0.11" +# Constant-time comparison of a presented secret against the stored one +# (`swarm-nats-auth::agent_token`). Already in the tree through the TLS stack. +subtle = "2.6" # The NATS protocol client, for the swarm queue's auth-callout responder. # `default-features = false` because the default set is broad - jetstream, kv, # object-store, websockets, service - and a callout responder speaks none of diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 4bcb348d..a02f8a82 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -32,6 +32,7 @@ ./glue-grafana-oidc-client.nix ./glue-matrix-bao-token.nix ./glue-matrix-ctl-bao-identity.nix + ./glue-nats-auth-bao-identity.nix ./glue-nats-bao-identity.nix ./glue-queue-agent-credential.nix ./glue-secret-publisher-bao-identity.nix diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index d410d9f9..179e22fc 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -260,6 +260,12 @@ in # leaf logs in with. Stays on this host; see its default above. [ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \ ${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth + + # The queue's auth-callout responder, which reads agent queue + # credentials. Minted here for the matrix-ctl leaf's reason: the queue + # is a swarm singleton, so elsewhere this is the file an operator copies. + [ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \ + ${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth ''; }; }; diff --git a/nix/host-modules/glue-nats-auth-bao-identity.nix b/nix/host-modules/glue-nats-auth-bao-identity.nix new file mode 100644 index 00000000..7c6cd7cb --- /dev/null +++ b/nix/host-modules/glue-nats-auth-bao-identity.nix @@ -0,0 +1,37 @@ +# Glue: point the queue's auth-callout responder at the bao leaf minted for it. +# +# ONE PAIRING PER FILE — the swarm-nats-auth principal ← bao, and nothing else. +# Deleting this leaves a responder with no store identity unless the operator +# names one: every agent token is then denied, and OIDC clients are unaffected. +# +# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the +# leaf. What belongs here is the pairing: which paths the responder presents. +# +# ⚠️ Gated on the leaf existing, not on the store being enabled, for the reason +# ./glue-nats-bao-identity.nix states: the hive hosting the queue need not be +# the hive hosting the store. +# +# Everything is `mkDefault`. An operator naming their own paths wins. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoDeploy = deployCfg.bao; + + # Where ./glue-bao-tls.nix puts the leaves, derived from the reader's own path + # rather than repeating that file's directory literal. + haveMintedPki = baoDeploy.clientCertFile != null; + pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null; +in +{ + config = lib.mkIf (hyperhiveCfg.enable && deployCfg.nats.enable && haveMintedPki) { + services.hyperhive.deploy.nats = { + authBaoClientCertFile = lib.mkDefault "${pkiDir}/nats-auth.pem"; + authBaoClientKeyFile = lib.mkDefault "${pkiDir}/nats-auth-key.pem"; + }; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 988520b0..bd2ee06e 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -754,6 +754,18 @@ let } ]; + # The queue's auth-callout responder, which checks the secret an agent + # presents against the one stored for it. One leaf under every agent: `+` is + # one path segment, where `*` globs only at the end and would reach every + # other credential an agent holds. + natsAuthReaders = [ + { + name = "swarm-nats-auth"; + cn = baoDeploy.natsAuthCommonName; + policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue"; + } + ]; + # The role name IS the policy name, as for the three service principals # above: the role attaches the policy by spelling it identically, and one # string for both objects removes the way they drift apart. @@ -1438,6 +1450,24 @@ in ''; }; + natsAuthCommonName = lib.mkOption { + type = lib.types.str; + default = "swarm-nats-auth"; + description = '' + Subject the store's `swarm-nats-auth` cert-auth role accepts: the + identity the queue's auth-callout responder presents to read agent + queue credentials, `swarm/agents//queue` and nothing else under + an agent. + + Its own principal rather than + {option}`services.hyperhive.deploy.bao.natsCommonName`: that one + issues the queue's TLS leaf from a host unit, this one reads secrets + from inside the queue's container. + + ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + matrixCtlHiveName = lib.mkOption { type = lib.types.str; default = toString hyperhiveCfg.hiveName; @@ -2021,6 +2051,7 @@ in "swarm-bao-services-issuer-policy" "swarm-bao-nats-tls-policy" "swarm-bao-agent-pki" + "swarm-bao-nats-auth-policy" ]; # 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is @@ -2723,6 +2754,8 @@ in systemd.services.swarm-bao-forwarder-oidc-policy = readerPolicyUnit "write the store forwarder's OIDC-secret-reader bao policy and cert-auth role" forwarderOidcReaders; + systemd.services.swarm-bao-nats-auth-policy = readerPolicyUnit "write the queue responder's agent-credential-reader bao policy and cert-auth role" natsAuthReaders; + # A FOURTH sibling, same shape and same reasons as the two above. This # one is what turns `swarm-services-issuer` from a declaration into a # grant: a bao policy reaches nothing until a login role hands it to a diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 91cde3b3..fe1bff9b 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -219,6 +219,25 @@ let tlsKeyCredential = "tls-key"; tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}"; + # The responder's own store identity, for reading agent queue credentials. + # Without it the responder denies every agent token; agents then fall back to + # their hive's OIDC client. + authStoreActive = + deployCfg.nats.authBaoClientCertFile != null && deployCfg.nats.authBaoClientKeyFile != null; + # The role ./swarm-bao.nix writes on the store's `cert` mount, by the same name. + authCertRole = "swarm-nats-auth"; + # Store identity files the copy unit delivers, as credential id → host source. + authStoreFiles = lib.optionalAttrs authStoreActive ( + { + "bao-client.pem" = deployCfg.nats.authBaoClientCertFile; + "bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + "bao-ca.pem" = baoDeploy.serverCaFile; + } + ); + authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}"; + # Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats` # grants it (./swarm-bao.nix), so the daily timer below has two weeks of # retries before it lapses. @@ -498,6 +517,34 @@ in A path, never a value. ''; }; + + authBaoClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/nats-auth.pem"; + description = '' + Client certificate the auth-callout responder presents to the swarm's + secret store to read agent queue credentials. Its subject must be + {option}`services.hyperhive.deploy.bao.natsAuthCommonName`. + + No default. ./glue-nats-auth-bao-identity.nix points it at the leaf + ./glue-bao-tls.nix mints, where this host mints one. Unset, or set to + a file that does not exist, the responder denies every agent token and + admits OIDC clients as before. + + A path, never a value. + ''; + }; + + authBaoClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/nats-auth-key.pem"; + description = '' + Private key for {option}`services.hyperhive.deploy.nats.authBaoClientCertFile`. + A path, never a value. + ''; + }; }; config = lib.mkIf deployCfg.nats.enable { @@ -905,6 +952,11 @@ in # an append-only row stream, a header is one current value # republished on change. "--agent-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{hive}.>"}" + # What an agent that proved its own credential may publish to: + # the same two streams, keyed on the agent alone. + "--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{agent}"}" + "--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{agent}"}" + "--store-cert-role ${lib.escapeShellArg authCertRole}" ]; # Every credential arrives by `LoadCredential` and is named # on the command line only as a **path** — `argv` is @@ -914,12 +966,25 @@ in "callout-user.seed:${inContainer "callout-user.seed"}" "issuer.seed:${inContainer "issuer.seed"}" "oidc-client.secret:${inContainer "oidc-client.secret"}" - ]; + ] + ++ lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles; DynamicUser = true; Restart = "on-failure"; RestartSec = "5s"; SyslogIdentifier = "swarm-nats-auth"; }; + # The store the responder reads agent credentials from. Absent + # without an identity, which the responder reads as "no store". + environment = lib.optionalAttrs authStoreActive ( + { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = authCredential "bao-client.pem"; + BAO_CLIENT_KEY = authCredential "bao-client-key.pem"; + } + // lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") { + BAO_CACERT = authCredential "bao-ca.pem"; + } + ); }; # The server binary, so an operator with a shell in here can @@ -957,7 +1022,10 @@ in # being up says nothing about whether its in-container secrets unit # has finished. The wait in the script is what actually closes it; # this only stops us spinning for the full timeout on every boot. - ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"; + ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service" + # Where the store's PKI is minted on this host, the responder's leaf is + # one of its files. Ordering only: elsewhere the unit does not exist. + ++ lib.optional authStoreActive "swarm-bao-pki.service"; requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"; serviceConfig = { Type = "oneshot"; @@ -1006,7 +1074,22 @@ in install -m 0400 "$secret" \ ${lib.escapeShellArg (hostPath "oidc-client.secret")} - ''; + '' + # The store identity is optional to the responder, so an absent source + # is delivered as an empty file rather than failing this unit: a missing + # `LoadCredential` source stops the responder starting, and a responder + # that does not start denies every client. An empty identity fails only + # the store login, which denies agent tokens. + + lib.concatStrings ( + lib.mapAttrsToList (id: source: '' + if [ -s ${lib.escapeShellArg source} ]; then + install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)} + else + echo "${source} is absent; the queue responder denies agent tokens until it exists" >&2 + install -m 0400 /dev/null ${lib.escapeShellArg (hostPath id)} + fi + '') authStoreFiles + ); }; # ⚠️ Minted on the HOST, not in the container, because the responder is diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index b7ac396e..86d90dd5 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -53,6 +53,7 @@ let deployCfg.bao.servicesIssuerCommonName deployCfg.bao.natsCommonName deployCfg.bao.granterCommonName + deployCfg.bao.natsAuthCommonName ] # The two per-hive readers' subjects, spelled out per hive rather than as the # prefix. The prefix alone would reserve the wrong string: the role for hive diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 4f175725..a32e0348 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -151,7 +151,7 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The eleven units that write a `swarm-*` grant, by name, for the discovery + # The twelve units that write a `swarm-*` grant, by name, for the discovery # control below. grantingUnitNames = [ "swarm-bao-controller-policy" @@ -165,6 +165,7 @@ let "swarm-bao-services-issuer-policy" "swarm-bao-nats-tls-policy" "swarm-bao-agent-pki" + "swarm-bao-nats-auth-policy" ]; # Comment lines dropped first: both the HCL and the scripts explain @@ -561,6 +562,33 @@ let && !(lib.hasInfix "swarm-grafana" s) && !(lib.hasInfix "sys/policies/acl" s); } + { + # `+` is one path segment, so this reaches `swarm/agents//queue` + # and no other credential an agent holds; `swarm/agents/*` would reach + # all of them. + name = "the queue responder's grant is every agent's queue credential and nothing else"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-nats-auth-policy.script; + in + lib.hasInfix "path \"secret/data/swarm/agents/+/queue\" {" s + && lib.hasInfix "capabilities = [\"read\"]" s + && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1 + && !(lib.hasInfix "secret/data/swarm/agents/*" s) + && !(lib.hasInfix "secret/data/swarm/hives" s) + && !(lib.hasInfix "secret/data/swarm/services" s) + && lib.hasInfix "auth/cert/certs/swarm-nats-auth" s + && lib.hasInfix "allowed_common_names=swarm-nats-auth" s + && lib.hasInfix "token_policies=swarm-nats-auth" s; + } + { + name = "the PKI unit signs the queue responder's leaf under its own subject"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "/nats-auth.pem ]" s && lib.hasInfix "swarm-nats-auth \"\" clientAuth" s; + } { # 🩸 The half that makes the policies above bind: a policy grants only # through a token that carries it, and a token is minted by a cert-auth @@ -748,24 +776,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the eleven units may exist. Without this arm + # way, so none of the twelve units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the eleven granting units render"; + name = "without the granter's pair none of the twelve granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all eleven. + # The control: the same store with the pair renders all twelve. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the eleven still + # 🩸 What replaced the silent skip. With no bootstrap token the twelve still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the eleven, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the twelve, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -1129,8 +1157,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all eleven units, and each yields calls. - name = "the granter-policy check sees all eleven granting units, and parses calls from each"; + # certificate reaches all twelve units, and each yields calls. + name = "the granter-policy check sees all twelve granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index cd72aa1b..cce9c1aa 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -60,6 +60,28 @@ let swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; }; + # The queue with a store identity for its responder, placed by hand: the + # queue host that is not the store's. + natsWithStoreIdentity = hive { + deploy.nats.enable = true; + deploy.nats.autoGenerateCallout = false; + deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; + deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; + deploy.nats.authPackage = pkgs.emptyDirectory; + deploy.nats.authBaoClientCertFile = "/etc/pki/nats-auth.pem"; + deploy.nats.authBaoClientKeyFile = "/etc/pki/nats-auth-key.pem"; + }; + + # The queue on the store's own host, where the PKI glue mints every leaf. + natsOnStoreHost = hive { + deploy.bao.enable = true; + deploy.nats.enable = true; + }; + + responderOf = h: h.containers.swarm-nats.config.systemd.services.swarm-nats-auth; + # A hive running NOTHING of the swarm's own services — no IdP here, no # `swarm.authelia.url` set by hand. The whole point of the fixture is what it # does *not* say: it is the shape whose IdP address used to be null, and @@ -117,6 +139,73 @@ let in lib.hasInfix "--agent-publish-subject '$$SWARM.agent-state.{hive}.>'" exec; } + { + # The per-agent grant: the same two streams keyed on the agent alone, + # with the same doubled dollar, and the role the store writes for it. + name = "the responder grants a verified agent its own hive-free subjects"; + ok = + let + exec = (responderOf natsOldPath).serviceConfig.ExecStart; + in + lib.hasInfix "--agent-token-publish-subject '$$SWARM.term.{agent}'" exec + && lib.hasInfix "--agent-token-publish-subject '$$SWARM.agent-state.{agent}'" exec + && lib.hasInfix "--store-cert-role swarm-nats-auth" exec; + } + { + # Each credential by `LoadCredential`, and the environment naming where + # the unit sees it: a DynamicUser cannot read the copies directly. + name = "a responder with a store identity loads it and is pointed at the store"; + ok = + let + u = responderOf natsWithStoreIdentity; + creds = u.serviceConfig.LoadCredential; + in + lib.elem "bao-client.pem:/var/lib/swarm-nats-auth/bao-client.pem" creds + && lib.elem "bao-client-key.pem:/var/lib/swarm-nats-auth/bao-client-key.pem" creds + && u.environment.BAO_CLIENT_CERT == "/run/credentials/swarm-nats-auth.service/bao-client.pem" + && u.environment.BAO_CLIENT_KEY == "/run/credentials/swarm-nats-auth.service/bao-client-key.pem" + && lib.hasPrefix "https://" u.environment.BAO_ADDR; + } + { + # An absent leaf must not stop the responder, which would deny every + # client: it is delivered empty, and the three credentials the responder + # cannot run without are delivered too. + name = "the copy unit delivers the store identity, and an absent one as an empty file"; + ok = + let + s = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script; + in + lib.hasInfix "install -m 0400 /etc/pki/nats-auth.pem " s + && lib.hasInfix "install -m 0400 /etc/pki/nats-auth-key.pem " s + && lib.hasInfix "install -m 0400 /dev/null " s + && lib.hasInfix "oidc-client.secret" s; + } + { + # The control: no identity, no store wiring, and the responder's + # credentials are exactly the three it cannot run without. + name = "a responder with no store identity is not pointed at a store"; + ok = + let + u = responderOf natsOldPath; + in + !(u.environment ? BAO_ADDR) + && !(u.environment ? BAO_CLIENT_CERT) + && lib.length u.serviceConfig.LoadCredential == 3; + } + { + # On the store's host the glue pairs the responder with a leaf of its + # own, never the queue's TLS-issuing one or the hive's. + name = "on the store's host the responder presents its own leaf"; + ok = + let + n = natsOnStoreHost.services.hyperhive.deploy.nats; + b = natsOnStoreHost.services.hyperhive.deploy.bao; + in + n.authBaoClientCertFile == "/var/lib/swarm-bao-pki/nats-auth.pem" + && n.authBaoClientKeyFile == "/var/lib/swarm-bao-pki/nats-auth-key.pem" + && n.authBaoClientCertFile != n.baoClientCertFile + && n.authBaoClientCertFile != b.clientCertFile; + } { # Not a rename test. `hostClientSecretDir` is `readOnly`, so the fixture # cannot define it; what can break is a reader left pointing at the diff --git a/swarm-nats-auth/Cargo.toml b/swarm-nats-auth/Cargo.toml index 20a52c4f..a921fb20 100644 --- a/swarm-nats-auth/Cargo.toml +++ b/swarm-nats-auth/Cargo.toml @@ -22,12 +22,17 @@ serde.workspace = true serde_json.workspace = true # The jti digest: base32hex(sha256(claims)) over every JWT this crate signs. sha2.workspace = true +# Comparing an agent's presented secret with the stored one. +subtle.workspace = true # For `status::BUCKET` and `notices::STREAM` - the subjects a hive may # publish to are derived from these names, and every end that touches them # must agree on the same one. Deliberately WITHOUT the `kv` feature: this # crate derives subject strings, it never opens the bucket. `notices` # is name-only too (no `jetstream`/`kv` surface), same reason. swarm-queue-client = { workspace = true, features = ["notices"] } +# The agent-token spelling the agent also uses, and the read of the stored +# credential it is checked against. +swarm-secret-client.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/swarm-nats-auth/src/agent_token.rs b/swarm-nats-auth/src/agent_token.rs new file mode 100644 index 00000000..4a929df4 --- /dev/null +++ b/swarm-nats-auth/src/agent_token.rs @@ -0,0 +1,382 @@ +//! Verifying an agent's own credential, presented as `auth_token`. +//! +//! The spelling is `swarm_queue_client::agent_token`'s, shared with the agent +//! that presents it: a prefix, the agent's name, and its secret. The name is only a +//! claim. It becomes an identity when the secret equals the one stored at +//! `swarm/agents//queue`, and nothing in this module grants on the name +//! alone: every path that does not reach that comparison, and every lookup +//! that fails, is a denial. +//! +//! A token without the prefix is not this module's: it goes to introspection +//! unchanged. + +use std::future::Future; + +use anyhow::Context; +use subtle::ConstantTimeEq; +use swarm_queue_client::agent_token::{AgentToken, Malformed, parse_agent_token}; +use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings}; +use swarm_secret_client::queue::{self, AgentCredential}; + +use crate::policy::{Permissions, Policy}; + +/// What a presented `auth_token` is. +pub enum Presented<'a> { + /// An agent's own credential, to be checked against the store. + Agent(AgentToken<'a>), + /// Carries the agent-token prefix but is not well-formed. Denied without a + /// lookup, and never handed to introspection: it holds a secret meant for + /// the store, not for the `IdP`. + Malformed(Malformed), + /// Anything else, which is an OIDC access token for introspection. + Bearer(&'a str), +} + +/// Sort `token` onto the agent path or the introspection path. +pub fn classify(token: &str) -> Presented<'_> { + match parse_agent_token(token) { + Some(Ok(agent)) => Presented::Agent(agent), + Some(Err(e)) => Presented::Malformed(e), + None => Presented::Bearer(token), + } +} + +/// Where the stored credential for an agent comes from. +pub trait CredentialSource { + /// The object at `agent`'s queue path, `None` when nothing is stored + /// there, or an error when the store could not answer. + fn lookup( + &self, + agent: &str, + ) -> impl Future>> + Send; +} + +/// The swarm secret store, reached with this responder's own certificate. +pub struct Store { + settings: Settings, + cert_role: String, +} + +impl Store { + /// The store named by the `BAO_*` environment, or `None` when that is + /// unset: a responder with no store identity denies every agent token and + /// serves the OIDC path unchanged. + pub fn from_env(cert_role: String) -> Option { + match Settings::from_env() { + Ok(settings) => Some(Self { + settings, + cert_role, + }), + Err(e) => { + tracing::info!(reason = %e, "no secret store configured; agent tokens are denied"); + None + } + } + } +} + +impl CredentialSource for Store { + /// Logs in per lookup. An agent connects once per boot and per reconnect, + /// so a login each time costs little, and it leaves no token to expire + /// inside a long-lived process. + async fn lookup(&self, agent: &str) -> anyhow::Result> { + let path = queue::agent_queue_path(agent)?; + let store = swarm_secret_client::SecretStore::connect( + &self.settings, + &self.cert_role, + DEFAULT_CERT_MOUNT, + ) + .await + .context("logging in to the secret store")?; + store + .read_optional(&path) + .await + .with_context(|| format!("reading {path}")) + } +} + +/// Whether `token`'s secret is the one stored for the agent it names. +/// +/// The lookup is bounded by introspection's budget, under the server's +/// `authorization.timeout`. The callout loop answers one request at a time, so +/// a store that does not answer holds every request behind it, OIDC ones +/// included, for at most that long, and this then denies. +async fn verify(source: &impl CredentialSource, token: &AgentToken<'_>) -> bool { + let agent = token.agent; + let stored = match tokio::time::timeout( + crate::introspect::INTROSPECTION_TIMEOUT, + source.lookup(agent), + ) + .await + { + Ok(Ok(Some(stored))) => stored, + Ok(Ok(None)) => { + tracing::warn!( + agent, + "no queue credential is stored for this agent; denying" + ); + return false; + } + Ok(Err(e)) => { + tracing::warn!( + agent, + error = format!("{e:#}"), + "credential lookup failed; denying" + ); + return false; + } + Err(_) => { + tracing::warn!(agent, "credential lookup timed out; denying"); + return false; + } + }; + if stored.agent != agent { + tracing::warn!( + agent, + stored_agent = %stored.agent, + "the stored credential names a different agent than its path; denying" + ); + return false; + } + // Constant time, so the time to refuse says nothing about how much of the + // secret was right. A length mismatch returns early; the length is not + // secret, every minted secret has the same one. + stored + .value + .as_bytes() + .ct_eq(token.secret.as_bytes()) + .into() +} + +/// The grant for an agent token, or `None` for a denial. +/// +/// `source` is `None` when this responder has no store identity, which denies. +pub async fn authorize( + policy: &Policy, + source: Option<&S>, + token: &AgentToken<'_>, +) -> Option { + let Some(source) = source else { + tracing::warn!( + agent = token.agent, + "agent token presented, but this responder has no secret store; denying" + ); + return None; + }; + if !verify(source, token).await { + return None; + } + let permissions = policy.agent_token_permissions(token.agent); + if permissions.is_none() { + tracing::warn!( + agent = token.agent, + "verified agent token, but no --agent-token-publish-subject is configured; denying" + ); + } + permissions +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A store holding at most one credential, or failing outright. + enum Fake { + /// `credential` is stored at `path_agent`'s queue path. + Holds { + path_agent: &'static str, + credential: AgentCredential, + }, + Fails, + /// A store that accepts the request and never answers. + Hangs, + } + + impl CredentialSource for Fake { + async fn lookup(&self, agent: &str) -> anyhow::Result> { + match self { + Self::Holds { + path_agent, + credential, + } => Ok((agent == *path_agent).then(|| credential.clone())), + Self::Fails => anyhow::bail!("store unreachable"), + Self::Hangs => std::future::pending().await, + } + } + } + + fn stored(path_agent: &'static str, named: &str) -> Fake { + Fake::Holds { + path_agent, + credential: AgentCredential { + value: "Ab9_-zSECRET".to_owned(), + agent: named.to_owned(), + }, + } + } + + fn atlas() -> Fake { + stored("atlas", "atlas") + } + + fn policy() -> Policy { + Policy::new( + "hive-".to_owned(), + "-agent".to_owned(), + "hive-status".to_owned(), + vec!["swarm-controller".to_owned()], + vec![], + vec!["$SWARM.term.{hive}.>".to_owned()], + ) + .expect("valid") + .with_agent_token_subjects(vec![ + "$SWARM.term.{agent}".to_owned(), + "$SWARM.agent-state.{agent}".to_owned(), + ]) + .expect("valid") + } + + async fn grant(source: Option<&Fake>, token: &str) -> Option { + let Presented::Agent(token) = classify(token) else { + panic!("{token:?} must classify as an agent token"); + }; + authorize(&policy(), source, &token).await + } + + #[tokio::test] + async fn a_valid_agent_token_is_granted_exactly_its_own_subjects() { + let g = grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECRET") + .await + .expect("granted"); + assert_eq!( + g.publish, + vec![ + "$SWARM.term.atlas".to_owned(), + "$SWARM.agent-state.atlas".to_owned(), + ] + ); + } + + #[tokio::test] + async fn a_wrong_secret_is_denied() { + assert!( + grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECREX") + .await + .is_none() + ); + assert!( + grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECRE") + .await + .is_none() + ); + } + + /// The secret check is what stops one agent claiming another's name: the + /// right secret under someone else's name finds that agent's credential, + /// or none. + #[tokio::test] + async fn another_agents_name_with_this_agents_secret_is_denied() { + assert!( + grant(Some(&atlas()), "swarm-agent.argus.Ab9_-zSECRET") + .await + .is_none() + ); + } + + #[tokio::test] + async fn an_agent_with_nothing_stored_is_denied() { + assert!( + grant( + Some(&stored("argus", "argus")), + "swarm-agent.atlas.Ab9_-zSECRET" + ) + .await + .is_none() + ); + } + + #[tokio::test] + async fn a_failed_lookup_is_denied() { + assert!( + grant(Some(&Fake::Fails), "swarm-agent.atlas.Ab9_-zSECRET") + .await + .is_none() + ); + } + + /// The callout loop answers one request at a time, so a store that never + /// answers must cost at most the bound, and then deny. + #[tokio::test] + async fn a_store_that_never_answers_is_denied_within_the_bound() { + let bound = crate::introspect::INTROSPECTION_TIMEOUT; + let started = std::time::Instant::now(); + assert!( + grant(Some(&Fake::Hangs), "swarm-agent.atlas.Ab9_-zSECRET") + .await + .is_none() + ); + let took = started.elapsed(); + assert!(took >= bound, "denied before the bound: {took:?}"); + assert!( + took < bound + std::time::Duration::from_millis(250), + "denied well after the bound: {took:?}" + ); + } + + #[tokio::test] + async fn no_store_is_denied() { + assert!( + grant(None, "swarm-agent.atlas.Ab9_-zSECRET") + .await + .is_none() + ); + } + + #[tokio::test] + async fn a_stored_object_naming_another_agent_is_denied() { + assert!( + grant( + Some(&stored("argus", "atlas")), + "swarm-agent.argus.Ab9_-zSECRET" + ) + .await + .is_none() + ); + } + + #[tokio::test] + async fn a_verified_agent_with_no_subjects_configured_is_denied() { + let bare = Policy::new( + "hive-".to_owned(), + "-agent".to_owned(), + "hive-status".to_owned(), + vec![], + vec![], + vec![], + ) + .expect("valid"); + let Presented::Agent(token) = classify("swarm-agent.atlas.Ab9_-zSECRET") else { + panic!("an agent token"); + }; + assert!(authorize(&bare, Some(&atlas()), &token).await.is_none()); + } + + /// Everything without the prefix reaches introspection as it arrived. + #[test] + fn a_token_without_the_prefix_goes_to_introspection_unchanged() { + for token in ["authelia_at_abc.def", "atlas.Ab9_-zSECRET"] { + let Presented::Bearer(t) = classify(token) else { + panic!("{token:?} is not an agent token"); + }; + assert_eq!(t, token); + } + } + + #[test] + fn a_malformed_agent_token_goes_nowhere() { + assert!(matches!( + classify("swarm-agent.atlas"), + Presented::Malformed(_) + )); + } +} diff --git a/swarm-nats-auth/src/main.rs b/swarm-nats-auth/src/main.rs index bfee3d5e..d88a88ed 100644 --- a/swarm-nats-auth/src/main.rs +++ b/swarm-nats-auth/src/main.rs @@ -8,7 +8,8 @@ //! It connects as the one callout-exempt user (by nkey, never by name — the //! server refuses to start if that entry carries a username), subscribes to //! `$SYS.REQ.USER.AUTH`, validates the presented bearer token against -//! authelia's introspection endpoint, and answers with a NATS user JWT signed +//! authelia's introspection endpoint (or, for an agent's own token, against the +//! secret store: see `agent_token`), and answers with a NATS user JWT signed //! by the account key. A rejection is answered explicitly: silence is //! indistinguishable from the responder being down, and the queue is the //! swarm's control path. @@ -27,6 +28,7 @@ use anyhow::Context; use clap::Parser; use futures_util::StreamExt; +mod agent_token; mod introspect; mod policy; mod request; @@ -89,7 +91,7 @@ struct Args { /// config change plus a reload. /// /// So this identity says which hive an agent belongs to and never which - /// agent: two agents on one hive are indistinguishable to this responder. + /// agent: two agents on one hive are indistinguishable under it. /// /// A suffix on the hive's id rather than a prefix of its own, because /// `agent-` reads as *the agent called ``* — the one thing @@ -127,6 +129,18 @@ struct Args { /// is refused, which is loud rather than silently over-broad. #[arg(long = "agent-publish-subject")] agent_publish_subjects: Vec, + + /// Subjects an agent that presented its own credential may publish to, + /// with `{agent}` standing for its name. Repeatable, empty by default, + /// which denies every agent token. + #[arg(long = "agent-token-publish-subject")] + agent_token_publish_subjects: Vec, + + /// Role on the secret store's `cert` auth mount this responder logs in + /// as to read agent credentials. The store is found through the `BAO_*` + /// environment; with none, every agent token is denied. + #[arg(long, default_value = "swarm-nats-auth")] + store_cert_role: String, } /// Read a secret file and strip surrounding whitespace. @@ -177,7 +191,9 @@ async fn main() -> anyhow::Result<()> { args.reader_clients.clone(), args.hive_publish_subjects.clone(), args.agent_publish_subjects.clone(), - )?; + )? + .with_agent_token_subjects(args.agent_token_publish_subjects.clone())?; + let store = agent_token::Store::from_env(args.store_cert_role.clone()); let http = reqwest::Client::new(); let issuer = nkeys::KeyPair::from_seed(&read_secret(&args.issuer_seed_file)?) .context("parse the account signing seed")?; @@ -210,44 +226,33 @@ async fn main() -> anyhow::Result<()> { }; // No token is a denial, not an error: an anonymous connect is a // normal thing for a client to attempt and an abnormal thing to - // grant. Introspection is only reached once something was presented. - // - // The caller is an identity or nothing — see `introspect`'s module - // docs. There is no "admitted, identity unknown" branch to write here - // because there is no such value to receive. - let caller = match &req.connect_opts.auth_token { - Some(token) => introspect::identify_caller( - &http, - &args.introspection_url, - &args.client_id, - &client_secret, - token, - ) - .await - // An introspection that could not be *made* is a denial too. The - // failure modes of an HTTP call are exactly the conditions under - // which an attacker would most like this to fall open. - .unwrap_or_else(|e| { - tracing::warn!(error = ?e, "introspection failed; denying"); - None - }), - None => None, + // grant. Neither the store nor introspection is reached until + // something was presented. + let (caller, permissions) = match req + .connect_opts + .auth_token + .as_deref() + .map(agent_token::classify) + { + // The caller is the name the token claims, logged whether or not + // the secret proved it; `granted` says which. + Some(agent_token::Presented::Agent(token)) => ( + Some(format!("agent:{}", token.agent)), + agent_token::authorize(&policy, store.as_ref(), &token).await, + ), + Some(agent_token::Presented::Malformed(e)) => { + tracing::warn!(error = %e, "malformed agent token; denying"); + (None, None) + } + Some(agent_token::Presented::Bearer(token)) => { + introspected(&policy, &http, &args, &client_secret, token).await + } + None => (None, None), }; - // Admission said who; the policy says what. A caller the `IdP` - // vouches for but no rule matches is denied — see `policy`'s module - // docs for why that is deny and not "connect with nothing". - let permissions = caller.as_deref().and_then(|id| policy.permissions(id)); - if let (Some(id), None) = (caller.as_deref(), permissions.as_ref()) { - // Loud, and the one case an operator has to be able to find: a - // valid credential refused by our own policy. The alternative is - // a client that authenticates fine and mysteriously cannot work. - tracing::warn!( - caller = %id, - "authenticated client matches no policy rule; denying" - ); - } - // The client id is an identifier, not a credential, and it is the - // only thing tying a connection in this log to a hive. + // The caller is an identifier, not a credential: a client id, or + // `agent:` for an agent token. One line per auth request, a connect + // or a reconnect, so `hive--agent` lines count requests made on a + // hive's shared credential, not agents. tracing::info!( user_nkey = %req.user_nkey, server_id = %req.server_id.id, @@ -281,6 +286,50 @@ async fn main() -> anyhow::Result<()> { anyhow::bail!("subscription to {AUTH_SUBJECT} ended") } +/// The OIDC path: who the `IdP` says presented `token`, and what the policy +/// grants that client. `None` permissions is a denial. +/// +/// The caller is an identity or nothing — see `introspect`'s module docs. +/// There is no "admitted, identity unknown" branch to write here because +/// there is no such value to receive. +async fn introspected( + policy: &policy::Policy, + http: &reqwest::Client, + args: &Args, + client_secret: &str, + token: &str, +) -> (Option, Option) { + let caller = introspect::identify_caller( + http, + &args.introspection_url, + &args.client_id, + client_secret, + token, + ) + .await + // An introspection that could not be *made* is a denial too. The + // failure modes of an HTTP call are exactly the conditions under + // which an attacker would most like this to fall open. + .unwrap_or_else(|e| { + tracing::warn!(error = ?e, "introspection failed; denying"); + None + }); + // Admission said who; the policy says what. A caller the `IdP` + // vouches for but no rule matches is denied — see `policy`'s module + // docs for why that is deny and not "connect with nothing". + let permissions = caller.as_deref().and_then(|id| policy.permissions(id)); + if let (Some(id), None) = (caller.as_deref(), permissions.as_ref()) { + // Loud, and the one case an operator has to be able to find: a + // valid credential refused by our own policy. The alternative is + // a client that authenticates fine and mysteriously cannot work. + tracing::warn!( + caller = %id, + "authenticated client matches no policy rule; denying" + ); + } + (caller, permissions) +} + #[cfg(test)] mod tests { use super::*; @@ -306,9 +355,16 @@ mod tests { "hive-", "--agent-client-suffix", "-agent", + "--agent-token-publish-subject", + "$SWARM.term.{agent}", + "--agent-token-publish-subject", + "$SWARM.agent-state.{agent}", + "--store-cert-role", + "swarm-nats-auth", ]) .expect("the unit's own argument vector must parse"); assert_eq!(args.agent_client_suffix, "-agent"); + assert_eq!(args.agent_token_publish_subjects.len(), 2); } /// The control for the case above: an ordinary value parses through the diff --git a/swarm-nats-auth/src/policy.rs b/swarm-nats-auth/src/policy.rs index 0edd6536..3da625eb 100644 --- a/swarm-nats-auth/src/policy.rs +++ b/swarm-nats-auth/src/policy.rs @@ -45,12 +45,16 @@ pub struct Policy { readers: Vec, extra_hive_subjects: Vec, extra_agent_subjects: Vec, + agent_token_subjects: Vec, } /// Placeholder replaced with the hive's own name in `extra_hive_subjects` and /// `extra_agent_subjects`. const HIVE_PLACEHOLDER: &str = "{hive}"; +/// Placeholder replaced with the agent's own name in `agent_token_subjects`. +const AGENT_PLACEHOLDER: &str = "{agent}"; + impl Policy { /// `hive_prefix` is the client-id prefix that marks a hive and /// `agent_suffix` what a hive's agent containers carry **on top of** it — @@ -130,9 +134,43 @@ impl Policy { readers, extra_hive_subjects, extra_agent_subjects, + agent_token_subjects: Vec::new(), }) } + /// Set the subjects an agent that proved its own credential may publish + /// to, with `{agent}` standing for its name. + /// + /// # Errors + /// + /// A template with no `{agent}` in it is refused: it would be one subject + /// shared by every agent in the swarm rather than the agent's own. + pub fn with_agent_token_subjects(mut self, subjects: Vec) -> anyhow::Result { + if let Some(bad) = subjects.iter().find(|s| !s.contains(AGENT_PLACEHOLDER)) { + anyhow::bail!( + "--agent-token-publish-subject {bad:?} contains no {AGENT_PLACEHOLDER}: every \ + agent in the swarm would be granted that exact subject" + ); + } + self.agent_token_subjects = subjects; + Ok(self) + } + + /// The permissions for an agent whose own credential was verified, or + /// `None` when none are configured. + /// + /// Keyed on the agent alone: its identity is not tied to a hive. `agent` + /// must already be a single `[A-Za-z0-9_-]` segment, which the token parse + /// guarantees, so it cannot widen a subject with `.`, `*` or `>`. + pub fn agent_token_permissions(&self, agent: &str) -> Option { + let publish: Vec = self + .agent_token_subjects + .iter() + .map(|s| s.replace(AGENT_PLACEHOLDER, agent)) + .collect(); + (!publish.is_empty()).then_some(Permissions { publish }) + } + /// The permissions for `client_id`, or `None` when no rule matches. /// /// `None` is a denial. It is not "grant nothing and let them connect": @@ -1175,4 +1213,35 @@ mod tests { "an empty hive name must never be expanded into a subject: {g:?}" ); } + + #[test] + fn an_agent_token_grant_is_the_agents_own_subjects_and_nothing_else() { + let p = policy_with_agent_subject() + .with_agent_token_subjects(vec![ + "$SWARM.term.{agent}".to_owned(), + "$SWARM.agent-state.{agent}".to_owned(), + ]) + .expect("per-agent templates are valid"); + let g = p.agent_token_permissions("atlas").expect("configured"); + assert_eq!( + g.publish, + vec![ + "$SWARM.term.atlas".to_owned(), + "$SWARM.agent-state.atlas".to_owned(), + ] + ); + } + + #[test] + fn an_agent_token_subject_without_the_placeholder_is_refused() { + let err = policy() + .with_agent_token_subjects(vec!["$SWARM.term.all".to_owned()]) + .expect_err("a subject shared by every agent is not the agent's own"); + assert!(format!("{err}").contains("$SWARM.term.all"), "{err}"); + } + + #[test] + fn with_no_agent_token_subject_configured_an_agent_token_is_refused() { + assert!(policy().agent_token_permissions("atlas").is_none()); + } }