diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 0ec0292b..6309fd69 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -59,6 +59,7 @@ strategy for every credential, including the mTLS leaf. | `swarm/agents//matrix/main` | `swarm-controller`, with the swarm's appservice token, at agent creation and in a five-minute pass | the agent container itself, under the certificate its hive passed in | the pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | | `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | | `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl`, inside the `hive-matrix` container, once | `swarm-controller`, under its own certificate | none: the container keeps its copy and republishes it when the store's differs | +| `swarm/controller/swarm-controller/oidc/client` | authelia, at its first boot, where the controller registers its client; `swarm-secret-publish` copies it in | `swarm-controller`, under its own certificate, once at start | none: authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit, and the controller holds the old value until it restarts | | `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), which generates the key, at `swarm-controller`'s request at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | must be stated | | `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent container itself, under its own certificate — the identity it presents to the swarm queue, naming that one agent rather than its hive | none: the secret is fixed for the life of the agent and is revoked by deleting the path. A rotation mechanism is tracked as separate work, because rotating this credential needs a reconnect path — a queue client holding a revoked secret doesn't find out until it reconnects | | `swarm/agents//forge-token` | `swarm-controller`, at agent creation and in a pass every 5 minutes over every agent with a store identity | the agent container itself, under its own certificate, fetched to `/run/hive-agent-forge-token/token` | the controller re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes); the agent re-fetches on a 10-minute timer | diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 11e1b1d0..edb47924 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -53,6 +53,7 @@ neither is a renaming of the other. | OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | | the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | | Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | +| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | | wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | | queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when the operator sets `deploy.nats.autoGenerateCallout` | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | diff --git a/hive-agent/src/swarm_queue.rs b/hive-agent/src/swarm_queue.rs index 3613e02a..b7aa4bc1 100644 --- a/hive-agent/src/swarm_queue.rs +++ b/hive-agent/src/swarm_queue.rs @@ -33,7 +33,7 @@ use std::sync::OnceLock; use anyhow::Context as _; use tokio::sync::OnceCell; -use swarm_queue_client::QueueConfig; +use swarm_queue_client::{ClientSecret, QueueConfig}; /// Variable prefix for this agent's coordinates. Distinct from `HIVE_C0RE`'s /// on purpose: an agent authenticates as its own client, not as its hive. @@ -208,7 +208,7 @@ fn decide(env: &QueueEnv, client_id: Option) -> Resolution { url: url.clone(), token_endpoint: token_endpoint.clone(), client_id, - client_secret_file: secret.into(), + client_secret: ClientSecret::File(secret.into()), ca_file: env.ca_file.as_ref().map(Into::into), })), None => { @@ -377,8 +377,8 @@ mod tests { use std::path::PathBuf; use super::{ - AgentPath, QueueEnv, Resolution, decide, decide_agent_path, decide_agent_secret, - read_client_id, + AgentPath, ClientSecret, QueueEnv, Resolution, decide, decide_agent_path, + decide_agent_secret, read_client_id, }; fn env(parts: [Option<&str>; 4]) -> QueueEnv { @@ -432,10 +432,13 @@ mod tests { }; assert_eq!(cfg.url, "nats://10.42.0.1:4222"); assert_eq!(cfg.client_id, "hive-h1-agent"); + let ClientSecret::File(path) = &cfg.client_secret else { + panic!("the secret stays a path: {:?}", cfg.client_secret); + }; assert!( - cfg.client_secret_file.ends_with("hive-queue-agent-secret"), + path.ends_with("hive-queue-agent-secret"), "the secret stays a path: {}", - cfg.client_secret_file.display() + path.display() ); } diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 34bd3f3c..d5b22ea1 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -77,9 +77,15 @@ in [ "services" "hyperhive" "swarm" "controller" "authBridgeUrl" ] [ "services" "hyperhive" "deploy" "swarm-controller" "authBridgeUrl" ] ) - (lib.mkRenamedOptionModule + (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "controller" "queue" "clientSecretFile" ] - [ "services" "hyperhive" "deploy" "swarm-controller" "queue" "clientSecretFile" ] + '' + The controller reads its queue client secret from the swarm's secret + store, under its own certificate (deploy.swarm-controller.baoClientCertFile), + and swarm-secret-publish on the host that runs authelia puts it there. + Remove this definition; the file it named is read by nothing now and can + be deleted. + '' ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "ui" "enable" ] diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index 3c6d80e1..5f91fac0 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -127,15 +127,4 @@ in config.services.hyperhive.deploy.bao.bootstrapTokenFile = lib.mkIf cfg.deploy.singleHostSwarm ( lib.mkDefault "/var/lib/swarm-bao-bootstrap/grant.token" ); - - # Same reason, one option later: the queue secret is a path on THIS host, - # so it moved to `deploy.*` with the rest of the controller's credentials. - # It has to sit out here rather than in the `swarm` attrset above — a bare - # `controller.` prefix in there means `swarm.controller`, which is now only - # a rename shim, so the definition would still resolve and warn on every - # evaluation of a single-host swarm. - config.services.hyperhive.deploy.swarm-controller.queue.clientSecretFile = - lib.mkIf cfg.deploy.singleHostSwarm ( - lib.mkDefault "${config.services.hyperhive.deploy.authelia.hostClientSecretDir}/swarm-controller.secret" - ); } diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 065fc780..3033a2df 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -360,9 +360,9 @@ let # re-run keeps the value a live agent already holds instead of rotating it — # the read is required, not incidental. # - # The swarm appservice token, read-only: the controller creates agents' - # matrix accounts with it and never writes it. matrix-ctl mints and publishes - # it (`matrixCtlPolicyText` below). + # The swarm appservice token and its own OIDC client secret, read-only: it + # uses both and writes neither. matrix-ctl publishes the token + # (`matrixCtlPolicyText` below), the publisher the secret. # # The agent PKI grant is its only path on that mount: it can ask the one role # for a certificate, not write that role or reach the issuer. @@ -391,6 +391,10 @@ let capabilities = ["read"] } + path "${credentialMountPath}/data/${controllerClientLeaf}" { + capabilities = ["read"] + } + path "${agentPkiMountPath}/issue/${agentPkiRoleName}" { capabilities = ["update"] } @@ -403,17 +407,18 @@ let secretPublisherPolicyName = "swarm-secret-publisher"; secretPublisherCn = baoDeploy.secretPublisherCommonName; - # Two grants, and every narrowing in each is load-bearing. + # Three grants, and every narrowing in each is load-bearing. # # `secret/data/` is KV v2's ACL prefix, inserted by the engine rather than # written by the caller — same trap as the controller's grant above. # - # Two prefixes and not `swarm/*`: this principal has no business with an - # agent's credentials or the controller's, and these two are the only paths - # it produces. It grew the `services/` one when the publisher gained a swarm - # service's OIDC secret to copy, which is the rule ../module-eval.nix states - # for the controller's side of the same wall — a grant widens when a path - # gains a WRITER, not when a kind is declared. + # Two prefixes and one leaf, not `swarm/*`: this principal has no business + # with an agent's credentials, and these are the only paths it produces. + # Under `controller/` it writes the controller's OIDC client alone, spelled + # to the leaf so the appservice token beside it stays out of reach. The rule + # ../module-eval.nix states for the controller's side of the same wall holds + # here too — a grant widens when a path gains a WRITER, not when a kind is + # declared. # # Write-only. It copies secrets in and never reads one back; a read # capability would let a file-copier recover every hive's credentials. @@ -425,6 +430,10 @@ let path "${credentialMountPath}/data/swarm/services/*" { capabilities = ["create", "update"] } + + path "${credentialMountPath}/data/${controllerClientLeaf}" { + capabilities = ["create", "update"] + } ''; # The identity the matrix container's `swarm-matrix-ctl` presents. Named outside `hive-*` @@ -489,6 +498,11 @@ let # homeserver's admin. swarmAppserviceTokenLeaf = "swarm/controller/swarm-controller/matrix/appservice-token"; + # The controller's own OIDC client secret, the nix half of + # `swarm_secret_client::queue::controller_client_path`. The publisher writes + # it; the controller reads it and holds no other copy. + controllerClientLeaf = "swarm/controller/swarm-controller/oidc/client"; + # The KV v2 engine the controller writes agent credentials through. Named # once because the grant above and the `secrets enable` in the bootstrap unit # have to agree: a policy pointing at a mount nobody created is precisely the diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 6091d8d4..47449c95 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -121,11 +121,9 @@ let # the swarm has one. queueClientId = cfg.queueClientId; - # `LoadCredential` and not a copy-oneshot, which is where this deliberately - # differs from the callout responder: that one delivers INTO a container, - # so it has to copy across a filesystem boundary. The controller is a plain - # host unit, so systemd can hand it the file directly — fewer moving parts, - # and the secret never gains a second on-disk copy to forget about. + # No secret here. The daemon reads its client secret from the store at + # start, under `baoEnv`'s identity, from the path `swarm-secret-publish` + # writes it to on authelia's host (`queue_identity.rs`). # # Every value here comes from an option rather than from what happens to # run on this host. The queue is not optional for a controller, but @@ -138,11 +136,6 @@ let SWARM_CONTROLLER_NATS_URL = cfg.queue.natsUrl; SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = cfg.queue.tokenEndpoint; SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId; - # `%d` is systemd's credentials directory: root reads the plaintext at - # unit start and the daemon's own user sees it 0400, without the unit - # ever being able to read the rest of whatever directory the secret - # came from. - SWARM_CONTROLLER_OIDC_CLIENT_SECRET_FILE = "%d/queue-client.secret"; }; # Independent of `queueEnv` on purpose, and now for a simpler reason @@ -160,8 +153,8 @@ let # shape for the other. forgeEnv = lib.optionalAttrs (deployCfg.swarm-controller.forgeTokenFile != null) { SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}"; - # Same `%d` shape as the queue secret above — root reads the plaintext - # at unit start, the daemon's own user sees a 0400 copy. + # `%d` is systemd's credentials directory: root reads the plaintext at + # unit start, the daemon's own user sees a 0400 copy. SWARM_CONTROLLER_FORGE_TOKEN_FILE = "%d/forge-token"; # `agent-configs` org avatar for the forge-objects pass # (`forge/objects.rs`). Only meaningful with forge access, hence here. @@ -297,6 +290,16 @@ in [ "services" "hyperhive" "c0re" "orgAvatarPng" ] [ "services" "hyperhive" "deploy" "swarm-controller" "configOrgAvatarPng" ] ) + (lib.mkRemovedOptionModule + [ "services" "hyperhive" "deploy" "swarm-controller" "queue" "clientSecretFile" ] + '' + The controller reads its queue client secret from the swarm's secret + store, under its own certificate (deploy.swarm-controller.baoClientCertFile), + and swarm-secret-publish on the host that runs authelia puts it there. + Remove this definition; the file it named is read by nothing now and can + be deleted. + '' + ) ]; options.services.hyperhive.swarm.controller = { @@ -549,7 +552,7 @@ in No new credential to configure: the bearer token presented to the bridge is minted from THIS daemon's own existing queue OIDC identity — its endpoints stay in `swarm.controller.queue`, its secret is - `queue.clientSecretFile` below — "one identity per principal" + read from the swarm's secret store — "one identity per principal" already covers it. `null` means no agent-identity support: `CreateIdentity` jobs fail with a clear "no auth bridge configured here" error rather than the daemon refusing to @@ -637,38 +640,10 @@ in before anyone has onboarded a hive. ''; }; - - queue = { - clientSecretFile = lib.mkOption { - type = lib.types.str; - default = ""; - example = "/var/lib/secrets/swarm-controller-queue.secret"; - description = '' - Path on **this** host holding the plaintext of the controller's - OAuth2 client secret. A path, never a value: the secret would - otherwise land in the world-readable nix store. - - The controller cannot mint its own — minting happens inside - authelia's state directory during its first boot — so away from - that host the operator places the secret and names it here. - `singleHostSwarm` points this at the minted file, which - is exactly the case where one exists locally. - - Read by `LoadCredential`, so it needs to be readable by root at - unit start and nothing more; the daemon's own user never sees - the original path. - ''; - }; - }; }; config = lib.mkIf deployCfg.swarm-controller.enable { - # The daemon and the oneshot that mints its credential — the second one - # failing leaves the first running and unable to authenticate anywhere. - services.hyperhive.swarm.otel.journaldUnits = [ - "swarm-controller" - "swarm-controller-credential" - ]; + services.hyperhive.swarm.otel.journaldUnits = [ "swarm-controller" ]; users.users.swarm-controller = { isSystemUser = true; @@ -743,16 +718,17 @@ in ''; } { - assertion = deployCfg.swarm-controller.queue.clientSecretFile != ""; + assertion = haveBaoIdentity; message = '' - services.hyperhive.deploy.swarm-controller.queue.clientSecretFile - is unset. + services.hyperhive.deploy.swarm-controller.baoClientCertFile and + baoClientKeyFile must both be set. - It defaults to the file authelia's first-boot generator mints, - which only exists when authelia runs on this host. Elsewhere the - operator places the secret and names it here — the controller - cannot mint its own, because minting happens inside authelia's - state directory. + The controller reads its queue client secret from the swarm's + secret store, logging in with this certificate. They default to + the leaf the store mints when it runs on this host; elsewhere, + issue a leaf whose CN is + services.hyperhive.deploy.bao.controllerCommonName and name it + in both options. ''; } ]; @@ -765,63 +741,28 @@ in serviceConfig = { ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller"; - # The two differ on purpose. The queue credential is - # unconditional — the assertions above make its path a value that - # always exists by the time this renders, so there is no "queue is - # off here" case left for a `mkIf` to express. The forge token - # stays optional: a controller with no forge access still serves - # its HTTP surface, and that IS a supported shape. - # - # ⚠️ "the path is a value" is not "the file is on disk". The - # co-located secret is minted by authelia's FIRST BOOT, in another - # container, and `hostClientSecretDir`'s own description says a - # consumer has to wait for it. A `LoadCredential=` naming an - # absolute path that is not there yet is fatal (`243/CREDENTIALS`), - # so the daemon spent three of systemd's five default starts losing - # that race on a real boot — two seconds more and it would have hit - # `start-limit-hit`, which does not self-heal. - LoadCredential = [ - "queue-client.secret:${deployCfg.swarm-controller.queue.clientSecretFile}" - ] - ++ lib.optional ( - deployCfg.swarm-controller.forgeTokenFile != null - ) "forge-token:${deployCfg.swarm-controller.forgeTokenFile}" - # The store identity, same shape and same reason as hive-c0re's: the - # key is root-owned `0600` and this daemon runs as `swarm-controller`, - # so it never gets read access to the original file. - ++ lib.optionals haveBaoIdentity [ - "bao-client.pem:${deployCfg.swarm-controller.baoClientCertFile}" - "bao-client-key.pem:${deployCfg.swarm-controller.baoClientKeyFile}" - ] - ++ lib.optional ( - haveBaoIdentity && deployCfg.bao.serverCaFile != null - ) "bao-ca.pem:${deployCfg.bao.serverCaFile}" - ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"; + # The forge token is optional: a controller with no forge access + # still serves its HTTP surface, and that IS a supported shape. + LoadCredential = + lib.optional ( + deployCfg.swarm-controller.forgeTokenFile != null + ) "forge-token:${deployCfg.swarm-controller.forgeTokenFile}" + # The store identity, same shape and same reason as hive-c0re's: the + # key is root-owned `0600` and this daemon runs as `swarm-controller`, + # so it never gets read access to the original file. + ++ lib.optionals haveBaoIdentity [ + "bao-client.pem:${deployCfg.swarm-controller.baoClientCertFile}" + "bao-client-key.pem:${deployCfg.swarm-controller.baoClientKeyFile}" + ] + ++ lib.optional ( + haveBaoIdentity && deployCfg.bao.serverCaFile != null + ) "bao-ca.pem:${deployCfg.bao.serverCaFile}" + ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"; - # The placeholder default that makes the above non-fatal. - # `LoadCredential=` takes priority over `SetCredential=`, so this is - # only ever seen when the file is missing — and in that case systemd - # starts the unit instead of refusing to. The controller then serves - # its HTTP surface with the queue unconfigured, which is a supported - # shape it already knows how to report. - # - # ⚠️ THE VALUE MUST BE NON-EMPTY. `SetCredential=:` with an empty - # value is rejected by systemd's parser — *"Invalid syntax, ignoring"* - # — so the whole line is dropped and the fail-soft above silently does - # not exist. Measured with `systemd-analyze verify`: empty is refused, - # any non-empty value is accepted. This shipped broken and only looked - # fine because the credential file happened to be present. - # - # The word is deliberate rather than arbitrary: it reaches the token - # request as the client secret, so authelia refuses it and the journal - # says so in terms an operator can act on. - # - # Safe in a unit file precisely because it is not a secret: - # `SetCredential=` values are readable by unprivileged processes over - # IPC, so real key material must never appear here. - SetCredential = [ "queue-client.secret:placeholder-no-secret-file" ]; User = "swarm-controller"; Group = "swarm-controller"; + # Also what retries a start that gave up waiting for the secret store + # (`queue_identity.rs`). Restart = "on-failure"; RestartSec = "5s"; @@ -925,49 +866,5 @@ in // baoEnv // otelEnv; }; - - # A systemd credential is a SNAPSHOT: it is materialised into `%d` once, - # at unit start, and never re-read. That is invisible until the file - # underneath it changes — and two ordinary things change it. - # - # - it ARRIVES LATE. The co-located secret is minted by authelia's - # first boot, in another container, which a host unit cannot order - # against. Before this, the unit died at `243/CREDENTIALS` and was - # rescued only by burning restarts until the file showed up. - # - it is ROTATED. `mint_token` deliberately reads the secret file on - # every call so a rotation takes effect without a restart — a - # snapshot in `%d` quietly defeats that, and nothing reports it. - # - # Watching the file closes both: on close-after-write, restart the - # daemon so it re-snapshots. `PathChanged=` and not `PathExists=`, - # measured against the semantics rather than guessed — `PathExists=` - # activates immediately whenever the file is *already there* at unit - # start, which would restart a perfectly healthy daemon on every boot. - # `PathChanged=` requires a write, so it cannot do that and cannot spin. - # - # ⚠️ Known gap, stated rather than papered over: a secret appearing in - # the sub-second window between the daemon starting and this unit - # watching is missed until the next write. Closing it needs - # `PathExists=`, whose cost is the spurious per-boot restart above. - systemd.paths.swarm-controller-credential = { - description = "watch the swarm controller's queue credential"; - wantedBy = [ "multi-user.target" ]; - pathConfig = { - PathChanged = deployCfg.swarm-controller.queue.clientSecretFile; - Unit = "swarm-controller-credential.service"; - }; - }; - - # `try-restart`, not `restart`: if the daemon is stopped — masked, - # disabled, or deliberately down — a secret rotation is not a reason to - # start it. Rotating a credential should never be how a service comes - # back to life. - systemd.services.swarm-controller-credential = { - description = "restart the swarm controller after its queue credential changed"; - serviceConfig = { - Type = "oneshot"; - ExecStart = "${pkgs.systemd}/bin/systemctl try-restart swarm-controller.service"; - }; - }; }; } diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index 36e1f732..144c05f2 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -1,7 +1,7 @@ # The unit that puts swarm-level secrets into the swarm's secret store, so -# whoever needs one can read it there: a hive its agents' credential, a swarm -# service its own, a hive its matrix appservice token. The OIDC secrets it only -# copies; the appservice token it MINTS, having had no swarm-side producer. +# whoever needs one can read it there: a hive its agents' credential, a +# service or the controller its own, a hive its appservice token. OIDC secrets +# it only copies; the appservice token it MINTS, having no swarm-side producer. # # ⚠️ "Whoever", including a reader on THIS host. A swarm service's secret goes # into the store even when the service runs beside authelia, because its @@ -80,6 +80,11 @@ let hyperhiveCfg.swarm.bao.otel.clientId ]; + # The swarm controller's OIDC client, which it reads back from the store + # under its own certificate. + controllerClientId = hyperhiveCfg.swarm.controller.queueClientId; + controllerSecretPath = "secret/swarm/controller/swarm-controller/oidc/client"; + # Where this unit keeps the appservice tokens it minted, and the whole reason # a re-publish is idempotent. The store cannot be that record: ./swarm-bao.nix # grants this principal `create`/`update` and deliberately no `read`, so "does @@ -104,8 +109,8 @@ in description = '' Publish the OIDC client secrets this host mints into the swarm's secret store, so a hive that does not run authelia can read its - agents' credential and a swarm service can read its own — from - wherever it runs, this host included. Also mints each hive's matrix + agents' credential and a swarm service or the controller can read its + own — from wherever it runs, this host included. Also mints each hive's matrix appservice token, which has no other swarm-side producer, and publishes it the same way. @@ -120,10 +125,10 @@ in identity's job (see `baoClientCertFile`), not this option's. Turning it off leaves every hive but this one without its agents' - credential, the swarm's Grafana without any login at all, its - collector pushing unauthenticated, and every hive falling back to the - appservice token its own first boot minted — so two hives never agree - on one. Each secret has exactly one route and this is the producer's + credential, the swarm controller unable to start, the swarm's Grafana + without any login at all, its collector pushing unauthenticated, and + every hive falling back to the appservice token its own first boot + minted — so two hives never agree on one. Each secret has exactly one route and this is the producer's end of it, so the honest reason to set it false is a deployment delivering those secrets by some other mechanism it owns. ''; @@ -261,6 +266,22 @@ in fi '') serviceClientIds} + # The controller's own client, under `controller/` rather than + # `services/`: every hive reads `services/*`, and this client may read + # every hive's status. The nix half of + # `swarm_secret_client::queue::controller_client_path`. + src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${controllerClientId}.secret"} + if [ -s "$src" ]; then + bao kv put ${lib.escapeShellArg controllerSecretPath} \ + value=@"$src" + published=$((published + 1)) + else + # Authelia mints it only where the controller registers its client, + # which is where the controller runs beside it. + echo "no minted secret at $src yet; the path unit will re-run this" >&2 + skipped=$((skipped + 1)) + fi + # The matrix appservice token, per hive. Unlike the two loops above # there is nothing to copy: authelia never minted this one, and each # homeserver's own host minted its own — which is exactly why two hives diff --git a/nix/module-eval/bao-controller.nix b/nix/module-eval/bao-controller.nix index 493c3923..f7bf14eb 100644 --- a/nix/module-eval/bao-controller.nix +++ b/nix/module-eval/bao-controller.nix @@ -85,7 +85,21 @@ let baoWrapper = lib.findFirst (p: (p.name or "") == "bao-hive") null baoHostPackages; baoWrapperCmd = if baoWrapper == null then "" else (baoWrapper.buildCommand or ""); + + refusedWithoutIdentity = + m: + lib.any ( + a: !a.assertion && lib.hasInfix "swarm-controller.baoClientCertFile and" a.message + ) m.assertions; cases = [ + { + # The queue client secret lives only in the store, so a controller that + # cannot log in there cannot reach the queue. Refused at eval rather than + # left to start and never connect. The control is the co-located + # controller, which is handed the minted leaf and passes. + name = "a controller with no store identity is refused at eval, one given the minted leaf is not"; + ok = refusedWithoutIdentity controllerNoStore && !(refusedWithoutIdentity baoControllerHere); + } { # No hive mints an agent's matrix account any more, so a controller that # did not know the homeserver would create every agent without one. The diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index df191daa..9f19273e 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -364,18 +364,22 @@ let !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-secret-publisher-policy); } { - # The whole point of a second principal. The two prefixes it publishes to - # and not `swarm/`, so it cannot touch an agent's credentials; and no - # `read`, so a unit whose job is copying a file cannot recover what is - # already there. Pinned as the full capability list per prefix, because an - # added capability is exactly what a presence check misses. - name = "the publisher's grant is write-only and reaches the hive and service prefixes alone"; + # The whole point of a second principal. The two prefixes and one leaf it + # publishes to and not `swarm/`, so it cannot touch an agent's + # credentials or the appservice token beside the controller's client; + # and no `read`, so a unit whose job is copying a file cannot recover + # what is already there. Pinned as the full capability list per path, + # because an added capability is exactly what a presence check misses. + name = "the publisher's grant is write-only and reaches the hive and service prefixes and the controller's client alone"; ok = let s = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/*\" {\n capabilities = [\"create\", \"update\"]" s && lib.hasInfix "path \"secret/data/swarm/services/*\" {\n capabilities = [\"create\", \"update\"]" s + && lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/oidc/client\" {\n capabilities = [\"create\", \"update\"]\n}" s + && !(lib.hasInfix "secret/data/swarm/controller/*" s) + && !(lib.hasInfix "appservice-token" s) && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/*" s) && !(lib.hasInfix "sys/policies/acl" s); @@ -1114,6 +1118,13 @@ let name = "the controller reads the swarm appservice token and cannot write it"; ok = lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/matrix/appservice-token\" {\n capabilities = [\"read\"]\n}" baoGrantHere.systemd.services.swarm-bao-controller-policy.script; } + { + # The controller's own OIDC client secret, which the publisher writes and + # the controller reads at start. Pinned as the whole stanza, so an added + # capability fails. + name = "the controller reads its own client secret and cannot write it"; + ok = lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/oidc/client\" {\n capabilities = [\"read\"]\n}" baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + } { # Every role lives under a mount nothing else creates, and the granter # holds no `sys/auth`, so the token-holding unit creates it — otherwise diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 7a6a7827..54b38fda 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -398,14 +398,14 @@ let # `? swarm-controller`: ./host-modules/hive-tls.nix defines an # environment key on that unit name, which leaves the attr # present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive - # that has a CA — see the comment there. The credential oneshot has no + # that has a CA — see the comment there. The daemon's system user has no # second definer, so its absence is the unambiguous half. name = "the swarm controller does not run unless this host is told to run it"; ok = let inert = machine: - !(machine.systemd.services ? swarm-controller-credential) + !(machine.users.users ? swarm-controller) && !( (machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart ); diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index 817837b2..8dce5e68 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -49,12 +49,14 @@ let # address set by hand: what every hive but one in a multi-host swarm looks # like. The controller is on too, since it may run away from the queue. # - # The two secrets are the ones a hive away from authelia already has to be - # handed, and neither is an address; without them this hive would fail - # assertions that have nothing to do with the queue. + # The forge's secret and the controller's store identity are what a hive + # away from authelia and the store already has to be handed, and none is an + # address; without them this hive would fail assertions that have nothing to + # do with the queue. remoteSecrets = { deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret"; - deploy.swarm-controller.queue.clientSecretFile = "/var/lib/secrets/swarm-controller.secret"; + deploy.swarm-controller.baoClientCertFile = "/var/lib/swarm-controller/bao-client.pem"; + deploy.swarm-controller.baoClientKeyFile = "/var/lib/swarm-controller/bao-client-key.pem"; }; remote = hive (lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; }); diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index fd343356..14263ddd 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -85,6 +85,20 @@ let deploy.matrix.enable = true; }; cases = [ + { + # The controller reads this exact path + # (`swarm_secret_client::queue::controller_client_path`, pinned by that + # crate's own test). Not under `services/`, which every hive reads: this + # client may read every hive's status. + name = "the publisher writes the controller's client under controller/, never services/"; + ok = + let + s = secretPublisherHere.systemd.services.swarm-secret-publish.script; + in + lib.hasInfix "secret/swarm/controller/swarm-controller/oidc/client" s + && lib.hasInfix "/swarm-controller.secret" s + && !(lib.hasInfix "secret/swarm/services/swarm-controller" s); + } { # Both ends of a wire nothing at eval time carries end to end: the # publisher on authelia's host writes the path the reader on Grafana's host diff --git a/swarm-controller/src/auth.rs b/swarm-controller/src/auth.rs index db34118a..72a070f3 100644 --- a/swarm-controller/src/auth.rs +++ b/swarm-controller/src/auth.rs @@ -3,7 +3,7 @@ //! README for why the file cannot be touched from here). //! //! Authenticated with THIS daemon's own queue OIDC identity -//! (`SWARM_CONTROLLER_OIDC_*`, the same one `swarm-queue-client` mints for +//! ([`crate::queue_identity`], the same one `swarm-queue-client` mints for //! the queue connection) — "one identity per principal" means a second //! op that needs to prove who this process is reuses the identity it //! already has rather than provisioning a new one. A fresh token is @@ -41,7 +41,7 @@ pub struct AuthBridge { impl AuthBridge { /// Read `SWARM_CONTROLLER_AUTH_BRIDGE_URL`; `Ok(None)` when unset. /// - /// The queue identity (`SWARM_CONTROLLER_OIDC_*`) is not optional once + /// The queue identity ([`crate::queue_identity`]) is not optional once /// the bridge URL is set: the nix module sets `queueEnv` unconditionally /// for every controller (the queue is required, not just co-located /// service), so a bridge URL with no queue identity to authenticate @@ -51,15 +51,13 @@ impl AuthBridge { let Ok(base_url) = std::env::var("SWARM_CONTROLLER_AUTH_BRIDGE_URL") else { return Ok(None); }; - let queue_cfg = swarm_queue_client::QueueConfig::from_env("SWARM_CONTROLLER") - .context("reading the queue OIDC identity the auth bridge authenticates with")? - .ok_or_else(|| { - anyhow::anyhow!( - "SWARM_CONTROLLER_AUTH_BRIDGE_URL is set but SWARM_CONTROLLER_OIDC_* is \ + let queue_cfg = crate::queue_identity::get().cloned().ok_or_else(|| { + anyhow::anyhow!( + "SWARM_CONTROLLER_AUTH_BRIDGE_URL is set but SWARM_CONTROLLER_OIDC_* is \ not — the bridge is authenticated with this daemon's queue identity, so \ that identity must exist first" - ) - })?; + ) + })?; let http = reqwest::Client::builder() .connect_timeout(HTTP_CONNECT_TIMEOUT) .timeout(HTTP_TIMEOUT) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 63a2f6b7..777a04c0 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -50,6 +50,7 @@ mod forge; mod issue_report; mod matrix_account; mod otel_http_client; +mod queue_identity; mod read_policy; mod status; mod store; @@ -2220,11 +2221,12 @@ fn register_swarm_webhooks(forge: Option>, secret: Option Result>> { - let Some(cfg) = swarm_queue_client::QueueConfig::from_env("SWARM_CONTROLLER")? else { + let Some(cfg) = queue_identity::get().cloned() else { tracing::info!("no swarm queue configured; status aggregation is off"); return Ok(None); }; @@ -2304,6 +2306,9 @@ async fn main() -> Result<()> { .with_context(|| format!("chmod {}", path.display()))?; tracing::info!(socket = %path.display(), "swarm-controller listening"); + // Before every user of the identity: the queue, the bridge and both OTLP + // exporters read it from `queue_identity::get`. + queue_identity::init().await?; let status = connect_status_reader().await?; // Same "not fatal, log and carry on" shape as the queue connect above: diff --git a/swarm-controller/src/queue_identity.rs b/swarm-controller/src/queue_identity.rs new file mode 100644 index 00000000..840c71c4 --- /dev/null +++ b/swarm-controller/src/queue_identity.rs @@ -0,0 +1,250 @@ +//! This daemon's OIDC identity: the one it connects to the queue with and +//! mints the auth bridge's and the OTLP receiver's bearer tokens from. +//! +//! The queue's address, the token endpoint and the client id come from the +//! environment. The client secret comes from the swarm's secret store, at +//! `swarm_secret_client::queue::controller_client_path`, read once at start +//! under this daemon's own certificate and held only in memory — it is never +//! on this host's disk. `swarm-secret-publish`, on the host that runs +//! authelia, puts it there. +//! +//! [`init`] runs once in `main`, before anything calls [`get`]. + +use std::future::Future; +use std::path::PathBuf; +use std::sync::OnceLock; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use swarm_queue_client::{ClientSecret, QueueConfig}; +use swarm_secret_client::queue::{self, ControllerCredential}; + +/// The environment prefix the nix module's `queueEnv` sets. +const PREFIX: &str = "SWARM_CONTROLLER"; + +/// Seconds to wait after each failed fetch before the next one. About a minute +/// in all, which rides out a store restarting alongside this daemon. A longer +/// outage fails the start, and the unit's `Restart=` tries again from the top. +const RETRY_DELAYS_S: [u64; 6] = [1, 2, 4, 8, 16, 30]; + +static IDENTITY: OnceLock> = OnceLock::new(); + +/// Everything in a [`QueueConfig`] except the secret. +#[derive(Debug, PartialEq, Eq)] +struct Coordinates { + url: String, + token_endpoint: String, + client_id: String, + ca_file: Option, +} + +impl Coordinates { + fn with_secret(self, secret: String) -> QueueConfig { + QueueConfig { + url: self.url, + token_endpoint: self.token_endpoint, + client_id: self.client_id, + client_secret: ClientSecret::Value(secret), + ca_file: self.ca_file, + } + } +} + +/// Resolve the identity and hold it for [`get`]. +/// +/// # Errors +/// A half-set environment, which is a deployment bug and not an absent queue; +/// a secret the store still would not give up after [`RETRY_DELAYS_S`]; or a +/// second call. +pub async fn init() -> Result<()> { + let identity = match coordinates_from(|name| std::env::var(name).ok())? { + None => None, + Some(coordinates) => { + let delays = RETRY_DELAYS_S.map(Duration::from_secs); + Some(coordinates.with_secret(fetch_with_retry(&delays, fetch_secret).await?)) + } + }; + IDENTITY + .set(identity) + .map_err(|_| anyhow::anyhow!("the queue identity was initialised twice")) +} + +/// The identity, or `None` when this deployment wired no queue up. +pub fn get() -> Option<&'static QueueConfig> { + IDENTITY.get().and_then(Option::as_ref) +} + +/// `_NATS_URL`, `_OIDC_TOKEN_ENDPOINT` and `_OIDC_CLIENT_ID`, all or +/// none; `_OIDC_CA_FILE` optionally. +fn coordinates_from(var: impl Fn(&str) -> Option) -> Result> { + let url = var(&format!("{PREFIX}_NATS_URL")); + let token_endpoint = var(&format!("{PREFIX}_OIDC_TOKEN_ENDPOINT")); + let client_id = var(&format!("{PREFIX}_OIDC_CLIENT_ID")); + let ca_file = var(&format!("{PREFIX}_OIDC_CA_FILE")).map(PathBuf::from); + match (url, token_endpoint, client_id) { + (None, None, None) => Ok(None), + (Some(url), Some(token_endpoint), Some(client_id)) => Ok(Some(Coordinates { + url, + token_endpoint, + client_id, + ca_file, + })), + _ => bail!( + "swarm queue is half-configured: {PREFIX}_NATS_URL, {PREFIX}_OIDC_TOKEN_ENDPOINT \ + and {PREFIX}_OIDC_CLIENT_ID must be set together or not at all" + ), + } +} + +/// One login and one read. An empty value is refused like an absent one: it +/// would reach authelia as the secret and come back as `invalid_client`, which +/// names neither the store nor the publisher. +async fn fetch_secret() -> Result { + let path = queue::controller_client_path()?; + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + let stored: Option = store + .read_optional(&path) + .await + .with_context(|| format!("reading {path}"))?; + match stored { + Some(c) if !c.value.trim().is_empty() => Ok(c.value.trim().to_owned()), + _ => bail!( + "nothing at {path}: swarm-secret-publish, on the host that runs authelia, has not \ + published this daemon's client secret" + ), + } +} + +/// Run `attempt`, waiting each of `delays` after a failure, then once more. +/// Answers with the first success, or the last attempt's error. +async fn fetch_with_retry(delays: &[Duration], mut attempt: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + for delay in delays { + match attempt().await { + Ok(secret) => return Ok(secret), + Err(e) => { + tracing::warn!( + error = %format!("{e:#}"), + retry_in = ?delay, + "fetching the queue client secret from the swarm secret store failed" + ); + tokio::time::sleep(*delay).await; + } + } + } + attempt().await.with_context(|| { + format!( + "fetching the queue client secret from the swarm secret store, after {} attempts", + delays.len() + 1 + ) + }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let map: HashMap = pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + move |name| map.get(name).cloned() + } + + const FULL: [(&str, &str); 3] = [ + ("SWARM_CONTROLLER_NATS_URL", "tls://nats.example:4222"), + ( + "SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT", + "https://auth.example/api/oidc/token", + ), + ("SWARM_CONTROLLER_OIDC_CLIENT_ID", "swarm-controller"), + ]; + + #[test] + fn no_coordinates_is_no_queue() { + assert_eq!( + coordinates_from(env(&[])).expect("absent is not an error"), + None + ); + } + + #[test] + fn a_half_set_environment_is_an_error_naming_the_variables() { + let err = coordinates_from(env(&FULL[..2])).expect_err("a partial set is not absent"); + assert!( + format!("{err}").contains("SWARM_CONTROLLER_OIDC_CLIENT_ID"), + "{err}" + ); + } + + #[test] + fn a_full_environment_resolves_with_its_optional_ca() { + let mut with_ca = FULL.to_vec(); + with_ca.push(("SWARM_CONTROLLER_OIDC_CA_FILE", "/etc/ca.pem")); + let c = coordinates_from(env(&with_ca)) + .expect("complete") + .expect("configured"); + assert_eq!(c.client_id, "swarm-controller"); + assert_eq!(c.ca_file, Some(PathBuf::from("/etc/ca.pem"))); + + let c = coordinates_from(env(&FULL)) + .expect("complete") + .expect("configured"); + assert_eq!(c.ca_file, None); + } + + #[test] + fn the_fetched_secret_is_held_as_a_value_not_a_path() { + let cfg = coordinates_from(env(&FULL)) + .expect("complete") + .expect("configured") + .with_secret("s3cret".to_owned()); + assert!( + matches!(&cfg.client_secret, ClientSecret::Value(v) if v == "s3cret"), + "{:?}", + cfg.client_secret + ); + } + + #[tokio::test] + async fn a_store_that_comes_back_within_the_window_is_waited_for() { + let mut calls = 0; + let got = fetch_with_retry(&[Duration::ZERO; 3], || { + calls += 1; + let n = calls; + async move { + if n < 3 { + bail!("store sealed") + } + Ok("s3cret".to_owned()) + } + }) + .await + .expect("the third attempt succeeds"); + assert_eq!(got, "s3cret"); + assert_eq!(calls, 3); + } + + #[tokio::test] + async fn a_store_that_stays_down_fails_after_one_attempt_per_delay_plus_one() { + let mut calls = 0; + let err = fetch_with_retry(&[Duration::ZERO; 3], || { + calls += 1; + async { bail!("store sealed") } + }) + .await + .expect_err("every attempt fails"); + assert_eq!(calls, 4); + let msg = format!("{err:#}"); + assert!(msg.contains("after 4 attempts"), "{msg}"); + assert!(msg.contains("store sealed"), "{msg}"); + } +} diff --git a/swarm-controller/src/vcs_metrics.rs b/swarm-controller/src/vcs_metrics.rs index 96a10eb5..94b50130 100644 --- a/swarm-controller/src/vcs_metrics.rs +++ b/swarm-controller/src/vcs_metrics.rs @@ -138,7 +138,7 @@ fn build_provider(interval: Duration) -> Result { /// Build the [`crate::otel_http_client::AuthenticatedHttpClient`] this /// exporter pushes through. /// -/// The queue identity (`SWARM_CONTROLLER_OIDC_*`) is read fresh here rather +/// The queue identity is read from [`crate::queue_identity::get`] rather /// than threaded in from a caller — by the time this runs, [`endpoint`] has /// already confirmed OTEL is configured for this host, and `queueEnv` is set /// **unconditionally** for every `swarm-controller` (the nix module's own @@ -162,15 +162,13 @@ fn build_provider(interval: Duration) -> Result { /// the same fact stated twice. pub(crate) fn authenticated_http_client() -> Result { - let cfg = swarm_queue_client::QueueConfig::from_env("SWARM_CONTROLLER") - .context("reading the queue identity this daemon authenticates its OTLP push with")? - .ok_or_else(|| { - anyhow::anyhow!( - "SWARM_CONTROLLER_NATS_URL and friends are unset, but OTEL_EXPORTER_OTLP_ENDPOINT \ + let cfg = crate::queue_identity::get().cloned().ok_or_else(|| { + anyhow::anyhow!( + "SWARM_CONTROLLER_NATS_URL and friends are unset, but OTEL_EXPORTER_OTLP_ENDPOINT \ is — the queue is required for every controller, so this combination is a \ deployment bug, not a supported partial config" - ) - })?; + ) + })?; let audience = std::env::var("SWARM_CONTROLLER_OTEL_AUDIENCE").context( "SWARM_CONTROLLER_OTEL_AUDIENCE is unset, but OTEL_EXPORTER_OTLP_ENDPOINT is — the nix \ module sets both together", diff --git a/swarm-logs/src/auth.rs b/swarm-logs/src/auth.rs index 40368b93..6007068c 100644 --- a/swarm-logs/src/auth.rs +++ b/swarm-logs/src/auth.rs @@ -24,7 +24,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; -use swarm_queue_client::QueueConfig; +use swarm_queue_client::{ClientSecret, QueueConfig}; /// Variable prefix for this binary's coordinates. /// @@ -113,7 +113,7 @@ impl Config { url: String::from("unused: swarm-logs mints a token and never connects"), token_endpoint, client_id, - client_secret_file: PathBuf::from(client_secret_file), + client_secret: ClientSecret::File(PathBuf::from(client_secret_file)), ca_file, }, }) diff --git a/swarm-queue-client/README.md b/swarm-queue-client/README.md index 8ce8b07a..60754188 100644 --- a/swarm-queue-client/README.md +++ b/swarm-queue-client/README.md @@ -43,10 +43,13 @@ environment is a hard error, because the failure it would otherwise produce is the expensive kind — the process comes up "fine", never connects, and the data it was supposed to move silently stops. -The client secret is a **path, not a value**: putting it in the environment -would publish it to anything that can read `/proc//environ`. It is read -per token request rather than cached, so a rotation the operator believes took -effect actually did. +The client secret is never in the environment, which would publish it to +anything that can read `/proc//environ`. `from_env` takes it as a +**path** (`ClientSecret::File`), read per token request rather than cached, so +a rotation the operator believes took effect actually did. A caller that +fetched the secret from somewhere else builds the config itself with +`ClientSecret::Value`, held in memory; `swarm-controller` does this with the +secret it reads from the swarm's secret store. ## Token request shape: HTTP Basic, `audience`, `scope` diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index 05fed75e..af3ce9e5 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -347,12 +347,9 @@ pub struct QueueConfig { pub token_endpoint: String, /// The controller's own `OAuth2` client id. pub client_id: String, - /// File holding the client secret's PLAINTEXT. - /// - /// A path and not a value: the secret is minted on the authelia host and - /// read here, and putting it in the environment would publish it to - /// anything that can read `/proc//environ`. - pub client_secret_file: PathBuf, + /// Where the client secret's plaintext comes from. Never the environment, + /// which would publish it to anything that can read `/proc//environ`. + pub client_secret: ClientSecret, /// Extra trust anchor for the token endpoint and the queue itself, when /// they are not signed by a publicly-trusted CA. /// @@ -368,6 +365,55 @@ pub struct QueueConfig { pub ca_file: Option, } +/// Where a [`QueueConfig`]'s client secret comes from. +#[derive(Clone)] +pub enum ClientSecret { + /// A file holding the plaintext, read at every token mint so a rotated + /// file takes effect without a restart. + File(PathBuf), + /// The plaintext itself, held only in this process's memory. + Value(String), +} + +/// Redacts [`ClientSecret::Value`]: a [`QueueConfig`] is `Debug`, and printing +/// one must not put the secret in a log. +impl std::fmt::Debug for ClientSecret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::File(path) => f.debug_tuple("File").field(path).finish(), + Self::Value(_) => f.write_str("Value()"), + } + } +} + +impl ClientSecret { + async fn read(&self) -> Result { + match self { + Self::File(path) => { + tokio::fs::read_to_string(path) + .await + .map_err(|source| Error::ClientSecret { + path: path.display().to_string(), + source, + }) + } + Self::Value(value) => Ok(value.clone()), + } + } + + fn read_blocking(&self) -> Result { + match self { + Self::File(path) => { + std::fs::read_to_string(path).map_err(|source| Error::ClientSecret { + path: path.display().to_string(), + source, + }) + } + Self::Value(value) => Ok(value.clone()), + } + } +} + impl QueueConfig { /// Read the config from `_NATS_URL`, `_OIDC_TOKEN_ENDPOINT`, /// `_OIDC_CLIENT_ID` and `_OIDC_CLIENT_SECRET_FILE`, or @@ -404,7 +450,7 @@ impl QueueConfig { url, token_endpoint, client_id, - client_secret_file: PathBuf::from(secret), + client_secret: ClientSecret::File(PathBuf::from(secret)), ca_file, })), // A partially-set environment is a deployment bug, and the failure @@ -433,12 +479,7 @@ async fn mint_token( ) -> Result { // Read per call rather than caching: the file is small, and a cached // secret would survive a rotation that the operator believes took effect. - let secret = tokio::fs::read_to_string(&cfg.client_secret_file) - .await - .map_err(|source| Error::ClientSecret { - path: cfg.client_secret_file.display().to_string(), - source, - })?; + let secret = cfg.client_secret.read().await?; let response = token_request(http, cfg, secret.trim(), audience, scope) .send() @@ -590,11 +631,7 @@ pub fn mint_token_for_blocking( // Read per call rather than caching — see `mint_token`'s identical // comment on the async path; the reasoning does not change with the // client type. - let secret = - std::fs::read_to_string(&cfg.client_secret_file).map_err(|source| Error::ClientSecret { - path: cfg.client_secret_file.display().to_string(), - source, - })?; + let secret = cfg.client_secret.read_blocking()?; let mut form = vec![("grant_type", "client_credentials")]; if let Some(audience) = audience { @@ -869,12 +906,32 @@ mod tests { } } + #[test] + fn a_held_secret_is_read_back_as_itself() { + let secret = ClientSecret::Value("s3cret".to_owned()); + assert_eq!( + secret.read_blocking().expect("a value needs no I/O"), + "s3cret" + ); + } + + #[test] + fn a_debug_print_of_a_config_does_not_carry_a_held_secret() { + let cfg = QueueConfig { + client_secret: ClientSecret::Value("s3cret".to_owned()), + ..token_cfg() + }; + let printed = format!("{cfg:?}"); + assert!(!printed.contains("s3cret"), "{printed}"); + assert!(printed.contains(""), "{printed}"); + } + fn token_cfg() -> QueueConfig { QueueConfig { url: "nats://127.0.0.1:4222".to_owned(), token_endpoint: "https://auth.example.com/api/oidc/token".to_owned(), client_id: "hive-alpha".to_owned(), - client_secret_file: PathBuf::from("/nonexistent"), + client_secret: ClientSecret::File(PathBuf::from("/nonexistent")), ca_file: None, } } diff --git a/swarm-secret-client/src/matrix.rs b/swarm-secret-client/src/matrix.rs index 352546cf..c653ef84 100644 --- a/swarm-secret-client/src/matrix.rs +++ b/swarm-secret-client/src/matrix.rs @@ -95,7 +95,7 @@ pub fn appservice_token_path(hive: &str) -> Result { /// The name segment of the controller's own subtree, and the cert-auth role it /// logs in under (`swarm-controller`'s `store::CERT_ROLE`). -const CONTROLLER: &str = "swarm-controller"; +pub(crate) const CONTROLLER: &str = "swarm-controller"; /// The path holding the **swarm's** appservice token: the one registration /// on the swarm's homeserver that is not a hive's, whose sender is promoted diff --git a/swarm-secret-client/src/queue.rs b/swarm-secret-client/src/queue.rs index 852906e0..c1a9e479 100644 --- a/swarm-secret-client/src/queue.rs +++ b/swarm-secret-client/src/queue.rs @@ -60,6 +60,33 @@ pub fn agent_queue_path(agent: &str) -> Result { Ok(format!("{prefix}/queue")) } +/// The path holding `swarm-controller`'s own OIDC client secret: the identity +/// it connects to the queue with, and mints its other bearer tokens from. +/// +/// Under [`Kind::Controller`] because no hive's policy reads that kind. The +/// client may read every hive's status, so a copy under `services/`, which +/// every hive reads, would hand that reach to every hive. +/// +/// The nix half is `swarm-secret-publisher.nix`, which writes it, and +/// `swarm-bao.nix`'s `controllerPolicyText`, which grants the read. +/// +/// # Errors +/// Never in practice: the name segment is a constant. The `Result` is +/// [`principal_prefix`]'s. +pub fn controller_client_path() -> Result { + let prefix = principal_prefix(Kind::Controller, crate::matrix::CONTROLLER)?; + Ok(format!("{prefix}/oidc/client")) +} + +/// What [`controller_client_path`] holds. No client id beside the value: the +/// controller's id is a swarm-wide option both the writer and the reader +/// already have. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ControllerCredential { + /// The secret itself, under the field name every kind here uses. + pub value: String, +} + /// What [`agent_queue_path`] holds: the secret, and the agent it proves. /// /// No hive: an agent's identity is not tied to one, and the subjects the @@ -149,6 +176,43 @@ mod tests { assert_eq!(json["client_id"], "hive-alpha-agent"); } + /// Spelled out in full because `swarm-secret-publisher.nix` and + /// `swarm-bao.nix` write the same string by hand. + #[test] + fn the_controller_client_lands_under_the_controller() { + assert_eq!( + controller_client_path().expect("a constant segment"), + "swarm/controller/swarm-controller/oidc/client" + ); + } + + #[test] + fn no_hive_policy_reaches_the_controller_client() { + let doc = crate::policy::render("alpha").expect("a plain name is legal"); + // Every stanza is `path "secret/data/*" { … }`. + let granted: Vec<&str> = doc + .lines() + .filter_map(|l| l.strip_prefix("path \"secret/data/")) + .filter_map(|p| p.strip_suffix("*\" {")) + .collect(); + let reads = |path: &str| granted.iter().any(|g| path.starts_with(g)); + let path = controller_client_path().expect("a constant segment"); + assert!(!reads(&path), "{path} is readable under {granted:?}"); + // The control: the hive's own agent client IS under a hive stanza, so + // the assertion above can fail at all. + let own = agent_client_path("alpha").expect("legal"); + assert!(reads(&own), "{own} should be readable under {granted:?}"); + } + + #[test] + fn the_controller_objects_field_name_is_the_one_the_publisher_writes() { + let json = serde_json::to_value(ControllerCredential { + value: "s3cr3t".to_owned(), + }) + .expect("serialises"); + assert_eq!(json, serde_json::json!({ "value": "s3cr3t" })); + } + #[test] fn an_agent_name_lands_under_its_own_principal_prefix() { assert_eq!(