From 4aa982cc2a2ed941ff7d1110957a3b051e2d6167 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 13 Sep 2026 18:14:11 +0200 Subject: [PATCH 1/2] swarm-grafana: deliver the OIDC client secret through the secret store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grafana's OIDC client secret only existed where authelia did. One `ssoLocal` gate — `grafana.enable && authelia.enable` — decided the client registration, the minted secret's delivery and the whole `auth.generic_oauth` block, so a swarm whose authelia runs on another host got Grafana with no SSO wiring at all. The local login form is disabled unconditionally, so that is no way in. Split the one gate into the two questions it was conflating: - `ssoConfigured` — does this SWARM have an identity provider (`swarm.authelia.url`, which is swarm-wide and whose own description makes null mean "no SSO configured"). With a delivery route present this is what emits Grafana's OIDC block. - `ssoLocal` — is authelia on THIS host, now spelled as the forge and matrix modules spell it. It decides only which unit delivers the secret. Where authelia is elsewhere, `swarm-bao-grafana-oidc.service` reads the secret from the swarm secret store, shaped after glue-queue-agent-credential.nix: cert login fails loudly because a retry fixes every state it fails on, the read degrades quietly because no retry turns "no value there" into a value, and nothing writes a stand-in. The producer is the publisher that already runs on authelia's host, which gains the swarm's service clients beside the per-hive ones at `swarm/services//oidc/client` — with the write grant in swarm-bao.nix and the hive read grant in `policy::render` to match. Registration moved to glue-grafana-oidc-client.nix. It has to be declared where authelia's config is rendered, and swarm-grafana.nix's config block hangs off this host running Grafana. Two judgement calls stated rather than buried: a hive's read policy now grants the whole `services` prefix, because a service's path names the service and nothing swarm-wide records which hive runs it (cost recorded in docs/trust-boundary/security.md); and the client is registered on any authelia host, because no swarm-wide "this swarm has a Grafana" fact exists to gate it on. Refs #4234 Co-Authored-By: Claude Opus 5 --- docs/swarm/secrets.md | 59 +++- docs/trust-boundary/security.md | 20 +- nix/host-modules/default.nix | 1 + nix/host-modules/glue-grafana-oidc-client.nix | 51 ++++ nix/host-modules/swarm-bao.nix | 15 +- nix/host-modules/swarm-grafana.nix | 288 +++++++++++++++--- nix/host-modules/swarm-secret-publisher.nix | 41 ++- nix/module-eval.nix | 135 +++++++- swarm-secret-client/src/policy.rs | 52 +++- 9 files changed, 577 insertions(+), 85 deletions(-) create mode 100644 nix/host-modules/glue-grafana-oidc-client.nix diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index afce2a45..48b5d03a 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -41,22 +41,23 @@ neither is a renaming of the other. ## Swarm-level — one of each per swarm -| secret | generated by | lives at | hive elsewhere | -| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| swarm root CA cert | `swarm-ca.nix` first-boot unit, when `autoConfigure` is set | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | -| swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | -| swarm-services sub-CA (cert + key) | `swarm-ca.nix`, signed by the root | `/var/lib/swarm-ca/services-ca{,-key}.pem` | issued where the root lives | -| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | -| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | -| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | -| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `deploy.swarm-otel.clientSecretFile` — the collector need not share a host with authelia | -| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | -| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | +| secret | generated by | lives at | hive elsewhere | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| swarm root CA cert | `swarm-ca.nix` first-boot unit, when `autoConfigure` is set | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | +| swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | +| swarm-services sub-CA (cert + key) | `swarm-ca.nix`, signed by the root | `/var/lib/swarm-ca/services-ca{,-key}.pem` | issued where the root lives | +| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | +| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | +| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | +| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | +| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `deploy.swarm-otel.clientSecretFile` — the collector need not share a host with authelia | +| Grafana's copy of its OIDC secret | `swarm-grafana-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store instead — nothing for an operator to place, see below | +| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | +| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | Authelia mints the three keys for itself, in-container, precisely because nothing outside that container ever reads them. **That's the test worth @@ -183,6 +184,32 @@ files aren't there the daemon forwards nothing and says so in its journal, and the harness logs that it has no queue coordinates — the same absent-and-legal state, twice, rather than a container that refuses to start. +The **third reader** is Grafana's OIDC client secret, and it's the first one +that's a _swarm service's_ own credential rather than a hive's. +`swarm-grafana.nix` asks two questions where it used to ask one: whether this +_swarm_ has an IdP (`swarm.authelia.url`) decides whether Grafana gets SSO +settings at all, and whether authelia is on _this_ host decides only which of +two units delivers the secret — the host copy out of authelia's tree, or +`swarm-bao-grafana-oidc.service` reading `swarm/services//oidc/client` +out of the store. Exactly one of the two exists in any deployment, and where +neither can (no authelia here, no store identity here) the module warns and +emits no OIDC settings rather than pointing Grafana at a file nothing writes. +Registration moved to `glue-grafana-oidc-client.nix` for a related reason: a +client is a row in authelia's own config, so it has to be declared on the host +that runs authelia — which is not necessarily the host that runs Grafana. + +A service's secret is one value for the whole swarm rather than one per hive, so +it lives under the `services` prefix, and a hive's read policy grants that prefix +whole. That's because nothing in a swarm's configuration records which hive runs +a given service — placement is a `deploy.*` fact, per-host by definition — so +there is nothing to scope the grant to. Every hive can therefore read every swarm +service's client secret; that's the cost of a shape where the service's host has +no store identity of its own to present, and it's stated in +`swarm-secret-client`'s `policy` module beside the grant itself. Absence behaves +as it does for the other two readers: a store that says "not here" leaves the +file alone and says so, since there is no such thing as a locally valid OIDC +client secret to stand in. + ⚠️ **Service↔store mTLS is its own trust domain.** A credential you must already hold to authenticate can't be fetched from the thing it authenticates you to, so the store's identity can't come from an authority the store diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index 852108d8..cc455653 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -60,10 +60,22 @@ advertises a boundary it doesn't actually hold. A wide grant that says what it is beats a narrow one that only looks narrow. What still holds: the grant is **read-only** (a hive can't write an agent's -credential, so it can't hand itself an agent's identity), and it reaches two -prefixes and nothing else in the store — every agent's credentials, and the -reader's _own_ entry under the hive namespace, which names the hive asking and -so widens nothing between them. +credential, so it can't hand itself an agent's identity), and it reaches three +prefixes and nothing else in the store — every agent's credentials, the reader's +_own_ entry under the hive namespace, which names the hive asking and so widens +nothing between them, and every **swarm service's** OIDC client secret. + +That third prefix has the same shape of reason as the first, and the same honest +cost. A swarm service (Grafana is the one there today) registers **one** client +for the whole swarm, so its credential's path names the service and never the +host — and which hive runs a given service is a `deploy.*` fact, per-host by +definition, so nothing swarm-wide exists to scope the grant to. The host running +such a service has no store identity of its own either; it reads with the +certificate of the hive it is. So **any hive can read any swarm service's client +secret**, which is worth what it buys: a swarm service whose identity provider +lives on another machine can get its secret at all, instead of only working +co-located. Giving such a service its own store identity is what would remove +this rather than re-scope it. **A tracked follow-up** narrows this, with the two candidate directions: scope the grant per hive (and pay for the re-emission), or give each agent container diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 7b57fee7..5c626d84 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -25,6 +25,7 @@ ./otel.nix ./glue-bao-tls.nix ./glue-controller-bao-identity.nix + ./glue-grafana-oidc-client.nix ./glue-matrix-bao-token.nix ./glue-queue-agent-credential.nix ./glue-secret-publisher-bao-identity.nix diff --git a/nix/host-modules/glue-grafana-oidc-client.nix b/nix/host-modules/glue-grafana-oidc-client.nix new file mode 100644 index 00000000..53767650 --- /dev/null +++ b/nix/host-modules/glue-grafana-oidc-client.nix @@ -0,0 +1,51 @@ +# Glue: register the swarm's Grafana as an OIDC client wherever authelia runs. +# +# ONE PAIRING PER FILE — grafana ← authelia, and nothing else. Deleting this +# leaves a swarm whose metrics UI is not a client authelia has ever heard of, so +# no login against it can complete and nothing minted its secret either. +# +# ⚠️ Gated on authelia being HERE, and deliberately NOT on this host running +# Grafana. A client is a row in THIS host's provider config, so it can only be +# declared where that config is rendered — and ./swarm-grafana.nix's whole +# `config` block hangs off `deploy.grafana.enable`, so a swarm with Grafana and +# authelia on different hosts registered the client nowhere at all. +# ./hive-forge/default.nix is already on the right side of that line: its module +# is gated on `hyperhive.enable` and only the registration asks about authelia. +# This file puts Grafana there without moving the rest of its module. +# +# ⚠️ Registered whether or not the swarm has a Grafana, because nothing in +# `swarm.*` records that — `deploy.grafana.enable` answers "does THIS host run +# it", and a swarm-wide answer does not exist. The cost is one unused client +# and one unused minted secret in a swarm with no metrics UI. The alternative +# was a new swarm-wide option an operator must set before a split deployment +# works, which leaves the reported failure in place for everyone who does not +# know to set it. ./swarm-otel.nix made the same call for its own client when it +# dropped the published-scrape-target guard: a client that may go unused beats a +# guard that reads as done and renders nothing. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + grafanaCfg = hyperhiveCfg.swarm.grafana; +in +{ + config = lib.mkIf (hyperhiveCfg.enable && deployCfg.authelia.enable) { + # One declaration, two readers. Grafana's callback URL is format-locked to + # its own root URL, and the read-only option it is taken from is where that + # format is spelled — restating it here would be a second source of truth + # for a string whose mismatch is a silently rejected login. + # + # `kind` is left at its `interactive` default: a person logs in here. + services.hyperhive.swarm.authelia.oidc.clients = [ + { + id = grafanaCfg.oidc.clientId; + description = "HyperHive swarm metrics"; + redirectUris = [ grafanaCfg.oidc.redirectUri ]; + } + ]; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 52590ec5..a1d2b5bf 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -210,14 +210,17 @@ let secretPublisherPolicyName = "swarm-secret-publisher"; secretPublisherCn = baoDeploy.secretPublisherCommonName; - # One grant, and every narrowing in it is load-bearing. + # Two grants, and every narrowing in each is load-bearing. # # `secret/data/` is KV v2's ACL prefix, inserted by the engine rather than # written by the caller — same trap as the controller's grant above. # - # `hives/` and not `swarm/*`: this principal has no business with an agent's - # or a service's credentials, and the hive prefix is the only one whose paths - # it produces. + # Two prefixes and not `swarm/*`: this principal has no business with an + # agent's credentials or the controller's, and these two are the only paths + # it produces. It grew the `services/` one when the publisher gained a swarm + # service's OIDC secret to copy, which is the rule ../module-eval.nix states + # for the controller's side of the same wall — a grant widens when a path + # gains a WRITER, not when a kind is declared. # # Write-only. It copies secrets in and never reads one back; a read # capability would let a file-copier recover every hive's credentials. @@ -225,6 +228,10 @@ let path "${credentialMountPath}/data/swarm/hives/*" { capabilities = ["create", "update"] } + + path "${credentialMountPath}/data/swarm/services/*" { + capabilities = ["create", "update"] + } ''; # The KV v2 engine the controller writes agent credentials through. Named diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 6c6d37f4..50d53fda 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -21,6 +21,8 @@ let hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; autheliaCfg = hyperhiveCfg.swarm.authelia; + baoCfg = hyperhiveCfg.swarm.bao; + baoDeploy = deployCfg.bao; vmCfg = hyperhiveCfg.swarm.victoriametrics; vlCfg = hyperhiveCfg.swarm.victorialogs; swarmDomain = hyperhiveCfg.swarm.domain; @@ -78,10 +80,41 @@ let # operator sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; - # The all-local case: this host runs BOTH Grafana and the swarm's authelia, - # so the minted secret can be moved without an operator. Same split the - # forge and matrix modules document. - ssoLocal = deployCfg.grafana.enable && deployCfg.authelia.enable; + # Is SSO configured for this SWARM. Swarm-wide by construction — `swarm.*` is + # identical on every host — and the option's own description is what makes + # this the right question to ask: a null URL means "no SSO configured". + ssoConfigured = autheliaCfg.url != null; + + # Is authelia on THIS host, spelled exactly as the forge and matrix modules + # spell it. All locality decides is where the minted plaintext is copied + # FROM: authelia's own tree here, the swarm secret store otherwise. + # + # 🩸 This used to be `grafana.enable && authelia.enable` and was the ONLY + # gate: registration, the mint's delivery and the OIDC block all hung off + # it, so a swarm whose authelia ran elsewhere got Grafana with no SSO wiring + # — and the login form is disabled whatever happens, so with nothing else. + ssoLocal = deployCfg.authelia.enable; + + # A reader of the store is defined by holding a certificate the store + # accepts, never by standing next to it — the rule + # ./glue-matrix-bao-token.nix states in full. + haveClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null; + + # Whether Grafana gets an OIDC block: the swarm-wide question, AND that + # something on this host delivers the secret that block names. The second + # half is not redundant — a block naming a file no unit here writes cannot + # produce a working login either way, and what Grafana does with an + # unreadable `$__file{}` target (refuse to start, or start and fail every + # login) is not worth betting the container on when not emitting the block is + # available and leaves this host as it was. + ssoWired = ssoConfigured && (ssoLocal || haveClientIdentity); + + # The delivery that is not a copy: authelia is elsewhere, so the same minted + # plaintext arrives out of the store that the publisher on authelia's host + # wrote it into. Exclusive with the copy unit by construction — two units + # writing one file is a race over which secret Grafana parses. + storeDelivery = ssoWired && !ssoLocal; + autheliaUrl = toString autheliaCfg.url; # Where the plaintext lands inside the container. Under /var/lib rather @@ -90,6 +123,23 @@ let # intermittent one. secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret"; + # The same file seen from the host, which is where both delivery units write + # it. Spelled once: two units landing a secret at two spellings of one path + # is a Grafana that reads whichever of them it was configured with. + hostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${secretPath}"; + hostSecretDir = builtins.dirOf hostSecretPath; + + # Where the publisher on authelia's host leaves this client's secret. The + # `services` segment is `swarm-secret-client`'s `path::Kind::Service` — a + # swarm service's client is registered once for the whole swarm, so its + # secret is one value and not one per hive. Both ends compose this from the + # same swarm-wide client id, so a rename cannot leave one of them behind. + # + # ⚠️ The prefix is not decoration: it is what this host's read grant covers, + # so a path outside it answers 403 rather than a miss, however correct it + # looks. + storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client"; + # Grafana's own datasource-encryption key. Generated in-container (see the # unit below) because nothing outside the container ever reads it — unlike # the OIDC secret above, whose other reader is authelia's container. @@ -104,11 +154,6 @@ let nginxGid = config.ids.gids.nginx; grafanaUid = config.ids.uids.grafana; - # Format-locked by Grafana: the generic OAuth callback is always - # `/login/generic_oauth`. Declared once here and read by both - # the authelia client and Grafana itself. - redirectUri = "https://${cfg.domain}/login/generic_oauth"; - in { # `enable` moved to `services.hyperhive.deploy.grafana.enable` — see @@ -186,6 +231,25 @@ in ''; }; + redirectUri = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "https://${cfg.domain}/login/generic_oauth"; + defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.grafana.domain}/login/generic_oauth"''; + description = '' + OAuth callback authelia sends the browser back to, and the URI it + matches **exactly**. + + Read-only, like {option}`services.hyperhive.swarm.grafana.machine` + and for the same reason: Grafana derives it from its own + `root_url` (`/login/generic_oauth`), so it is a fact + other modules may read rather than a knob. The glue that registers + this client wherever authelia runs reads it from here instead of + restating the format — a second spelling of it is a silently + rejected login. + ''; + }; + role = lib.mkOption { type = lib.types.enum [ "Viewer" @@ -336,11 +400,16 @@ in # The secret oneshots as well as grafana itself: each runs before it and # fails in ways grafana then reports only as a login that does not work. + # + # Each delivery unit is listed only where it exists, the way + # ./hive-matrix.nix lists its own: a unit name that never renders is a + # journald scrape target matching nothing, which reads as a quiet unit. services.hyperhive.swarm.otel.journaldUnits = [ "grafana" - "swarm-grafana-oidc-secret" "swarm-grafana-secret-key" - ]; + ] + ++ lib.optional ssoLocal "swarm-grafana-oidc-secret" + ++ lib.optional storeDelivery "swarm-bao-grafana-oidc"; services.hyperhive.swarm.controller.links = [ { @@ -350,35 +419,40 @@ in } ]; - # One declaration, two readers. Grafana's callback URL is format-locked - # to its own root URL; making the operator restate it in authelia's - # client list would be a second source of truth for a string whose - # mismatch is a silently rejected login. - # - # `kind` is left at its `interactive` default: a person logs in here. - services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [ - { - id = cfg.oidc.clientId; - description = "HyperHive swarm metrics"; - redirectUris = [ redirectUri ]; - } - ]; + # Registering the client is NOT here any more: it has to happen on the + # host that runs authelia, and this whole block is gated on the host that + # runs Grafana. ./glue-grafana-oidc-client.nix is where it moved to. + + warnings = lib.optional (ssoConfigured && !ssoLocal && !haveClientIdentity) '' + services.hyperhive.swarm.grafana: this swarm has an IdP + (services.hyperhive.swarm.authelia.url) but authelia is not on this + host and this host holds no swarm-secret-store identity, so nothing + can deliver Grafana's OIDC client secret. Grafana will come up with + no OIDC login and no local login form, which is no way in at all. + + Name this host's store client leaf in + services.hyperhive.deploy.bao.clientCertFile and + services.hyperhive.deploy.bao.clientKeyFile — the same identity every + other reader of the store uses. See docs/swarm/secrets.md. + ''; assertions = [ { - # Grafana reaches the token endpoint server-to-server, so a null URL - # would become a request to `null/api/oidc/token` — a DNS failure - # several layers from its cause. Only reachable by enabling authelia - # and clearing its `url`, which is why it is an assertion and not a - # fallback. - assertion = !ssoLocal || autheliaCfg.url != null; + # The URL is what `ssoConfigured` above reads, so with it null this + # host mints and delivers a secret for an OIDC block it then does not + # emit — SSO silently absent on the one deployment that has every + # other piece of it. Only reachable by enabling authelia and clearing + # its `url`, which is why it is an assertion and not a fallback. + assertion = !ssoLocal || ssoConfigured; message = '' services.hyperhive.swarm.grafana requires services.hyperhive.swarm.authelia.url when authelia is enabled. Grafana exchanges its authorization code at - `''${url}/api/oidc/token` from inside its container. With the URL - null there is no endpoint to name. + `''${url}/api/oidc/token` from inside its container, so with the URL + null there is no endpoint to name — and a null URL is also how this + module reads "no SSO configured for this swarm", which would leave + Grafana with no login of any kind on the host that mints its secret. ''; } ]; @@ -450,11 +524,11 @@ in "d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" ]; - # The secret delivery. It runs on the HOST because that is the only place - # both container trees are addressable: they share this host's network - # namespace, which makes them feel co-located, but their filesystem roots - # are separate — Grafana cannot open a path inside authelia's tree - # however local the port looks. + # The local delivery, and the ONLY thing locality still decides. It runs on + # the HOST because that is the only place both container trees are + # addressable: they share this host's network namespace, which makes them + # feel co-located, but their filesystem roots are separate — Grafana cannot + # open a path inside authelia's tree however local the port looks. # # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container # refuses to start when a bind source is missing, and this secret does @@ -482,7 +556,7 @@ in set -euo pipefail src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${cfg.oidc.clientId}.secret"} - dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${secretPath}"} + dst=${lib.escapeShellArg hostSecretPath} # authelia's container is up, but its first-boot generator may still # be minting. Bounded wait, then fail: a silent skip here produces a @@ -513,6 +587,137 @@ in ''; }; + # The same secret, the same destination, for the deployment where authelia + # is NOT here: it arrives out of the swarm secret store, which the + # publisher on authelia's host wrote it into. Nothing above changes — this + # is the second source for one file, never a second file. + # + # Shaped after ./glue-queue-agent-credential.nix rather than after the copy + # above, because it is a store reader and those are the store's readers: a + # cert login that fails LOUDLY, since every state it fails on is one a + # retry fixes, then a read that degrades QUIETLY, since no retry turns "no + # value there" into a value. + # + # ⚠️ An ABSENT secret is "not published yet", not a fault of this host's. + # The publisher runs on the authelia host and authelia mints on its first + # boot, so early in a swarm's life there is legitimately nothing to read. + # Nothing here writes a stand-in: there is no such thing as a locally valid + # OIDC client secret, and a placeholder would turn a Grafana that cannot + # log anyone in into one whose token exchange is refused. + # + # ⇒ So the absence surfaces at Grafana rather than here, and this unit's job + # is to leave the journal line that says which of the three it was. That is + # the same division the copy unit above makes by failing rather than + # skipping: the secret not arriving is a Grafana nobody can log into, and a + # named cause is the only thing separating that from a mystery. + systemd.services.swarm-bao-grafana-oidc = lib.mkIf storeDelivery { + description = "fetch Grafana's OIDC client secret from the swarm secret store"; + # Every one of these names a unit that exists only where the store runs. + # `Requires=` on an absent unit fails the job outright, so the ordering + # is conditional even though the read is not: off-host there is nothing + # local to wait for, and the timeout below bounds the attempt instead. + after = lib.optionals baoDeploy.enable [ + "swarm-bao-pki.service" + "container@${baoCfg.machine}.service" + ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; + requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ + "multi-user.target" + "container@${cfg.machine}.service" + ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + # Sized for the race this loses, not for an unseal: `swarm-bao` comes up + # seconds before this unit asks, and the cert-auth role it logs in + # against is written seconds after, so a few short attempts cover it. An + # hours-long window would be a bet on a sealed store, and the degrade + # below is already correct for that. + # + # `StartLimit*` are `[Unit]` settings, so they go here and not in + # `serviceConfig` — systemd ignores them under `[Service]`. The window + # has to exceed `RestartSec × burst`. + startLimitBurst = 4; + startLimitIntervalSec = 300; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-bao-grafana-oidc"; + # What actually bounds the read below. Stated here rather than left to + # systemd's default, so the number a boot waits on is in the file that + # waits. + TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = baoDeploy.clientCertFile; + BAO_CLIENT_KEY = baoDeploy.clientKeyFile; + } + # Absent means the system trust store, which is what a deployment with a + # real CA wants and what a self-signed one must not be left with. + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + # `bao`'s own message is the only thing separating a missing value from + # a refused identity from an unreachable host. This unit's degraded + # mode is correct for all three, so it reports which one rather than + # asserting all three in a sentence of ours. + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*` + # variables above only decide which certificate the TLS handshake + # presents; without a token `bao` asks its token helper instead, and + # that is a `sh` this unit's `path` does not carry. `-token-only` + # answers on stdout and skips the helper on both sides. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to swarm-bao with this host's certificate; leaving Grafana's OIDC client secret as it is." >&2 + if [ -s "$err" ]; then + cat "$err" >&2 + else + echo "bao failed without writing a diagnostic." >&2 + fi + exit 1 + fi + export BAO_TOKEN + + if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then + echo "swarm-bao did not return ${storeSecretPath}; Grafana has no OIDC client secret yet." >&2 + if [ -s "$err" ]; then + cat "$err" >&2 + else + echo "bao failed without writing a diagnostic." >&2 + fi + exit 0 + fi + + if [ -z "$secret" ]; then + echo "swarm-bao returned an empty ${storeSecretPath}; leaving the file as it is." >&2 + exit 0 + fi + + # Written with a shell builtin, never handed to a program: `printf` is + # bash's own, so the plaintext never becomes an argument in /proc the + # way `install <<<"$secret"` or an `echo` from `path` would. + # + # Same uid, group and mode as the copy above, for the reasons stated + # there — this is the same file arriving by a different route. + install -d -m 0755 ${lib.escapeShellArg hostSecretDir} + umask 077 + printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath} + chown ${toString config.ids.uids.grafana}:0 ${lib.escapeShellArg hostSecretPath} + chmod 0400 ${lib.escapeShellArg hostSecretPath} + ''; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -710,12 +915,11 @@ in # # Deliberately NOT `ssoLocal`: whether a password box sits on a # public login page cannot depend on which host happens to run - # authelia. The OIDC block below is still gated on locality; - # making that follow the same swarm-wide question is a larger - # change tracked on its own. + # authelia. The OIDC block below now answers the same swarm-wide + # question, so the two no longer disagree. auth.disable_login_form = true; } - // lib.optionalAttrs ssoLocal { + // lib.optionalAttrs ssoWired { "auth.generic_oauth" = { enabled = true; name = "HyperHive"; diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index 85cf126d..0d7030b0 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -1,5 +1,6 @@ # The unit that copies authelia's minted OIDC client secrets into the swarm's -# secret store, so a hive that does not host authelia can read its own. +# secret store, so whoever needs one without hosting authelia can read it: a +# hive its agents' credential, a swarm service its own. # # ⚠️ IT RUNS WHERE AUTHELIA DOES, and that is the whole reason it exists as a # separate thing. `deploy.authelia.hostClientSecretDir`'s own description says @@ -47,6 +48,18 @@ let # ./swarm-authelia.nix composes it, from the same two read-only options, so a # rename there cannot leave this spelling behind. agentClientId = hive: "${autheliaCfg.hiveClientPrefix}${hive}${autheliaCfg.agentClientSuffix}"; + + # The swarm's own services, as opposed to its hives. One client for the whole + # swarm rather than one per hive, so one value in the store rather than a copy + # each: `swarm/services//oidc/client`, under the `services` kind + # `swarm-secret-client`'s `path::Kind` declares. + # + # The ids come from `swarm.*`, which is identical on every host — that is what + # lets this host name a service's client while running none of them, and it is + # the same read the service's own module registers the client with. A swarm + # that runs no Grafana mints no secret for it, so its entry skips below rather + # than needing a condition here. + serviceClientIds = [ hyperhiveCfg.swarm.grafana.oidc.clientId ]; in { options.services.hyperhive.deploy.swarm-secret-publisher = { @@ -56,7 +69,8 @@ in defaultText = lib.literalExpression "deploy.authelia.enable"; description = '' Publish the OIDC client secrets this host mints into the swarm's - secret store, so hives that do not run authelia can read their own. + secret store, so a hive that does not run authelia can read its + agents' credential and a swarm service elsewhere can read its own. Defaults to whether this host mints them, which is the only half of the question that is a property of *this* host. @@ -177,6 +191,29 @@ in fi '') hiveNames} + ${lib.concatMapStringsSep "\n" (id: '' + src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${id}.secret"} + if [ -s "$src" ]; then + # `value=@$src` for the same reason as the hive loop above: bao + # opens the file itself, so the plaintext is never an argument of + # this process. + # + # No `client_id` field beside it, unlike a hive's credential: that + # one is derived per hive and has to be reconstructable from the + # store alone, whereas a service's client id is the swarm-wide + # option both ends already read. + bao kv put ${lib.escapeShellArg "secret/swarm/services/${id}/oidc/client"} \ + value=@"$src" + published=$((published + 1)) + else + # Not an error, and the ordinary state of a swarm that runs this + # service nowhere: nothing registered the client, so authelia minted + # nothing to publish. + echo "no minted secret at $src yet; the path unit will re-run this" >&2 + skipped=$((skipped + 1)) + fi + '') serviceClientIds} + echo "published $published client secret(s), skipped $skipped" ''; }; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 2be72a41..afadac61 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -223,6 +223,37 @@ let swarm.grafana.package = pkgs.emptyDirectory; }; + # The metrics UI beside the IdP, which is the deployment whose secret was + # already delivered: a host copy out of authelia's own tree. + grafanaWithAuthelia = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + deploy.authelia.enable = true; + }; + # The same UI with the IdP on ANOTHER host and a store leaf placed by hand — + # the deployment that had no delivery path at all. Knowing an IdP is not + # running one: `swarm.authelia.url` is what says this swarm has SSO, and + # nothing about this host does. + grafanaRemoteAuthelia = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + swarm.authelia.url = "https://auth.example.invalid"; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + # The same again with the store identity taken away, which is the one shape + # that can deliver this secret by NO route. It separates "the swarm has an + # IdP" from "something here can fetch what it minted", and it is what keeps + # Grafana's config from naming a file nothing writes. + grafanaRemoteAutheliaNoIdentity = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + swarm.authelia.url = "https://auth.example.invalid"; + }; + # The first slice to leave options on BOTH sides of the split, so the # fixture sets all three of them through the paths an existing config uses: # the two movers via their rename entries, `tokenEndpoint` via the path it @@ -836,6 +867,97 @@ let ok = grafanaOldPath.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form; } + { + # The absence class this whole file is for, and the reported defect in one + # arm: the OIDC block hung off "authelia is on this host", so the split + # deployment got a Grafana with no SSO settings and no login form — no way + # in at all. The question it asks now is whether the SWARM has an IdP. + name = "grafana is wired for SSO against an IdP on another host"; + ok = + let + s = + grafanaRemoteAuthelia.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth"; + in + s.enabled && lib.hasInfix "https://auth.example.invalid/api/oidc/token" s.token_url; + } + { + # The control for the arm above, and the reason the gate is a conjunction: + # a block naming a file no unit on this host writes cannot produce a + # working login, so it is emitted only where a delivery route exists — + # leaving a host with neither exactly as it was rather than pointing + # Grafana's config at a secret that never arrives. + name = "grafana emits no OIDC block when nothing on this host can deliver the secret"; + ok = + !( + grafanaRemoteAutheliaNoIdentity.containers.swarm-grafana.config.services.grafana.settings + ? "auth.generic_oauth" + ); + } + { + # The other half of the arm above, and the only deliverable that shape + # has: silence there is a Grafana nobody can log into for a reason no log + # names. A warning is read back by nothing, so an unevaluated one is a + # warning nobody sees — the same reason the peers fixture exists. + name = "the undeliverable-secret shape warns instead of going quiet"; + ok = + lib.any ( + w: lib.hasInfix "no swarm-secret-store identity" w + ) grafanaRemoteAutheliaNoIdentity.warnings + && !(lib.any (w: lib.hasInfix "no swarm-secret-store identity" w) grafanaRemoteAuthelia.warnings); + } + { + # Exactly one delivery unit per deployment, and which one is the only + # thing locality decides. Two units writing one path is a race over which + # secret Grafana parses; zero is the defect this pair replaced. + name = "grafana's OIDC secret has one delivery unit, chosen by where authelia is"; + ok = + let + local = grafanaWithAuthelia.systemd.services; + remote = grafanaRemoteAuthelia.systemd.services; + in + local ? swarm-grafana-oidc-secret + && !(local ? swarm-bao-grafana-oidc) + && remote ? swarm-bao-grafana-oidc + && !(remote ? swarm-grafana-oidc-secret); + } + { + # Same 403-not-a-miss reason as the matrix and queue arms below: the + # reader's grant covers the `services` prefix, so a path outside it is + # refused rather than empty, however correct it reads. The negative arm is + # the rename this is exposed to — a secret filed under the hive that runs + # the service instead of under the service itself. + name = "grafana's OIDC secret is read from the prefix the publisher writes"; + ok = + let + s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script; + in + lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s + && !(lib.hasInfix "secret/swarm/hives/" s); + } + { + # Both ends of a wire nothing at eval time carries end to end: the + # publisher on authelia's host writes the path the reader on Grafana's host + # reads, and the two files agree only because both compose it from the same + # swarm-wide client id. + name = "the publisher writes the swarm service path grafana reads"; + ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" ( + secretPublisherHere.systemd.services.swarm-secret-publish.script + ); + } + { + # Registering the client cannot live where the rest of grafana's module + # lives: that block is gated on this host RUNNING grafana, so on the split + # deployment nothing registered the client, authelia minted no secret, and + # every layer below had nothing to carry. The second arm is the control — + # a host with no IdP registers nothing. + name = "the swarm's grafana client is registered wherever authelia runs"; + ok = + let + clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; + in + lib.elem "swarm-grafana" (clients secretPublisherHere) + && !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia)); + } { # Reads the daemon's rendered unit, not the options: the queue address # arrives as an env var whose whole attrset is guarded on `natsUrl`, and @@ -934,17 +1056,18 @@ let !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-secret-publisher-policy); } { - # The whole point of a second principal. `hives/` and not `swarm/`, so it - # cannot touch an agent's or a service's credentials; and no `read`, so a - # unit whose job is copying a file cannot recover what is already there. - # Pinned as the full capability list, because an added capability is - # exactly what a presence check misses. - name = "the publisher's grant is write-only and reaches the hive prefix alone"; + # The whole point of a second principal. The two prefixes it publishes to + # and not `swarm/`, so it cannot touch an agent's credentials; and no + # `read`, so a unit whose job is copying a file cannot recover what is + # already there. Pinned as the full capability list per prefix, because an + # added capability is exactly what a presence check misses. + name = "the publisher's grant is write-only and reaches the hive and service prefixes alone"; ok = let s = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/*\" {\n capabilities = [\"create\", \"update\"]" s + && lib.hasInfix "path \"secret/data/swarm/services/*\" {\n capabilities = [\"create\", \"update\"]" s && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/*" s) && !(lib.hasInfix "sys/policies/acl" s); diff --git a/swarm-secret-client/src/policy.rs b/swarm-secret-client/src/policy.rs index 62201f2d..2977a906 100644 --- a/swarm-secret-client/src/policy.rs +++ b/swarm-secret-client/src/policy.rs @@ -5,8 +5,8 @@ //! two have to agree on the same path or a delivery fails with a 403 that names //! nothing. //! -//! The document has one stanza per kind a hive reads, and the two are scoped -//! differently on purpose: +//! The document has one stanza per kind a hive reads, and they are not all +//! scoped alike — which is the point rather than an inconsistency: //! //! ⚠️ The **agent** stanza grants read on *every* agent's credentials rather //! than on the ones that hive hosts. That is a decision, not an oversight: an @@ -16,9 +16,14 @@ //! narrow one that only looks narrow. The narrower shapes, and what they would //! cost, are in `docs/trust-boundary/security.md`. //! +//! ⚠️ The **service** stanza is wide for the same shape of reason: a swarm +//! service's client is registered once per swarm, so its path names the service +//! and never the host, and which hive runs a service is a `deploy.*` fact with +//! no swarm-wide spelling to scope against. Cost: the same doc. +//! //! The **hive** stanza has no such problem and is therefore narrow: that path //! names its principal, so scoping it to the reader's own name costs nothing -//! and drifts nowhere. Do not widen it to match its neighbour — the asymmetry +//! and drifts nowhere. Do not widen it to match its neighbours — the asymmetry //! is the point. //! //! Rendering stays separate from writing so the text can be asserted with no @@ -52,8 +57,8 @@ fn read_stanza(path: &str) -> String { format!("path \"{path}\" {{\n capabilities = [\"read\"]\n}}\n") } -/// Render `hive`'s policy document: read on every agent's credentials, and on -/// this hive's own. +/// Render `hive`'s policy document: read on every agent's credentials, on this +/// hive's own, and on the swarm services'. /// /// The name is the only input, and it is deploy-time — so the document is /// still a deploy-time object rather than derived state with a re-emission to @@ -61,9 +66,15 @@ fn read_stanza(path: &str) -> String { /// /// Read-only: the controller mints these and never reads one back. /// -/// ⚠️ The service and controller kinds are deliberately absent: a hive has no -/// business reading a service's or the controller's credentials. Adding either -/// is a boundary decision, not a consequence of the namespace growing. +/// ⚠️ The controller kind is deliberately absent: a hive has no business +/// reading the credentials of the thing that provisions it. Adding it is a +/// boundary decision, not a consequence of the namespace growing. +/// +/// The **service** kind is granted, and that was such a decision rather than a +/// consequence: a service whose identity provider is on another host reads its +/// own OIDC client secret with the certificate of the hive it runs on, that +/// being the only identity such a host has — so every hive can read every +/// service's. Bought and paid for in `docs/trust-boundary/security.md`. /// /// The hive's *own* kind is granted, and that is the decision the agent-only /// version of this grant said had to be made rather than assumed: a hive holds @@ -85,7 +96,11 @@ pub fn render(hive: &str) -> Result { "{MOUNT}/data/{ROOT}/{}/{hive}/*", <&str>::from(Kind::Hive) )); - Ok(format!("{agents}{own}")) + let services = read_stanza(&format!( + "{MOUNT}/data/{ROOT}/{}/*", + <&str>::from(Kind::Service) + )); + Ok(format!("{agents}{own}{services}")) } #[cfg(test)] @@ -93,11 +108,26 @@ mod tests { use super::*; #[test] - fn the_document_grants_the_whole_agent_prefix_and_this_hive_alone() { + fn the_document_grants_two_whole_prefixes_and_this_hive_alone() { assert_eq!( render("pr1ma").expect("a plain name is legal"), "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"read\"]\n}\n\ - path \"secret/data/swarm/hives/pr1ma/*\" {\n capabilities = [\"read\"]\n}\n" + path \"secret/data/swarm/hives/pr1ma/*\" {\n capabilities = [\"read\"]\n}\n\ + path \"secret/data/swarm/services/*\" {\n capabilities = [\"read\"]\n}\n" + ); + } + + #[test] + fn the_service_stanza_covers_a_service_this_hive_was_never_named_beside() { + // The property the swarm-grafana delivery depends on: the host running a + // swarm service reads that service's client secret with its own hive + // certificate, and the path names the service rather than the host. A + // stanza narrowed to the reader's name would 403 every such read. + let p = render("pr1ma").expect("legal"); + assert!(p.contains("path \"secret/data/swarm/services/*\"")); + assert!( + !p.contains("services/pr1ma"), + "the service stanza is not scoped to the reader" ); } From 815f977d7c63c25ab82ff1eb69b2abf3a5bec004 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 13 Sep 2026 19:27:33 +0200 Subject: [PATCH 2/2] swarm-grafana: one delivery route for the OIDC client secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit left two delivery paths and a three-way gate: `swarm-grafana-oidc-secret.service` copied authelia's minted plaintext out of its host tree wherever the two were co-located, `swarm-bao-grafana-oidc.service` read the same value from the swarm secret store wherever they were not, and `ssoConfigured && (ssoLocal || haveClientIdentity)` decided whether Grafana got an OIDC block at all. Delete the co-located path. The store reader is now THE delivery unit, in every deployment — the publisher on authelia's host writes `swarm/services//oidc/client` whether the reader is a network away or in the container next door. The ruling behind it: the store exists so a host holds ONE out-of-band secret, its client certificate, and reads everything else with it. Skipping the store when the producer happens to be local saves a round trip and costs a second delivery unit, a second way for the file to be wrong, and a gate to choose between them. The gate goes too, and both of its questions become assertions, scoped to hosts that run Grafana: - `swarm.authelia.url` must be set. `auth.disable_login_form` is unconditional — Grafana ships an admin/admin account on a public vhost — so dropping the OIDC block when the swarm names no IdP produced a container with no SSO and no password box, silently. An eval-time refusal naming the option is the only report that reaches anyone, the shape swarm-nats.nix already uses for the same option. - `deploy.bao.clientCertFile` / `clientKeyFile` must be set. This replaces a warning that nothing reads back, and its message names both options and where the leaf comes from. Fixtures follow. `grafanaWithAuthelia` gains the cert pair, because a co-located host is a store reader like any other. The old `grafanaRemoteAutheliaNoIdentity` is kept rather than deleted, renamed `grafanaNoIdentity`: the shape is still reachable, only its deliverable changed from silence to a refusal, and an arm now reads that refusal back. Its mirror `grafanaNoSso` covers the other assertion, each fixture wrong in exactly one way so an arm can name which refusal fired. Every positive keeps an explicit negative — the one-delivery-unit arm asserts the deleted unit is absent in both topologies rather than merely that the store reader is present. Refs #4234 Co-Authored-By: Claude Opus 5 --- docs/swarm/secrets.md | 72 +++--- docs/trust-boundary/security.md | 10 +- nix/host-modules/swarm-grafana.nix | 266 +++++++++----------- nix/host-modules/swarm-secret-publisher.nix | 23 +- nix/module-eval.nix | 155 ++++++++---- 5 files changed, 294 insertions(+), 232 deletions(-) diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 48b5d03a..3a0a91e8 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -41,23 +41,23 @@ neither is a renaming of the other. ## Swarm-level — one of each per swarm -| secret | generated by | lives at | hive elsewhere | -| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| swarm root CA cert | `swarm-ca.nix` first-boot unit, when `autoConfigure` is set | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | -| swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | -| swarm-services sub-CA (cert + key) | `swarm-ca.nix`, signed by the root | `/var/lib/swarm-ca/services-ca{,-key}.pem` | issued where the root lives | -| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | -| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | -| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | -| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `deploy.swarm-otel.clientSecretFile` — the collector need not share a host with authelia | -| Grafana's copy of its OIDC secret | `swarm-grafana-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store instead — nothing for an operator to place, see below | -| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | -| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | +| secret | generated by | lives at | hive elsewhere | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| swarm root CA cert | `swarm-ca.nix` first-boot unit, when `autoConfigure` is set | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | +| swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | +| swarm-services sub-CA (cert + key) | `swarm-ca.nix`, signed by the root | `/var/lib/swarm-ca/services-ca{,-key}.pem` | issued where the root lives | +| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | +| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | +| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | +| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | +| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `deploy.swarm-otel.clientSecretFile` — the collector need not share a host with authelia | +| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | +| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | +| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | Authelia mints the three keys for itself, in-container, precisely because nothing outside that container ever reads them. **That's the test worth @@ -185,18 +185,32 @@ the harness logs that it has no queue coordinates — the same absent-and-legal rather than a container that refuses to start. The **third reader** is Grafana's OIDC client secret, and it's the first one -that's a _swarm service's_ own credential rather than a hive's. -`swarm-grafana.nix` asks two questions where it used to ask one: whether this -_swarm_ has an IdP (`swarm.authelia.url`) decides whether Grafana gets SSO -settings at all, and whether authelia is on _this_ host decides only which of -two units delivers the secret — the host copy out of authelia's tree, or -`swarm-bao-grafana-oidc.service` reading `swarm/services//oidc/client` -out of the store. Exactly one of the two exists in any deployment, and where -neither can (no authelia here, no store identity here) the module warns and -emits no OIDC settings rather than pointing Grafana at a file nothing writes. -Registration moved to `glue-grafana-oidc-client.nix` for a related reason: a -client is a row in authelia's own config, so it has to be declared on the host -that runs authelia — which is not necessarily the host that runs Grafana. +that's a _swarm service's_ own credential rather than a hive's. It has exactly +one delivery route: `swarm-bao-grafana-oidc.service` reads +`swarm/services//oidc/client` out of the store, on **every** host +that runs Grafana. Where authelia is doesn't enter into it — the publisher on +authelia's host writes that path whether the reader is a network away or in the +container next door. + +That's a deliberate refusal of the shortcut. Copying the plaintext +straight out of authelia's tree when the two happen to share a host is one +round trip cheaper and costs a second delivery unit, a second way for the file +to be wrong, and a gate deciding between them — and the gate is the expensive +part, because whatever it's wrong about is an outage nobody can read. The store +exists so a host holds **one** out-of-band secret, its client certificate, and +reads everything else with it. + +Two things follow, and `swarm-grafana.nix` asserts both rather than degrading: +running Grafana requires `swarm.authelia.url` (its local login form is disabled +unconditionally, so SSO isn't a feature of some topologies — it's the only way +in), and it requires this host's `deploy.bao.clientCertFile` / +`clientKeyFile`. Each refusal names the option to set. Both used to be silent: +a null URL dropped the OIDC block, a missing leaf produced a warning, and +either one left a Grafana with no SSO and no password box — nothing failed, so +nothing said anything. Registration is separate and stays where authelia is +(`glue-grafana-oidc-client.nix`): a client is a row in authelia's own config, +so it's declared on the host running authelia, which isn't necessarily the host +running Grafana. A service's secret is one value for the whole swarm rather than one per hive, so it lives under the `services` prefix, and a hive's read policy grants that prefix diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index cc455653..4d71ee7c 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -71,11 +71,11 @@ for the whole swarm, so its credential's path names the service and never the host — and which hive runs a given service is a `deploy.*` fact, per-host by definition, so nothing swarm-wide exists to scope the grant to. The host running such a service has no store identity of its own either; it reads with the -certificate of the hive it is. So **any hive can read any swarm service's client -secret**, which is worth what it buys: a swarm service whose identity provider -lives on another machine can get its secret at all, instead of only working -co-located. Giving such a service its own store identity is what would remove -this rather than re-scope it. +certificate of the hive it is. **Any hive can therefore read any swarm service's +client secret**, which is worth what it buys: every swarm service gets its +credential the same way from anywhere, instead of only where the identity +provider happens to sit. Giving such a service its own store identity is what +would remove this rather than re-scope it. **A tracked follow-up** narrows this, with the two candidate directions: scope the grant per hive (and pay for the re-emission), or give each agent container diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 50d53fda..f059a06e 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -83,38 +83,27 @@ let # Is SSO configured for this SWARM. Swarm-wide by construction — `swarm.*` is # identical on every host — and the option's own description is what makes # this the right question to ask: a null URL means "no SSO configured". - ssoConfigured = autheliaCfg.url != null; - - # Is authelia on THIS host, spelled exactly as the forge and matrix modules - # spell it. All locality decides is where the minted plaintext is copied - # FROM: authelia's own tree here, the swarm secret store otherwise. # - # 🩸 This used to be `grafana.enable && authelia.enable` and was the ONLY - # gate: registration, the mint's delivery and the OIDC block all hung off - # it, so a swarm whose authelia ran elsewhere got Grafana with no SSO wiring - # — and the login form is disabled whatever happens, so with nothing else. - ssoLocal = deployCfg.authelia.enable; + # 🩸 A subject of an assertion below, NOT a gate. Dropping the OIDC block when + # this is false looks conservative and is the outage: `disable_login_form` is + # unconditional a few hundred lines down, so a Grafana with no OIDC settings + # is a Grafana with no login of any kind, arrived at silently. SSO is a + # requirement of running this service, so an unconfigured swarm fails to + # build and says which option to set. + ssoConfigured = autheliaCfg.url != null; # A reader of the store is defined by holding a certificate the store # accepts, never by standing next to it — the rule # ./glue-matrix-bao-token.nix states in full. + # + # Also asserted rather than gating anything, and that is the ruling this + # module's second delivery route was deleted under: the store exists so that + # ONE secret is placed out of band per host — this certificate — and every + # other secret is read with it. A Grafana host without it has not been given + # its identity yet, which is a thing to say out loud rather than to route + # around by reaching into authelia's tree whenever it happens to be local. haveClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null; - # Whether Grafana gets an OIDC block: the swarm-wide question, AND that - # something on this host delivers the secret that block names. The second - # half is not redundant — a block naming a file no unit here writes cannot - # produce a working login either way, and what Grafana does with an - # unreadable `$__file{}` target (refuse to start, or start and fail every - # login) is not worth betting the container on when not emitting the block is - # available and leaves this host as it was. - ssoWired = ssoConfigured && (ssoLocal || haveClientIdentity); - - # The delivery that is not a copy: authelia is elsewhere, so the same minted - # plaintext arrives out of the store that the publisher on authelia's host - # wrote it into. Exclusive with the copy unit by construction — two units - # writing one file is a race over which secret Grafana parses. - storeDelivery = ssoWired && !ssoLocal; - autheliaUrl = toString autheliaCfg.url; # Where the plaintext lands inside the container. Under /var/lib rather @@ -123,9 +112,10 @@ let # intermittent one. secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret"; - # The same file seen from the host, which is where both delivery units write - # it. Spelled once: two units landing a secret at two spellings of one path - # is a Grafana that reads whichever of them it was configured with. + # The same file seen from the host, which is where the delivery unit writes + # it. Spelled once: the unit that writes it and the config that names it are + # a few hundred lines apart, and a Grafana reading a path nothing writes is + # a login that fails with nothing in any log about the file. hostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${secretPath}"; hostSecretDir = builtins.dirOf hostSecretPath; @@ -401,15 +391,16 @@ in # The secret oneshots as well as grafana itself: each runs before it and # fails in ways grafana then reports only as a login that does not work. # - # Each delivery unit is listed only where it exists, the way - # ./hive-matrix.nix lists its own: a unit name that never renders is a - # journald scrape target matching nothing, which reads as a quiet unit. + # Unconditional, because every unit named here now renders in every + # deployment. This list used to be assembled with `lib.optional` per + # delivery route, which was the right shape while there were two — a unit + # name that never renders is a journald scrape target matching nothing, + # which reads as a quiet unit rather than an absent one. services.hyperhive.swarm.otel.journaldUnits = [ "grafana" "swarm-grafana-secret-key" - ] - ++ lib.optional ssoLocal "swarm-grafana-oidc-secret" - ++ lib.optional storeDelivery "swarm-bao-grafana-oidc"; + "swarm-bao-grafana-oidc" + ]; services.hyperhive.swarm.controller.links = [ { @@ -423,36 +414,57 @@ in # host that runs authelia, and this whole block is gated on the host that # runs Grafana. ./glue-grafana-oidc-client.nix is where it moved to. - warnings = lib.optional (ssoConfigured && !ssoLocal && !haveClientIdentity) '' - services.hyperhive.swarm.grafana: this swarm has an IdP - (services.hyperhive.swarm.authelia.url) but authelia is not on this - host and this host holds no swarm-secret-store identity, so nothing - can deliver Grafana's OIDC client secret. Grafana will come up with - no OIDC login and no local login form, which is no way in at all. - - Name this host's store client leaf in - services.hyperhive.deploy.bao.clientCertFile and - services.hyperhive.deploy.bao.clientKeyFile — the same identity every - other reader of the store uses. See docs/swarm/secrets.md. - ''; - + # Both arms are what used to be a silent gate, and both fire only where + # Grafana runs — this whole block is under `deploy.grafana.enable`. The + # binding itself is asserted rather than a copy of its formula, the way + # ./swarm-nats.nix's own `autheliaUrl` arm does it: two spellings of one + # boolean is two places for a future edit to land in only one. assertions = [ { - # The URL is what `ssoConfigured` above reads, so with it null this - # host mints and delivers a secret for an OIDC block it then does not - # emit — SSO silently absent on the one deployment that has every - # other piece of it. Only reachable by enabling authelia and clearing - # its `url`, which is why it is an assertion and not a fallback. - assertion = !ssoLocal || ssoConfigured; + # SSO is not optional for this service, and the reason is a hundred + # lines below in `auth.disable_login_form = true`: Grafana ships an + # `admin`/`admin` account on a public vhost, so the password box is + # off whatever the topology. Emitting no OIDC block when the swarm + # names no IdP therefore produces a container with no way in at all — + # a state no log names, since nothing failed. Failing the build and + # naming the option is the only report that reaches anyone. + assertion = ssoConfigured; message = '' - services.hyperhive.swarm.grafana requires - services.hyperhive.swarm.authelia.url when authelia is enabled. + services.hyperhive.deploy.grafana.enable requires + services.hyperhive.swarm.authelia.url — Grafana's only login is SSO, + because its local login form is disabled unconditionally (it ships + an admin/admin account and its vhost is on the public gateway). - Grafana exchanges its authorization code at - `''${url}/api/oidc/token` from inside its container, so with the URL - null there is no endpoint to name — and a null URL is also how this - module reads "no SSO configured for this swarm", which would leave - Grafana with no login of any kind on the host that mints its secret. + It defaults to this host's own instance only when this host runs + authelia. A hive that federates with a swarm sets it explicitly to + wherever that provider lives. Grafana exchanges its authorization + code at `''${url}/api/oidc/token` from inside its container, so a + null URL leaves no endpoint to name. + ''; + } + { + # The other half of one login: the OIDC block names a `$__file{}` that + # `swarm-bao-grafana-oidc.service` below writes, and that unit reads + # the store with this host's client certificate. No certificate, no + # secret, and the same no-way-in Grafana as the arm above. + assertion = haveClientIdentity; + message = '' + services.hyperhive.deploy.grafana.enable requires this host to hold a + swarm-secret-store client identity: set both + + services.hyperhive.deploy.bao.clientCertFile + services.hyperhive.deploy.bao.clientKeyFile + + Grafana's OIDC client secret is minted by authelia and read out of + the store, on every host that runs Grafana — including the host that + runs authelia. That is one delivery route rather than two, and it is + what the store is for: this certificate is the single credential + placed out of band, and every other secret comes from the store with + it. + + On a hive that runs the store, glue-bao-tls.nix supplies both as + defaults and there is nothing to do. Elsewhere the leaf is issued + from that CA out of band and named here — see docs/swarm/secrets.md. ''; } ]; @@ -524,79 +536,24 @@ in "d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" ]; - # The local delivery, and the ONLY thing locality still decides. It runs on - # the HOST because that is the only place both container trees are - # addressable: they share this host's network namespace, which makes them - # feel co-located, but their filesystem roots are separate — Grafana cannot - # open a path inside authelia's tree however local the port looks. + # THE delivery unit — one route, in every deployment. The secret authelia + # minted arrives out of the swarm secret store, which the publisher on + # authelia's host wrote it into, and that is true whether authelia is a + # network away or in the container next door. # - # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container - # refuses to start when a bind source is missing, and this secret does - # not exist until authelia's first boot has minted it — so binding it - # would make Grafana wait on a file that waits on a container that starts - # after it. - systemd.services.swarm-grafana-oidc-secret = lib.mkIf ssoLocal { - description = "deliver Grafana's OIDC client secret from authelia"; - after = [ "container@${autheliaCfg.machine}.service" ]; - requires = [ "container@${autheliaCfg.machine}.service" ]; - before = [ "container@${cfg.machine}.service" ]; - wantedBy = [ "container@${cfg.machine}.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "swarm-grafana-oidc-secret"; - # ⚠️ Longer than the wait below, and that is the whole point: - # `DefaultTimeoutStartSec` is 90s, so a 120s bounded wait is killed - # by systemd at 90 — before it can emit the error naming the file it - # waited for. The timeout has to outlive the thing it is timing. - TimeoutStartSec = "180s"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -euo pipefail - - src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${cfg.oidc.clientId}.secret"} - dst=${lib.escapeShellArg hostSecretPath} - - # authelia's container is up, but its first-boot generator may still - # be minting. Bounded wait, then fail: a silent skip here produces a - # Grafana whose only login path dead-ends. - deadline=$(( SECONDS + 120 )) - while [ ! -s "$src" ]; do - if [ "$SECONDS" -ge "$deadline" ]; then - echo "authelia has not minted $src after 120s" >&2 - exit 1 - fi - sleep 2 - done - - # Owned by Grafana's own uid, unlike the matrix sibling which lands - # root-owned: tuwunel's secret is read by `LoadCredential` as root - # before the sandbox exists, whereas Grafana expands `$__file{}` - # itself, as itself, while parsing its config. These containers set - # no `privateUsers`, so the host uid is the container uid, and both - # sides take it from the same static NixOS id. - # - # Group is root, not grafana, and that is forced rather than chosen: - # `ids.uids.grafana` is a static id but there is no `ids.gids.grafana` - # — the group's gid is allocated at activation inside the container, - # so the host cannot know it at eval time. Harmless here because 0400 - # grants the group nothing; if this mode ever widens, the gid has to - # be discovered at runtime rather than assumed. - install -D -m 0400 -o ${toString config.ids.uids.grafana} -g 0 "$src" "$dst" - ''; - }; - - # The same secret, the same destination, for the deployment where authelia - # is NOT here: it arrives out of the swarm secret store, which the - # publisher on authelia's host wrote it into. Nothing above changes — this - # is the second source for one file, never a second file. + # 🩸 A second unit here used to copy the plaintext out of authelia's host + # tree wherever the two were co-located, with the OIDC block gated on + # either route being available. The ruling that deleted it: the store + # exists so a host holds ONE out-of-band secret — its client certificate — + # and reads everything else with it, so skipping the store when the + # producer is local buys a round trip and costs a second delivery path, a + # second way for the file to be wrong, and a gate whose false arm was a + # Grafana with no login. Recorded in docs/swarm/secrets.md. # - # Shaped after ./glue-queue-agent-credential.nix rather than after the copy - # above, because it is a store reader and those are the store's readers: a - # cert login that fails LOUDLY, since every state it fails on is one a - # retry fixes, then a read that degrades QUIETLY, since no retry turns "no - # value there" into a value. + # Shaped after ./glue-queue-agent-credential.nix, because it is a store + # reader and this is what the store's readers do: a cert login that fails + # LOUDLY, since every state it fails on is one a retry fixes, then a read + # that degrades QUIETLY, since no retry turns "no value there" into a value. # # ⚠️ An ABSENT secret is "not published yet", not a fault of this host's. # The publisher runs on the authelia host and authelia mints on its first @@ -606,11 +563,10 @@ in # log anyone in into one whose token exchange is refused. # # ⇒ So the absence surfaces at Grafana rather than here, and this unit's job - # is to leave the journal line that says which of the three it was. That is - # the same division the copy unit above makes by failing rather than - # skipping: the secret not arriving is a Grafana nobody can log into, and a - # named cause is the only thing separating that from a mystery. - systemd.services.swarm-bao-grafana-oidc = lib.mkIf storeDelivery { + # is to leave the journal line that says which of the three it was: the + # secret not arriving is a Grafana nobody can log into, and a named cause is + # the only thing separating that from a mystery. + systemd.services.swarm-bao-grafana-oidc = { description = "fetch Grafana's OIDC client secret from the swarm secret store"; # Every one of these names a unit that exists only where the store runs. # `Requires=` on an absent unit fails the job outright, so the ordering @@ -708,8 +664,19 @@ in # bash's own, so the plaintext never becomes an argument in /proc the # way `install <<<"$secret"` or an `echo` from `path` would. # - # Same uid, group and mode as the copy above, for the reasons stated - # there — this is the same file arriving by a different route. + # Owned by Grafana's own uid, unlike the matrix sibling which lands + # root-owned: tuwunel's secret is read by `LoadCredential` as root + # before the sandbox exists, whereas Grafana expands `$__file{}` + # itself, as itself, while parsing its config. This container sets no + # `privateUsers`, so the host uid is the container uid, and both sides + # take it from the same static NixOS id. + # + # Group is root, not grafana, and that is forced rather than chosen: + # `ids.uids.grafana` is a static id but there is no `ids.gids.grafana` + # — the group's gid is allocated at activation inside the container, so + # the host cannot know it at eval time. Harmless here because 0400 + # grants the group nothing; if this mode ever widens, the gid has to be + # discovered at runtime rather than assumed. install -d -m 0755 ${lib.escapeShellArg hostSecretDir} umask 077 printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath} @@ -913,13 +880,21 @@ in # public gateway, so SSO is the only way in — not merely the # preferred one. # - # Deliberately NOT `ssoLocal`: whether a password box sits on a - # public login page cannot depend on which host happens to run - # authelia. The OIDC block below now answers the same swarm-wide - # question, so the two no longer disagree. + # Not conditional on anything, and the block below is not + # either, which is what keeps the two from ever disagreeing. + # Whether a password box sits on a public login page cannot + # depend on which host happens to run authelia — and it used to, + # in a module where the OIDC block was gated and this line was + # not. auth.disable_login_form = true; - } - // lib.optionalAttrs ssoWired { + + # Emitted in every deployment, because SSO is a requirement of + # running this service rather than a feature of some topologies: + # the assertions above refuse to build a Grafana that lacks an + # IdP to point at or the identity to fetch its secret with. A + # `lib.optionalAttrs` here would be the silent half of that pair, + # dropping the only login this container has whenever a value + # went null. "auth.generic_oauth" = { enabled = true; name = "HyperHive"; @@ -930,6 +905,9 @@ in # `database.password`, `security.admin_password` and # datasource `secureJsonData`), so nothing but this comment # stands between a literal and the store. + # + # `swarm-bao-grafana-oidc.service` above is what writes this + # path, on every host that runs Grafana. client_secret = "$__file{${secretPath}}"; scopes = "openid profile email groups"; auth_url = "${autheliaUrl}/api/oidc/authorization"; diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index 0d7030b0..9033f24b 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -1,6 +1,13 @@ # The unit that copies authelia's minted OIDC client secrets into the swarm's -# secret store, so whoever needs one without hosting authelia can read it: a -# hive its agents' credential, a swarm service its own. +# secret store, so whoever needs one can read it there: a hive its agents' +# credential, a swarm service its own. +# +# ⚠️ "Whoever", including a reader on THIS host. A swarm service's secret goes +# into the store even when the service runs beside authelia, because its +# reader fetches it from the store in every deployment — ./swarm-grafana.nix +# states the ruling that made that the only route. Publishing "only when the +# reader is elsewhere" would be a second shape of this unit, gated on a fact +# about another host, to save a round trip on the one host that can afford it. # # ⚠️ IT RUNS WHERE AUTHELIA DOES, and that is the whole reason it exists as a # separate thing. `deploy.authelia.hostClientSecretDir`'s own description says @@ -70,7 +77,8 @@ in description = '' Publish the OIDC client secrets this host mints into the swarm's secret store, so a hive that does not run authelia can read its - agents' credential and a swarm service elsewhere can read its own. + agents' credential and a swarm service can read its own — from + wherever it runs, this host included. Defaults to whether this host mints them, which is the only half of the question that is a property of *this* host. @@ -82,10 +90,11 @@ in publishing nothing. Whether the wiring is complete is the client identity's job (see `baoClientCertFile`), not this option's. - Turning it off leaves every non-co-located hive without a delivery - path, which is the state this exists to end — so the honest reason - to set it false is a deployment delivering those secrets by some - other mechanism it owns. + Turning it off leaves every hive but this one without its agents' + credential, and the swarm's Grafana without any login at all — its + secret has exactly one route and this is the producer's end of it. So + the honest reason to set it false is a deployment delivering those + secrets by some other mechanism it owns. ''; }; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index afadac61..7956957b 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -223,18 +223,22 @@ let swarm.grafana.package = pkgs.emptyDirectory; }; - # The metrics UI beside the IdP, which is the deployment whose secret was - # already delivered: a host copy out of authelia's own tree. + # The metrics UI beside the IdP. It reads its secret out of the store like + # every other Grafana host, so it needs a store identity like every other + # Grafana host — the cert pair here is not scenery, it is the arm that would + # have caught the deleted co-located copy unit coming back. grafanaWithAuthelia = hive { deploy.grafana.enable = true; deploy.grafana.plugins = [ ]; deploy.grafana.package = pkgs.emptyDirectory; deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; }; - # The same UI with the IdP on ANOTHER host and a store leaf placed by hand — - # the deployment that had no delivery path at all. Knowing an IdP is not - # running one: `swarm.authelia.url` is what says this swarm has SSO, and - # nothing about this host does. + # The same UI with the IdP on ANOTHER host and a store leaf placed by hand. + # Knowing an IdP is not running one: `swarm.authelia.url` is what says this + # swarm has SSO, and nothing about this host does. Identical to the fixture + # above in everything the delivery path reads, which is the point. grafanaRemoteAuthelia = hive { deploy.grafana.enable = true; deploy.grafana.plugins = [ ]; @@ -243,16 +247,48 @@ let deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; }; - # The same again with the store identity taken away, which is the one shape - # that can deliver this secret by NO route. It separates "the swarm has an - # IdP" from "something here can fetch what it minted", and it is what keeps - # Grafana's config from naming a file nothing writes. - grafanaRemoteAutheliaNoIdentity = hive { + # A Grafana host holding no store identity. This used to be the shape the + # module went QUIET on — no OIDC block, a warning, and a container whose + # login form is off regardless, so no way in and nothing failed. It is kept + # rather than deleted because the shape is still reachable by an operator; + # what changed is the deliverable, from a warning nothing reads back to a + # refusal naming the two options to set. Only the identity is missing, so an + # arm below can name which refusal fired. + grafanaNoIdentity = hive { deploy.grafana.enable = true; deploy.grafana.plugins = [ ]; deploy.grafana.package = pkgs.emptyDirectory; swarm.authelia.url = "https://auth.example.invalid"; }; + # The mirror image: the identity is placed, and the swarm names no IdP. The + # other half of "SSO must always be configured", and isolated the same way — + # exactly one thing wrong, so the arm reads one refusal. + grafanaNoSso = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + + # Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its + # two reasons. An assertion is a config VALUE until something forces it — + # `.config` never throws — so a fixture in a state the module refuses is + # evaluable and the refusal is readable as data. That is what lets a case + # check that a misconfiguration is REPORTED, rather than only that it is not + # silently accepted. + # + # Matched on the option name the message names, not on its prose, so the + # wording stays rewordable: the option name is the part an operator has to + # act on, and a message that stopped naming it would be the actual defect. + grafanaRefusedFor = + m: option: + lib.any ( + a: + !a.assertion + && lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message + && lib.hasInfix option a.message + ) m.assertions; # The first slice to leave options on BOTH sides of the split, so the # fixture sets all three of them through the paths an existing config uses: @@ -871,55 +907,80 @@ let # The absence class this whole file is for, and the reported defect in one # arm: the OIDC block hung off "authelia is on this host", so the split # deployment got a Grafana with no SSO settings and no login form — no way - # in at all. The question it asks now is whether the SWARM has an IdP. - name = "grafana is wired for SSO against an IdP on another host"; + # in at all. The block is emitted in every deployment now, so the negative + # arm is not "no block elsewhere" but "the two do not name the same IdP": + # each host's block has to point at the URL the SWARM names, and a block + # built from `deploy.authelia` rather than `swarm.authelia.url` would pass + # a presence check on both fixtures while sending one of them nowhere. + name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs"; ok = let - s = - grafanaRemoteAuthelia.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth"; + oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth"; + remote = oauth grafanaRemoteAuthelia; + local = oauth grafanaWithAuthelia; in - s.enabled && lib.hasInfix "https://auth.example.invalid/api/oidc/token" s.token_url; + remote.enabled + && lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url + && local.enabled + && lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url + && !(lib.hasInfix "auth.example.invalid" local.token_url); } { - # The control for the arm above, and the reason the gate is a conjunction: - # a block naming a file no unit on this host writes cannot produce a - # working login, so it is emitted only where a delivery route exists — - # leaving a host with neither exactly as it was rather than pointing - # Grafana's config at a secret that never arrives. - name = "grafana emits no OIDC block when nothing on this host can deliver the secret"; - ok = - !( - grafanaRemoteAutheliaNoIdentity.containers.swarm-grafana.config.services.grafana.settings - ? "auth.generic_oauth" - ); - } - { - # The other half of the arm above, and the only deliverable that shape - # has: silence there is a Grafana nobody can log into for a reason no log - # names. A warning is read back by nothing, so an unevaluated one is a - # warning nobody sees — the same reason the peers fixture exists. - name = "the undeliverable-secret shape warns instead of going quiet"; - ok = - lib.any ( - w: lib.hasInfix "no swarm-secret-store identity" w - ) grafanaRemoteAutheliaNoIdentity.warnings - && !(lib.any (w: lib.hasInfix "no swarm-secret-store identity" w) grafanaRemoteAuthelia.warnings); - } - { - # Exactly one delivery unit per deployment, and which one is the only - # thing locality decides. Two units writing one path is a race over which - # secret Grafana parses; zero is the defect this pair replaced. - name = "grafana's OIDC secret has one delivery unit, chosen by where authelia is"; + # 🩸 The arm that guards the ruling this slice landed under. There is ONE + # delivery route: the store reader, on every host that runs Grafana. The + # negative names the deleted unit rather than a generic absence, because + # the way this regresses is someone re-adding the co-located copy as an + # optimisation — a second writer of one path, and a second shape of "the + # secret is wrong" to debug. + name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies"; ok = let local = grafanaWithAuthelia.systemd.services; remote = grafanaRemoteAuthelia.systemd.services; in - local ? swarm-grafana-oidc-secret - && !(local ? swarm-bao-grafana-oidc) + local ? swarm-bao-grafana-oidc && remote ? swarm-bao-grafana-oidc + && !(local ? swarm-grafana-oidc-secret) && !(remote ? swarm-grafana-oidc-secret); } + { + # What the deleted warning became. The shape is unchanged — a Grafana host + # holding no store leaf — but silence there is a container nobody can log + # into for a reason no log names, and a warning is read back by nothing. + # The second arm is what makes this a refusal about the IDENTITY: this + # fixture names an IdP, so a message about `swarm.authelia.url` here would + # mean the two assertions had been collapsed into one conjunction. + name = "a grafana host with no store identity is refused, naming the options to set"; + ok = + grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientCertFile" + && grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientKeyFile" + && !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url"); + } + { + # "SSO must always be configured", as an eval-time refusal rather than a + # gate. A null URL used to drop the OIDC block silently, and + # `disable_login_form` is unconditional a hundred lines below it, so that + # combination produced a Grafana with no SSO and no password box — an + # outage whose cause is a boolean that evaluated to false at build time + # and left no trace. Same isolation as the arm above, mirrored. + name = "a grafana host in a swarm with no IdP is refused, naming swarm.authelia.url"; + ok = + grafanaRefusedFor grafanaNoSso "services.hyperhive.swarm.authelia.url" + && !(grafanaRefusedFor grafanaNoSso "deploy.bao.clientCertFile"); + } + { + # Without this the two arms above prove nothing: a refusal that fires on + # every host is not a check, and both of these are hosts a swarm is + # expected to have. Read through the same helper, so a message that + # stopped naming its option would fail the arms above rather than pass + # this one by accident. + name = "neither grafana refusal fires on a correctly configured host, co-located or not"; + ok = + !(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url") + && !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.clientCertFile") + && !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url") + && !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.clientCertFile"); + } { # Same 403-not-a-miss reason as the matrix and queue arms below: the # reader's grant covers the `services` prefix, so a path outside it is