diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index afce2a45..48b5d03a 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -41,22 +41,23 @@ neither is a renaming of the other. ## Swarm-level — one of each per swarm -| secret | generated by | lives at | hive elsewhere | -| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| swarm root CA cert | `swarm-ca.nix` first-boot unit, when `autoConfigure` is set | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | -| swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | -| swarm-services sub-CA (cert + key) | `swarm-ca.nix`, signed by the root | `/var/lib/swarm-ca/services-ca{,-key}.pem` | issued where the root lives | -| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | -| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | -| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | -| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `deploy.swarm-otel.clientSecretFile` — the collector need not share a host with authelia | -| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | -| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | +| secret | generated by | lives at | hive elsewhere | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| swarm root CA cert | `swarm-ca.nix` first-boot unit, when `autoConfigure` is set | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | +| swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | +| swarm-services sub-CA (cert + key) | `swarm-ca.nix`, signed by the root | `/var/lib/swarm-ca/services-ca{,-key}.pem` | issued where the root lives | +| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | +| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | +| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | +| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | +| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `deploy.swarm-otel.clientSecretFile` — the collector need not share a host with authelia | +| Grafana's copy of its OIDC secret | `swarm-grafana-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store instead — nothing for an operator to place, see below | +| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | +| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | Authelia mints the three keys for itself, in-container, precisely because nothing outside that container ever reads them. **That's the test worth @@ -183,6 +184,32 @@ files aren't there the daemon forwards nothing and says so in its journal, and the harness logs that it has no queue coordinates — the same absent-and-legal state, twice, rather than a container that refuses to start. +The **third reader** is Grafana's OIDC client secret, and it's the first one +that's a _swarm service's_ own credential rather than a hive's. +`swarm-grafana.nix` asks two questions where it used to ask one: whether this +_swarm_ has an IdP (`swarm.authelia.url`) decides whether Grafana gets SSO +settings at all, and whether authelia is on _this_ host decides only which of +two units delivers the secret — the host copy out of authelia's tree, or +`swarm-bao-grafana-oidc.service` reading `swarm/services//oidc/client` +out of the store. Exactly one of the two exists in any deployment, and where +neither can (no authelia here, no store identity here) the module warns and +emits no OIDC settings rather than pointing Grafana at a file nothing writes. +Registration moved to `glue-grafana-oidc-client.nix` for a related reason: a +client is a row in authelia's own config, so it has to be declared on the host +that runs authelia — which is not necessarily the host that runs Grafana. + +A service's secret is one value for the whole swarm rather than one per hive, so +it lives under the `services` prefix, and a hive's read policy grants that prefix +whole. That's because nothing in a swarm's configuration records which hive runs +a given service — placement is a `deploy.*` fact, per-host by definition — so +there is nothing to scope the grant to. Every hive can therefore read every swarm +service's client secret; that's the cost of a shape where the service's host has +no store identity of its own to present, and it's stated in +`swarm-secret-client`'s `policy` module beside the grant itself. Absence behaves +as it does for the other two readers: a store that says "not here" leaves the +file alone and says so, since there is no such thing as a locally valid OIDC +client secret to stand in. + ⚠️ **Service↔store mTLS is its own trust domain.** A credential you must already hold to authenticate can't be fetched from the thing it authenticates you to, so the store's identity can't come from an authority the store diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index 852108d8..cc455653 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -60,10 +60,22 @@ advertises a boundary it doesn't actually hold. A wide grant that says what it is beats a narrow one that only looks narrow. What still holds: the grant is **read-only** (a hive can't write an agent's -credential, so it can't hand itself an agent's identity), and it reaches two -prefixes and nothing else in the store — every agent's credentials, and the -reader's _own_ entry under the hive namespace, which names the hive asking and -so widens nothing between them. +credential, so it can't hand itself an agent's identity), and it reaches three +prefixes and nothing else in the store — every agent's credentials, the reader's +_own_ entry under the hive namespace, which names the hive asking and so widens +nothing between them, and every **swarm service's** OIDC client secret. + +That third prefix has the same shape of reason as the first, and the same honest +cost. A swarm service (Grafana is the one there today) registers **one** client +for the whole swarm, so its credential's path names the service and never the +host — and which hive runs a given service is a `deploy.*` fact, per-host by +definition, so nothing swarm-wide exists to scope the grant to. The host running +such a service has no store identity of its own either; it reads with the +certificate of the hive it is. So **any hive can read any swarm service's client +secret**, which is worth what it buys: a swarm service whose identity provider +lives on another machine can get its secret at all, instead of only working +co-located. Giving such a service its own store identity is what would remove +this rather than re-scope it. **A tracked follow-up** narrows this, with the two candidate directions: scope the grant per hive (and pay for the re-emission), or give each agent container diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 7b57fee7..5c626d84 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -25,6 +25,7 @@ ./otel.nix ./glue-bao-tls.nix ./glue-controller-bao-identity.nix + ./glue-grafana-oidc-client.nix ./glue-matrix-bao-token.nix ./glue-queue-agent-credential.nix ./glue-secret-publisher-bao-identity.nix diff --git a/nix/host-modules/glue-grafana-oidc-client.nix b/nix/host-modules/glue-grafana-oidc-client.nix new file mode 100644 index 00000000..53767650 --- /dev/null +++ b/nix/host-modules/glue-grafana-oidc-client.nix @@ -0,0 +1,51 @@ +# Glue: register the swarm's Grafana as an OIDC client wherever authelia runs. +# +# ONE PAIRING PER FILE — grafana ← authelia, and nothing else. Deleting this +# leaves a swarm whose metrics UI is not a client authelia has ever heard of, so +# no login against it can complete and nothing minted its secret either. +# +# ⚠️ Gated on authelia being HERE, and deliberately NOT on this host running +# Grafana. A client is a row in THIS host's provider config, so it can only be +# declared where that config is rendered — and ./swarm-grafana.nix's whole +# `config` block hangs off `deploy.grafana.enable`, so a swarm with Grafana and +# authelia on different hosts registered the client nowhere at all. +# ./hive-forge/default.nix is already on the right side of that line: its module +# is gated on `hyperhive.enable` and only the registration asks about authelia. +# This file puts Grafana there without moving the rest of its module. +# +# ⚠️ Registered whether or not the swarm has a Grafana, because nothing in +# `swarm.*` records that — `deploy.grafana.enable` answers "does THIS host run +# it", and a swarm-wide answer does not exist. The cost is one unused client +# and one unused minted secret in a swarm with no metrics UI. The alternative +# was a new swarm-wide option an operator must set before a split deployment +# works, which leaves the reported failure in place for everyone who does not +# know to set it. ./swarm-otel.nix made the same call for its own client when it +# dropped the published-scrape-target guard: a client that may go unused beats a +# guard that reads as done and renders nothing. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + grafanaCfg = hyperhiveCfg.swarm.grafana; +in +{ + config = lib.mkIf (hyperhiveCfg.enable && deployCfg.authelia.enable) { + # One declaration, two readers. Grafana's callback URL is format-locked to + # its own root URL, and the read-only option it is taken from is where that + # format is spelled — restating it here would be a second source of truth + # for a string whose mismatch is a silently rejected login. + # + # `kind` is left at its `interactive` default: a person logs in here. + services.hyperhive.swarm.authelia.oidc.clients = [ + { + id = grafanaCfg.oidc.clientId; + description = "HyperHive swarm metrics"; + redirectUris = [ grafanaCfg.oidc.redirectUri ]; + } + ]; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 52590ec5..a1d2b5bf 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -210,14 +210,17 @@ let secretPublisherPolicyName = "swarm-secret-publisher"; secretPublisherCn = baoDeploy.secretPublisherCommonName; - # One grant, and every narrowing in it is load-bearing. + # Two grants, and every narrowing in each is load-bearing. # # `secret/data/` is KV v2's ACL prefix, inserted by the engine rather than # written by the caller — same trap as the controller's grant above. # - # `hives/` and not `swarm/*`: this principal has no business with an agent's - # or a service's credentials, and the hive prefix is the only one whose paths - # it produces. + # Two prefixes and not `swarm/*`: this principal has no business with an + # agent's credentials or the controller's, and these two are the only paths + # it produces. It grew the `services/` one when the publisher gained a swarm + # service's OIDC secret to copy, which is the rule ../module-eval.nix states + # for the controller's side of the same wall — a grant widens when a path + # gains a WRITER, not when a kind is declared. # # Write-only. It copies secrets in and never reads one back; a read # capability would let a file-copier recover every hive's credentials. @@ -225,6 +228,10 @@ let path "${credentialMountPath}/data/swarm/hives/*" { capabilities = ["create", "update"] } + + path "${credentialMountPath}/data/swarm/services/*" { + capabilities = ["create", "update"] + } ''; # The KV v2 engine the controller writes agent credentials through. Named diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 6c6d37f4..50d53fda 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -21,6 +21,8 @@ let hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; autheliaCfg = hyperhiveCfg.swarm.authelia; + baoCfg = hyperhiveCfg.swarm.bao; + baoDeploy = deployCfg.bao; vmCfg = hyperhiveCfg.swarm.victoriametrics; vlCfg = hyperhiveCfg.swarm.victorialogs; swarmDomain = hyperhiveCfg.swarm.domain; @@ -78,10 +80,41 @@ let # operator sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; - # The all-local case: this host runs BOTH Grafana and the swarm's authelia, - # so the minted secret can be moved without an operator. Same split the - # forge and matrix modules document. - ssoLocal = deployCfg.grafana.enable && deployCfg.authelia.enable; + # Is SSO configured for this SWARM. Swarm-wide by construction — `swarm.*` is + # identical on every host — and the option's own description is what makes + # this the right question to ask: a null URL means "no SSO configured". + ssoConfigured = autheliaCfg.url != null; + + # Is authelia on THIS host, spelled exactly as the forge and matrix modules + # spell it. All locality decides is where the minted plaintext is copied + # FROM: authelia's own tree here, the swarm secret store otherwise. + # + # 🩸 This used to be `grafana.enable && authelia.enable` and was the ONLY + # gate: registration, the mint's delivery and the OIDC block all hung off + # it, so a swarm whose authelia ran elsewhere got Grafana with no SSO wiring + # — and the login form is disabled whatever happens, so with nothing else. + ssoLocal = deployCfg.authelia.enable; + + # A reader of the store is defined by holding a certificate the store + # accepts, never by standing next to it — the rule + # ./glue-matrix-bao-token.nix states in full. + haveClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null; + + # Whether Grafana gets an OIDC block: the swarm-wide question, AND that + # something on this host delivers the secret that block names. The second + # half is not redundant — a block naming a file no unit here writes cannot + # produce a working login either way, and what Grafana does with an + # unreadable `$__file{}` target (refuse to start, or start and fail every + # login) is not worth betting the container on when not emitting the block is + # available and leaves this host as it was. + ssoWired = ssoConfigured && (ssoLocal || haveClientIdentity); + + # The delivery that is not a copy: authelia is elsewhere, so the same minted + # plaintext arrives out of the store that the publisher on authelia's host + # wrote it into. Exclusive with the copy unit by construction — two units + # writing one file is a race over which secret Grafana parses. + storeDelivery = ssoWired && !ssoLocal; + autheliaUrl = toString autheliaCfg.url; # Where the plaintext lands inside the container. Under /var/lib rather @@ -90,6 +123,23 @@ let # intermittent one. secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret"; + # The same file seen from the host, which is where both delivery units write + # it. Spelled once: two units landing a secret at two spellings of one path + # is a Grafana that reads whichever of them it was configured with. + hostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${secretPath}"; + hostSecretDir = builtins.dirOf hostSecretPath; + + # Where the publisher on authelia's host leaves this client's secret. The + # `services` segment is `swarm-secret-client`'s `path::Kind::Service` — a + # swarm service's client is registered once for the whole swarm, so its + # secret is one value and not one per hive. Both ends compose this from the + # same swarm-wide client id, so a rename cannot leave one of them behind. + # + # ⚠️ The prefix is not decoration: it is what this host's read grant covers, + # so a path outside it answers 403 rather than a miss, however correct it + # looks. + storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client"; + # Grafana's own datasource-encryption key. Generated in-container (see the # unit below) because nothing outside the container ever reads it — unlike # the OIDC secret above, whose other reader is authelia's container. @@ -104,11 +154,6 @@ let nginxGid = config.ids.gids.nginx; grafanaUid = config.ids.uids.grafana; - # Format-locked by Grafana: the generic OAuth callback is always - # `/login/generic_oauth`. Declared once here and read by both - # the authelia client and Grafana itself. - redirectUri = "https://${cfg.domain}/login/generic_oauth"; - in { # `enable` moved to `services.hyperhive.deploy.grafana.enable` — see @@ -186,6 +231,25 @@ in ''; }; + redirectUri = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "https://${cfg.domain}/login/generic_oauth"; + defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.grafana.domain}/login/generic_oauth"''; + description = '' + OAuth callback authelia sends the browser back to, and the URI it + matches **exactly**. + + Read-only, like {option}`services.hyperhive.swarm.grafana.machine` + and for the same reason: Grafana derives it from its own + `root_url` (`/login/generic_oauth`), so it is a fact + other modules may read rather than a knob. The glue that registers + this client wherever authelia runs reads it from here instead of + restating the format — a second spelling of it is a silently + rejected login. + ''; + }; + role = lib.mkOption { type = lib.types.enum [ "Viewer" @@ -336,11 +400,16 @@ in # The secret oneshots as well as grafana itself: each runs before it and # fails in ways grafana then reports only as a login that does not work. + # + # Each delivery unit is listed only where it exists, the way + # ./hive-matrix.nix lists its own: a unit name that never renders is a + # journald scrape target matching nothing, which reads as a quiet unit. services.hyperhive.swarm.otel.journaldUnits = [ "grafana" - "swarm-grafana-oidc-secret" "swarm-grafana-secret-key" - ]; + ] + ++ lib.optional ssoLocal "swarm-grafana-oidc-secret" + ++ lib.optional storeDelivery "swarm-bao-grafana-oidc"; services.hyperhive.swarm.controller.links = [ { @@ -350,35 +419,40 @@ in } ]; - # One declaration, two readers. Grafana's callback URL is format-locked - # to its own root URL; making the operator restate it in authelia's - # client list would be a second source of truth for a string whose - # mismatch is a silently rejected login. - # - # `kind` is left at its `interactive` default: a person logs in here. - services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [ - { - id = cfg.oidc.clientId; - description = "HyperHive swarm metrics"; - redirectUris = [ redirectUri ]; - } - ]; + # Registering the client is NOT here any more: it has to happen on the + # host that runs authelia, and this whole block is gated on the host that + # runs Grafana. ./glue-grafana-oidc-client.nix is where it moved to. + + warnings = lib.optional (ssoConfigured && !ssoLocal && !haveClientIdentity) '' + services.hyperhive.swarm.grafana: this swarm has an IdP + (services.hyperhive.swarm.authelia.url) but authelia is not on this + host and this host holds no swarm-secret-store identity, so nothing + can deliver Grafana's OIDC client secret. Grafana will come up with + no OIDC login and no local login form, which is no way in at all. + + Name this host's store client leaf in + services.hyperhive.deploy.bao.clientCertFile and + services.hyperhive.deploy.bao.clientKeyFile — the same identity every + other reader of the store uses. See docs/swarm/secrets.md. + ''; assertions = [ { - # Grafana reaches the token endpoint server-to-server, so a null URL - # would become a request to `null/api/oidc/token` — a DNS failure - # several layers from its cause. Only reachable by enabling authelia - # and clearing its `url`, which is why it is an assertion and not a - # fallback. - assertion = !ssoLocal || autheliaCfg.url != null; + # The URL is what `ssoConfigured` above reads, so with it null this + # host mints and delivers a secret for an OIDC block it then does not + # emit — SSO silently absent on the one deployment that has every + # other piece of it. Only reachable by enabling authelia and clearing + # its `url`, which is why it is an assertion and not a fallback. + assertion = !ssoLocal || ssoConfigured; message = '' services.hyperhive.swarm.grafana requires services.hyperhive.swarm.authelia.url when authelia is enabled. Grafana exchanges its authorization code at - `''${url}/api/oidc/token` from inside its container. With the URL - null there is no endpoint to name. + `''${url}/api/oidc/token` from inside its container, so with the URL + null there is no endpoint to name — and a null URL is also how this + module reads "no SSO configured for this swarm", which would leave + Grafana with no login of any kind on the host that mints its secret. ''; } ]; @@ -450,11 +524,11 @@ in "d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" ]; - # The secret delivery. It runs on the HOST because that is the only place - # both container trees are addressable: they share this host's network - # namespace, which makes them feel co-located, but their filesystem roots - # are separate — Grafana cannot open a path inside authelia's tree - # however local the port looks. + # The local delivery, and the ONLY thing locality still decides. It runs on + # the HOST because that is the only place both container trees are + # addressable: they share this host's network namespace, which makes them + # feel co-located, but their filesystem roots are separate — Grafana cannot + # open a path inside authelia's tree however local the port looks. # # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container # refuses to start when a bind source is missing, and this secret does @@ -482,7 +556,7 @@ in set -euo pipefail src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${cfg.oidc.clientId}.secret"} - dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${secretPath}"} + dst=${lib.escapeShellArg hostSecretPath} # authelia's container is up, but its first-boot generator may still # be minting. Bounded wait, then fail: a silent skip here produces a @@ -513,6 +587,137 @@ in ''; }; + # The same secret, the same destination, for the deployment where authelia + # is NOT here: it arrives out of the swarm secret store, which the + # publisher on authelia's host wrote it into. Nothing above changes — this + # is the second source for one file, never a second file. + # + # Shaped after ./glue-queue-agent-credential.nix rather than after the copy + # above, because it is a store reader and those are the store's readers: a + # cert login that fails LOUDLY, since every state it fails on is one a + # retry fixes, then a read that degrades QUIETLY, since no retry turns "no + # value there" into a value. + # + # ⚠️ An ABSENT secret is "not published yet", not a fault of this host's. + # The publisher runs on the authelia host and authelia mints on its first + # boot, so early in a swarm's life there is legitimately nothing to read. + # Nothing here writes a stand-in: there is no such thing as a locally valid + # OIDC client secret, and a placeholder would turn a Grafana that cannot + # log anyone in into one whose token exchange is refused. + # + # ⇒ So the absence surfaces at Grafana rather than here, and this unit's job + # is to leave the journal line that says which of the three it was. That is + # the same division the copy unit above makes by failing rather than + # skipping: the secret not arriving is a Grafana nobody can log into, and a + # named cause is the only thing separating that from a mystery. + systemd.services.swarm-bao-grafana-oidc = lib.mkIf storeDelivery { + description = "fetch Grafana's OIDC client secret from the swarm secret store"; + # Every one of these names a unit that exists only where the store runs. + # `Requires=` on an absent unit fails the job outright, so the ordering + # is conditional even though the read is not: off-host there is nothing + # local to wait for, and the timeout below bounds the attempt instead. + after = lib.optionals baoDeploy.enable [ + "swarm-bao-pki.service" + "container@${baoCfg.machine}.service" + ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; + requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ + "multi-user.target" + "container@${cfg.machine}.service" + ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + # Sized for the race this loses, not for an unseal: `swarm-bao` comes up + # seconds before this unit asks, and the cert-auth role it logs in + # against is written seconds after, so a few short attempts cover it. An + # hours-long window would be a bet on a sealed store, and the degrade + # below is already correct for that. + # + # `StartLimit*` are `[Unit]` settings, so they go here and not in + # `serviceConfig` — systemd ignores them under `[Service]`. The window + # has to exceed `RestartSec × burst`. + startLimitBurst = 4; + startLimitIntervalSec = 300; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-bao-grafana-oidc"; + # What actually bounds the read below. Stated here rather than left to + # systemd's default, so the number a boot waits on is in the file that + # waits. + TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = baoDeploy.clientCertFile; + BAO_CLIENT_KEY = baoDeploy.clientKeyFile; + } + # Absent means the system trust store, which is what a deployment with a + # real CA wants and what a self-signed one must not be left with. + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + # `bao`'s own message is the only thing separating a missing value from + # a refused identity from an unreachable host. This unit's degraded + # mode is correct for all three, so it reports which one rather than + # asserting all three in a sentence of ours. + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*` + # variables above only decide which certificate the TLS handshake + # presents; without a token `bao` asks its token helper instead, and + # that is a `sh` this unit's `path` does not carry. `-token-only` + # answers on stdout and skips the helper on both sides. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to swarm-bao with this host's certificate; leaving Grafana's OIDC client secret as it is." >&2 + if [ -s "$err" ]; then + cat "$err" >&2 + else + echo "bao failed without writing a diagnostic." >&2 + fi + exit 1 + fi + export BAO_TOKEN + + if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then + echo "swarm-bao did not return ${storeSecretPath}; Grafana has no OIDC client secret yet." >&2 + if [ -s "$err" ]; then + cat "$err" >&2 + else + echo "bao failed without writing a diagnostic." >&2 + fi + exit 0 + fi + + if [ -z "$secret" ]; then + echo "swarm-bao returned an empty ${storeSecretPath}; leaving the file as it is." >&2 + exit 0 + fi + + # Written with a shell builtin, never handed to a program: `printf` is + # bash's own, so the plaintext never becomes an argument in /proc the + # way `install <<<"$secret"` or an `echo` from `path` would. + # + # Same uid, group and mode as the copy above, for the reasons stated + # there — this is the same file arriving by a different route. + install -d -m 0755 ${lib.escapeShellArg hostSecretDir} + umask 077 + printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath} + chown ${toString config.ids.uids.grafana}:0 ${lib.escapeShellArg hostSecretPath} + chmod 0400 ${lib.escapeShellArg hostSecretPath} + ''; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -710,12 +915,11 @@ in # # Deliberately NOT `ssoLocal`: whether a password box sits on a # public login page cannot depend on which host happens to run - # authelia. The OIDC block below is still gated on locality; - # making that follow the same swarm-wide question is a larger - # change tracked on its own. + # authelia. The OIDC block below now answers the same swarm-wide + # question, so the two no longer disagree. auth.disable_login_form = true; } - // lib.optionalAttrs ssoLocal { + // lib.optionalAttrs ssoWired { "auth.generic_oauth" = { enabled = true; name = "HyperHive"; diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index 85cf126d..0d7030b0 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -1,5 +1,6 @@ # The unit that copies authelia's minted OIDC client secrets into the swarm's -# secret store, so a hive that does not host authelia can read its own. +# secret store, so whoever needs one without hosting authelia can read it: a +# hive its agents' credential, a swarm service its own. # # ⚠️ IT RUNS WHERE AUTHELIA DOES, and that is the whole reason it exists as a # separate thing. `deploy.authelia.hostClientSecretDir`'s own description says @@ -47,6 +48,18 @@ let # ./swarm-authelia.nix composes it, from the same two read-only options, so a # rename there cannot leave this spelling behind. agentClientId = hive: "${autheliaCfg.hiveClientPrefix}${hive}${autheliaCfg.agentClientSuffix}"; + + # The swarm's own services, as opposed to its hives. One client for the whole + # swarm rather than one per hive, so one value in the store rather than a copy + # each: `swarm/services//oidc/client`, under the `services` kind + # `swarm-secret-client`'s `path::Kind` declares. + # + # The ids come from `swarm.*`, which is identical on every host — that is what + # lets this host name a service's client while running none of them, and it is + # the same read the service's own module registers the client with. A swarm + # that runs no Grafana mints no secret for it, so its entry skips below rather + # than needing a condition here. + serviceClientIds = [ hyperhiveCfg.swarm.grafana.oidc.clientId ]; in { options.services.hyperhive.deploy.swarm-secret-publisher = { @@ -56,7 +69,8 @@ in defaultText = lib.literalExpression "deploy.authelia.enable"; description = '' Publish the OIDC client secrets this host mints into the swarm's - secret store, so hives that do not run authelia can read their own. + secret store, so a hive that does not run authelia can read its + agents' credential and a swarm service elsewhere can read its own. Defaults to whether this host mints them, which is the only half of the question that is a property of *this* host. @@ -177,6 +191,29 @@ in fi '') hiveNames} + ${lib.concatMapStringsSep "\n" (id: '' + src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${id}.secret"} + if [ -s "$src" ]; then + # `value=@$src` for the same reason as the hive loop above: bao + # opens the file itself, so the plaintext is never an argument of + # this process. + # + # No `client_id` field beside it, unlike a hive's credential: that + # one is derived per hive and has to be reconstructable from the + # store alone, whereas a service's client id is the swarm-wide + # option both ends already read. + bao kv put ${lib.escapeShellArg "secret/swarm/services/${id}/oidc/client"} \ + value=@"$src" + published=$((published + 1)) + else + # Not an error, and the ordinary state of a swarm that runs this + # service nowhere: nothing registered the client, so authelia minted + # nothing to publish. + echo "no minted secret at $src yet; the path unit will re-run this" >&2 + skipped=$((skipped + 1)) + fi + '') serviceClientIds} + echo "published $published client secret(s), skipped $skipped" ''; }; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 2be72a41..afadac61 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -223,6 +223,37 @@ let swarm.grafana.package = pkgs.emptyDirectory; }; + # The metrics UI beside the IdP, which is the deployment whose secret was + # already delivered: a host copy out of authelia's own tree. + grafanaWithAuthelia = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + deploy.authelia.enable = true; + }; + # The same UI with the IdP on ANOTHER host and a store leaf placed by hand — + # the deployment that had no delivery path at all. Knowing an IdP is not + # running one: `swarm.authelia.url` is what says this swarm has SSO, and + # nothing about this host does. + grafanaRemoteAuthelia = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + swarm.authelia.url = "https://auth.example.invalid"; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + # The same again with the store identity taken away, which is the one shape + # that can deliver this secret by NO route. It separates "the swarm has an + # IdP" from "something here can fetch what it minted", and it is what keeps + # Grafana's config from naming a file nothing writes. + grafanaRemoteAutheliaNoIdentity = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + swarm.authelia.url = "https://auth.example.invalid"; + }; + # The first slice to leave options on BOTH sides of the split, so the # fixture sets all three of them through the paths an existing config uses: # the two movers via their rename entries, `tokenEndpoint` via the path it @@ -836,6 +867,97 @@ let ok = grafanaOldPath.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form; } + { + # The absence class this whole file is for, and the reported defect in one + # arm: the OIDC block hung off "authelia is on this host", so the split + # deployment got a Grafana with no SSO settings and no login form — no way + # in at all. The question it asks now is whether the SWARM has an IdP. + name = "grafana is wired for SSO against an IdP on another host"; + ok = + let + s = + grafanaRemoteAuthelia.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth"; + in + s.enabled && lib.hasInfix "https://auth.example.invalid/api/oidc/token" s.token_url; + } + { + # The control for the arm above, and the reason the gate is a conjunction: + # a block naming a file no unit on this host writes cannot produce a + # working login, so it is emitted only where a delivery route exists — + # leaving a host with neither exactly as it was rather than pointing + # Grafana's config at a secret that never arrives. + name = "grafana emits no OIDC block when nothing on this host can deliver the secret"; + ok = + !( + grafanaRemoteAutheliaNoIdentity.containers.swarm-grafana.config.services.grafana.settings + ? "auth.generic_oauth" + ); + } + { + # The other half of the arm above, and the only deliverable that shape + # has: silence there is a Grafana nobody can log into for a reason no log + # names. A warning is read back by nothing, so an unevaluated one is a + # warning nobody sees — the same reason the peers fixture exists. + name = "the undeliverable-secret shape warns instead of going quiet"; + ok = + lib.any ( + w: lib.hasInfix "no swarm-secret-store identity" w + ) grafanaRemoteAutheliaNoIdentity.warnings + && !(lib.any (w: lib.hasInfix "no swarm-secret-store identity" w) grafanaRemoteAuthelia.warnings); + } + { + # Exactly one delivery unit per deployment, and which one is the only + # thing locality decides. Two units writing one path is a race over which + # secret Grafana parses; zero is the defect this pair replaced. + name = "grafana's OIDC secret has one delivery unit, chosen by where authelia is"; + ok = + let + local = grafanaWithAuthelia.systemd.services; + remote = grafanaRemoteAuthelia.systemd.services; + in + local ? swarm-grafana-oidc-secret + && !(local ? swarm-bao-grafana-oidc) + && remote ? swarm-bao-grafana-oidc + && !(remote ? swarm-grafana-oidc-secret); + } + { + # Same 403-not-a-miss reason as the matrix and queue arms below: the + # reader's grant covers the `services` prefix, so a path outside it is + # refused rather than empty, however correct it reads. The negative arm is + # the rename this is exposed to — a secret filed under the hive that runs + # the service instead of under the service itself. + name = "grafana's OIDC secret is read from the prefix the publisher writes"; + ok = + let + s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script; + in + lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s + && !(lib.hasInfix "secret/swarm/hives/" s); + } + { + # Both ends of a wire nothing at eval time carries end to end: the + # publisher on authelia's host writes the path the reader on Grafana's host + # reads, and the two files agree only because both compose it from the same + # swarm-wide client id. + name = "the publisher writes the swarm service path grafana reads"; + ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" ( + secretPublisherHere.systemd.services.swarm-secret-publish.script + ); + } + { + # Registering the client cannot live where the rest of grafana's module + # lives: that block is gated on this host RUNNING grafana, so on the split + # deployment nothing registered the client, authelia minted no secret, and + # every layer below had nothing to carry. The second arm is the control — + # a host with no IdP registers nothing. + name = "the swarm's grafana client is registered wherever authelia runs"; + ok = + let + clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; + in + lib.elem "swarm-grafana" (clients secretPublisherHere) + && !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia)); + } { # Reads the daemon's rendered unit, not the options: the queue address # arrives as an env var whose whole attrset is guarded on `natsUrl`, and @@ -934,17 +1056,18 @@ let !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-secret-publisher-policy); } { - # The whole point of a second principal. `hives/` and not `swarm/`, so it - # cannot touch an agent's or a service's credentials; and no `read`, so a - # unit whose job is copying a file cannot recover what is already there. - # Pinned as the full capability list, because an added capability is - # exactly what a presence check misses. - name = "the publisher's grant is write-only and reaches the hive prefix alone"; + # The whole point of a second principal. The two prefixes it publishes to + # and not `swarm/`, so it cannot touch an agent's credentials; and no + # `read`, so a unit whose job is copying a file cannot recover what is + # already there. Pinned as the full capability list per prefix, because an + # added capability is exactly what a presence check misses. + name = "the publisher's grant is write-only and reaches the hive and service prefixes alone"; ok = let s = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/*\" {\n capabilities = [\"create\", \"update\"]" s + && lib.hasInfix "path \"secret/data/swarm/services/*\" {\n capabilities = [\"create\", \"update\"]" s && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/*" s) && !(lib.hasInfix "sys/policies/acl" s); diff --git a/swarm-secret-client/src/policy.rs b/swarm-secret-client/src/policy.rs index 62201f2d..2977a906 100644 --- a/swarm-secret-client/src/policy.rs +++ b/swarm-secret-client/src/policy.rs @@ -5,8 +5,8 @@ //! two have to agree on the same path or a delivery fails with a 403 that names //! nothing. //! -//! The document has one stanza per kind a hive reads, and the two are scoped -//! differently on purpose: +//! The document has one stanza per kind a hive reads, and they are not all +//! scoped alike — which is the point rather than an inconsistency: //! //! ⚠️ The **agent** stanza grants read on *every* agent's credentials rather //! than on the ones that hive hosts. That is a decision, not an oversight: an @@ -16,9 +16,14 @@ //! narrow one that only looks narrow. The narrower shapes, and what they would //! cost, are in `docs/trust-boundary/security.md`. //! +//! ⚠️ The **service** stanza is wide for the same shape of reason: a swarm +//! service's client is registered once per swarm, so its path names the service +//! and never the host, and which hive runs a service is a `deploy.*` fact with +//! no swarm-wide spelling to scope against. Cost: the same doc. +//! //! The **hive** stanza has no such problem and is therefore narrow: that path //! names its principal, so scoping it to the reader's own name costs nothing -//! and drifts nowhere. Do not widen it to match its neighbour — the asymmetry +//! and drifts nowhere. Do not widen it to match its neighbours — the asymmetry //! is the point. //! //! Rendering stays separate from writing so the text can be asserted with no @@ -52,8 +57,8 @@ fn read_stanza(path: &str) -> String { format!("path \"{path}\" {{\n capabilities = [\"read\"]\n}}\n") } -/// Render `hive`'s policy document: read on every agent's credentials, and on -/// this hive's own. +/// Render `hive`'s policy document: read on every agent's credentials, on this +/// hive's own, and on the swarm services'. /// /// The name is the only input, and it is deploy-time — so the document is /// still a deploy-time object rather than derived state with a re-emission to @@ -61,9 +66,15 @@ fn read_stanza(path: &str) -> String { /// /// Read-only: the controller mints these and never reads one back. /// -/// ⚠️ The service and controller kinds are deliberately absent: a hive has no -/// business reading a service's or the controller's credentials. Adding either -/// is a boundary decision, not a consequence of the namespace growing. +/// ⚠️ The controller kind is deliberately absent: a hive has no business +/// reading the credentials of the thing that provisions it. Adding it is a +/// boundary decision, not a consequence of the namespace growing. +/// +/// The **service** kind is granted, and that was such a decision rather than a +/// consequence: a service whose identity provider is on another host reads its +/// own OIDC client secret with the certificate of the hive it runs on, that +/// being the only identity such a host has — so every hive can read every +/// service's. Bought and paid for in `docs/trust-boundary/security.md`. /// /// The hive's *own* kind is granted, and that is the decision the agent-only /// version of this grant said had to be made rather than assumed: a hive holds @@ -85,7 +96,11 @@ pub fn render(hive: &str) -> Result { "{MOUNT}/data/{ROOT}/{}/{hive}/*", <&str>::from(Kind::Hive) )); - Ok(format!("{agents}{own}")) + let services = read_stanza(&format!( + "{MOUNT}/data/{ROOT}/{}/*", + <&str>::from(Kind::Service) + )); + Ok(format!("{agents}{own}{services}")) } #[cfg(test)] @@ -93,11 +108,26 @@ mod tests { use super::*; #[test] - fn the_document_grants_the_whole_agent_prefix_and_this_hive_alone() { + fn the_document_grants_two_whole_prefixes_and_this_hive_alone() { assert_eq!( render("pr1ma").expect("a plain name is legal"), "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"read\"]\n}\n\ - path \"secret/data/swarm/hives/pr1ma/*\" {\n capabilities = [\"read\"]\n}\n" + path \"secret/data/swarm/hives/pr1ma/*\" {\n capabilities = [\"read\"]\n}\n\ + path \"secret/data/swarm/services/*\" {\n capabilities = [\"read\"]\n}\n" + ); + } + + #[test] + fn the_service_stanza_covers_a_service_this_hive_was_never_named_beside() { + // The property the swarm-grafana delivery depends on: the host running a + // swarm service reads that service's client secret with its own hive + // certificate, and the path names the service rather than the host. A + // stanza narrowed to the reader's name would 403 every such read. + let p = render("pr1ma").expect("legal"); + assert!(p.contains("path \"secret/data/swarm/services/*\"")); + assert!( + !p.contains("services/pr1ma"), + "the service stanza is not scoped to the reader" ); }