diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index e3c2e0ce..53bce3ad 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -64,7 +64,8 @@ because nothing outside that container ever reads them. **That's the test worth applying to any secret added here** — and the client secret's plaintext half is the one row that fails it, which is the entire reason a delivery step exists. -Two telemetry collectors exist, and they land on opposite sides of that test. +Three telemetry collectors exist, and they don't land on the same side of +that test. The **hive's** collector needs no delivery step. It authenticates to the swarm's collector as its own hive, and it's a host unit rather than a container, so on @@ -94,6 +95,27 @@ uses for its non-optional one. The operator can still deliver the secret by hand, at `services.hyperhive.deploy.swarm-otel.clientSecretFile`, on a collector this unit never reaches. +The **secret store's own** collector — the forwarder inside the `swarm-bao` +container — needs a delivery step too, and it takes the same route with one +principal of its own: `swarm-bao-forwarder-oidc.service` reads +`swarm/services//oidc/client` under this host's certificate and +lands it in the container's tree, where `LoadCredential` hands it to the +collector. The client id is its own +(`services.hyperhive.swarm.bao.otel.clientId`), registered by +`glue-swarm-bao-otel-oidc-client.nix` wherever authelia runs, and the swarm +collector admits it on a receiver of its own — one identity per principal, so +this forwarder borrows neither the swarm collector's client nor +`swarm-controller`'s. + +⚠️ **This one has no degraded mode, and the difference is structural.** The +route it exports over accepts an authenticated request and nothing else, so a +forwarder without the secret has nothing to fall back to: the unit fails and +retries, and the collector refuses to start until the file arrives rather than +exporting into a 401 while reporting healthy. Its ordering differs from the +unit above it for the same structural reason — it reads the store inside the +container it delivers into, so it runs **after** that container rather than +before it, and only a first boot ever waits. + ### Minting the queue's callout nkeys `deploy.nats.autoGenerateCallout` mints both keypairs on the host before the queue @@ -254,6 +276,16 @@ doesn't render the reading unit without them, the shape `glue-matrix-bao-token.nix` and `glue-queue-agent-credential.nix` use for their own optional reads. +The **fifth reader** is the secret store forwarder's OIDC client secret: +`swarm-bao-forwarder-oidc.service`, on the store's own host, registered by +`glue-swarm-bao-otel-oidc-client.nix` on authelia's. It differs from the +fourth in both directions. It renders unconditionally, because the export it +authenticates has no unauthenticated mode to degrade into — and it orders +itself **after** `container@swarm-bao`, because the store it reads runs in the +container it delivers into. Nothing circular sits behind that: the identity it +logs in with is this host's static `swarm-bao-pki` leaf, not anything the store +mints. + A service's secret is one value for the whole swarm rather than one per hive, so it lives under the `services` prefix, and a hive's read policy grants that prefix whole. That's because nothing in a swarm's configuration records which hive runs diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index b4f1e7e8..01030d24 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,6 +30,7 @@ ./glue-matrix-ctl-bao-identity.nix ./glue-queue-agent-credential.nix ./glue-secret-publisher-bao-identity.nix + ./glue-swarm-bao-otel-oidc-client.nix ./glue-swarm-otel-oidc-client.nix ./swarm-authelia.nix ./swarm-bao.nix diff --git a/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix b/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix new file mode 100644 index 00000000..6e9d850a --- /dev/null +++ b/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix @@ -0,0 +1,56 @@ +# Glue: register the secret store's own journal forwarder as an OIDC client +# wherever authelia runs. +# +# ONE PAIRING PER FILE — swarm-bao's forwarder ← authelia, and nothing else. +# Deleting this leaves a forwarder authelia has never heard of: the token +# endpoint refuses it, nothing is minted, the publisher has nothing to copy +# into the store, and the collector's export is a 401 nobody asked for. +# +# ⚠️ A SECOND client beside ./glue-swarm-otel-oidc-client.nix's, not a reuse +# of it. That one is the swarm collector's identity for what IT pushes; this +# one belongs to the collector inside the store's container, which is a +# different principal on a different host — one identity per principal, the +# rule `swarm-controller.nix` states over its own `queueClientId`. +# +# ⚠️ Gated on authelia being HERE, and deliberately NOT on this host running +# the store. A client is a row in THIS host's provider config, so it can only +# be declared where that config is rendered, and ./swarm-bao.nix's `config` +# hangs off `deploy.bao.enable` — the split ./glue-grafana-oidc-client.nix and +# ./glue-swarm-otel-oidc-client.nix each made for the same reason. Read either +# file's own comment for the property this one shares with them. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoCfg = hyperhiveCfg.swarm.bao; +in +{ + config = lib.mkIf (hyperhiveCfg.enable && deployCfg.authelia.enable) { + # One declaration, two readers: `clientId` is a read-only option + # ./swarm-bao.nix owns, and ./swarm-otel.nix builds this principal's + # authenticator audience from the same option. + services.hyperhive.swarm.authelia.oidc.clients = [ + { + id = baoCfg.otel.clientId; + description = "HyperHive secret store journal forwarder"; + kind = "machine"; + redirectUris = [ ]; + # Its own id as its own permitted audience — the self-referential + # form `swarm-authelia.nix`'s `hiveClients` uses, which is the shape + # to copy here: this forwarder's token lands on an `oidc/*` receiver + # of the swarm collector, exactly as a hive's does. + audience = [ baoCfg.otel.clientId ]; + # ⚠️ What makes the token READABLE by that receiver at all. Authelia's + # default is an opaque handle, and an `oidc` extension verifies + # offline against `/jwks.json` — so without this the export fails with + # a message about the verifier rather than about the token. Same + # value, same reason, as every hive's client. + accessTokenSignedResponseAlg = "RS256"; + } + ]; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 5f88a6c0..34676bda 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -429,19 +429,23 @@ let // metricsListener; # Metrics get their own listener rather than a flag on the one above, and - # that follows from what a scraper can express: `swarm.otel.scrapeTargets` - # carries no scheme and no credential, while the API listener is TLS and — - # once a client CA is set — demands a client certificate. The collector - # cannot reach it at all. + # that follows from what a scraper can express: a prometheus scrape config + # carries no client certificate, while the API listener is TLS and — once a + # client CA is set — demands one. A scraper cannot reach it at all. # # `metrics_only` narrows this one to the metrics path (every other path 404s) # and the unauthenticated access is confined to loopback. **Deliberately - # unauthenticated for now** — a tracked follow-up owns giving the collector a - # credential, since no scrape option can carry one today. + # unauthenticated** — the only process that can dial 127.0.0.1 here is one + # already inside this netns, which is the container's own collector and the + # store itself. Widening this address, or dropping + # `tls_require_and_verify_client_cert` on the API listener to scrape THAT + # instead, would each trade that property away for nothing. # - # Exists only where a collector does: an endpoint with no reader is exposure - # bought for nothing. - metricsListener = lib.optionalAttrs scrapeHere { + # Unconditional, and that is the change this carries: the reader is now the + # collector in this very container (below), which renders wherever the store + # does. There is no longer a host where this endpoint has no reader, so there + # is nothing left for it to be a function of. + metricsListener = { metrics = { type = "tcp"; address = "127.0.0.1:${toString baoDeploy.metricsPort}"; @@ -453,21 +457,32 @@ let }; }; - scrapeHere = deployCfg.swarm-otel.enable; + # What the container's own collector dials. Loopback inside a netns this + # container SHARES with the host (`privateNetwork = false` below), so the + # scrape crosses no network and needs no bind address wider than the one + # above. + metricsScrapeTarget = "127.0.0.1:${toString baoDeploy.metricsPort}"; - # ⚠️ The forwarder below has NO condition of its own, and `scrapeHere` above - # is not a template for one — the two are different tiers on purpose. A - # loopback metrics listener works only where the scraper is, which is what - # `Here` in that name says; forwarding reaches the swarm's collector BY NAME, - # from wherever this store runs. + # ⚠️ Neither the listener above nor the forwarder below has a condition of + # its own, and they lost theirs for the same reason at different times. # - # There is nothing left to gate on. "Does the swarm have a collector" is not - # a question that can be false (mara: "the swarm always has at least one - # instance of all the swarm level services"), so both earlier gates — - # `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong - # host and rendered no forwarder wherever the answer happened to be no. The - # only condition that belongs here is the one already enclosing this block: - # the store is deployed on this host. + # The listener used to be gated on `scrapeHere = deployCfg.swarm-otel.enable` + # — "does the swarm's collector happen to run on THIS host" — because the + # thing reading it was that collector, over loopback, which only works when + # the two land together. mara's ruling (*"move the existing scraper + # to the local collector"*) removes the premise rather than the symptom: the + # scraper is now in this container, so the reader is wherever the store is, + # and the metrics path stops being silently dead on every host that does not + # also run swarm-otel. + # + # The forwarder never had a defensible one. "Does the swarm have a collector" + # is not a question that can be false (mara: "the swarm always has at least + # one instance of all the swarm level services"), so both earlier gates — + # `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong host + # and rendered no forwarder wherever the answer happened to be no. + # + # The only condition that belongs on either is the one already enclosing this + # block: the store is deployed on this host. # The swarm collector's own name, served by the gateway — the address # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host @@ -479,7 +494,49 @@ let # The port suffix follows ./hive-forge/default.nix: elided on the canonical # 443 so this renders byte for byte like the tier's other two consumers of # this name, present when an operator moved the gateway's TLS port. - otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.producerName}"; + otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.storeProducerName}"; + + # The forwarder's own credential, and the three names it takes on the way + # in. The shape is ./swarm-otel.nix's `swarm-bao-otel-oidc` — the unit that + # already reads an OIDC client secret out of the store with this host's + # certificate and lands it in a collector's container — because that route + # is proven and there is no second one worth inventing. + # + # Where the publisher on authelia's host leaves it. The `services` segment + # is `swarm-secret-client`'s `path::Kind::Service`: a swarm service's client + # is registered once for the whole swarm, so its secret is one value. The + # prefix is also exactly what a hive certificate's read grant covers, so a + # path outside it answers 403 however correct it looks. + forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client"; + # At rest in the container's tree, written by the host unit below — under + # /var/lib rather than /run, because a secret that evaporates on reboot + # turns a working export into an intermittent one. + forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret"; + forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}"; + forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath; + # The collector runs under `DynamicUser` and opens `client_secret_file` + # itself, so there is no uid to hand a 0400 file to. `LoadCredential` reads + # it as root before the sandbox exists and re-exposes it under a path that + # does not depend on which uid the unit got; the collector expands + # `${env:CREDENTIALS_DIRECTORY}` at load, so this id is the only spelling. + forwarderCredentialId = "oidc-client-secret"; + # Named once and read three times — the extension that defines it, the + # exporter that names it, and `service.extensions`, where an omission makes + # it inert rather than an error. + forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}"; + autheliaCfg = hyperhiveCfg.swarm.authelia; + + # Outbound TLS for that collector: it dials the swarm collector's gateway + # name and authelia's token endpoint, both leaves of the runtime-generated + # hive CA, which cannot be baked into a derivation. Same helper, same two + # halves and the same `SSL_CERT_FILE` consumption as ./swarm-otel.nix's own + # container — a Go process whose trust variable REPLACES the store, which is + # what the bundle (system CAs ++ our anchors) is for. + caTrust = import ./lib/hive-ca-trust.nix { + inherit lib; + tlsCfg = deployCfg.hive-controller.tls; + gatewayCfg = hyperhiveCfg.gateway; + }; swarmOtelCfg = hyperhiveCfg.swarm.otel; gatewayPortSuffix = @@ -490,8 +547,9 @@ let # Non-zero is what SERVES the endpoint at all — the switch is a duration, not # a boolean, so a zero here is an openbao that answers 404 on a listener - # configured to do nothing else. - telemetry = lib.optionalAttrs scrapeHere { + # configured to do nothing else. Unconditional for the same reason the + # listener above is: its reader ships with the store now. + telemetry = { telemetry = { prometheus_retention_time = "24h"; disable_hostname = true; @@ -885,6 +943,29 @@ in ''; }; + otel.clientId = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-bao-collector"; + description = '' + OAuth2 client id the collector inside the store's container + authenticates as, and — self-referentially, the shape every hive's + client already uses — the audience it asks its token for. + + **Its own, not the swarm collector's and not + `swarm-controller`'s.** One identity per principal: this forwarder + runs wherever the store runs, which is not where either of those + two runs, and the receiver it pushes to + (`swarm.otel.storeProducerName`) admits this id alone. + + Swarm-wide and read-only because three hosts have to agree on it: + authelia registers the client + (`glue-swarm-bao-otel-oidc-client.nix`), the swarm collector checks + the audience (`swarm-otel.nix`), and the store's host reads the + minted secret back out of the store under a path composed from it. + Two spellings present as a healthy-looking 401. + ''; + }; }; # ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than @@ -973,15 +1054,21 @@ in "openbao" "swarm-bao-certs" "swarm-bao-token" + "swarm-bao-forwarder-oidc" ]; - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere { - # Path and query, not just `host:port`: openbao serves no `/metrics` - # at all, and `/v1/sys/metrics` answers JSON unless the format is - # asked for. A scrape of the default path 404s, which reads as a - # dead exporter rather than a wrong address. - bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus"; - }; + # 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is + # the deliverable rather than a tidy-up. That option is read only by the + # SWARM collector, over loopback, so declaring the store there meant its + # metrics existed exactly on the hosts that also ran swarm-otel and + # nowhere else — silently, since a store with no entry looks identical to + # one nothing scrapes. The scrape moved into this container's own + # collector (see `receivers.prometheus` below), which travels with the + # store. + # + # Logs are untouched by that move: `journaldUnits` above still names this + # store's units for the shared collector, and the two are retired + # together once every sibling swarm container has a collector of its own. # ⚠️ The one nginx exception to this file's header, and it is one because # it never terminates. `ssl_preread` reads the SNI off the ClientHello @@ -1015,6 +1102,113 @@ in # still proves nothing — bao answers nothing without a client # certificate its CA signed. services.hyperhive.network.exposeHostPorts = [ cfg.port ]; + + # THE delivery unit for the forwarder's own OIDC client secret, copied + # in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login + # that fails LOUDLY, since every state it fails on is one a retry fixes, + # then a read, then a root-owned file in the container's tree that + # `LoadCredential` re-exposes to the collector's dynamic user. + # + # ⚠️ ORDERED AFTER the store's container and deliberately NOT before it, + # which is the one line that differs from the unit it copies. That one + # reads a store in the container NEXT DOOR; this one reads the store in + # the very container it is delivering into, so "before" is a wait on a + # process that cannot start until this finishes. There is no bootstrap + # cycle behind it — the identity used here is this host's static + # `swarm-bao-pki` certificate, not anything the store mints — only an + # ordering one, and the cost is bounded: the file is under /var/lib, so + # it survives reboots and only a FIRST boot has the collector starting + # before it exists. `LoadCredential` refuses to start a unit whose + # source is missing, so that boot is a collector that restarts, says so + # each time, and comes up the moment this lands. Loud and self-healing + # rather than silently exporting without a credential. + systemd.services.swarm-bao-forwarder-oidc = { + description = "fetch the secret store forwarder's OIDC client secret from the store"; + after = [ + "swarm-bao-pki.service" + "container@${cfg.machine}.service" + ]; + wants = [ "container@${cfg.machine}.service" ]; + requires = [ "swarm-bao-pki.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + # Sized for the race this loses: the secret is minted by authelia and + # copied in by the publisher, both of which may be a host away and + # neither of which this boot waits on. `StartLimit*` are `[Unit]` + # settings — systemd ignores them under `[Service]` — and the window + # has to exceed `RestartSec × burst`. + startLimitBurst = 4; + startLimitIntervalSec = 300; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-bao-forwarder-oidc"; + TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; + }; + environment = { + BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}"; + BAO_CLIENT_CERT = baoDeploy.clientCertFile; + BAO_CLIENT_KEY = baoDeploy.clientKeyFile; + } + # Absent means the system trust store, which is what a deployment with + # a real CA wants and what a self-signed one must not be left with. + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + # `bao`'s own message is the only thing separating a missing value + # from a refused identity from an unreachable store, and this unit + # retries on all three — so it reports which one rather than + # asserting all three in a sentence of ours. + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + # Cert auth is a login, not a transport setting: the `BAO_CLIENT_*` + # variables only decide which certificate the handshake presents. + # Without a token `bao` asks its token helper, and that is a `sh` + # this unit's `path` does not carry — `-token-only` answers on + # stdout and skips the helper on both sides. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to the swarm secret store with this host's certificate." >&2 + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + + # ⚠️ Non-zero, where the sibling unit one module over exits 0 on this + # branch. That collector treats a missing secret as a supported + # degrade; this one has no such mode — the receiver it pushes to + # takes an authenticated request and nothing else — so "not + # published yet" is a state to retry out of, not to settle into. + if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then + echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2 + cat "$err" >&2 + exit 1 + fi + + if [ -z "$secret" ]; then + echo "the store returned an empty ${forwarderStoreSecretPath}." >&2 + exit 1 + fi + + # root-owned 0400, written with a shell builtin and never handed to + # a program: `printf` is bash's own, so the plaintext never becomes + # an argument in /proc the way `install <<<"$secret"` or an `echo` + # from `path` would. + install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir} + umask 077 + printf '%s\n' "$secret" > ${lib.escapeShellArg forwarderHostSecretPath} + chown root:root ${lib.escapeShellArg forwarderHostSecretPath} + chmod 0400 ${lib.escapeShellArg forwarderHostSecretPath} + ''; + }; }) (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) { @@ -1399,6 +1593,11 @@ in ''; }; + # The CA bind source is written at runtime by a host unit, so the + # container has to start after it — otherwise nspawn sets up a mount + # over a file that does not exist yet. + systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -1448,7 +1647,11 @@ in hostPath = bootstrapTokenDir; isReadOnly = true; }; - }; + } + # The hive CA, for the forwarder's two outbound calls: the swarm + # collector's gateway name and authelia's token endpoint. Empty when + # the gateway is not self-signed, so the whole path drops out. + // caTrust.bindMount; # The seal talks to the TPM through the kernel's resource manager, so # the device has to cross the container boundary or the store cannot @@ -1471,7 +1674,27 @@ in imports = [ (import ./swarm-container-resolver.nix { inherit (networkCfg) bridgeIp; - dnsConsumers = [ "openbao.service" ]; + # The forwarder resolves two swarm names of its own — the + # collector it exports to and the identity provider it mints + # a token at — so it belongs in this list for the reason the + # module states: an unordered resolver write is a cold-boot + # race that shows up arbitrarily far from its cause. + dnsConsumers = [ + "openbao.service" + "opentelemetry-collector.service" + ]; + }) + ] + # `SSL_CERT_FILE` REPLACES the trust store rather than adding to + # it, so a failed assembly yields an empty pool and every TLS call + # fails while the unit looks healthy. That is why this is the + # shared helper — it carries the `Requires` and the non-empty + # check — and not a local `cat`. + ++ [ + (caTrust.trustBundle { + inherit pkgs; + name = cfg.machine; + consumers = [ "opentelemetry-collector" ]; }) ]; @@ -1708,6 +1931,34 @@ in storage = "file_storage"; }; + # The scrape that used to belong to the swarm collector, now + # one hop shorter and one host less fussy — mara: + # *"move the existing scraper to the local collector"*. + # + # ⚠️ `metrics_path` and `params` are STATED, and both have to + # be: openbao serves no `/metrics` at all, and + # `/v1/sys/metrics` answers JSON unless the format is asked + # for. A scrape of the default path 404s and a scrape without + # the query parses as nothing — each reads as a dead exporter + # rather than as a wrong request, which is how the previous + # owner of this scrape learned to spell it + # (./swarm-otel.nix's `loopbackScrapeConfig`, still the + # renderer for every other service's target). + # + # No credential, unlike the swarm collector's + # `publishedScrapeTargets` jobs: the target is the loopback + # listener above, inside this container's own netns, which is + # what lets the scrape stay unauthenticated without the API + # listener's client-cert requirement being touched. + receivers.prometheus.config.scrape_configs = [ + { + job_name = "bao"; + static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; + metrics_path = "/v1/sys/metrics"; + params.format = [ "prometheus" ]; + } + ]; + # Identity the hop above cannot supply: a host-side reader can # say which machine a line came from, and only a collector # inside this container can say it was the secret store's. No @@ -1728,7 +1979,34 @@ in # protocol is, which is the same reason # ./hive-c0re/environment.nix pins the protocol it hands out. # `endpoint` is a BASE the exporter appends `/v1/logs` to. - exporters.otlphttp.endpoint = otelFirstHop; + exporters.otlphttp = { + endpoint = otelFirstHop; + # The receiver at that address accepts an OIDC token and + # nothing else (./swarm-otel.nix's `oidc/*` authenticators), + # so this is what makes the hop deliver rather than collect + # 401s — unconditional, because a forwarder that exports + # unauthenticated has no working mode to fall back to. + auth.authenticator = forwarderAuthName; + }; + + # The push side, opposite in direction to the receiver's + # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one + # to send. Hence `oauth2client`, and hence a client id and + # secret rather than an issuer and an audience to check. + extensions.${forwarderAuthName} = { + client_id = cfg.otel.clientId; + # A path, never a value — and one systemd resolves at + # runtime, so neither the secret nor the credentials + # directory is rendered into the nix store. + client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; + token_url = "${toString autheliaCfg.url}/api/oidc/token"; + # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. + # Registering it on the client only makes it permissible; a + # token minted without asking carries `aud: []` and the + # receiver refuses it — with a config that reads perfectly + # at both ends. + endpoint_params.audience = cfg.otel.clientId; + }; service.pipelines.logs = { receivers = [ "journald" ]; @@ -1736,12 +2014,53 @@ in exporters = [ "otlphttp" ]; }; + # A SECOND pipeline, not a second receiver on the logs one: a + # pipeline is typed, and a prometheus receiver in `logs` is a + # config the collector rejects at startup rather than one that + # quietly ships metrics as log records. + # + # Same `resource` processor and same `otlphttp` exporter, so + # the store's metrics carry the same `service.name` its logs + # do and leave by the same hop — no otel bypass, one exit from + # this container. The exporter treats `endpoint` as a base and + # appends the signal's own path, so `/v1/metrics` here and + # `/v1/logs` above come from the one address. + service.pipelines.metrics = { + receivers = [ "prometheus" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; + # An extension configured but not listed here is INERT — the # receiver's `storage: file_storage` above would name a - # component the collector never starts. - service.extensions = [ "file_storage" ]; + # component the collector never starts, and the exporter's + # authenticator would push unauthenticated while the + # collector reports healthy. + service.extensions = [ + "file_storage" + forwarderAuthName + ]; }; }; + + # How the secret crosses the sandbox. The host unit + # `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this + # path in the container's tree; systemd reads it as root before + # the sandbox exists and re-exposes it to whichever uid + # `DynamicUser` picked, under the directory the extension above + # names through `${env:CREDENTIALS_DIRECTORY}`. + # + # ⚠️ Unconditional, and a missing source is a unit that refuses to + # start. That is the intended shape rather than an oversight: this + # forwarder has one way to deliver, so a boot where the secret has + # not arrived yet is a collector that says so and retries, not one + # that quietly exports into a 401. The restart is what closes the + # first-boot window the host unit's own comment describes. + systemd.services.opentelemetry-collector.serviceConfig = { + LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ]; + Restart = "on-failure"; + RestartSec = 15; + }; }; }; }) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 305f8bad..945f79ea 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -47,6 +47,24 @@ let # bound once here (rather than copy-pasted at both sites) is the only way # the two can't drift apart. swarmTierName = cfg.producerName; + + # The SECOND swarm-tier producer: the collector inside the secret store's + # container (`swarm-bao.nix`). It gets an owner of its own rather than + # pushing into `swarmTierName`'s route, because the audience that route + # checks is `swarm-controller`'s own client id — one identity per principal, + # so a second principal brings its own client, its own audience and + # therefore its own authenticator and receiver. + # + # Named after the store's container, so `otlp/swarm-bao` reads as the thing + # that pushes into it. No `reserved-names.nix` entry is needed for it the + # way `swarm` has one: the name CONTAINS `swarm`, which + # `nix/reserved-hive-fragments.nix` forbids as a substring of any hive name, + # so no hive can ever own these components. The assertion below checks that + # fragment is still listed rather than trusting it. + storeProducerName = baoCfg.machine; + storeClientId = baoCfg.otel.clientId; + reservedHiveFragments = import ../reserved-hive-fragments.nix; + # Every `` no hive may take. Read from `nix/reserved-names.nix`, the # same file the daemons are handed as `HIVE_RESERVED_NAMES`, because agent # names and hive names are ONE namespace going forward — a locally-owned @@ -363,6 +381,50 @@ in ''; }; + storeProducerPort = lib.mkOption { + type = lib.types.port; + default = 4391; + description = '' + Port the receiver for the **secret store's own forwarder** listens + on, at `127.0.0.1`. Reached through the gateway at + `https://''${domain}/''${storeProducerName}/`, the same + path-per-producer shape a hive and the swarm-tier producer above + both use. + + Its own receiver, and not a second sender into + {option}`services.hyperhive.swarm.otel.producerPort`, because an + `oidc` authenticator checks exactly ONE audience: that receiver + admits the audience `swarm-controller` registers for itself, and + the store's forwarder is a different principal with a client of its + own. Sharing the route would mean sharing that client — which is + the thing the store exists to make unnecessary. + + ⚠️ A reserved value rather than an offset from + {option}`services.hyperhive.swarm.otel.port`, for the reason stated + on `producerPort`: that range grows with the hive count and would + eventually walk into any fixed offset from it. The collision + assertion below covers this port too. + ''; + }; + + storeProducerName = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = baoCfg.machine; + defaultText = lib.literalExpression "services.hyperhive.swarm.bao.machine"; + description = '' + Path and component name of the store forwarder's own receiver + (`otlp/''${storeProducerName}`, `oidc/''${storeProducerName}`). + Published so `swarm-bao.nix` addresses the receiver by name rather + than repeating the string, exactly as `producerName` above is + published for `swarm-controller.nix`. + + Read-only and derived from the store's container name: two + spellings of a name both ends have to agree on is a 404 on a + request that authenticated perfectly. + ''; + }; + producerName = lib.mkOption { type = lib.types.str; readOnly = true; @@ -619,6 +681,9 @@ in # request reaches the receiver, which knows nothing about the # path it was found at. "/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/"; + # The secret store's forwarder, on its own route for its own + # audience — same trailing slashes, same reason. + "/${storeProducerName}/".proxyPass = "http://127.0.0.1:${toString cfg.storeProducerPort}/"; # There is no swarm-wide inbox, and a closed door is the honest # description of that. Every other route into this collector # belongs to exactly one hive or the swarm-tier producer above. @@ -873,6 +938,25 @@ in Put it back, or give this module a different swarmTierName. ''; } + { + # The store forwarder's components are named after a container whose + # name merely CONTAINS the reserved word, so what protects them is + # the substring list rather than the equality one. Same reason the + # assertion above exists: the list lives in another file, and a guard + # whose subject can be edited elsewhere has to assert its own case is + # still covered. + assertion = lib.any (f: lib.hasInfix f storeProducerName) reservedHiveFragments; + message = '' + nix/reserved-hive-fragments.nix forbids no substring of + '${storeProducerName}', which the swarm collector needs kept out of + the hive namespace: it names components `/`, so a hive + called '${storeProducerName}' would replace the secret store + forwarder's own receiver and pipeline entry. + + Put the fragment back, or give the store's container a name one of + them covers. + ''; + } { # A published target that is not an `https://host/path` url. Without # this the split returns null and the failure surfaces as @@ -974,6 +1058,7 @@ in others = [ cfg.telemetryPort cfg.producerPort + cfg.storeProducerPort otelCfg.collector.port ] ++ lib.optional deployCfg.victoriametrics.enable vmCfg.port; @@ -1119,6 +1204,15 @@ in auth.authenticator = "oidc/${swarmTierName}"; }; } + # The secret store's forwarder, which is a swarm-tier + # producer with a principal of its own — a receiver per + # audience, because an `oidc` authenticator checks one. + // { + "otlp/${storeProducerName}".protocols.http = { + endpoint = "127.0.0.1:${toString cfg.storeProducerPort}"; + auth.authenticator = "oidc/${storeProducerName}"; + }; + } # MERGED with the per-hive receivers, never assigned over # them. A plain assignment here would drop every hive's # receiver and still render a valid config that starts @@ -1303,7 +1397,13 @@ in # same reasoning as its receiver above: `otlp/${swarmTierName}` # always exists, so the authenticator it names must too, or an # extension-not-listed startup failure follows every deploy. - ++ [ "oidc/${swarmTierName}" ] + ++ [ + "oidc/${swarmTierName}" + # Unconditional for the same reason, and one level sharper: + # the store runs in every swarm, so its forwarder's + # receiver is as permanent as the swarm tier's own. + "oidc/${storeProducerName}" + ] # The push side's authenticators. Same rule as above: one an # exporter names but this list omits is INERT — the collector # starts clean and pushes unauthenticated. @@ -1347,6 +1447,16 @@ in "metrics/${swarmTierName}" = { receivers = [ "otlp/${swarmTierName}" + # The store forwarder's receiver joins the swarm + # tier's pipeline rather than bringing one of its own: + # the stamp it needs is `resource/${swarmTierName}`'s + # exact stamp — the swarm's name and deliberately no + # `hive` — and a second pipeline would be that same + # processor under a second name to keep in step. What + # a producer is separated FOR is the credential it + # proves, and that is the receiver's job, not the + # pipeline's. + "otlp/${storeProducerName}" ] ++ lib.optional (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) "prometheus"; processors = [ "resource/${swarmTierName}" ]; @@ -1386,7 +1496,14 @@ in # `hive` value to put on them. // lib.optionalAttrs collectLogs { "logs/${swarmTierName}" = { - receivers = [ "journald" ]; + # This host's own journal, and the store container's, + # which arrives over the receiver above rather than off + # a disk — the store's forwarder reads a journal no + # reader on another host can see. + receivers = [ + "journald" + "otlp/${storeProducerName}" + ]; processors = [ "resource/${swarmTierName}" ]; exporters = logExporterNames; }; @@ -1450,6 +1567,18 @@ in audience = config.services.hyperhive.swarm.controller.queueClientId; }; } + # The secret store forwarder's own authenticator. Same shape + # again, and the audience is that principal's own client id + # — the self-referential form `swarm-authelia.nix` registers + # a hive client under and `swarm-controller` uses above, + # read from the option `glue-swarm-bao-otel-oidc-client.nix` + # registers so the two ends cannot spell it differently. + // { + "oidc/${storeProducerName}" = { + issuer_url = hyperhiveCfg.swarm.authelia.url; + audience = storeClientId; + }; + } # The push side. Opposite direction to every `oidc/*` above — # those VALIDATE a token arriving; these OBTAIN one to send. # Hence `oauth2client` rather than `oidc`, and hence a client diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index 6f1a51e6..cfe8f5d1 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -71,6 +71,13 @@ let serviceClientIds = [ hyperhiveCfg.swarm.grafana.oidc.clientId hyperhiveCfg.swarm.otel.clientId + # The secret store's own journal forwarder. A third swarm service by this + # list's own rule — a client registered once for the whole swarm, whose + # secret one host mints and another reads — even though the principal + # behind it runs inside the store's container rather than beside it. + # Missing from here, authelia mints the value and nothing carries it, so + # the reader waits on a path that is never written. + hyperhiveCfg.swarm.bao.otel.clientId ]; # Where this unit keeps the appservice tokens it minted, and the whole reason diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index f6d9b83f..ebd1d198 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -26,7 +26,10 @@ let deploy.bao.enable = true; deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; # Pinned, not incidental: the case counting these listeners is about the - # declared addresses, and a collector on this host would add one of its own. + # declared API addresses. ⚠️ Since the store got its own collector, this no longer keeps the METRICS + # listener out of the count — that one is a function of the store, not of + # a collector's placement — so the case below subtracts it by name rather + # than by leaving the swarm tier off. deploy.swarm-otel.enable = false; }; @@ -72,10 +75,20 @@ let # The scrape list prometheus is handed, not the option a service declared: # the address, the path and the query are one string on the way in and three # fields on the way out, and only the second shape is what gets requested. - scrapeJob = - machine: job: - lib.findFirst (c: c.job_name == job) null - machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; + # + # ⚠️ Two of these now, aimed at two different containers, and the split is + # the point: the store's own job left the swarm collector for the + # one inside its container, so a case that looked in the old place would + # find `null` and a case that only looked in the new one would not notice + # the store still being declared to both. + scrapeJobIn = + container: machine: job: + lib.findFirst (c: c.job_name == job) null ( + machine.containers.${container}.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs + or [ ] + ); + scrapeJob = scrapeJobIn "swarm-otel"; + baoScrapeJob = scrapeJobIn "swarm-bao"; cases = [ { # Same gap one tier up, and it needs its own arm: this collector @@ -102,9 +115,14 @@ let } { # Control for the case above: these settings are rendered per deployment, - # not constants a passing case could be indifferent to. + # not constants a passing case could be indifferent to. The metrics + # listener is excluded by name — it answers on a port of its own and is + # not one of the API addresses this case is counting. name = "a declared extra address renders a second listener beside loopback"; - ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2; + ok = + builtins.length ( + builtins.filter (n: n != "metrics") (builtins.attrNames (baoSettings baoTwoAddresses).listener) + ) == 2; } { # Retention is what serves the endpoint at all, so the listener alone @@ -119,13 +137,56 @@ let { # openbao serves no `/metrics` at all, so a scrape of the default path # 404s: the store looks like a dead exporter, and every panel built on - # it renders empty rather than erroring. + # it renders empty rather than erroring. The query is the other half — + # `/v1/sys/metrics` answers JSON without it, which parses as no series + # rather than as an error. name = "the store's scrape asks for the path openbao serves"; ok = let - j = scrapeJob baoWithCollector "bao"; + j = baoScrapeJob baoWithCollector "bao"; in - (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; + j != null + && (j.metrics_path or "") == "/v1/sys/metrics" + && (j.params.format or [ ]) == [ "prometheus" ]; + } + { + # 🎯 This move's deliverable, and it needs BOTH halves or it is not a move: + # the job is in the store's own container, and it is no longer declared + # to the swarm collector. Only the second half can catch a "move" that + # left the original behind — two scrapers of one endpoint, double-counted + # and neither obviously wrong. + name = "the store's metrics are scraped by its own collector, not the swarm's"; + ok = baoScrapeJob baoWithCollector "bao" != null && scrapeJob baoWithCollector "bao" == null; + } + { + # The scrape target is the loopback listener, not the API one: dialing + # the API listener would need a client certificate the collector has no + # way to present, and the fix for that would be relaxing + # `tls_require_and_verify_client_cert` — which is the trade this address + # exists to avoid. Pinned so a later edit has to argue with it. + name = "the store's collector scrapes loopback, never a wider address"; + ok = + let + j = baoScrapeJob baoWithCollector "bao"; + s = baoSettings baoWithCollector; + in + j.static_configs == [ { targets = [ "127.0.0.1:8202" ]; } ] + && s.listener.metrics.address == "127.0.0.1:8202"; + } + { + # A pipeline is typed: a prometheus receiver named in `logs` is rejected + # at startup, and metrics with no pipeline at all are collected and + # dropped. The exporter is pinned alongside because "no otel bypass" + # means both signals leave by the one hop. + name = "the store's collector has a metrics pipeline out the same hop as its logs"; + ok = + let + s = (baoForwarder baoWithCollector).settings; + m = s.service.pipelines.metrics; + in + m.receivers == [ "prometheus" ] + && m.exporters == [ "otlphttp" ] + && m.exporters == s.service.pipelines.logs.exporters; } { # Presence control for the case above: both fields are omitted rather @@ -139,14 +200,28 @@ let j != null && !(j ? metrics_path) && !(j ? params); } { - # Absence arm. Unauthenticated by design, so it must not exist where - # nothing reads it. - name = "a store with no collector beside it serves no metrics"; + # 🩸 This case USED to be the absence arm — "a store with no collector + # beside it serves no metrics" — and this inverts it deliberately + # rather than deleting it, because the condition it asserted is the bug. + # + # The old reasoning was sound for the old reader: the endpoint is + # unauthenticated, so it should not exist where nothing reads it, and + # the only reader was a swarm collector that had to be on this host. + # What that rendered on every OTHER host was a store whose metrics went + # nowhere, indistinguishably from a store nobody had asked to scrape. + # + # The reader now ships inside the container, so "no collector beside it" + # describes no host that exists. The endpoint is still not exposed to + # anything new: it is loopback in a netns whose only other occupants are + # the store and its collector. + name = "a store whose swarm collector is elsewhere still serves and scrapes its metrics"; ok = let s = baoSettings baoOtelElsewhere; in - !(s.listener ? metrics) && !(s ? telemetry); + s.listener.metrics.address == "127.0.0.1:8202" + && (s.telemetry.prometheus_retention_time or "0s") != "0s" + && baoScrapeJob baoOtelElsewhere "bao" != null; } { # The store's journal reaches a reader through a collector of its own, @@ -162,7 +237,11 @@ let in s.receivers.journald.directory == "/var/log/journal" && s.receivers.journald.storage == "file_storage" - && s.service.extensions == [ "file_storage" ] + && + s.service.extensions == [ + "file_storage" + "oauth2client/swarm-bao" + ] && p.receivers == [ "journald" ] && p.exporters == [ "otlphttp" ] && s ? exporters.otlphttp; @@ -188,8 +267,8 @@ let vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain}; in (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint - == "https://${otel.domain}/${otel.producerName}" - && vhost.locations ? "/${otel.producerName}/"; + == "https://${otel.domain}/${otel.storeProducerName}" + && vhost.locations ? "/${otel.storeProducerName}/"; } { # The case the old hive-tier gate got wrong: this host runs the swarm's @@ -219,7 +298,7 @@ let ok = let otel = baoOtelElsewhere.services.hyperhive.swarm.otel; - route = "https://${otel.domain}/${otel.producerName}"; + route = "https://${otel.domain}/${otel.storeProducerName}"; in !(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain}) && (baoForwarder baoOtelElsewhere).enable @@ -229,6 +308,129 @@ let && (baoForwarder baoWithHiveOtel).enable && (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route; } + { + # 🎯 The credential half of this move, and the arm that says the hop + # DELIVERS rather than merely resolves. The receiver at the far end + # takes an OIDC token and nothing else, so an exporter with no + # authenticator is a forwarder that retries into 401s while every other + # case here passes. Both halves: the extension has to exist AND the + # exporter has to name it. + name = "the store's forwarder presents a credential on the hop it exports over"; + ok = + let + s = (baoForwarder baoWithCollector).settings; + auth = "oauth2client/swarm-bao"; + in + s.exporters.otlphttp.auth.authenticator == auth + && s.extensions ? ${auth} + && lib.elem auth s.service.extensions; + } + { + # Its OWN client, never the swarm collector's and never the + # controller's: one identity per principal, and this one runs on a host + # neither of those two does. The audience is the same id because the + # receiver checks exactly that — a token minted without ASKING for it + # carries `aud: []` and is refused with a config that reads correctly at + # both ends. + name = "the forwarder authenticates as a principal of its own"; + ok = + let + bao = baoWithCollector.services.hyperhive.swarm.bao; + e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao"; + in + e.client_id == bao.otel.clientId + && e.endpoint_params.audience == bao.otel.clientId + && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.otel.clientId + && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.controller.queueClientId; + } + { + # A path systemd resolves at runtime, never a value and never a store + # path: the collector runs under `DynamicUser` and opens this file + # itself, so the credential arrives through `LoadCredential` and the + # config names only the directory systemd exports. + name = "the forwarder's client secret reaches it as a credential, not a value"; + ok = + let + container = baoWithCollector.containers.swarm-bao.config; + e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao"; + unit = container.systemd.services.opentelemetry-collector.serviceConfig; + in + e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret" + && + unit.LoadCredential == [ + "oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" + ]; + } + { + # Same 403-not-a-miss property the grafana and matrix readers are pinned + # for: the reader's grant covers the `services` prefix, so a secret + # filed under the hive that happens to run the store would be refused + # rather than missing, however correct the path reads. + name = "the forwarder's secret is read from the prefix the publisher writes"; + ok = + let + s = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc.script; + in + lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" s + && !(lib.hasInfix "secret/swarm/hives/" s); + } + { + # The delivery unit is ordered AFTER the store's container, not before + # it: the store it reads lives in the container it delivers into, so a + # `before` edge is a wait on a process that cannot start until this + # finishes. Pinned because the unit it was copied from does the + # opposite, and copying that line too would deadlock every cold boot. + name = "the delivery unit waits for the store rather than blocking it"; + ok = + let + u = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc; + in + lib.elem "container@swarm-bao.service" u.after + && !(lib.elem "container@swarm-bao.service" (u.before or [ ])) + && lib.elem "swarm-bao-pki.service" u.requires; + } + { + # The far end of the credential. An authenticator checks ONE audience, + # so the store's forwarder gets a receiver of its own rather than a + # second sender into the route whose audience belongs to + # `swarm-controller` — and every one of these four is silent when + # wrong: an unlisted extension is inert, a receiver in no pipeline + # drops what it accepts, and a mismatched audience is a healthy 401. + name = "the swarm collector admits this principal on a receiver of its own"; + ok = + let + otel = baoWithCollector.services.hyperhive.swarm.otel; + s = baoWithCollector.containers.swarm-otel.config.services.opentelemetry-collector.settings; + name = otel.storeProducerName; + in + s.receivers."otlp/${name}".protocols.http.auth.authenticator == "oidc/${name}" + && + s.extensions."oidc/${name}".audience == baoWithCollector.services.hyperhive.swarm.bao.otel.clientId + && lib.elem "oidc/${name}" s.service.extensions + && lib.elem "otlp/${name}" s.service.pipelines."metrics/${otel.producerName}".receivers + && lib.elem "otlp/${name}" s.service.pipelines."logs/${otel.producerName}".receivers; + } + { + # A client authelia has never heard of mints nothing, so the publisher + # has nothing to copy and the delivery unit reads an empty path + # forever. Registered where AUTHELIA runs, which is why it is a glue + # module and not a line in the store's own config — this fixture runs + # no store at all and still has to register it. + name = "the forwarder's client is registered wherever authelia runs"; + ok = + let + autheliaOnly = hive { deploy.authelia.enable = true; }; + clients = autheliaOnly.services.hyperhive.swarm.authelia.oidc.clients; + c = lib.findFirst (c: c.id == "swarm-bao-collector") null clients; + in + !autheliaOnly.services.hyperhive.deploy.bao.enable + && c != null + && c.audience == [ "swarm-bao-collector" ] + # ⚠️ What makes the token READABLE by the receiver at all: authelia's + # default is an opaque handle, and an `oidc` extension verifies + # offline against the provider's JWKS. + && c.accessTokenSignedResponseAlg == "RS256"; + } ]; in runGroup "bao-otel-collector" cases diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index 7c2f9f6d..fd343356 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -132,6 +132,17 @@ let secretPublisherHere.systemd.services.swarm-secret-publish.script ); } + { + # The third swarm service on that list, and the one whose absence is + # hardest to see: the store's own forwarder holds a client authelia + # registers and mints for, so every layer above looks complete while + # the one hop that CARRIES the value skips it and the reader on the + # store's host waits forever on a path nothing writes. + name = "the publisher carries the store forwarder's secret too"; + ok = lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" ( + secretPublisherHere.systemd.services.swarm-secret-publish.script + ); + } { # The same hole the controller's case above names, open a second time: the # PKI script grew a third leaf and no case read it.