diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 53bce3ad..e3c2e0ce 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -64,8 +64,7 @@ because nothing outside that container ever reads them. **That's the test worth applying to any secret added here** — and the client secret's plaintext half is the one row that fails it, which is the entire reason a delivery step exists. -Three telemetry collectors exist, and they don't land on the same side of -that test. +Two telemetry collectors exist, and they land on opposite sides of that test. The **hive's** collector needs no delivery step. It authenticates to the swarm's collector as its own hive, and it's a host unit rather than a container, so on @@ -95,27 +94,6 @@ uses for its non-optional one. The operator can still deliver the secret by hand, at `services.hyperhive.deploy.swarm-otel.clientSecretFile`, on a collector this unit never reaches. -The **secret store's own** collector — the forwarder inside the `swarm-bao` -container — needs a delivery step too, and it takes the same route with one -principal of its own: `swarm-bao-forwarder-oidc.service` reads -`swarm/services//oidc/client` under this host's certificate and -lands it in the container's tree, where `LoadCredential` hands it to the -collector. The client id is its own -(`services.hyperhive.swarm.bao.otel.clientId`), registered by -`glue-swarm-bao-otel-oidc-client.nix` wherever authelia runs, and the swarm -collector admits it on a receiver of its own — one identity per principal, so -this forwarder borrows neither the swarm collector's client nor -`swarm-controller`'s. - -⚠️ **This one has no degraded mode, and the difference is structural.** The -route it exports over accepts an authenticated request and nothing else, so a -forwarder without the secret has nothing to fall back to: the unit fails and -retries, and the collector refuses to start until the file arrives rather than -exporting into a 401 while reporting healthy. Its ordering differs from the -unit above it for the same structural reason — it reads the store inside the -container it delivers into, so it runs **after** that container rather than -before it, and only a first boot ever waits. - ### Minting the queue's callout nkeys `deploy.nats.autoGenerateCallout` mints both keypairs on the host before the queue @@ -276,16 +254,6 @@ doesn't render the reading unit without them, the shape `glue-matrix-bao-token.nix` and `glue-queue-agent-credential.nix` use for their own optional reads. -The **fifth reader** is the secret store forwarder's OIDC client secret: -`swarm-bao-forwarder-oidc.service`, on the store's own host, registered by -`glue-swarm-bao-otel-oidc-client.nix` on authelia's. It differs from the -fourth in both directions. It renders unconditionally, because the export it -authenticates has no unauthenticated mode to degrade into — and it orders -itself **after** `container@swarm-bao`, because the store it reads runs in the -container it delivers into. Nothing circular sits behind that: the identity it -logs in with is this host's static `swarm-bao-pki` leaf, not anything the store -mints. - A service's secret is one value for the whole swarm rather than one per hive, so it lives under the `services` prefix, and a hive's read policy grants that prefix whole. That's because nothing in a swarm's configuration records which hive runs diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 01030d24..b4f1e7e8 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,7 +30,6 @@ ./glue-matrix-ctl-bao-identity.nix ./glue-queue-agent-credential.nix ./glue-secret-publisher-bao-identity.nix - ./glue-swarm-bao-otel-oidc-client.nix ./glue-swarm-otel-oidc-client.nix ./swarm-authelia.nix ./swarm-bao.nix diff --git a/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix b/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix deleted file mode 100644 index 6e9d850a..00000000 --- a/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix +++ /dev/null @@ -1,56 +0,0 @@ -# Glue: register the secret store's own journal forwarder as an OIDC client -# wherever authelia runs. -# -# ONE PAIRING PER FILE — swarm-bao's forwarder ← authelia, and nothing else. -# Deleting this leaves a forwarder authelia has never heard of: the token -# endpoint refuses it, nothing is minted, the publisher has nothing to copy -# into the store, and the collector's export is a 401 nobody asked for. -# -# ⚠️ A SECOND client beside ./glue-swarm-otel-oidc-client.nix's, not a reuse -# of it. That one is the swarm collector's identity for what IT pushes; this -# one belongs to the collector inside the store's container, which is a -# different principal on a different host — one identity per principal, the -# rule `swarm-controller.nix` states over its own `queueClientId`. -# -# ⚠️ Gated on authelia being HERE, and deliberately NOT on this host running -# the store. A client is a row in THIS host's provider config, so it can only -# be declared where that config is rendered, and ./swarm-bao.nix's `config` -# hangs off `deploy.bao.enable` — the split ./glue-grafana-oidc-client.nix and -# ./glue-swarm-otel-oidc-client.nix each made for the same reason. Read either -# file's own comment for the property this one shares with them. -{ - lib, - config, - ... -}: -let - hyperhiveCfg = config.services.hyperhive; - deployCfg = hyperhiveCfg.deploy; - baoCfg = hyperhiveCfg.swarm.bao; -in -{ - config = lib.mkIf (hyperhiveCfg.enable && deployCfg.authelia.enable) { - # One declaration, two readers: `clientId` is a read-only option - # ./swarm-bao.nix owns, and ./swarm-otel.nix builds this principal's - # authenticator audience from the same option. - services.hyperhive.swarm.authelia.oidc.clients = [ - { - id = baoCfg.otel.clientId; - description = "HyperHive secret store journal forwarder"; - kind = "machine"; - redirectUris = [ ]; - # Its own id as its own permitted audience — the self-referential - # form `swarm-authelia.nix`'s `hiveClients` uses, which is the shape - # to copy here: this forwarder's token lands on an `oidc/*` receiver - # of the swarm collector, exactly as a hive's does. - audience = [ baoCfg.otel.clientId ]; - # ⚠️ What makes the token READABLE by that receiver at all. Authelia's - # default is an opaque handle, and an `oidc` extension verifies - # offline against `/jwks.json` — so without this the export fails with - # a message about the verifier rather than about the token. Same - # value, same reason, as every hive's client. - accessTokenSignedResponseAlg = "RS256"; - } - ]; - }; -} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 34676bda..702e5e72 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -429,23 +429,19 @@ let // metricsListener; # Metrics get their own listener rather than a flag on the one above, and - # that follows from what a scraper can express: a prometheus scrape config - # carries no client certificate, while the API listener is TLS and — once a - # client CA is set — demands one. A scraper cannot reach it at all. + # that follows from what a scraper can express: `swarm.otel.scrapeTargets` + # carries no scheme and no credential, while the API listener is TLS and — + # once a client CA is set — demands a client certificate. The collector + # cannot reach it at all. # # `metrics_only` narrows this one to the metrics path (every other path 404s) # and the unauthenticated access is confined to loopback. **Deliberately - # unauthenticated** — the only process that can dial 127.0.0.1 here is one - # already inside this netns, which is the container's own collector and the - # store itself. Widening this address, or dropping - # `tls_require_and_verify_client_cert` on the API listener to scrape THAT - # instead, would each trade that property away for nothing. + # unauthenticated for now** — a tracked follow-up owns giving the collector a + # credential, since no scrape option can carry one today. # - # Unconditional, and that is the change this carries: the reader is now the - # collector in this very container (below), which renders wherever the store - # does. There is no longer a host where this endpoint has no reader, so there - # is nothing left for it to be a function of. - metricsListener = { + # Exists only where a collector does: an endpoint with no reader is exposure + # bought for nothing. + metricsListener = lib.optionalAttrs scrapeHere { metrics = { type = "tcp"; address = "127.0.0.1:${toString baoDeploy.metricsPort}"; @@ -457,99 +453,12 @@ let }; }; - # What the container's own collector dials. Loopback inside a netns this - # container SHARES with the host (`privateNetwork = false` below), so the - # scrape crosses no network and needs no bind address wider than the one - # above. - metricsScrapeTarget = "127.0.0.1:${toString baoDeploy.metricsPort}"; - - # ⚠️ Neither the listener above nor the forwarder below has a condition of - # its own, and they lost theirs for the same reason at different times. - # - # The listener used to be gated on `scrapeHere = deployCfg.swarm-otel.enable` - # — "does the swarm's collector happen to run on THIS host" — because the - # thing reading it was that collector, over loopback, which only works when - # the two land together. mara's ruling (*"move the existing scraper - # to the local collector"*) removes the premise rather than the symptom: the - # scraper is now in this container, so the reader is wherever the store is, - # and the metrics path stops being silently dead on every host that does not - # also run swarm-otel. - # - # The forwarder never had a defensible one. "Does the swarm have a collector" - # is not a question that can be false (mara: "the swarm always has at least - # one instance of all the swarm level services"), so both earlier gates — - # `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong host - # and rendered no forwarder wherever the answer happened to be no. - # - # The only condition that belongs on either is the one already enclosing this - # block: the store is deployed on this host. - - # The swarm collector's own name, served by the gateway — the address - # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host - # and over the real network otherwise, with no loopback-vs-remote knob to get - # wrong. The path selects WHICH receiver: this container is a swarm-level - # producer and not a hive, so it is the swarm tier's own route, the same one - # ./swarm-controller.nix exports to. `/` on that vhost answers 404. - # - # The port suffix follows ./hive-forge/default.nix: elided on the canonical - # 443 so this renders byte for byte like the tier's other two consumers of - # this name, present when an operator moved the gateway's TLS port. - otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.storeProducerName}"; - - # The forwarder's own credential, and the three names it takes on the way - # in. The shape is ./swarm-otel.nix's `swarm-bao-otel-oidc` — the unit that - # already reads an OIDC client secret out of the store with this host's - # certificate and lands it in a collector's container — because that route - # is proven and there is no second one worth inventing. - # - # Where the publisher on authelia's host leaves it. The `services` segment - # is `swarm-secret-client`'s `path::Kind::Service`: a swarm service's client - # is registered once for the whole swarm, so its secret is one value. The - # prefix is also exactly what a hive certificate's read grant covers, so a - # path outside it answers 403 however correct it looks. - forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client"; - # At rest in the container's tree, written by the host unit below — under - # /var/lib rather than /run, because a secret that evaporates on reboot - # turns a working export into an intermittent one. - forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret"; - forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}"; - forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath; - # The collector runs under `DynamicUser` and opens `client_secret_file` - # itself, so there is no uid to hand a 0400 file to. `LoadCredential` reads - # it as root before the sandbox exists and re-exposes it under a path that - # does not depend on which uid the unit got; the collector expands - # `${env:CREDENTIALS_DIRECTORY}` at load, so this id is the only spelling. - forwarderCredentialId = "oidc-client-secret"; - # Named once and read three times — the extension that defines it, the - # exporter that names it, and `service.extensions`, where an omission makes - # it inert rather than an error. - forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}"; - autheliaCfg = hyperhiveCfg.swarm.authelia; - - # Outbound TLS for that collector: it dials the swarm collector's gateway - # name and authelia's token endpoint, both leaves of the runtime-generated - # hive CA, which cannot be baked into a derivation. Same helper, same two - # halves and the same `SSL_CERT_FILE` consumption as ./swarm-otel.nix's own - # container — a Go process whose trust variable REPLACES the store, which is - # what the bundle (system CAs ++ our anchors) is for. - caTrust = import ./lib/hive-ca-trust.nix { - inherit lib; - tlsCfg = deployCfg.hive-controller.tls; - gatewayCfg = hyperhiveCfg.gateway; - }; - - swarmOtelCfg = hyperhiveCfg.swarm.otel; - gatewayPortSuffix = - let - p = hyperhiveCfg.gateway.httpsPort; - in - lib.optionalString (p != 443) ":${toString p}"; + scrapeHere = deployCfg.swarm-otel.enable; # Non-zero is what SERVES the endpoint at all — the switch is a duration, not # a boolean, so a zero here is an openbao that answers 404 on a listener - # configured to do nothing else. Unconditional for the same reason the - # listener above is: its reader ships with the store now. - telemetry = { + # configured to do nothing else. + telemetry = lib.optionalAttrs scrapeHere { telemetry = { prometheus_retention_time = "24h"; disable_hostname = true; @@ -943,29 +852,6 @@ in ''; }; - otel.clientId = lib.mkOption { - type = lib.types.str; - readOnly = true; - default = "swarm-bao-collector"; - description = '' - OAuth2 client id the collector inside the store's container - authenticates as, and — self-referentially, the shape every hive's - client already uses — the audience it asks its token for. - - **Its own, not the swarm collector's and not - `swarm-controller`'s.** One identity per principal: this forwarder - runs wherever the store runs, which is not where either of those - two runs, and the receiver it pushes to - (`swarm.otel.storeProducerName`) admits this id alone. - - Swarm-wide and read-only because three hosts have to agree on it: - authelia registers the client - (`glue-swarm-bao-otel-oidc-client.nix`), the swarm collector checks - the audience (`swarm-otel.nix`), and the store's host reads the - minted secret back out of the store under a path composed from it. - Two spellings present as a healthy-looking 401. - ''; - }; }; # ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than @@ -1054,21 +940,15 @@ in "openbao" "swarm-bao-certs" "swarm-bao-token" - "swarm-bao-forwarder-oidc" ]; - # 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is - # the deliverable rather than a tidy-up. That option is read only by the - # SWARM collector, over loopback, so declaring the store there meant its - # metrics existed exactly on the hosts that also ran swarm-otel and - # nowhere else — silently, since a store with no entry looks identical to - # one nothing scrapes. The scrape moved into this container's own - # collector (see `receivers.prometheus` below), which travels with the - # store. - # - # Logs are untouched by that move: `journaldUnits` above still names this - # store's units for the shared collector, and the two are retired - # together once every sibling swarm container has a collector of its own. + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere { + # Path and query, not just `host:port`: openbao serves no `/metrics` + # at all, and `/v1/sys/metrics` answers JSON unless the format is + # asked for. A scrape of the default path 404s, which reads as a + # dead exporter rather than a wrong address. + bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus"; + }; # ⚠️ The one nginx exception to this file's header, and it is one because # it never terminates. `ssl_preread` reads the SNI off the ClientHello @@ -1102,113 +982,6 @@ in # still proves nothing — bao answers nothing without a client # certificate its CA signed. services.hyperhive.network.exposeHostPorts = [ cfg.port ]; - - # THE delivery unit for the forwarder's own OIDC client secret, copied - # in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login - # that fails LOUDLY, since every state it fails on is one a retry fixes, - # then a read, then a root-owned file in the container's tree that - # `LoadCredential` re-exposes to the collector's dynamic user. - # - # ⚠️ ORDERED AFTER the store's container and deliberately NOT before it, - # which is the one line that differs from the unit it copies. That one - # reads a store in the container NEXT DOOR; this one reads the store in - # the very container it is delivering into, so "before" is a wait on a - # process that cannot start until this finishes. There is no bootstrap - # cycle behind it — the identity used here is this host's static - # `swarm-bao-pki` certificate, not anything the store mints — only an - # ordering one, and the cost is bounded: the file is under /var/lib, so - # it survives reboots and only a FIRST boot has the collector starting - # before it exists. `LoadCredential` refuses to start a unit whose - # source is missing, so that boot is a collector that restarts, says so - # each time, and comes up the moment this lands. Loud and self-healing - # rather than silently exporting without a credential. - systemd.services.swarm-bao-forwarder-oidc = { - description = "fetch the secret store forwarder's OIDC client secret from the store"; - after = [ - "swarm-bao-pki.service" - "container@${cfg.machine}.service" - ]; - wants = [ "container@${cfg.machine}.service" ]; - requires = [ "swarm-bao-pki.service" ]; - wantedBy = [ "multi-user.target" ]; - path = [ - baoDeploy.package - pkgs.coreutils - ]; - # Sized for the race this loses: the secret is minted by authelia and - # copied in by the publisher, both of which may be a host away and - # neither of which this boot waits on. `StartLimit*` are `[Unit]` - # settings — systemd ignores them under `[Service]` — and the window - # has to exceed `RestartSec × burst`. - startLimitBurst = 4; - startLimitIntervalSec = 300; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "swarm-bao-forwarder-oidc"; - TimeoutStartSec = 30; - Restart = "on-failure"; - RestartSec = 15; - }; - environment = { - BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}"; - BAO_CLIENT_CERT = baoDeploy.clientCertFile; - BAO_CLIENT_KEY = baoDeploy.clientKeyFile; - } - # Absent means the system trust store, which is what a deployment with - # a real CA wants and what a self-signed one must not be left with. - // lib.optionalAttrs (baoDeploy.serverCaFile != null) { - BAO_CACERT = baoDeploy.serverCaFile; - }; - script = '' - set -euo pipefail - - # `bao`'s own message is the only thing separating a missing value - # from a refused identity from an unreachable store, and this unit - # retries on all three — so it reports which one rather than - # asserting all three in a sentence of ours. - err="$(mktemp)" - trap 'rm -f "$err"' EXIT - - # Cert auth is a login, not a transport setting: the `BAO_CLIENT_*` - # variables only decide which certificate the handshake presents. - # Without a token `bao` asks its token helper, and that is a `sh` - # this unit's `path` does not carry — `-token-only` answers on - # stdout and skips the helper on both sides. - if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - echo "could not log in to the swarm secret store with this host's certificate." >&2 - cat "$err" >&2 - exit 1 - fi - export BAO_TOKEN - - # ⚠️ Non-zero, where the sibling unit one module over exits 0 on this - # branch. That collector treats a missing secret as a supported - # degrade; this one has no such mode — the receiver it pushes to - # takes an authenticated request and nothing else — so "not - # published yet" is a state to retry out of, not to settle into. - if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then - echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2 - cat "$err" >&2 - exit 1 - fi - - if [ -z "$secret" ]; then - echo "the store returned an empty ${forwarderStoreSecretPath}." >&2 - exit 1 - fi - - # root-owned 0400, written with a shell builtin and never handed to - # a program: `printf` is bash's own, so the plaintext never becomes - # an argument in /proc the way `install <<<"$secret"` or an `echo` - # from `path` would. - install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir} - umask 077 - printf '%s\n' "$secret" > ${lib.escapeShellArg forwarderHostSecretPath} - chown root:root ${lib.escapeShellArg forwarderHostSecretPath} - chmod 0400 ${lib.escapeShellArg forwarderHostSecretPath} - ''; - }; }) (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) { @@ -1593,11 +1366,6 @@ in ''; }; - # The CA bind source is written at runtime by a host unit, so the - # container has to start after it — otherwise nspawn sets up a mount - # over a file that does not exist yet. - systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; - containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -1647,11 +1415,7 @@ in hostPath = bootstrapTokenDir; isReadOnly = true; }; - } - # The hive CA, for the forwarder's two outbound calls: the swarm - # collector's gateway name and authelia's token endpoint. Empty when - # the gateway is not self-signed, so the whole path drops out. - // caTrust.bindMount; + }; # The seal talks to the TPM through the kernel's resource manager, so # the device has to cross the container boundary or the store cannot @@ -1664,37 +1428,12 @@ in ]; config = - # `config` is the CONTAINER's, shadowing the host's inside this - # function — nothing in here reads the host's, and the journal - # assertion below has to read the setting on this side of the - # boundary. `lib` and `pkgs` still come from the enclosing scope, as - # every other expression in this block already does. - { config, ... }: + { ... }: { imports = [ (import ./swarm-container-resolver.nix { inherit (networkCfg) bridgeIp; - # The forwarder resolves two swarm names of its own — the - # collector it exports to and the identity provider it mints - # a token at — so it belongs in this list for the reason the - # module states: an unordered resolver write is a cold-boot - # race that shows up arbitrarily far from its cause. - dnsConsumers = [ - "openbao.service" - "opentelemetry-collector.service" - ]; - }) - ] - # `SSL_CERT_FILE` REPLACES the trust store rather than adding to - # it, so a failed assembly yields an empty pool and every TLS call - # fails while the unit looks healthy. That is why this is the - # shared helper — it carries the `Requires` and the non-empty - # check — and not a local `cat`. - ++ [ - (caTrust.trustBundle { - inherit pkgs; - name = cfg.machine; - consumers = [ "opentelemetry-collector" ]; + dnsConsumers = [ "openbao.service" ]; }) ]; @@ -1847,220 +1586,6 @@ in # Restarting is an operator action with an unseal on the far side of # it, which is why nothing here tries to be clever about it. - # This container's own journal forwarder, copied from an agent - # container's (nix/agent-modules/otel.nix) because every container - # is supposed to ship its own logs to the next hop rather than - # leave a *different* container's collector to find them. - # - # That older shape is what `--link-journal=host` above and the - # `swarm.otel.journaldUnits` entry near the top of this file serve: - # the journal lands in the host tree, where the swarm collector — - # itself a container — reads it through a unit allowlist. Both stay - # for now, because every sibling swarm container still rides that - # collector; they come out once each of them has one of these. - # - # ⚠️ NO `units` allowlist here, and that is the point rather than a - # simplification. A shared collector needs one because the journal - # it reads holds six containers' units plus the host's own; this - # one reads only openbao and the two oneshots beside it, so there - # is nothing foreign to separate out — and a list of unit names is - # a thing to get wrong, which ships nothing while looking healthy. - assertions = [ - { - # Sibling of the agent forwarder's identical assertion, and it - # exists because the failure is silent at every layer: with a - # volatile journal the receiver below finds an empty directory, - # reads nothing, and the collector starts clean and stays - # healthy forever. - assertion = - !(lib.elem config.services.journald.storage [ - "volatile" - "none" - ]); - message = '' - ${cfg.machine} forwards its own journal to the swarm's - collector — but services.journald.storage is - "${config.services.journald.storage}" in this container. - - The forwarder reads /var/log/journal, which journald only writes - when it stores persistently: with "volatile" the journal lives in - /run/log/journal and with "none" there is none at all. Either way - the forwarder would ship nothing while looking healthy. - ''; - } - ]; - - services.opentelemetry-collector = { - enable = true; - # Contrib, and not a preference: `journald` is a contrib - # receiver. The upstream default build has no way to read a - # journal at all. - package = pkgs.opentelemetry-collector-contrib; - # Runs `otelcol validate` at build time. ⚠️ A parser, not a - # wiring check — it accepts a pipeline naming a component the - # build lacks, and the collector then dies at startup. Same - # caveat as every other tier; see nix/host-modules/otel.nix for - # the measurement. - validateConfigFile = true; - settings = { - # `journalctl --follow --lines=0` (what the receiver runs under - # the hood) ships only what is written *after* it starts, so - # every collector start has a silent gap at the front. This - # persists the read cursor, so a restart resumes instead of - # re-opening that gap; `start_at` stays `end` (its default), - # since a cursor covers every run after the first and - # `beginning` without one would re-ship the whole journal on - # every restart. `create_directory` is required for a - # build-time reason rather than a convenience one: - # `validateConfigFile` above runs `otelcol validate` in the nix - # sandbox, before systemd's `StateDirectory=` has created the - # path, and the validator refuses with "directory must exist". - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; - - receivers.journald = { - # ⚠️ STATED, and it must stay stated: the receiver's own - # default is the RUNTIME journal (`/run/log/journal`), and - # every entry here is under /var/log/journal — which - # `--link-journal=host` above makes the host's directory for - # this machine id. Dropping this line leaves a collector that - # validates, starts, reports healthy and forwards nothing. - directory = "/var/log/journal"; - storage = "file_storage"; - }; - - # The scrape that used to belong to the swarm collector, now - # one hop shorter and one host less fussy — mara: - # *"move the existing scraper to the local collector"*. - # - # ⚠️ `metrics_path` and `params` are STATED, and both have to - # be: openbao serves no `/metrics` at all, and - # `/v1/sys/metrics` answers JSON unless the format is asked - # for. A scrape of the default path 404s and a scrape without - # the query parses as nothing — each reads as a dead exporter - # rather than as a wrong request, which is how the previous - # owner of this scrape learned to spell it - # (./swarm-otel.nix's `loopbackScrapeConfig`, still the - # renderer for every other service's target). - # - # No credential, unlike the swarm collector's - # `publishedScrapeTargets` jobs: the target is the loopback - # listener above, inside this container's own netns, which is - # what lets the scrape stay unauthenticated without the API - # listener's client-cert requirement being touched. - receivers.prometheus.config.scrape_configs = [ - { - job_name = "bao"; - static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; - metrics_path = "/v1/sys/metrics"; - params.format = [ "prometheus" ]; - } - ]; - - # Identity the hop above cannot supply: a host-side reader can - # say which machine a line came from, and only a collector - # inside this container can say it was the secret store's. No - # `hive` or `swarm` label — the swarm tier upserts `hive` from - # the receiver that accepted the record, precisely so it comes - # from something the sender cannot write. - processors.resource.attributes = [ - { - key = "service.name"; - value = cfg.machine; - action = "upsert"; - } - ]; - - # `otlphttp` unconditionally, unlike the agent forwarder's - # protocol-derived exporter name: the swarm receiver speaks - # OTLP/HTTP protobuf whatever this tier's *upstream* - # protocol is, which is the same reason - # ./hive-c0re/environment.nix pins the protocol it hands out. - # `endpoint` is a BASE the exporter appends `/v1/logs` to. - exporters.otlphttp = { - endpoint = otelFirstHop; - # The receiver at that address accepts an OIDC token and - # nothing else (./swarm-otel.nix's `oidc/*` authenticators), - # so this is what makes the hop deliver rather than collect - # 401s — unconditional, because a forwarder that exports - # unauthenticated has no working mode to fall back to. - auth.authenticator = forwarderAuthName; - }; - - # The push side, opposite in direction to the receiver's - # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one - # to send. Hence `oauth2client`, and hence a client id and - # secret rather than an issuer and an audience to check. - extensions.${forwarderAuthName} = { - client_id = cfg.otel.clientId; - # A path, never a value — and one systemd resolves at - # runtime, so neither the secret nor the credentials - # directory is rendered into the nix store. - client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; - token_url = "${toString autheliaCfg.url}/api/oidc/token"; - # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. - # Registering it on the client only makes it permissible; a - # token minted without asking carries `aud: []` and the - # receiver refuses it — with a config that reads perfectly - # at both ends. - endpoint_params.audience = cfg.otel.clientId; - }; - - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; - - # A SECOND pipeline, not a second receiver on the logs one: a - # pipeline is typed, and a prometheus receiver in `logs` is a - # config the collector rejects at startup rather than one that - # quietly ships metrics as log records. - # - # Same `resource` processor and same `otlphttp` exporter, so - # the store's metrics carry the same `service.name` its logs - # do and leave by the same hop — no otel bypass, one exit from - # this container. The exporter treats `endpoint` as a base and - # appends the signal's own path, so `/v1/metrics` here and - # `/v1/logs` above come from the one address. - service.pipelines.metrics = { - receivers = [ "prometheus" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; - - # An extension configured but not listed here is INERT — the - # receiver's `storage: file_storage` above would name a - # component the collector never starts, and the exporter's - # authenticator would push unauthenticated while the - # collector reports healthy. - service.extensions = [ - "file_storage" - forwarderAuthName - ]; - }; - }; - - # How the secret crosses the sandbox. The host unit - # `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this - # path in the container's tree; systemd reads it as root before - # the sandbox exists and re-exposes it to whichever uid - # `DynamicUser` picked, under the directory the extension above - # names through `${env:CREDENTIALS_DIRECTORY}`. - # - # ⚠️ Unconditional, and a missing source is a unit that refuses to - # start. That is the intended shape rather than an oversight: this - # forwarder has one way to deliver, so a boot where the secret has - # not arrived yet is a collector that says so and retries, not one - # that quietly exports into a 401. The restart is what closes the - # first-boot window the host unit's own comment describes. - systemd.services.opentelemetry-collector.serviceConfig = { - LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ]; - Restart = "on-failure"; - RestartSec = 15; - }; }; }; }) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 945f79ea..305f8bad 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -47,24 +47,6 @@ let # bound once here (rather than copy-pasted at both sites) is the only way # the two can't drift apart. swarmTierName = cfg.producerName; - - # The SECOND swarm-tier producer: the collector inside the secret store's - # container (`swarm-bao.nix`). It gets an owner of its own rather than - # pushing into `swarmTierName`'s route, because the audience that route - # checks is `swarm-controller`'s own client id — one identity per principal, - # so a second principal brings its own client, its own audience and - # therefore its own authenticator and receiver. - # - # Named after the store's container, so `otlp/swarm-bao` reads as the thing - # that pushes into it. No `reserved-names.nix` entry is needed for it the - # way `swarm` has one: the name CONTAINS `swarm`, which - # `nix/reserved-hive-fragments.nix` forbids as a substring of any hive name, - # so no hive can ever own these components. The assertion below checks that - # fragment is still listed rather than trusting it. - storeProducerName = baoCfg.machine; - storeClientId = baoCfg.otel.clientId; - reservedHiveFragments = import ../reserved-hive-fragments.nix; - # Every `` no hive may take. Read from `nix/reserved-names.nix`, the # same file the daemons are handed as `HIVE_RESERVED_NAMES`, because agent # names and hive names are ONE namespace going forward — a locally-owned @@ -381,50 +363,6 @@ in ''; }; - storeProducerPort = lib.mkOption { - type = lib.types.port; - default = 4391; - description = '' - Port the receiver for the **secret store's own forwarder** listens - on, at `127.0.0.1`. Reached through the gateway at - `https://''${domain}/''${storeProducerName}/`, the same - path-per-producer shape a hive and the swarm-tier producer above - both use. - - Its own receiver, and not a second sender into - {option}`services.hyperhive.swarm.otel.producerPort`, because an - `oidc` authenticator checks exactly ONE audience: that receiver - admits the audience `swarm-controller` registers for itself, and - the store's forwarder is a different principal with a client of its - own. Sharing the route would mean sharing that client — which is - the thing the store exists to make unnecessary. - - ⚠️ A reserved value rather than an offset from - {option}`services.hyperhive.swarm.otel.port`, for the reason stated - on `producerPort`: that range grows with the hive count and would - eventually walk into any fixed offset from it. The collision - assertion below covers this port too. - ''; - }; - - storeProducerName = lib.mkOption { - type = lib.types.str; - readOnly = true; - default = baoCfg.machine; - defaultText = lib.literalExpression "services.hyperhive.swarm.bao.machine"; - description = '' - Path and component name of the store forwarder's own receiver - (`otlp/''${storeProducerName}`, `oidc/''${storeProducerName}`). - Published so `swarm-bao.nix` addresses the receiver by name rather - than repeating the string, exactly as `producerName` above is - published for `swarm-controller.nix`. - - Read-only and derived from the store's container name: two - spellings of a name both ends have to agree on is a 404 on a - request that authenticated perfectly. - ''; - }; - producerName = lib.mkOption { type = lib.types.str; readOnly = true; @@ -681,9 +619,6 @@ in # request reaches the receiver, which knows nothing about the # path it was found at. "/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/"; - # The secret store's forwarder, on its own route for its own - # audience — same trailing slashes, same reason. - "/${storeProducerName}/".proxyPass = "http://127.0.0.1:${toString cfg.storeProducerPort}/"; # There is no swarm-wide inbox, and a closed door is the honest # description of that. Every other route into this collector # belongs to exactly one hive or the swarm-tier producer above. @@ -938,25 +873,6 @@ in Put it back, or give this module a different swarmTierName. ''; } - { - # The store forwarder's components are named after a container whose - # name merely CONTAINS the reserved word, so what protects them is - # the substring list rather than the equality one. Same reason the - # assertion above exists: the list lives in another file, and a guard - # whose subject can be edited elsewhere has to assert its own case is - # still covered. - assertion = lib.any (f: lib.hasInfix f storeProducerName) reservedHiveFragments; - message = '' - nix/reserved-hive-fragments.nix forbids no substring of - '${storeProducerName}', which the swarm collector needs kept out of - the hive namespace: it names components `/`, so a hive - called '${storeProducerName}' would replace the secret store - forwarder's own receiver and pipeline entry. - - Put the fragment back, or give the store's container a name one of - them covers. - ''; - } { # A published target that is not an `https://host/path` url. Without # this the split returns null and the failure surfaces as @@ -1058,7 +974,6 @@ in others = [ cfg.telemetryPort cfg.producerPort - cfg.storeProducerPort otelCfg.collector.port ] ++ lib.optional deployCfg.victoriametrics.enable vmCfg.port; @@ -1204,15 +1119,6 @@ in auth.authenticator = "oidc/${swarmTierName}"; }; } - # The secret store's forwarder, which is a swarm-tier - # producer with a principal of its own — a receiver per - # audience, because an `oidc` authenticator checks one. - // { - "otlp/${storeProducerName}".protocols.http = { - endpoint = "127.0.0.1:${toString cfg.storeProducerPort}"; - auth.authenticator = "oidc/${storeProducerName}"; - }; - } # MERGED with the per-hive receivers, never assigned over # them. A plain assignment here would drop every hive's # receiver and still render a valid config that starts @@ -1397,13 +1303,7 @@ in # same reasoning as its receiver above: `otlp/${swarmTierName}` # always exists, so the authenticator it names must too, or an # extension-not-listed startup failure follows every deploy. - ++ [ - "oidc/${swarmTierName}" - # Unconditional for the same reason, and one level sharper: - # the store runs in every swarm, so its forwarder's - # receiver is as permanent as the swarm tier's own. - "oidc/${storeProducerName}" - ] + ++ [ "oidc/${swarmTierName}" ] # The push side's authenticators. Same rule as above: one an # exporter names but this list omits is INERT — the collector # starts clean and pushes unauthenticated. @@ -1447,16 +1347,6 @@ in "metrics/${swarmTierName}" = { receivers = [ "otlp/${swarmTierName}" - # The store forwarder's receiver joins the swarm - # tier's pipeline rather than bringing one of its own: - # the stamp it needs is `resource/${swarmTierName}`'s - # exact stamp — the swarm's name and deliberately no - # `hive` — and a second pipeline would be that same - # processor under a second name to keep in step. What - # a producer is separated FOR is the credential it - # proves, and that is the receiver's job, not the - # pipeline's. - "otlp/${storeProducerName}" ] ++ lib.optional (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) "prometheus"; processors = [ "resource/${swarmTierName}" ]; @@ -1496,14 +1386,7 @@ in # `hive` value to put on them. // lib.optionalAttrs collectLogs { "logs/${swarmTierName}" = { - # This host's own journal, and the store container's, - # which arrives over the receiver above rather than off - # a disk — the store's forwarder reads a journal no - # reader on another host can see. - receivers = [ - "journald" - "otlp/${storeProducerName}" - ]; + receivers = [ "journald" ]; processors = [ "resource/${swarmTierName}" ]; exporters = logExporterNames; }; @@ -1567,18 +1450,6 @@ in audience = config.services.hyperhive.swarm.controller.queueClientId; }; } - # The secret store forwarder's own authenticator. Same shape - # again, and the audience is that principal's own client id - # — the self-referential form `swarm-authelia.nix` registers - # a hive client under and `swarm-controller` uses above, - # read from the option `glue-swarm-bao-otel-oidc-client.nix` - # registers so the two ends cannot spell it differently. - // { - "oidc/${storeProducerName}" = { - issuer_url = hyperhiveCfg.swarm.authelia.url; - audience = storeClientId; - }; - } # The push side. Opposite direction to every `oidc/*` above — # those VALIDATE a token arriving; these OBTAIN one to send. # Hence `oauth2client` rather than `oidc`, and hence a client diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index cfe8f5d1..6f1a51e6 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -71,13 +71,6 @@ let serviceClientIds = [ hyperhiveCfg.swarm.grafana.oidc.clientId hyperhiveCfg.swarm.otel.clientId - # The secret store's own journal forwarder. A third swarm service by this - # list's own rule — a client registered once for the whole swarm, whose - # secret one host mints and another reads — even though the principal - # behind it runs inside the store's container rather than beside it. - # Missing from here, authelia mints the value and nothing carries it, so - # the reader waits on a path that is never written. - hyperhiveCfg.swarm.bao.otel.clientId ]; # Where this unit keeps the appservice tokens it minted, and the whole reason diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index ebd1d198..4558d17b 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -26,10 +26,7 @@ let deploy.bao.enable = true; deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; # Pinned, not incidental: the case counting these listeners is about the - # declared API addresses. ⚠️ Since the store got its own collector, this no longer keeps the METRICS - # listener out of the count — that one is a function of the store, not of - # a collector's placement — so the case below subtracts it by name rather - # than by leaving the swarm tier off. + # declared addresses, and a collector on this host would add one of its own. deploy.swarm-otel.enable = false; }; @@ -45,50 +42,18 @@ let swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; - # The SPLIT topology, and the reason it is not called `baoNoCollector` any - # more: a swarm always runs every swarm-level service somewhere, so this host - # having `deploy.swarm-otel.enable = false` means the collector is on ANOTHER - # host, never that the swarm has none. Both of those read the same locally, - # and only one of them is a real deployment — which is what the two gates - # this PR removed each mistook for "there is nothing to forward to". - baoOtelElsewhere = hive { + baoNoCollector = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = false; }; - # The same split, plus a HIVE collector on the store's host — the pairing the - # first gate read as permission to forward. It is a third collector with a - # different job, so it changes neither whether the store forwards nor where. - # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion - # demands of any hive with the tier on. - baoWithHiveOtel = hive { - deploy.bao.enable = true; - otel.enable = true; - otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; - }; - - # The forwarder INSIDE the store's container, not the host's collector and - # not the swarm tier's — three collectors in this tree, and only this one can - # see the store's journal. - baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector; - # The scrape list prometheus is handed, not the option a service declared: # the address, the path and the query are one string on the way in and three # fields on the way out, and only the second shape is what gets requested. - # - # ⚠️ Two of these now, aimed at two different containers, and the split is - # the point: the store's own job left the swarm collector for the - # one inside its container, so a case that looked in the old place would - # find `null` and a case that only looked in the new one would not notice - # the store still being declared to both. - scrapeJobIn = - container: machine: job: - lib.findFirst (c: c.job_name == job) null ( - machine.containers.${container}.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs - or [ ] - ); - scrapeJob = scrapeJobIn "swarm-otel"; - baoScrapeJob = scrapeJobIn "swarm-bao"; + scrapeJob = + machine: job: + lib.findFirst (c: c.job_name == job) null + machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; cases = [ { # Same gap one tier up, and it needs its own arm: this collector @@ -115,14 +80,9 @@ let } { # Control for the case above: these settings are rendered per deployment, - # not constants a passing case could be indifferent to. The metrics - # listener is excluded by name — it answers on a port of its own and is - # not one of the API addresses this case is counting. + # not constants a passing case could be indifferent to. name = "a declared extra address renders a second listener beside loopback"; - ok = - builtins.length ( - builtins.filter (n: n != "metrics") (builtins.attrNames (baoSettings baoTwoAddresses).listener) - ) == 2; + ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2; } { # Retention is what serves the endpoint at all, so the listener alone @@ -137,56 +97,13 @@ let { # openbao serves no `/metrics` at all, so a scrape of the default path # 404s: the store looks like a dead exporter, and every panel built on - # it renders empty rather than erroring. The query is the other half — - # `/v1/sys/metrics` answers JSON without it, which parses as no series - # rather than as an error. + # it renders empty rather than erroring. name = "the store's scrape asks for the path openbao serves"; ok = let - j = baoScrapeJob baoWithCollector "bao"; + j = scrapeJob baoWithCollector "bao"; in - j != null - && (j.metrics_path or "") == "/v1/sys/metrics" - && (j.params.format or [ ]) == [ "prometheus" ]; - } - { - # 🎯 This move's deliverable, and it needs BOTH halves or it is not a move: - # the job is in the store's own container, and it is no longer declared - # to the swarm collector. Only the second half can catch a "move" that - # left the original behind — two scrapers of one endpoint, double-counted - # and neither obviously wrong. - name = "the store's metrics are scraped by its own collector, not the swarm's"; - ok = baoScrapeJob baoWithCollector "bao" != null && scrapeJob baoWithCollector "bao" == null; - } - { - # The scrape target is the loopback listener, not the API one: dialing - # the API listener would need a client certificate the collector has no - # way to present, and the fix for that would be relaxing - # `tls_require_and_verify_client_cert` — which is the trade this address - # exists to avoid. Pinned so a later edit has to argue with it. - name = "the store's collector scrapes loopback, never a wider address"; - ok = - let - j = baoScrapeJob baoWithCollector "bao"; - s = baoSettings baoWithCollector; - in - j.static_configs == [ { targets = [ "127.0.0.1:8202" ]; } ] - && s.listener.metrics.address == "127.0.0.1:8202"; - } - { - # A pipeline is typed: a prometheus receiver named in `logs` is rejected - # at startup, and metrics with no pipeline at all are collected and - # dropped. The exporter is pinned alongside because "no otel bypass" - # means both signals leave by the one hop. - name = "the store's collector has a metrics pipeline out the same hop as its logs"; - ok = - let - s = (baoForwarder baoWithCollector).settings; - m = s.service.pipelines.metrics; - in - m.receivers == [ "prometheus" ] - && m.exporters == [ "otlphttp" ] - && m.exporters == s.service.pipelines.logs.exporters; + (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; } { # Presence control for the case above: both fields are omitted rather @@ -200,236 +117,14 @@ let j != null && !(j ? metrics_path) && !(j ? params); } { - # 🩸 This case USED to be the absence arm — "a store with no collector - # beside it serves no metrics" — and this inverts it deliberately - # rather than deleting it, because the condition it asserted is the bug. - # - # The old reasoning was sound for the old reader: the endpoint is - # unauthenticated, so it should not exist where nothing reads it, and - # the only reader was a swarm collector that had to be on this host. - # What that rendered on every OTHER host was a store whose metrics went - # nowhere, indistinguishably from a store nobody had asked to scrape. - # - # The reader now ships inside the container, so "no collector beside it" - # describes no host that exists. The endpoint is still not exposed to - # anything new: it is loopback in a netns whose only other occupants are - # the store and its collector. - name = "a store whose swarm collector is elsewhere still serves and scrapes its metrics"; + # Absence arm. Unauthenticated by design, so it must not exist where + # nothing reads it. + name = "a store with no collector beside it serves no metrics"; ok = let - s = baoSettings baoOtelElsewhere; + s = baoSettings baoNoCollector; in - s.listener.metrics.address == "127.0.0.1:8202" - && (s.telemetry.prometheus_retention_time or "0s") != "0s" - && baoScrapeJob baoOtelElsewhere "bao" != null; - } - { - # The store's journal reaches a reader through a collector of its own, - # and every one of these fields is silent when wrong: the runtime journal - # is the receiver's own default and is empty here, an unlisted extension - # is inert so the cursor silently stops persisting, and a pipeline is - # free to name none of it. - name = "the store's container forwards its own journal"; - ok = - let - s = (baoForwarder baoWithCollector).settings; - p = s.service.pipelines.logs; - in - s.receivers.journald.directory == "/var/log/journal" - && s.receivers.journald.storage == "file_storage" - && - s.service.extensions == [ - "file_storage" - "oauth2client/swarm-bao" - ] - && p.receivers == [ "journald" ] - && p.exporters == [ "otlphttp" ] - && s ? exporters.otlphttp; - } - { - # The whole journal, which is what the shared collector's unit allowlist - # is not. A `units` list here would render and deploy perfectly while - # shipping only the units someone remembered to name — the failure this - # forwarder exists to end. - name = "the store's forwarder filters no units"; - ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units); - } - { - # Both ends of the hop, because a mismatch between them is silent in both - # directions: the exporter retries into a 404 and the receiver never - # hears from it. Not "on one host" any more — the far end is the gateway - # vhost the swarm collector serves its name on, which is the whole point - # of addressing it by name rather than by a bridge address. - name = "the store's forwarder exports to the swarm collector's own route"; - ok = - let - otel = baoWithCollector.services.hyperhive.swarm.otel; - vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain}; - in - (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint - == "https://${otel.domain}/${otel.storeProducerName}" - && vhost.locations ? "/${otel.storeProducerName}/"; - } - { - # The case the old hive-tier gate got wrong: this host runs the swarm's - # collector and no hive collector at all, and the forwarder still exists - # with an address that resolves. Gated on `otel.enable` it rendered - # nothing here, so the store's journal left no trace anywhere. - name = "a store on a host with no hive collector still forwards"; - ok = - !baoWithCollector.services.hyperhive.otel.enable - && (baoForwarder baoWithCollector).enable - && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != ""; - } - { - # 🎯 The split topology, which is what both removed gates got wrong and - # neither earlier test could see: the swarm's collector runs on ANOTHER - # host, so nothing local says it exists — and it exists anyway, because - # every swarm-level service runs somewhere in the swarm. The forwarder - # has to render here and has to be addressed by the collector's swarm - # name, which is reachable from a host that serves no vhost for it. - # - # Both halves matter. Without `enable` this is the old bug; without the - # endpoint it is a forwarder that renders unconditionally and points at - # nothing. The `!(vhost ? domain)` conjunct is what makes the fixture - # genuinely remote rather than a co-located host in disguise — the - # premise the co-located fixtures cannot test. - name = "a store forwards by name from a host that does not run the swarm collector"; - ok = - let - otel = baoOtelElsewhere.services.hyperhive.swarm.otel; - route = "https://${otel.domain}/${otel.storeProducerName}"; - in - !(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain}) - && (baoForwarder baoOtelElsewhere).enable - && (baoForwarder baoOtelElsewhere).settings.exporters.otlphttp.endpoint == route - # …and a hive collector standing beside the store neither supplies the - # reason to forward nor changes the addressee: same route, same tier. - && (baoForwarder baoWithHiveOtel).enable - && (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route; - } - { - # 🎯 The credential half of this move, and the arm that says the hop - # DELIVERS rather than merely resolves. The receiver at the far end - # takes an OIDC token and nothing else, so an exporter with no - # authenticator is a forwarder that retries into 401s while every other - # case here passes. Both halves: the extension has to exist AND the - # exporter has to name it. - name = "the store's forwarder presents a credential on the hop it exports over"; - ok = - let - s = (baoForwarder baoWithCollector).settings; - auth = "oauth2client/swarm-bao"; - in - s.exporters.otlphttp.auth.authenticator == auth - && s.extensions ? ${auth} - && lib.elem auth s.service.extensions; - } - { - # Its OWN client, never the swarm collector's and never the - # controller's: one identity per principal, and this one runs on a host - # neither of those two does. The audience is the same id because the - # receiver checks exactly that — a token minted without ASKING for it - # carries `aud: []` and is refused with a config that reads correctly at - # both ends. - name = "the forwarder authenticates as a principal of its own"; - ok = - let - bao = baoWithCollector.services.hyperhive.swarm.bao; - e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao"; - in - e.client_id == bao.otel.clientId - && e.endpoint_params.audience == bao.otel.clientId - && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.otel.clientId - && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.controller.queueClientId; - } - { - # A path systemd resolves at runtime, never a value and never a store - # path: the collector runs under `DynamicUser` and opens this file - # itself, so the credential arrives through `LoadCredential` and the - # config names only the directory systemd exports. - name = "the forwarder's client secret reaches it as a credential, not a value"; - ok = - let - container = baoWithCollector.containers.swarm-bao.config; - e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao"; - unit = container.systemd.services.opentelemetry-collector.serviceConfig; - in - e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret" - && - unit.LoadCredential == [ - "oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" - ]; - } - { - # Same 403-not-a-miss property the grafana and matrix readers are pinned - # for: the reader's grant covers the `services` prefix, so a secret - # filed under the hive that happens to run the store would be refused - # rather than missing, however correct the path reads. - name = "the forwarder's secret is read from the prefix the publisher writes"; - ok = - let - s = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc.script; - in - lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" s - && !(lib.hasInfix "secret/swarm/hives/" s); - } - { - # The delivery unit is ordered AFTER the store's container, not before - # it: the store it reads lives in the container it delivers into, so a - # `before` edge is a wait on a process that cannot start until this - # finishes. Pinned because the unit it was copied from does the - # opposite, and copying that line too would deadlock every cold boot. - name = "the delivery unit waits for the store rather than blocking it"; - ok = - let - u = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc; - in - lib.elem "container@swarm-bao.service" u.after - && !(lib.elem "container@swarm-bao.service" (u.before or [ ])) - && lib.elem "swarm-bao-pki.service" u.requires; - } - { - # The far end of the credential. An authenticator checks ONE audience, - # so the store's forwarder gets a receiver of its own rather than a - # second sender into the route whose audience belongs to - # `swarm-controller` — and every one of these four is silent when - # wrong: an unlisted extension is inert, a receiver in no pipeline - # drops what it accepts, and a mismatched audience is a healthy 401. - name = "the swarm collector admits this principal on a receiver of its own"; - ok = - let - otel = baoWithCollector.services.hyperhive.swarm.otel; - s = baoWithCollector.containers.swarm-otel.config.services.opentelemetry-collector.settings; - name = otel.storeProducerName; - in - s.receivers."otlp/${name}".protocols.http.auth.authenticator == "oidc/${name}" - && - s.extensions."oidc/${name}".audience == baoWithCollector.services.hyperhive.swarm.bao.otel.clientId - && lib.elem "oidc/${name}" s.service.extensions - && lib.elem "otlp/${name}" s.service.pipelines."metrics/${otel.producerName}".receivers - && lib.elem "otlp/${name}" s.service.pipelines."logs/${otel.producerName}".receivers; - } - { - # A client authelia has never heard of mints nothing, so the publisher - # has nothing to copy and the delivery unit reads an empty path - # forever. Registered where AUTHELIA runs, which is why it is a glue - # module and not a line in the store's own config — this fixture runs - # no store at all and still has to register it. - name = "the forwarder's client is registered wherever authelia runs"; - ok = - let - autheliaOnly = hive { deploy.authelia.enable = true; }; - clients = autheliaOnly.services.hyperhive.swarm.authelia.oidc.clients; - c = lib.findFirst (c: c.id == "swarm-bao-collector") null clients; - in - !autheliaOnly.services.hyperhive.deploy.bao.enable - && c != null - && c.audience == [ "swarm-bao-collector" ] - # ⚠️ What makes the token READABLE by the receiver at all: authelia's - # default is an opaque handle, and an `oidc` extension verifies - # offline against the provider's JWKS. - && c.accessTokenSignedResponseAlg == "RS256"; + !(s.listener ? metrics) && !(s ? telemetry); } ]; in diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index fd343356..7c2f9f6d 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -132,17 +132,6 @@ let secretPublisherHere.systemd.services.swarm-secret-publish.script ); } - { - # The third swarm service on that list, and the one whose absence is - # hardest to see: the store's own forwarder holds a client authelia - # registers and mints for, so every layer above looks complete while - # the one hop that CARRIES the value skips it and the reader on the - # store's host waits forever on a path nothing writes. - name = "the publisher carries the store forwarder's secret too"; - ok = lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" ( - secretPublisherHere.systemd.services.swarm-secret-publish.script - ); - } { # The same hole the controller's case above names, open a second time: the # PKI script grew a third leaf and no case read it.