From e5224a67258dae5a7aa8a9197954f0b8175b5883 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 19 Sep 2026 15:39:24 +0200 Subject: [PATCH 1/4] nix: give swarm-bao its own otel collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every container is supposed to run a collector that passes its logs and metrics to the next hop. swarm-bao did not: its journal reached the store only because `--link-journal=host` puts it in the host tree, where the swarm collector — a different container — reads it through a unit allowlist. That is the topology being retired, and in this deployment it delivers nothing: no `_SYSTEMD_UNIT` value in the seven-day store mentions openbao at all. So the store's container now runs its own journal forwarder, copied from an agent container's (nix/agent-modules/otel.nix): the whole journal, no unit allowlist, pushed to the same first hop every agent on the host already exports to. A local collector reads the local journal, so there is nothing for a list of unit names to disagree with. The `swarm.otel.journaldUnits` entry and `--link-journal=host` both stay. Every sibling swarm container still rides the shared collector, and they come out once each of them has a forwarder of its own. Closes #4526 --- nix/host-modules/swarm-bao.nix | 140 ++++++++++++++++++++++++- nix/module-eval/bao-otel-collector.nix | 61 +++++++++++ 2 files changed, 200 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 702e5e72..fc5aa664 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -455,6 +455,21 @@ let scrapeHere = deployCfg.swarm-otel.enable; + # Whether this container forwards its own journal, and it is gated on the + # HIVE collector rather than the swarm one: the forwarder below pushes to the + # first hop, and with `otel.enable` off nothing listens at that address at + # all — a collector aimed at it would retry forever while looking healthy. + shipJournal = hyperhiveCfg.otel.enable; + + # The first hop for anything running on this host, byte for byte what + # hive-c0re hands every agent (`firstHop` in ./hive-c0re/environment.nix). + # Plain http with no credential is the tier boundary working as designed, not + # an omission: the hive's collector is the only thing here that holds one, + # and presenting it to the swarm is its job. This container shares the host + # netns (`privateNetwork = false` below), so the bridge address is reachable + # from inside it without the firewall hole an agent container needs. + otelFirstHop = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.otel.collector.port}"; + # Non-zero is what SERVES the endpoint at all — the switch is a duration, not # a boolean, so a zero here is an openbao that answers 404 on a listener # configured to do nothing else. @@ -1428,7 +1443,12 @@ in ]; config = - { ... }: + # `config` is the CONTAINER's, shadowing the host's inside this + # function — nothing in here reads the host's, and the journal + # assertion below has to read the setting on this side of the + # boundary. `lib` and `pkgs` still come from the enclosing scope, as + # every other expression in this block already does. + { config, ... }: { imports = [ (import ./swarm-container-resolver.nix { @@ -1586,6 +1606,124 @@ in # Restarting is an operator action with an unseal on the far side of # it, which is why nothing here tries to be clever about it. + # This container's own journal forwarder, copied from an agent + # container's (nix/agent-modules/otel.nix) because every container + # is supposed to ship its own logs to the next hop rather than + # leave a *different* container's collector to find them. + # + # That older shape is what `--link-journal=host` above and the + # `swarm.otel.journaldUnits` entry near the top of this file serve: + # the journal lands in the host tree, where the swarm collector — + # itself a container — reads it through a unit allowlist. Both stay + # for now, because every sibling swarm container still rides that + # collector; they come out once each of them has one of these. + # + # ⚠️ NO `units` allowlist here, and that is the point rather than a + # simplification. A shared collector needs one because the journal + # it reads holds six containers' units plus the host's own; this + # one reads only openbao and the two oneshots beside it, so there + # is nothing foreign to separate out — and a list of unit names is + # a thing to get wrong, which ships nothing while looking healthy. + assertions = lib.optionals shipJournal [ + { + # Sibling of the agent forwarder's identical assertion, and it + # exists because the failure is silent at every layer: with a + # volatile journal the receiver below finds an empty directory, + # reads nothing, and the collector starts clean and stays + # healthy forever. + assertion = + !(lib.elem config.services.journald.storage [ + "volatile" + "none" + ]); + message = '' + services.hyperhive.otel.enable is on, so ${cfg.machine} forwards its + own journal — but services.journald.storage is + "${config.services.journald.storage}" in this container. + + The forwarder reads /var/log/journal, which journald only writes + when it stores persistently: with "volatile" the journal lives in + /run/log/journal and with "none" there is none at all. Either way + the forwarder would ship nothing while looking healthy. + ''; + } + ]; + + services.opentelemetry-collector = lib.mkIf shipJournal { + enable = true; + # Contrib, and not a preference: `journald` is a contrib + # receiver. The upstream default build has no way to read a + # journal at all. + package = pkgs.opentelemetry-collector-contrib; + # Runs `otelcol validate` at build time. ⚠️ A parser, not a + # wiring check — it accepts a pipeline naming a component the + # build lacks, and the collector then dies at startup. Same + # caveat as every other tier; see nix/host-modules/otel.nix for + # the measurement. + validateConfigFile = true; + settings = { + # `journalctl --follow --lines=0` (what the receiver runs under + # the hood) ships only what is written *after* it starts, so + # every collector start has a silent gap at the front. This + # persists the read cursor, so a restart resumes instead of + # re-opening that gap; `start_at` stays `end` (its default), + # since a cursor covers every run after the first and + # `beginning` without one would re-ship the whole journal on + # every restart. `create_directory` is required for a + # build-time reason rather than a convenience one: + # `validateConfigFile` above runs `otelcol validate` in the nix + # sandbox, before systemd's `StateDirectory=` has created the + # path, and the validator refuses with "directory must exist". + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; + + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own + # default is the RUNTIME journal (`/run/log/journal`), and + # every entry here is under /var/log/journal — which + # `--link-journal=host` above makes the host's directory for + # this machine id. Dropping this line leaves a collector that + # validates, starts, reports healthy and forwards nothing. + directory = "/var/log/journal"; + storage = "file_storage"; + }; + + # Identity the hop above cannot supply: a host-side reader can + # say which machine a line came from, and only a collector + # inside this container can say it was the secret store's. No + # `hive` or `swarm` label — the swarm tier upserts `hive` from + # the receiver that accepted the record, precisely so it comes + # from something the sender cannot write. + processors.resource.attributes = [ + { + key = "service.name"; + value = cfg.machine; + action = "upsert"; + } + ]; + + # `otlphttp` unconditionally, unlike the agent forwarder's + # protocol-derived exporter name: the first hop's receiver + # speaks OTLP/HTTP protobuf whatever the hive's *upstream* + # protocol is, which is the same reason + # ./hive-c0re/environment.nix pins the protocol it hands out. + # `endpoint` is a BASE the exporter appends `/v1/logs` to. + exporters.otlphttp.endpoint = otelFirstHop; + + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; + + # An extension configured but not listed here is INERT — the + # receiver's `storage: file_storage` above would name a + # component the collector never starts. + service.extensions = [ "file_storage" ]; + }; + }; }; }; }) diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 4558d17b..8345f2a8 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -47,6 +47,24 @@ let deploy.swarm-otel.enable = false; }; + # The store on a host whose HIVE collector is running — the only state in + # which the container forwards its own journal, since that collector is the + # hop it forwards to. `clientSecretFile` is what ../host-modules/otel.nix's + # identity assertion demands of any hive with the tier on. `swarm-otel` is + # deliberately left off: the forwarder is a function of the first hop + # existing, and pairing the two fixtures would make a case that passed on the + # wrong condition. + baoWithHiveOtel = hive { + deploy.bao.enable = true; + otel.enable = true; + otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; + }; + + # The forwarder INSIDE the store's container, not the host's collector and + # not the swarm tier's — three collectors in this tree, and only this one can + # see the store's journal. + baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector; + # The scrape list prometheus is handed, not the option a service declared: # the address, the path and the query are one string on the way in and three # fields on the way out, and only the second shape is what gets requested. @@ -126,6 +144,49 @@ let in !(s.listener ? metrics) && !(s ? telemetry); } + { + # The store's journal reaches a reader through a collector of its own, + # and every one of these fields is silent when wrong: the runtime journal + # is the receiver's own default and is empty here, an unlisted extension + # is inert so the cursor silently stops persisting, and a pipeline is + # free to name none of it. + name = "the store's container forwards its own journal"; + ok = + let + s = (baoForwarder baoWithHiveOtel).settings; + p = s.service.pipelines.logs; + in + s.receivers.journald.directory == "/var/log/journal" + && s.receivers.journald.storage == "file_storage" + && s.service.extensions == [ "file_storage" ] + && p.receivers == [ "journald" ] + && p.exporters == [ "otlphttp" ] + && s ? exporters.otlphttp; + } + { + # The whole journal, which is what the shared collector's unit allowlist + # is not. A `units` list here would render and deploy perfectly while + # shipping only the units someone remembered to name — the failure this + # forwarder exists to end. + name = "the store's forwarder filters no units"; + ok = !((baoForwarder baoWithHiveOtel).settings.receivers.journald ? units); + } + { + # Both ends of the first hop, on ONE host, because a mismatch between + # them is silent in both directions: the exporter retries into nothing + # and the receiver never hears from it. + name = "the store's forwarder exports to this hive's own collector"; + ok = + (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint + == "http://${baoWithHiveOtel.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint}"; + } + { + # Absence arm — `baoNoCollector` runs neither tier, so there is no first + # hop on this host at all. A forwarder rendered anyway would start, find + # nothing listening, and retry forever while reporting healthy. + name = "a store with no hive collector renders no forwarder"; + ok = !(baoForwarder baoNoCollector).enable; + } ]; in runGroup "bao-otel-collector" cases From ca8fc4ca64e3a631fdb9341fe85ab2bfe7832857 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 19 Sep 2026 18:32:13 +0200 Subject: [PATCH 2/4] nix: address swarm-bao's journal forwarder by swarm name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forwarder pointed at the hive bridge address and was gated on the hive's `otel.enable`, so it existed only where a hive collector stood beside it. It now exports to `swarm.otel.domain` — the gateway-served name that resolves locally when co-located and over the network otherwise — on the swarm tier's own producer route, and is gated on `deploy.swarm-otel.enable` like its sibling `scrapeHere`. Refs #4526 --- nix/host-modules/swarm-bao.nix | 43 ++++++++++++------- nix/module-eval/bao-otel-collector.nix | 58 +++++++++++++++++--------- 2 files changed, 65 insertions(+), 36 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index fc5aa664..3f6e956b 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -455,20 +455,31 @@ let scrapeHere = deployCfg.swarm-otel.enable; - # Whether this container forwards its own journal, and it is gated on the - # HIVE collector rather than the swarm one: the forwarder below pushes to the - # first hop, and with `otel.enable` off nothing listens at that address at - # all — a collector aimed at it would retry forever while looking healthy. - shipJournal = hyperhiveCfg.otel.enable; + # Whether this container forwards its own journal, gated exactly like + # `scrapeHere` above: the store is a swarm-tier component, so what decides + # this is the swarm collector existing, never which hive happens to stand + # beside it. On the hive flag it rendered no forwarder at all on a host that + # runs the swarm's own collector but not a hive's. + shipJournal = deployCfg.swarm-otel.enable; - # The first hop for anything running on this host, byte for byte what - # hive-c0re hands every agent (`firstHop` in ./hive-c0re/environment.nix). - # Plain http with no credential is the tier boundary working as designed, not - # an omission: the hive's collector is the only thing here that holds one, - # and presenting it to the swarm is its job. This container shares the host - # netns (`privateNetwork = false` below), so the bridge address is reachable - # from inside it without the firewall hole an agent container needs. - otelFirstHop = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.otel.collector.port}"; + # The swarm collector's own name, served by the gateway — the address + # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host + # and over the real network otherwise, with no loopback-vs-remote knob to get + # wrong. The path selects WHICH receiver: this container is a swarm-level + # producer and not a hive, so it is the swarm tier's own route, the same one + # ./swarm-controller.nix exports to. `/` on that vhost answers 404. + # + # The port suffix follows ./hive-forge/default.nix: elided on the canonical + # 443 so this renders byte for byte like the tier's other two consumers of + # this name, present when an operator moved the gateway's TLS port. + otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.producerName}"; + + swarmOtelCfg = hyperhiveCfg.swarm.otel; + gatewayPortSuffix = + let + p = hyperhiveCfg.gateway.httpsPort; + in + lib.optionalString (p != 443) ":${toString p}"; # Non-zero is what SERVES the endpoint at all — the switch is a duration, not # a boolean, so a zero here is an openbao that answers 404 on a listener @@ -1637,7 +1648,7 @@ in "none" ]); message = '' - services.hyperhive.otel.enable is on, so ${cfg.machine} forwards its + services.hyperhive.deploy.swarm-otel.enable is on, so ${cfg.machine} forwards its own journal — but services.journald.storage is "${config.services.journald.storage}" in this container. @@ -1705,8 +1716,8 @@ in ]; # `otlphttp` unconditionally, unlike the agent forwarder's - # protocol-derived exporter name: the first hop's receiver - # speaks OTLP/HTTP protobuf whatever the hive's *upstream* + # protocol-derived exporter name: the swarm receiver speaks + # OTLP/HTTP protobuf whatever this tier's *upstream* # protocol is, which is the same reason # ./hive-c0re/environment.nix pins the protocol it hands out. # `endpoint` is a BASE the exporter appends `/v1/logs` to. diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 8345f2a8..cae6b220 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -47,13 +47,11 @@ let deploy.swarm-otel.enable = false; }; - # The store on a host whose HIVE collector is running — the only state in - # which the container forwards its own journal, since that collector is the - # hop it forwards to. `clientSecretFile` is what ../host-modules/otel.nix's - # identity assertion demands of any hive with the tier on. `swarm-otel` is - # deliberately left off: the forwarder is a function of the first hop - # existing, and pairing the two fixtures would make a case that passed on the - # wrong condition. + # The store on a host whose HIVE collector is running and whose SWARM one is + # not. The forwarder is a function of the swarm tier, so this fixture pins + # the negative: a hive collector beside the store buys it nothing. + # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion + # demands of any hive with the tier on. baoWithHiveOtel = hive { deploy.bao.enable = true; otel.enable = true; @@ -153,7 +151,7 @@ let name = "the store's container forwards its own journal"; ok = let - s = (baoForwarder baoWithHiveOtel).settings; + s = (baoForwarder baoWithCollector).settings; p = s.service.pipelines.logs; in s.receivers.journald.directory == "/var/log/journal" @@ -169,23 +167,43 @@ let # shipping only the units someone remembered to name — the failure this # forwarder exists to end. name = "the store's forwarder filters no units"; - ok = !((baoForwarder baoWithHiveOtel).settings.receivers.journald ? units); + ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units); } { - # Both ends of the first hop, on ONE host, because a mismatch between - # them is silent in both directions: the exporter retries into nothing - # and the receiver never hears from it. - name = "the store's forwarder exports to this hive's own collector"; + # Both ends of the hop, because a mismatch between them is silent in both + # directions: the exporter retries into a 404 and the receiver never + # hears from it. Not "on one host" any more — the far end is the gateway + # vhost the swarm collector serves its name on, which is the whole point + # of addressing it by name rather than by a bridge address. + name = "the store's forwarder exports to the swarm collector's own route"; ok = - (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint - == "http://${baoWithHiveOtel.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint}"; + let + otel = baoWithCollector.services.hyperhive.swarm.otel; + vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain}; + in + (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint + == "https://${otel.domain}/${otel.producerName}" + && vhost.locations ? "/${otel.producerName}/"; } { - # Absence arm — `baoNoCollector` runs neither tier, so there is no first - # hop on this host at all. A forwarder rendered anyway would start, find - # nothing listening, and retry forever while reporting healthy. - name = "a store with no hive collector renders no forwarder"; - ok = !(baoForwarder baoNoCollector).enable; + # The case the old hive-tier gate got wrong: this host runs the swarm's + # collector and no hive collector at all, and the forwarder still exists + # with an address that resolves. Gated on `otel.enable` it rendered + # nothing here, so the store's journal left no trace anywhere. + name = "a store on a host with no hive collector still forwards"; + ok = + !baoWithCollector.services.hyperhive.otel.enable + && (baoForwarder baoWithCollector).enable + && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != ""; + } + { + # Absence arm — no swarm collector, so nothing this side of the gateway + # serves that name. A forwarder rendered anyway would start, find nothing + # listening, and retry forever while reporting healthy. `baoWithHiveOtel` + # is the pointed half: a hive collector beside the store is not a reason + # to forward, which is exactly what the old gate assumed. + name = "a store with no swarm collector renders no forwarder"; + ok = !(baoForwarder baoNoCollector).enable && !(baoForwarder baoWithHiveOtel).enable; } ]; in From d4313fc34d30cab3821e114ff06105b85bff7437 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 19 Sep 2026 20:23:43 +0200 Subject: [PATCH 3/4] nix: the store's journal forwarder has no gate to have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both earlier versions asked the wrong host. `hyperhive.otel.enable` asked whether this host runs a HIVE collector; `deploy.swarm-otel.enable` asked whether this host runs the SWARM one. Neither answers the question the forwarder actually has — "is there a collector to forward to" — and that question cannot be false: a swarm always runs at least one instance of every swarm-level service. So the forwarder renders under the condition already enclosing it, that the store is deployed here, and nothing else. `scrapeHere` deliberately keeps its `deploy.*` gate one line up. It is a loopback metrics listener, which genuinely only works where the scraper is — the two are different tiers, and the name says so. The module-eval case that pins it is the split topology: the swarm collector on another host, nothing local naming it, and the forwarder still enabled and still addressed at `swarm.otel.domain`'s route. Both removed gates render nothing in that fixture, which the co-located ones they shipped with could not show. --- nix/host-modules/swarm-bao.nix | 27 +++++++++------ nix/module-eval/bao-otel-collector.nix | 48 +++++++++++++++++++------- 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 3f6e956b..5f88a6c0 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -455,12 +455,19 @@ let scrapeHere = deployCfg.swarm-otel.enable; - # Whether this container forwards its own journal, gated exactly like - # `scrapeHere` above: the store is a swarm-tier component, so what decides - # this is the swarm collector existing, never which hive happens to stand - # beside it. On the hive flag it rendered no forwarder at all on a host that - # runs the swarm's own collector but not a hive's. - shipJournal = deployCfg.swarm-otel.enable; + # ⚠️ The forwarder below has NO condition of its own, and `scrapeHere` above + # is not a template for one — the two are different tiers on purpose. A + # loopback metrics listener works only where the scraper is, which is what + # `Here` in that name says; forwarding reaches the swarm's collector BY NAME, + # from wherever this store runs. + # + # There is nothing left to gate on. "Does the swarm have a collector" is not + # a question that can be false (mara: "the swarm always has at least one + # instance of all the swarm level services"), so both earlier gates — + # `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong + # host and rendered no forwarder wherever the answer happened to be no. The + # only condition that belongs here is the one already enclosing this block: + # the store is deployed on this host. # The swarm collector's own name, served by the gateway — the address # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host @@ -1635,7 +1642,7 @@ in # one reads only openbao and the two oneshots beside it, so there # is nothing foreign to separate out — and a list of unit names is # a thing to get wrong, which ships nothing while looking healthy. - assertions = lib.optionals shipJournal [ + assertions = [ { # Sibling of the agent forwarder's identical assertion, and it # exists because the failure is silent at every layer: with a @@ -1648,8 +1655,8 @@ in "none" ]); message = '' - services.hyperhive.deploy.swarm-otel.enable is on, so ${cfg.machine} forwards its - own journal — but services.journald.storage is + ${cfg.machine} forwards its own journal to the swarm's + collector — but services.journald.storage is "${config.services.journald.storage}" in this container. The forwarder reads /var/log/journal, which journald only writes @@ -1660,7 +1667,7 @@ in } ]; - services.opentelemetry-collector = lib.mkIf shipJournal { + services.opentelemetry-collector = { enable = true; # Contrib, and not a preference: `journald` is a contrib # receiver. The upstream default build has no way to read a diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index cae6b220..f6d9b83f 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -42,14 +42,20 @@ let swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; - baoNoCollector = hive { + # The SPLIT topology, and the reason it is not called `baoNoCollector` any + # more: a swarm always runs every swarm-level service somewhere, so this host + # having `deploy.swarm-otel.enable = false` means the collector is on ANOTHER + # host, never that the swarm has none. Both of those read the same locally, + # and only one of them is a real deployment — which is what the two gates + # this PR removed each mistook for "there is nothing to forward to". + baoOtelElsewhere = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = false; }; - # The store on a host whose HIVE collector is running and whose SWARM one is - # not. The forwarder is a function of the swarm tier, so this fixture pins - # the negative: a hive collector beside the store buys it nothing. + # The same split, plus a HIVE collector on the store's host — the pairing the + # first gate read as permission to forward. It is a third collector with a + # different job, so it changes neither whether the store forwards nor where. # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion # demands of any hive with the tier on. baoWithHiveOtel = hive { @@ -138,7 +144,7 @@ let name = "a store with no collector beside it serves no metrics"; ok = let - s = baoSettings baoNoCollector; + s = baoSettings baoOtelElsewhere; in !(s.listener ? metrics) && !(s ? telemetry); } @@ -197,13 +203,31 @@ let && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != ""; } { - # Absence arm — no swarm collector, so nothing this side of the gateway - # serves that name. A forwarder rendered anyway would start, find nothing - # listening, and retry forever while reporting healthy. `baoWithHiveOtel` - # is the pointed half: a hive collector beside the store is not a reason - # to forward, which is exactly what the old gate assumed. - name = "a store with no swarm collector renders no forwarder"; - ok = !(baoForwarder baoNoCollector).enable && !(baoForwarder baoWithHiveOtel).enable; + # 🎯 The split topology, which is what both removed gates got wrong and + # neither earlier test could see: the swarm's collector runs on ANOTHER + # host, so nothing local says it exists — and it exists anyway, because + # every swarm-level service runs somewhere in the swarm. The forwarder + # has to render here and has to be addressed by the collector's swarm + # name, which is reachable from a host that serves no vhost for it. + # + # Both halves matter. Without `enable` this is the old bug; without the + # endpoint it is a forwarder that renders unconditionally and points at + # nothing. The `!(vhost ? domain)` conjunct is what makes the fixture + # genuinely remote rather than a co-located host in disguise — the + # premise the co-located fixtures cannot test. + name = "a store forwards by name from a host that does not run the swarm collector"; + ok = + let + otel = baoOtelElsewhere.services.hyperhive.swarm.otel; + route = "https://${otel.domain}/${otel.producerName}"; + in + !(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain}) + && (baoForwarder baoOtelElsewhere).enable + && (baoForwarder baoOtelElsewhere).settings.exporters.otlphttp.endpoint == route + # …and a hive collector standing beside the store neither supplies the + # reason to forward nor changes the addressee: same route, same tier. + && (baoForwarder baoWithHiveOtel).enable + && (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route; } ]; in From bb0afcd256b5e2b8a746c4f129bc4d1ef1aac019 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 20 Sep 2026 18:19:06 +0200 Subject: [PATCH 4/4] nix: the store's own collector scrapes its metrics listener MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bao's metrics were scraped by the SWARM collector over loopback, via a `swarm.otel.scrapeTargets.bao` entry gated on `deploy.swarm-otel.enable` — "does the swarm's collector run on THIS host". It had to be: loopback only reaches a reader that landed on the same host. What that rendered everywhere else was nothing at all. Off that host the metrics listener was not emitted, so the store's metrics reached the store nowhere, and a host with no entry is indistinguishable from a host nobody asked to scrape. Moves the scrape into the collector this container already runs, per mara on #4537: "move the existing scraper to the local collector". The container shares the host netns (privateNetwork = false), so the scrape still dials 127.0.0.1 — the listener keeps its address, its `metrics_only` narrowing and its loopback-only bind, and the API listener's `tls_require_and_verify_client_cert` is untouched. The listener and its `prometheus_retention_time` lose their gate: the reader ships with the store now, so there is no host where the endpoint has none. The metrics pipeline reuses the logs pipeline's `resource` processor and `otlphttp` exporter, so both signals carry the same `service.name` and leave by the one hop. Logs are unaffected: `journaldUnits` and --link-journal=host stay until every sibling swarm container has a collector of its own. The module-eval absence arm "a store with no collector beside it serves no metrics" is inverted rather than dropped — the condition it asserted is the bug. Three cases join it: the job is in swarm-bao AND gone from swarm-otel (a move, not a copy), the scrape target and listener are both pinned to loopback, and the metrics pipeline shares its exporter with the logs one. --- docs/swarm/secrets.md | 34 +- nix/host-modules/default.nix | 1 + .../glue-swarm-bao-otel-oidc-client.nix | 56 +++ nix/host-modules/swarm-bao.nix | 393 ++++++++++++++++-- nix/host-modules/swarm-otel.nix | 133 +++++- nix/host-modules/swarm-secret-publisher.nix | 7 + nix/module-eval/bao-otel-collector.nix | 238 ++++++++++- nix/module-eval/secret-publisher.nix | 11 + 8 files changed, 815 insertions(+), 58 deletions(-) create mode 100644 nix/host-modules/glue-swarm-bao-otel-oidc-client.nix diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index e3c2e0ce..53bce3ad 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -64,7 +64,8 @@ because nothing outside that container ever reads them. **That's the test worth applying to any secret added here** — and the client secret's plaintext half is the one row that fails it, which is the entire reason a delivery step exists. -Two telemetry collectors exist, and they land on opposite sides of that test. +Three telemetry collectors exist, and they don't land on the same side of +that test. The **hive's** collector needs no delivery step. It authenticates to the swarm's collector as its own hive, and it's a host unit rather than a container, so on @@ -94,6 +95,27 @@ uses for its non-optional one. The operator can still deliver the secret by hand, at `services.hyperhive.deploy.swarm-otel.clientSecretFile`, on a collector this unit never reaches. +The **secret store's own** collector — the forwarder inside the `swarm-bao` +container — needs a delivery step too, and it takes the same route with one +principal of its own: `swarm-bao-forwarder-oidc.service` reads +`swarm/services//oidc/client` under this host's certificate and +lands it in the container's tree, where `LoadCredential` hands it to the +collector. The client id is its own +(`services.hyperhive.swarm.bao.otel.clientId`), registered by +`glue-swarm-bao-otel-oidc-client.nix` wherever authelia runs, and the swarm +collector admits it on a receiver of its own — one identity per principal, so +this forwarder borrows neither the swarm collector's client nor +`swarm-controller`'s. + +⚠️ **This one has no degraded mode, and the difference is structural.** The +route it exports over accepts an authenticated request and nothing else, so a +forwarder without the secret has nothing to fall back to: the unit fails and +retries, and the collector refuses to start until the file arrives rather than +exporting into a 401 while reporting healthy. Its ordering differs from the +unit above it for the same structural reason — it reads the store inside the +container it delivers into, so it runs **after** that container rather than +before it, and only a first boot ever waits. + ### Minting the queue's callout nkeys `deploy.nats.autoGenerateCallout` mints both keypairs on the host before the queue @@ -254,6 +276,16 @@ doesn't render the reading unit without them, the shape `glue-matrix-bao-token.nix` and `glue-queue-agent-credential.nix` use for their own optional reads. +The **fifth reader** is the secret store forwarder's OIDC client secret: +`swarm-bao-forwarder-oidc.service`, on the store's own host, registered by +`glue-swarm-bao-otel-oidc-client.nix` on authelia's. It differs from the +fourth in both directions. It renders unconditionally, because the export it +authenticates has no unauthenticated mode to degrade into — and it orders +itself **after** `container@swarm-bao`, because the store it reads runs in the +container it delivers into. Nothing circular sits behind that: the identity it +logs in with is this host's static `swarm-bao-pki` leaf, not anything the store +mints. + A service's secret is one value for the whole swarm rather than one per hive, so it lives under the `services` prefix, and a hive's read policy grants that prefix whole. That's because nothing in a swarm's configuration records which hive runs diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index b4f1e7e8..01030d24 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,6 +30,7 @@ ./glue-matrix-ctl-bao-identity.nix ./glue-queue-agent-credential.nix ./glue-secret-publisher-bao-identity.nix + ./glue-swarm-bao-otel-oidc-client.nix ./glue-swarm-otel-oidc-client.nix ./swarm-authelia.nix ./swarm-bao.nix diff --git a/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix b/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix new file mode 100644 index 00000000..6e9d850a --- /dev/null +++ b/nix/host-modules/glue-swarm-bao-otel-oidc-client.nix @@ -0,0 +1,56 @@ +# Glue: register the secret store's own journal forwarder as an OIDC client +# wherever authelia runs. +# +# ONE PAIRING PER FILE — swarm-bao's forwarder ← authelia, and nothing else. +# Deleting this leaves a forwarder authelia has never heard of: the token +# endpoint refuses it, nothing is minted, the publisher has nothing to copy +# into the store, and the collector's export is a 401 nobody asked for. +# +# ⚠️ A SECOND client beside ./glue-swarm-otel-oidc-client.nix's, not a reuse +# of it. That one is the swarm collector's identity for what IT pushes; this +# one belongs to the collector inside the store's container, which is a +# different principal on a different host — one identity per principal, the +# rule `swarm-controller.nix` states over its own `queueClientId`. +# +# ⚠️ Gated on authelia being HERE, and deliberately NOT on this host running +# the store. A client is a row in THIS host's provider config, so it can only +# be declared where that config is rendered, and ./swarm-bao.nix's `config` +# hangs off `deploy.bao.enable` — the split ./glue-grafana-oidc-client.nix and +# ./glue-swarm-otel-oidc-client.nix each made for the same reason. Read either +# file's own comment for the property this one shares with them. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoCfg = hyperhiveCfg.swarm.bao; +in +{ + config = lib.mkIf (hyperhiveCfg.enable && deployCfg.authelia.enable) { + # One declaration, two readers: `clientId` is a read-only option + # ./swarm-bao.nix owns, and ./swarm-otel.nix builds this principal's + # authenticator audience from the same option. + services.hyperhive.swarm.authelia.oidc.clients = [ + { + id = baoCfg.otel.clientId; + description = "HyperHive secret store journal forwarder"; + kind = "machine"; + redirectUris = [ ]; + # Its own id as its own permitted audience — the self-referential + # form `swarm-authelia.nix`'s `hiveClients` uses, which is the shape + # to copy here: this forwarder's token lands on an `oidc/*` receiver + # of the swarm collector, exactly as a hive's does. + audience = [ baoCfg.otel.clientId ]; + # ⚠️ What makes the token READABLE by that receiver at all. Authelia's + # default is an opaque handle, and an `oidc` extension verifies + # offline against `/jwks.json` — so without this the export fails with + # a message about the verifier rather than about the token. Same + # value, same reason, as every hive's client. + accessTokenSignedResponseAlg = "RS256"; + } + ]; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 5f88a6c0..34676bda 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -429,19 +429,23 @@ let // metricsListener; # Metrics get their own listener rather than a flag on the one above, and - # that follows from what a scraper can express: `swarm.otel.scrapeTargets` - # carries no scheme and no credential, while the API listener is TLS and — - # once a client CA is set — demands a client certificate. The collector - # cannot reach it at all. + # that follows from what a scraper can express: a prometheus scrape config + # carries no client certificate, while the API listener is TLS and — once a + # client CA is set — demands one. A scraper cannot reach it at all. # # `metrics_only` narrows this one to the metrics path (every other path 404s) # and the unauthenticated access is confined to loopback. **Deliberately - # unauthenticated for now** — a tracked follow-up owns giving the collector a - # credential, since no scrape option can carry one today. + # unauthenticated** — the only process that can dial 127.0.0.1 here is one + # already inside this netns, which is the container's own collector and the + # store itself. Widening this address, or dropping + # `tls_require_and_verify_client_cert` on the API listener to scrape THAT + # instead, would each trade that property away for nothing. # - # Exists only where a collector does: an endpoint with no reader is exposure - # bought for nothing. - metricsListener = lib.optionalAttrs scrapeHere { + # Unconditional, and that is the change this carries: the reader is now the + # collector in this very container (below), which renders wherever the store + # does. There is no longer a host where this endpoint has no reader, so there + # is nothing left for it to be a function of. + metricsListener = { metrics = { type = "tcp"; address = "127.0.0.1:${toString baoDeploy.metricsPort}"; @@ -453,21 +457,32 @@ let }; }; - scrapeHere = deployCfg.swarm-otel.enable; + # What the container's own collector dials. Loopback inside a netns this + # container SHARES with the host (`privateNetwork = false` below), so the + # scrape crosses no network and needs no bind address wider than the one + # above. + metricsScrapeTarget = "127.0.0.1:${toString baoDeploy.metricsPort}"; - # ⚠️ The forwarder below has NO condition of its own, and `scrapeHere` above - # is not a template for one — the two are different tiers on purpose. A - # loopback metrics listener works only where the scraper is, which is what - # `Here` in that name says; forwarding reaches the swarm's collector BY NAME, - # from wherever this store runs. + # ⚠️ Neither the listener above nor the forwarder below has a condition of + # its own, and they lost theirs for the same reason at different times. # - # There is nothing left to gate on. "Does the swarm have a collector" is not - # a question that can be false (mara: "the swarm always has at least one - # instance of all the swarm level services"), so both earlier gates — - # `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong - # host and rendered no forwarder wherever the answer happened to be no. The - # only condition that belongs here is the one already enclosing this block: - # the store is deployed on this host. + # The listener used to be gated on `scrapeHere = deployCfg.swarm-otel.enable` + # — "does the swarm's collector happen to run on THIS host" — because the + # thing reading it was that collector, over loopback, which only works when + # the two land together. mara's ruling (*"move the existing scraper + # to the local collector"*) removes the premise rather than the symptom: the + # scraper is now in this container, so the reader is wherever the store is, + # and the metrics path stops being silently dead on every host that does not + # also run swarm-otel. + # + # The forwarder never had a defensible one. "Does the swarm have a collector" + # is not a question that can be false (mara: "the swarm always has at least + # one instance of all the swarm level services"), so both earlier gates — + # `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong host + # and rendered no forwarder wherever the answer happened to be no. + # + # The only condition that belongs on either is the one already enclosing this + # block: the store is deployed on this host. # The swarm collector's own name, served by the gateway — the address # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host @@ -479,7 +494,49 @@ let # The port suffix follows ./hive-forge/default.nix: elided on the canonical # 443 so this renders byte for byte like the tier's other two consumers of # this name, present when an operator moved the gateway's TLS port. - otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.producerName}"; + otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.storeProducerName}"; + + # The forwarder's own credential, and the three names it takes on the way + # in. The shape is ./swarm-otel.nix's `swarm-bao-otel-oidc` — the unit that + # already reads an OIDC client secret out of the store with this host's + # certificate and lands it in a collector's container — because that route + # is proven and there is no second one worth inventing. + # + # Where the publisher on authelia's host leaves it. The `services` segment + # is `swarm-secret-client`'s `path::Kind::Service`: a swarm service's client + # is registered once for the whole swarm, so its secret is one value. The + # prefix is also exactly what a hive certificate's read grant covers, so a + # path outside it answers 403 however correct it looks. + forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client"; + # At rest in the container's tree, written by the host unit below — under + # /var/lib rather than /run, because a secret that evaporates on reboot + # turns a working export into an intermittent one. + forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret"; + forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}"; + forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath; + # The collector runs under `DynamicUser` and opens `client_secret_file` + # itself, so there is no uid to hand a 0400 file to. `LoadCredential` reads + # it as root before the sandbox exists and re-exposes it under a path that + # does not depend on which uid the unit got; the collector expands + # `${env:CREDENTIALS_DIRECTORY}` at load, so this id is the only spelling. + forwarderCredentialId = "oidc-client-secret"; + # Named once and read three times — the extension that defines it, the + # exporter that names it, and `service.extensions`, where an omission makes + # it inert rather than an error. + forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}"; + autheliaCfg = hyperhiveCfg.swarm.authelia; + + # Outbound TLS for that collector: it dials the swarm collector's gateway + # name and authelia's token endpoint, both leaves of the runtime-generated + # hive CA, which cannot be baked into a derivation. Same helper, same two + # halves and the same `SSL_CERT_FILE` consumption as ./swarm-otel.nix's own + # container — a Go process whose trust variable REPLACES the store, which is + # what the bundle (system CAs ++ our anchors) is for. + caTrust = import ./lib/hive-ca-trust.nix { + inherit lib; + tlsCfg = deployCfg.hive-controller.tls; + gatewayCfg = hyperhiveCfg.gateway; + }; swarmOtelCfg = hyperhiveCfg.swarm.otel; gatewayPortSuffix = @@ -490,8 +547,9 @@ let # Non-zero is what SERVES the endpoint at all — the switch is a duration, not # a boolean, so a zero here is an openbao that answers 404 on a listener - # configured to do nothing else. - telemetry = lib.optionalAttrs scrapeHere { + # configured to do nothing else. Unconditional for the same reason the + # listener above is: its reader ships with the store now. + telemetry = { telemetry = { prometheus_retention_time = "24h"; disable_hostname = true; @@ -885,6 +943,29 @@ in ''; }; + otel.clientId = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-bao-collector"; + description = '' + OAuth2 client id the collector inside the store's container + authenticates as, and — self-referentially, the shape every hive's + client already uses — the audience it asks its token for. + + **Its own, not the swarm collector's and not + `swarm-controller`'s.** One identity per principal: this forwarder + runs wherever the store runs, which is not where either of those + two runs, and the receiver it pushes to + (`swarm.otel.storeProducerName`) admits this id alone. + + Swarm-wide and read-only because three hosts have to agree on it: + authelia registers the client + (`glue-swarm-bao-otel-oidc-client.nix`), the swarm collector checks + the audience (`swarm-otel.nix`), and the store's host reads the + minted secret back out of the store under a path composed from it. + Two spellings present as a healthy-looking 401. + ''; + }; }; # ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than @@ -973,15 +1054,21 @@ in "openbao" "swarm-bao-certs" "swarm-bao-token" + "swarm-bao-forwarder-oidc" ]; - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere { - # Path and query, not just `host:port`: openbao serves no `/metrics` - # at all, and `/v1/sys/metrics` answers JSON unless the format is - # asked for. A scrape of the default path 404s, which reads as a - # dead exporter rather than a wrong address. - bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus"; - }; + # 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is + # the deliverable rather than a tidy-up. That option is read only by the + # SWARM collector, over loopback, so declaring the store there meant its + # metrics existed exactly on the hosts that also ran swarm-otel and + # nowhere else — silently, since a store with no entry looks identical to + # one nothing scrapes. The scrape moved into this container's own + # collector (see `receivers.prometheus` below), which travels with the + # store. + # + # Logs are untouched by that move: `journaldUnits` above still names this + # store's units for the shared collector, and the two are retired + # together once every sibling swarm container has a collector of its own. # ⚠️ The one nginx exception to this file's header, and it is one because # it never terminates. `ssl_preread` reads the SNI off the ClientHello @@ -1015,6 +1102,113 @@ in # still proves nothing — bao answers nothing without a client # certificate its CA signed. services.hyperhive.network.exposeHostPorts = [ cfg.port ]; + + # THE delivery unit for the forwarder's own OIDC client secret, copied + # in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login + # that fails LOUDLY, since every state it fails on is one a retry fixes, + # then a read, then a root-owned file in the container's tree that + # `LoadCredential` re-exposes to the collector's dynamic user. + # + # ⚠️ ORDERED AFTER the store's container and deliberately NOT before it, + # which is the one line that differs from the unit it copies. That one + # reads a store in the container NEXT DOOR; this one reads the store in + # the very container it is delivering into, so "before" is a wait on a + # process that cannot start until this finishes. There is no bootstrap + # cycle behind it — the identity used here is this host's static + # `swarm-bao-pki` certificate, not anything the store mints — only an + # ordering one, and the cost is bounded: the file is under /var/lib, so + # it survives reboots and only a FIRST boot has the collector starting + # before it exists. `LoadCredential` refuses to start a unit whose + # source is missing, so that boot is a collector that restarts, says so + # each time, and comes up the moment this lands. Loud and self-healing + # rather than silently exporting without a credential. + systemd.services.swarm-bao-forwarder-oidc = { + description = "fetch the secret store forwarder's OIDC client secret from the store"; + after = [ + "swarm-bao-pki.service" + "container@${cfg.machine}.service" + ]; + wants = [ "container@${cfg.machine}.service" ]; + requires = [ "swarm-bao-pki.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + # Sized for the race this loses: the secret is minted by authelia and + # copied in by the publisher, both of which may be a host away and + # neither of which this boot waits on. `StartLimit*` are `[Unit]` + # settings — systemd ignores them under `[Service]` — and the window + # has to exceed `RestartSec × burst`. + startLimitBurst = 4; + startLimitIntervalSec = 300; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-bao-forwarder-oidc"; + TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; + }; + environment = { + BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}"; + BAO_CLIENT_CERT = baoDeploy.clientCertFile; + BAO_CLIENT_KEY = baoDeploy.clientKeyFile; + } + # Absent means the system trust store, which is what a deployment with + # a real CA wants and what a self-signed one must not be left with. + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + # `bao`'s own message is the only thing separating a missing value + # from a refused identity from an unreachable store, and this unit + # retries on all three — so it reports which one rather than + # asserting all three in a sentence of ours. + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + # Cert auth is a login, not a transport setting: the `BAO_CLIENT_*` + # variables only decide which certificate the handshake presents. + # Without a token `bao` asks its token helper, and that is a `sh` + # this unit's `path` does not carry — `-token-only` answers on + # stdout and skips the helper on both sides. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to the swarm secret store with this host's certificate." >&2 + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + + # ⚠️ Non-zero, where the sibling unit one module over exits 0 on this + # branch. That collector treats a missing secret as a supported + # degrade; this one has no such mode — the receiver it pushes to + # takes an authenticated request and nothing else — so "not + # published yet" is a state to retry out of, not to settle into. + if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then + echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2 + cat "$err" >&2 + exit 1 + fi + + if [ -z "$secret" ]; then + echo "the store returned an empty ${forwarderStoreSecretPath}." >&2 + exit 1 + fi + + # root-owned 0400, written with a shell builtin and never handed to + # a program: `printf` is bash's own, so the plaintext never becomes + # an argument in /proc the way `install <<<"$secret"` or an `echo` + # from `path` would. + install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir} + umask 077 + printf '%s\n' "$secret" > ${lib.escapeShellArg forwarderHostSecretPath} + chown root:root ${lib.escapeShellArg forwarderHostSecretPath} + chmod 0400 ${lib.escapeShellArg forwarderHostSecretPath} + ''; + }; }) (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) { @@ -1399,6 +1593,11 @@ in ''; }; + # The CA bind source is written at runtime by a host unit, so the + # container has to start after it — otherwise nspawn sets up a mount + # over a file that does not exist yet. + systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -1448,7 +1647,11 @@ in hostPath = bootstrapTokenDir; isReadOnly = true; }; - }; + } + # The hive CA, for the forwarder's two outbound calls: the swarm + # collector's gateway name and authelia's token endpoint. Empty when + # the gateway is not self-signed, so the whole path drops out. + // caTrust.bindMount; # The seal talks to the TPM through the kernel's resource manager, so # the device has to cross the container boundary or the store cannot @@ -1471,7 +1674,27 @@ in imports = [ (import ./swarm-container-resolver.nix { inherit (networkCfg) bridgeIp; - dnsConsumers = [ "openbao.service" ]; + # The forwarder resolves two swarm names of its own — the + # collector it exports to and the identity provider it mints + # a token at — so it belongs in this list for the reason the + # module states: an unordered resolver write is a cold-boot + # race that shows up arbitrarily far from its cause. + dnsConsumers = [ + "openbao.service" + "opentelemetry-collector.service" + ]; + }) + ] + # `SSL_CERT_FILE` REPLACES the trust store rather than adding to + # it, so a failed assembly yields an empty pool and every TLS call + # fails while the unit looks healthy. That is why this is the + # shared helper — it carries the `Requires` and the non-empty + # check — and not a local `cat`. + ++ [ + (caTrust.trustBundle { + inherit pkgs; + name = cfg.machine; + consumers = [ "opentelemetry-collector" ]; }) ]; @@ -1708,6 +1931,34 @@ in storage = "file_storage"; }; + # The scrape that used to belong to the swarm collector, now + # one hop shorter and one host less fussy — mara: + # *"move the existing scraper to the local collector"*. + # + # ⚠️ `metrics_path` and `params` are STATED, and both have to + # be: openbao serves no `/metrics` at all, and + # `/v1/sys/metrics` answers JSON unless the format is asked + # for. A scrape of the default path 404s and a scrape without + # the query parses as nothing — each reads as a dead exporter + # rather than as a wrong request, which is how the previous + # owner of this scrape learned to spell it + # (./swarm-otel.nix's `loopbackScrapeConfig`, still the + # renderer for every other service's target). + # + # No credential, unlike the swarm collector's + # `publishedScrapeTargets` jobs: the target is the loopback + # listener above, inside this container's own netns, which is + # what lets the scrape stay unauthenticated without the API + # listener's client-cert requirement being touched. + receivers.prometheus.config.scrape_configs = [ + { + job_name = "bao"; + static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; + metrics_path = "/v1/sys/metrics"; + params.format = [ "prometheus" ]; + } + ]; + # Identity the hop above cannot supply: a host-side reader can # say which machine a line came from, and only a collector # inside this container can say it was the secret store's. No @@ -1728,7 +1979,34 @@ in # protocol is, which is the same reason # ./hive-c0re/environment.nix pins the protocol it hands out. # `endpoint` is a BASE the exporter appends `/v1/logs` to. - exporters.otlphttp.endpoint = otelFirstHop; + exporters.otlphttp = { + endpoint = otelFirstHop; + # The receiver at that address accepts an OIDC token and + # nothing else (./swarm-otel.nix's `oidc/*` authenticators), + # so this is what makes the hop deliver rather than collect + # 401s — unconditional, because a forwarder that exports + # unauthenticated has no working mode to fall back to. + auth.authenticator = forwarderAuthName; + }; + + # The push side, opposite in direction to the receiver's + # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one + # to send. Hence `oauth2client`, and hence a client id and + # secret rather than an issuer and an audience to check. + extensions.${forwarderAuthName} = { + client_id = cfg.otel.clientId; + # A path, never a value — and one systemd resolves at + # runtime, so neither the secret nor the credentials + # directory is rendered into the nix store. + client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; + token_url = "${toString autheliaCfg.url}/api/oidc/token"; + # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. + # Registering it on the client only makes it permissible; a + # token minted without asking carries `aud: []` and the + # receiver refuses it — with a config that reads perfectly + # at both ends. + endpoint_params.audience = cfg.otel.clientId; + }; service.pipelines.logs = { receivers = [ "journald" ]; @@ -1736,12 +2014,53 @@ in exporters = [ "otlphttp" ]; }; + # A SECOND pipeline, not a second receiver on the logs one: a + # pipeline is typed, and a prometheus receiver in `logs` is a + # config the collector rejects at startup rather than one that + # quietly ships metrics as log records. + # + # Same `resource` processor and same `otlphttp` exporter, so + # the store's metrics carry the same `service.name` its logs + # do and leave by the same hop — no otel bypass, one exit from + # this container. The exporter treats `endpoint` as a base and + # appends the signal's own path, so `/v1/metrics` here and + # `/v1/logs` above come from the one address. + service.pipelines.metrics = { + receivers = [ "prometheus" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; + # An extension configured but not listed here is INERT — the # receiver's `storage: file_storage` above would name a - # component the collector never starts. - service.extensions = [ "file_storage" ]; + # component the collector never starts, and the exporter's + # authenticator would push unauthenticated while the + # collector reports healthy. + service.extensions = [ + "file_storage" + forwarderAuthName + ]; }; }; + + # How the secret crosses the sandbox. The host unit + # `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this + # path in the container's tree; systemd reads it as root before + # the sandbox exists and re-exposes it to whichever uid + # `DynamicUser` picked, under the directory the extension above + # names through `${env:CREDENTIALS_DIRECTORY}`. + # + # ⚠️ Unconditional, and a missing source is a unit that refuses to + # start. That is the intended shape rather than an oversight: this + # forwarder has one way to deliver, so a boot where the secret has + # not arrived yet is a collector that says so and retries, not one + # that quietly exports into a 401. The restart is what closes the + # first-boot window the host unit's own comment describes. + systemd.services.opentelemetry-collector.serviceConfig = { + LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ]; + Restart = "on-failure"; + RestartSec = 15; + }; }; }; }) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 305f8bad..945f79ea 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -47,6 +47,24 @@ let # bound once here (rather than copy-pasted at both sites) is the only way # the two can't drift apart. swarmTierName = cfg.producerName; + + # The SECOND swarm-tier producer: the collector inside the secret store's + # container (`swarm-bao.nix`). It gets an owner of its own rather than + # pushing into `swarmTierName`'s route, because the audience that route + # checks is `swarm-controller`'s own client id — one identity per principal, + # so a second principal brings its own client, its own audience and + # therefore its own authenticator and receiver. + # + # Named after the store's container, so `otlp/swarm-bao` reads as the thing + # that pushes into it. No `reserved-names.nix` entry is needed for it the + # way `swarm` has one: the name CONTAINS `swarm`, which + # `nix/reserved-hive-fragments.nix` forbids as a substring of any hive name, + # so no hive can ever own these components. The assertion below checks that + # fragment is still listed rather than trusting it. + storeProducerName = baoCfg.machine; + storeClientId = baoCfg.otel.clientId; + reservedHiveFragments = import ../reserved-hive-fragments.nix; + # Every `` no hive may take. Read from `nix/reserved-names.nix`, the # same file the daemons are handed as `HIVE_RESERVED_NAMES`, because agent # names and hive names are ONE namespace going forward — a locally-owned @@ -363,6 +381,50 @@ in ''; }; + storeProducerPort = lib.mkOption { + type = lib.types.port; + default = 4391; + description = '' + Port the receiver for the **secret store's own forwarder** listens + on, at `127.0.0.1`. Reached through the gateway at + `https://''${domain}/''${storeProducerName}/`, the same + path-per-producer shape a hive and the swarm-tier producer above + both use. + + Its own receiver, and not a second sender into + {option}`services.hyperhive.swarm.otel.producerPort`, because an + `oidc` authenticator checks exactly ONE audience: that receiver + admits the audience `swarm-controller` registers for itself, and + the store's forwarder is a different principal with a client of its + own. Sharing the route would mean sharing that client — which is + the thing the store exists to make unnecessary. + + ⚠️ A reserved value rather than an offset from + {option}`services.hyperhive.swarm.otel.port`, for the reason stated + on `producerPort`: that range grows with the hive count and would + eventually walk into any fixed offset from it. The collision + assertion below covers this port too. + ''; + }; + + storeProducerName = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = baoCfg.machine; + defaultText = lib.literalExpression "services.hyperhive.swarm.bao.machine"; + description = '' + Path and component name of the store forwarder's own receiver + (`otlp/''${storeProducerName}`, `oidc/''${storeProducerName}`). + Published so `swarm-bao.nix` addresses the receiver by name rather + than repeating the string, exactly as `producerName` above is + published for `swarm-controller.nix`. + + Read-only and derived from the store's container name: two + spellings of a name both ends have to agree on is a 404 on a + request that authenticated perfectly. + ''; + }; + producerName = lib.mkOption { type = lib.types.str; readOnly = true; @@ -619,6 +681,9 @@ in # request reaches the receiver, which knows nothing about the # path it was found at. "/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/"; + # The secret store's forwarder, on its own route for its own + # audience — same trailing slashes, same reason. + "/${storeProducerName}/".proxyPass = "http://127.0.0.1:${toString cfg.storeProducerPort}/"; # There is no swarm-wide inbox, and a closed door is the honest # description of that. Every other route into this collector # belongs to exactly one hive or the swarm-tier producer above. @@ -873,6 +938,25 @@ in Put it back, or give this module a different swarmTierName. ''; } + { + # The store forwarder's components are named after a container whose + # name merely CONTAINS the reserved word, so what protects them is + # the substring list rather than the equality one. Same reason the + # assertion above exists: the list lives in another file, and a guard + # whose subject can be edited elsewhere has to assert its own case is + # still covered. + assertion = lib.any (f: lib.hasInfix f storeProducerName) reservedHiveFragments; + message = '' + nix/reserved-hive-fragments.nix forbids no substring of + '${storeProducerName}', which the swarm collector needs kept out of + the hive namespace: it names components `/`, so a hive + called '${storeProducerName}' would replace the secret store + forwarder's own receiver and pipeline entry. + + Put the fragment back, or give the store's container a name one of + them covers. + ''; + } { # A published target that is not an `https://host/path` url. Without # this the split returns null and the failure surfaces as @@ -974,6 +1058,7 @@ in others = [ cfg.telemetryPort cfg.producerPort + cfg.storeProducerPort otelCfg.collector.port ] ++ lib.optional deployCfg.victoriametrics.enable vmCfg.port; @@ -1119,6 +1204,15 @@ in auth.authenticator = "oidc/${swarmTierName}"; }; } + # The secret store's forwarder, which is a swarm-tier + # producer with a principal of its own — a receiver per + # audience, because an `oidc` authenticator checks one. + // { + "otlp/${storeProducerName}".protocols.http = { + endpoint = "127.0.0.1:${toString cfg.storeProducerPort}"; + auth.authenticator = "oidc/${storeProducerName}"; + }; + } # MERGED with the per-hive receivers, never assigned over # them. A plain assignment here would drop every hive's # receiver and still render a valid config that starts @@ -1303,7 +1397,13 @@ in # same reasoning as its receiver above: `otlp/${swarmTierName}` # always exists, so the authenticator it names must too, or an # extension-not-listed startup failure follows every deploy. - ++ [ "oidc/${swarmTierName}" ] + ++ [ + "oidc/${swarmTierName}" + # Unconditional for the same reason, and one level sharper: + # the store runs in every swarm, so its forwarder's + # receiver is as permanent as the swarm tier's own. + "oidc/${storeProducerName}" + ] # The push side's authenticators. Same rule as above: one an # exporter names but this list omits is INERT — the collector # starts clean and pushes unauthenticated. @@ -1347,6 +1447,16 @@ in "metrics/${swarmTierName}" = { receivers = [ "otlp/${swarmTierName}" + # The store forwarder's receiver joins the swarm + # tier's pipeline rather than bringing one of its own: + # the stamp it needs is `resource/${swarmTierName}`'s + # exact stamp — the swarm's name and deliberately no + # `hive` — and a second pipeline would be that same + # processor under a second name to keep in step. What + # a producer is separated FOR is the credential it + # proves, and that is the receiver's job, not the + # pipeline's. + "otlp/${storeProducerName}" ] ++ lib.optional (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) "prometheus"; processors = [ "resource/${swarmTierName}" ]; @@ -1386,7 +1496,14 @@ in # `hive` value to put on them. // lib.optionalAttrs collectLogs { "logs/${swarmTierName}" = { - receivers = [ "journald" ]; + # This host's own journal, and the store container's, + # which arrives over the receiver above rather than off + # a disk — the store's forwarder reads a journal no + # reader on another host can see. + receivers = [ + "journald" + "otlp/${storeProducerName}" + ]; processors = [ "resource/${swarmTierName}" ]; exporters = logExporterNames; }; @@ -1450,6 +1567,18 @@ in audience = config.services.hyperhive.swarm.controller.queueClientId; }; } + # The secret store forwarder's own authenticator. Same shape + # again, and the audience is that principal's own client id + # — the self-referential form `swarm-authelia.nix` registers + # a hive client under and `swarm-controller` uses above, + # read from the option `glue-swarm-bao-otel-oidc-client.nix` + # registers so the two ends cannot spell it differently. + // { + "oidc/${storeProducerName}" = { + issuer_url = hyperhiveCfg.swarm.authelia.url; + audience = storeClientId; + }; + } # The push side. Opposite direction to every `oidc/*` above — # those VALIDATE a token arriving; these OBTAIN one to send. # Hence `oauth2client` rather than `oidc`, and hence a client diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index 6f1a51e6..cfe8f5d1 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -71,6 +71,13 @@ let serviceClientIds = [ hyperhiveCfg.swarm.grafana.oidc.clientId hyperhiveCfg.swarm.otel.clientId + # The secret store's own journal forwarder. A third swarm service by this + # list's own rule — a client registered once for the whole swarm, whose + # secret one host mints and another reads — even though the principal + # behind it runs inside the store's container rather than beside it. + # Missing from here, authelia mints the value and nothing carries it, so + # the reader waits on a path that is never written. + hyperhiveCfg.swarm.bao.otel.clientId ]; # Where this unit keeps the appservice tokens it minted, and the whole reason diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index f6d9b83f..ebd1d198 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -26,7 +26,10 @@ let deploy.bao.enable = true; deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; # Pinned, not incidental: the case counting these listeners is about the - # declared addresses, and a collector on this host would add one of its own. + # declared API addresses. ⚠️ Since the store got its own collector, this no longer keeps the METRICS + # listener out of the count — that one is a function of the store, not of + # a collector's placement — so the case below subtracts it by name rather + # than by leaving the swarm tier off. deploy.swarm-otel.enable = false; }; @@ -72,10 +75,20 @@ let # The scrape list prometheus is handed, not the option a service declared: # the address, the path and the query are one string on the way in and three # fields on the way out, and only the second shape is what gets requested. - scrapeJob = - machine: job: - lib.findFirst (c: c.job_name == job) null - machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; + # + # ⚠️ Two of these now, aimed at two different containers, and the split is + # the point: the store's own job left the swarm collector for the + # one inside its container, so a case that looked in the old place would + # find `null` and a case that only looked in the new one would not notice + # the store still being declared to both. + scrapeJobIn = + container: machine: job: + lib.findFirst (c: c.job_name == job) null ( + machine.containers.${container}.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs + or [ ] + ); + scrapeJob = scrapeJobIn "swarm-otel"; + baoScrapeJob = scrapeJobIn "swarm-bao"; cases = [ { # Same gap one tier up, and it needs its own arm: this collector @@ -102,9 +115,14 @@ let } { # Control for the case above: these settings are rendered per deployment, - # not constants a passing case could be indifferent to. + # not constants a passing case could be indifferent to. The metrics + # listener is excluded by name — it answers on a port of its own and is + # not one of the API addresses this case is counting. name = "a declared extra address renders a second listener beside loopback"; - ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2; + ok = + builtins.length ( + builtins.filter (n: n != "metrics") (builtins.attrNames (baoSettings baoTwoAddresses).listener) + ) == 2; } { # Retention is what serves the endpoint at all, so the listener alone @@ -119,13 +137,56 @@ let { # openbao serves no `/metrics` at all, so a scrape of the default path # 404s: the store looks like a dead exporter, and every panel built on - # it renders empty rather than erroring. + # it renders empty rather than erroring. The query is the other half — + # `/v1/sys/metrics` answers JSON without it, which parses as no series + # rather than as an error. name = "the store's scrape asks for the path openbao serves"; ok = let - j = scrapeJob baoWithCollector "bao"; + j = baoScrapeJob baoWithCollector "bao"; in - (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; + j != null + && (j.metrics_path or "") == "/v1/sys/metrics" + && (j.params.format or [ ]) == [ "prometheus" ]; + } + { + # 🎯 This move's deliverable, and it needs BOTH halves or it is not a move: + # the job is in the store's own container, and it is no longer declared + # to the swarm collector. Only the second half can catch a "move" that + # left the original behind — two scrapers of one endpoint, double-counted + # and neither obviously wrong. + name = "the store's metrics are scraped by its own collector, not the swarm's"; + ok = baoScrapeJob baoWithCollector "bao" != null && scrapeJob baoWithCollector "bao" == null; + } + { + # The scrape target is the loopback listener, not the API one: dialing + # the API listener would need a client certificate the collector has no + # way to present, and the fix for that would be relaxing + # `tls_require_and_verify_client_cert` — which is the trade this address + # exists to avoid. Pinned so a later edit has to argue with it. + name = "the store's collector scrapes loopback, never a wider address"; + ok = + let + j = baoScrapeJob baoWithCollector "bao"; + s = baoSettings baoWithCollector; + in + j.static_configs == [ { targets = [ "127.0.0.1:8202" ]; } ] + && s.listener.metrics.address == "127.0.0.1:8202"; + } + { + # A pipeline is typed: a prometheus receiver named in `logs` is rejected + # at startup, and metrics with no pipeline at all are collected and + # dropped. The exporter is pinned alongside because "no otel bypass" + # means both signals leave by the one hop. + name = "the store's collector has a metrics pipeline out the same hop as its logs"; + ok = + let + s = (baoForwarder baoWithCollector).settings; + m = s.service.pipelines.metrics; + in + m.receivers == [ "prometheus" ] + && m.exporters == [ "otlphttp" ] + && m.exporters == s.service.pipelines.logs.exporters; } { # Presence control for the case above: both fields are omitted rather @@ -139,14 +200,28 @@ let j != null && !(j ? metrics_path) && !(j ? params); } { - # Absence arm. Unauthenticated by design, so it must not exist where - # nothing reads it. - name = "a store with no collector beside it serves no metrics"; + # 🩸 This case USED to be the absence arm — "a store with no collector + # beside it serves no metrics" — and this inverts it deliberately + # rather than deleting it, because the condition it asserted is the bug. + # + # The old reasoning was sound for the old reader: the endpoint is + # unauthenticated, so it should not exist where nothing reads it, and + # the only reader was a swarm collector that had to be on this host. + # What that rendered on every OTHER host was a store whose metrics went + # nowhere, indistinguishably from a store nobody had asked to scrape. + # + # The reader now ships inside the container, so "no collector beside it" + # describes no host that exists. The endpoint is still not exposed to + # anything new: it is loopback in a netns whose only other occupants are + # the store and its collector. + name = "a store whose swarm collector is elsewhere still serves and scrapes its metrics"; ok = let s = baoSettings baoOtelElsewhere; in - !(s.listener ? metrics) && !(s ? telemetry); + s.listener.metrics.address == "127.0.0.1:8202" + && (s.telemetry.prometheus_retention_time or "0s") != "0s" + && baoScrapeJob baoOtelElsewhere "bao" != null; } { # The store's journal reaches a reader through a collector of its own, @@ -162,7 +237,11 @@ let in s.receivers.journald.directory == "/var/log/journal" && s.receivers.journald.storage == "file_storage" - && s.service.extensions == [ "file_storage" ] + && + s.service.extensions == [ + "file_storage" + "oauth2client/swarm-bao" + ] && p.receivers == [ "journald" ] && p.exporters == [ "otlphttp" ] && s ? exporters.otlphttp; @@ -188,8 +267,8 @@ let vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain}; in (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint - == "https://${otel.domain}/${otel.producerName}" - && vhost.locations ? "/${otel.producerName}/"; + == "https://${otel.domain}/${otel.storeProducerName}" + && vhost.locations ? "/${otel.storeProducerName}/"; } { # The case the old hive-tier gate got wrong: this host runs the swarm's @@ -219,7 +298,7 @@ let ok = let otel = baoOtelElsewhere.services.hyperhive.swarm.otel; - route = "https://${otel.domain}/${otel.producerName}"; + route = "https://${otel.domain}/${otel.storeProducerName}"; in !(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain}) && (baoForwarder baoOtelElsewhere).enable @@ -229,6 +308,129 @@ let && (baoForwarder baoWithHiveOtel).enable && (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route; } + { + # 🎯 The credential half of this move, and the arm that says the hop + # DELIVERS rather than merely resolves. The receiver at the far end + # takes an OIDC token and nothing else, so an exporter with no + # authenticator is a forwarder that retries into 401s while every other + # case here passes. Both halves: the extension has to exist AND the + # exporter has to name it. + name = "the store's forwarder presents a credential on the hop it exports over"; + ok = + let + s = (baoForwarder baoWithCollector).settings; + auth = "oauth2client/swarm-bao"; + in + s.exporters.otlphttp.auth.authenticator == auth + && s.extensions ? ${auth} + && lib.elem auth s.service.extensions; + } + { + # Its OWN client, never the swarm collector's and never the + # controller's: one identity per principal, and this one runs on a host + # neither of those two does. The audience is the same id because the + # receiver checks exactly that — a token minted without ASKING for it + # carries `aud: []` and is refused with a config that reads correctly at + # both ends. + name = "the forwarder authenticates as a principal of its own"; + ok = + let + bao = baoWithCollector.services.hyperhive.swarm.bao; + e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao"; + in + e.client_id == bao.otel.clientId + && e.endpoint_params.audience == bao.otel.clientId + && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.otel.clientId + && bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.controller.queueClientId; + } + { + # A path systemd resolves at runtime, never a value and never a store + # path: the collector runs under `DynamicUser` and opens this file + # itself, so the credential arrives through `LoadCredential` and the + # config names only the directory systemd exports. + name = "the forwarder's client secret reaches it as a credential, not a value"; + ok = + let + container = baoWithCollector.containers.swarm-bao.config; + e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao"; + unit = container.systemd.services.opentelemetry-collector.serviceConfig; + in + e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret" + && + unit.LoadCredential == [ + "oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" + ]; + } + { + # Same 403-not-a-miss property the grafana and matrix readers are pinned + # for: the reader's grant covers the `services` prefix, so a secret + # filed under the hive that happens to run the store would be refused + # rather than missing, however correct the path reads. + name = "the forwarder's secret is read from the prefix the publisher writes"; + ok = + let + s = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc.script; + in + lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" s + && !(lib.hasInfix "secret/swarm/hives/" s); + } + { + # The delivery unit is ordered AFTER the store's container, not before + # it: the store it reads lives in the container it delivers into, so a + # `before` edge is a wait on a process that cannot start until this + # finishes. Pinned because the unit it was copied from does the + # opposite, and copying that line too would deadlock every cold boot. + name = "the delivery unit waits for the store rather than blocking it"; + ok = + let + u = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc; + in + lib.elem "container@swarm-bao.service" u.after + && !(lib.elem "container@swarm-bao.service" (u.before or [ ])) + && lib.elem "swarm-bao-pki.service" u.requires; + } + { + # The far end of the credential. An authenticator checks ONE audience, + # so the store's forwarder gets a receiver of its own rather than a + # second sender into the route whose audience belongs to + # `swarm-controller` — and every one of these four is silent when + # wrong: an unlisted extension is inert, a receiver in no pipeline + # drops what it accepts, and a mismatched audience is a healthy 401. + name = "the swarm collector admits this principal on a receiver of its own"; + ok = + let + otel = baoWithCollector.services.hyperhive.swarm.otel; + s = baoWithCollector.containers.swarm-otel.config.services.opentelemetry-collector.settings; + name = otel.storeProducerName; + in + s.receivers."otlp/${name}".protocols.http.auth.authenticator == "oidc/${name}" + && + s.extensions."oidc/${name}".audience == baoWithCollector.services.hyperhive.swarm.bao.otel.clientId + && lib.elem "oidc/${name}" s.service.extensions + && lib.elem "otlp/${name}" s.service.pipelines."metrics/${otel.producerName}".receivers + && lib.elem "otlp/${name}" s.service.pipelines."logs/${otel.producerName}".receivers; + } + { + # A client authelia has never heard of mints nothing, so the publisher + # has nothing to copy and the delivery unit reads an empty path + # forever. Registered where AUTHELIA runs, which is why it is a glue + # module and not a line in the store's own config — this fixture runs + # no store at all and still has to register it. + name = "the forwarder's client is registered wherever authelia runs"; + ok = + let + autheliaOnly = hive { deploy.authelia.enable = true; }; + clients = autheliaOnly.services.hyperhive.swarm.authelia.oidc.clients; + c = lib.findFirst (c: c.id == "swarm-bao-collector") null clients; + in + !autheliaOnly.services.hyperhive.deploy.bao.enable + && c != null + && c.audience == [ "swarm-bao-collector" ] + # ⚠️ What makes the token READABLE by the receiver at all: authelia's + # default is an opaque handle, and an `oidc` extension verifies + # offline against the provider's JWKS. + && c.accessTokenSignedResponseAlg == "RS256"; + } ]; in runGroup "bao-otel-collector" cases diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index 7c2f9f6d..fd343356 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -132,6 +132,17 @@ let secretPublisherHere.systemd.services.swarm-secret-publish.script ); } + { + # The third swarm service on that list, and the one whose absence is + # hardest to see: the store's own forwarder holds a client authelia + # registers and mints for, so every layer above looks complete while + # the one hop that CARRIES the value skips it and the reader on the + # store's host waits forever on a path nothing writes. + name = "the publisher carries the store forwarder's secret too"; + ok = lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" ( + secretPublisherHere.systemd.services.swarm-secret-publish.script + ); + } { # The same hole the controller's case above names, open a second time: the # PKI script grew a third leaf and no case read it.