diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 702e5e72..fc5aa664 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -455,6 +455,21 @@ let scrapeHere = deployCfg.swarm-otel.enable; + # Whether this container forwards its own journal, and it is gated on the + # HIVE collector rather than the swarm one: the forwarder below pushes to the + # first hop, and with `otel.enable` off nothing listens at that address at + # all — a collector aimed at it would retry forever while looking healthy. + shipJournal = hyperhiveCfg.otel.enable; + + # The first hop for anything running on this host, byte for byte what + # hive-c0re hands every agent (`firstHop` in ./hive-c0re/environment.nix). + # Plain http with no credential is the tier boundary working as designed, not + # an omission: the hive's collector is the only thing here that holds one, + # and presenting it to the swarm is its job. This container shares the host + # netns (`privateNetwork = false` below), so the bridge address is reachable + # from inside it without the firewall hole an agent container needs. + otelFirstHop = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.otel.collector.port}"; + # Non-zero is what SERVES the endpoint at all — the switch is a duration, not # a boolean, so a zero here is an openbao that answers 404 on a listener # configured to do nothing else. @@ -1428,7 +1443,12 @@ in ]; config = - { ... }: + # `config` is the CONTAINER's, shadowing the host's inside this + # function — nothing in here reads the host's, and the journal + # assertion below has to read the setting on this side of the + # boundary. `lib` and `pkgs` still come from the enclosing scope, as + # every other expression in this block already does. + { config, ... }: { imports = [ (import ./swarm-container-resolver.nix { @@ -1586,6 +1606,124 @@ in # Restarting is an operator action with an unseal on the far side of # it, which is why nothing here tries to be clever about it. + # This container's own journal forwarder, copied from an agent + # container's (nix/agent-modules/otel.nix) because every container + # is supposed to ship its own logs to the next hop rather than + # leave a *different* container's collector to find them. + # + # That older shape is what `--link-journal=host` above and the + # `swarm.otel.journaldUnits` entry near the top of this file serve: + # the journal lands in the host tree, where the swarm collector — + # itself a container — reads it through a unit allowlist. Both stay + # for now, because every sibling swarm container still rides that + # collector; they come out once each of them has one of these. + # + # ⚠️ NO `units` allowlist here, and that is the point rather than a + # simplification. A shared collector needs one because the journal + # it reads holds six containers' units plus the host's own; this + # one reads only openbao and the two oneshots beside it, so there + # is nothing foreign to separate out — and a list of unit names is + # a thing to get wrong, which ships nothing while looking healthy. + assertions = lib.optionals shipJournal [ + { + # Sibling of the agent forwarder's identical assertion, and it + # exists because the failure is silent at every layer: with a + # volatile journal the receiver below finds an empty directory, + # reads nothing, and the collector starts clean and stays + # healthy forever. + assertion = + !(lib.elem config.services.journald.storage [ + "volatile" + "none" + ]); + message = '' + services.hyperhive.otel.enable is on, so ${cfg.machine} forwards its + own journal — but services.journald.storage is + "${config.services.journald.storage}" in this container. + + The forwarder reads /var/log/journal, which journald only writes + when it stores persistently: with "volatile" the journal lives in + /run/log/journal and with "none" there is none at all. Either way + the forwarder would ship nothing while looking healthy. + ''; + } + ]; + + services.opentelemetry-collector = lib.mkIf shipJournal { + enable = true; + # Contrib, and not a preference: `journald` is a contrib + # receiver. The upstream default build has no way to read a + # journal at all. + package = pkgs.opentelemetry-collector-contrib; + # Runs `otelcol validate` at build time. ⚠️ A parser, not a + # wiring check — it accepts a pipeline naming a component the + # build lacks, and the collector then dies at startup. Same + # caveat as every other tier; see nix/host-modules/otel.nix for + # the measurement. + validateConfigFile = true; + settings = { + # `journalctl --follow --lines=0` (what the receiver runs under + # the hood) ships only what is written *after* it starts, so + # every collector start has a silent gap at the front. This + # persists the read cursor, so a restart resumes instead of + # re-opening that gap; `start_at` stays `end` (its default), + # since a cursor covers every run after the first and + # `beginning` without one would re-ship the whole journal on + # every restart. `create_directory` is required for a + # build-time reason rather than a convenience one: + # `validateConfigFile` above runs `otelcol validate` in the nix + # sandbox, before systemd's `StateDirectory=` has created the + # path, and the validator refuses with "directory must exist". + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; + + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own + # default is the RUNTIME journal (`/run/log/journal`), and + # every entry here is under /var/log/journal — which + # `--link-journal=host` above makes the host's directory for + # this machine id. Dropping this line leaves a collector that + # validates, starts, reports healthy and forwards nothing. + directory = "/var/log/journal"; + storage = "file_storage"; + }; + + # Identity the hop above cannot supply: a host-side reader can + # say which machine a line came from, and only a collector + # inside this container can say it was the secret store's. No + # `hive` or `swarm` label — the swarm tier upserts `hive` from + # the receiver that accepted the record, precisely so it comes + # from something the sender cannot write. + processors.resource.attributes = [ + { + key = "service.name"; + value = cfg.machine; + action = "upsert"; + } + ]; + + # `otlphttp` unconditionally, unlike the agent forwarder's + # protocol-derived exporter name: the first hop's receiver + # speaks OTLP/HTTP protobuf whatever the hive's *upstream* + # protocol is, which is the same reason + # ./hive-c0re/environment.nix pins the protocol it hands out. + # `endpoint` is a BASE the exporter appends `/v1/logs` to. + exporters.otlphttp.endpoint = otelFirstHop; + + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; + + # An extension configured but not listed here is INERT — the + # receiver's `storage: file_storage` above would name a + # component the collector never starts. + service.extensions = [ "file_storage" ]; + }; + }; }; }; }) diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 4558d17b..8345f2a8 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -47,6 +47,24 @@ let deploy.swarm-otel.enable = false; }; + # The store on a host whose HIVE collector is running — the only state in + # which the container forwards its own journal, since that collector is the + # hop it forwards to. `clientSecretFile` is what ../host-modules/otel.nix's + # identity assertion demands of any hive with the tier on. `swarm-otel` is + # deliberately left off: the forwarder is a function of the first hop + # existing, and pairing the two fixtures would make a case that passed on the + # wrong condition. + baoWithHiveOtel = hive { + deploy.bao.enable = true; + otel.enable = true; + otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; + }; + + # The forwarder INSIDE the store's container, not the host's collector and + # not the swarm tier's — three collectors in this tree, and only this one can + # see the store's journal. + baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector; + # The scrape list prometheus is handed, not the option a service declared: # the address, the path and the query are one string on the way in and three # fields on the way out, and only the second shape is what gets requested. @@ -126,6 +144,49 @@ let in !(s.listener ? metrics) && !(s ? telemetry); } + { + # The store's journal reaches a reader through a collector of its own, + # and every one of these fields is silent when wrong: the runtime journal + # is the receiver's own default and is empty here, an unlisted extension + # is inert so the cursor silently stops persisting, and a pipeline is + # free to name none of it. + name = "the store's container forwards its own journal"; + ok = + let + s = (baoForwarder baoWithHiveOtel).settings; + p = s.service.pipelines.logs; + in + s.receivers.journald.directory == "/var/log/journal" + && s.receivers.journald.storage == "file_storage" + && s.service.extensions == [ "file_storage" ] + && p.receivers == [ "journald" ] + && p.exporters == [ "otlphttp" ] + && s ? exporters.otlphttp; + } + { + # The whole journal, which is what the shared collector's unit allowlist + # is not. A `units` list here would render and deploy perfectly while + # shipping only the units someone remembered to name — the failure this + # forwarder exists to end. + name = "the store's forwarder filters no units"; + ok = !((baoForwarder baoWithHiveOtel).settings.receivers.journald ? units); + } + { + # Both ends of the first hop, on ONE host, because a mismatch between + # them is silent in both directions: the exporter retries into nothing + # and the receiver never hears from it. + name = "the store's forwarder exports to this hive's own collector"; + ok = + (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint + == "http://${baoWithHiveOtel.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint}"; + } + { + # Absence arm — `baoNoCollector` runs neither tier, so there is no first + # hop on this host at all. A forwarder rendered anyway would start, find + # nothing listening, and retry forever while reporting healthy. + name = "a store with no hive collector renders no forwarder"; + ok = !(baoForwarder baoNoCollector).enable; + } ]; in runGroup "bao-otel-collector" cases