From 07639fd364b4dea33873c7fc9a35d4804667b2ef Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 02:46:08 +0200 Subject: [PATCH] otel: evaluate the agent log forwarder in module-eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in this suite evaluated nix/agent-modules/ at all: every fixture was a host, so a typo in a rendered container config surfaced on a real deploy and nowhere else. This adds an `agent` constructor beside `hive`, off the same `nixosModules.agent-base` the meta flake hands a container. It also adds the suite's first two-hive fixture. Every existing one declares `swarm.hives.h1` alone, so a per-hive arm written against those passes on a hardcoded literal — which is exactly what the new per-hive logs pipeline needed covering. Eight cases, each paired with the control that makes it mean something: the absence arms with a presence half, the per-hive arm with a roster length check, because `lib.all` over an empty roster holds vacuously. Each was then shown to fail. Eight mutations across the three files — `directory` back to the runtime default, a pipeline naming no receiver, an exporter aimed at a loopback literal, an exporter name that stops reading `protocol`, a missing hive-tier logs pipeline, the metrics-only processor inside it, a per-hive pipeline hardcoded to one hive, and logs pipelines exporting to the metrics store — all caught, none survived, none skipped, each run's baseline green. Part of #3940. --- nix/module-eval.nix | 166 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 17e2e210..71931666 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -58,6 +58,32 @@ let ]; }).config; + # The other half of the tree. `nix/agent-modules/` is evaluated by nothing + # else in this suite — every fixture above is a host — so a rendered + # container config was only ever read by a real deploy. Same entry point + # the meta flake hands a container, so what this evaluates is what an + # agent gets. + # + # Note `hyperhive`, not `services.hyperhive`: an agent container's options + # live at the top level. + agent = + extra: + (nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + self.nixosModules.agent-base + { + fileSystems."/" = { + device = "/dev/null"; + fsType = "tmpfs"; + }; + boot.loader.grub.enable = false; + system.stateVersion = "25.11"; + hyperhive = lib.recursiveUpdate { user.name = "a1"; } extra; + } + ]; + }).config; + allLocal = hive { deploy.singleHostSwarm = true; }; bare = hive { }; withCi = hive { deploy.forgejo.ci.enable = true; }; @@ -373,6 +399,45 @@ let swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret"; }; + # The log path's three hops, one fixture each. Nothing carries a journal + # record end to end at eval time, so what these defend is the part no tier + # can check for itself: each hop's output is the next hop's input, and + # every mismatch between them is silent — a push accepted and routed + # nowhere, a receiver pointed at an empty directory, a pipeline that does + # not exist. + agentBridge = "http://10.42.0.1:4318"; + agentOtel = agent { + otel.enable = true; + otel.endpoint = agentBridge; + }; + # The same agent over the other wire protocol. An exporter's NAME is what + # selects it, so this is where a defined exporter and the pipeline's + # reference to it can drift apart. + agentOtelGrpc = agent { + otel.enable = true; + otel.endpoint = agentBridge; + otel.protocol = "grpc"; + }; + agentNoOtel = agent { }; + agentSettings = machine: machine.services.opentelemetry-collector.settings; + + # This hive's own collector, which is a HOST service — unlike the swarm + # tier's, which lives in a container and is read through `otelSettings`. + hiveOtel = hive { + otel.enable = true; + otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; + }; + hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines; + + # Two hives in the roster, which no other fixture here has: every one of + # them declares `swarm.hives.h1` alone, so a per-hive arm written against + # one of those passes on a hardcoded literal. + otelTwoHives = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + swarm.hives.h2.domain = "h2.t.local"; + }; + # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -1022,6 +1087,107 @@ let in named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } + { + # The journald receiver's own default directory is the RUNTIME + # journal, and a container that stores persistently leaves that + # empty. At the default the forwarder validates, starts, reports + # healthy and ships nothing, so this one literal is the difference + # between the path working and silently not. + name = "the agent forwarder reads the persistent journal, not the runtime one"; + ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; + } + { + # The hop's two ends: what it reads, and where what it reads goes. + # The endpoint is compared against the value the fixture handed the + # option rather than a literal spelled here, so an exporter that + # stopped reading the option fails instead of matching a constant + # that travelled beside it. + name = "the agent forwarder ships the journal to the endpoint its hive gave it"; + ok = + let + s = agentSettings agentOtel; + p = s.service.pipelines.logs; + in + p.receivers == [ "journald" ] + && p.exporters != [ ] + && lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters; + } + { + # Presence control for the two cases above: with the switch off there + # is no collector in the container at all, so their passing is about + # the wiring rather than about a unit that renders regardless. + name = "an agent that has not opted into telemetry runs no collector"; + ok = !agentNoOtel.services.opentelemetry-collector.enable; + } + { + # `otlp` and `otlphttp` are different components and the protocol + # option picks which one is defined. A pipeline left naming the other + # is a startup failure; an exporter no pipeline names is silence. + name = "the agent forwarder's exporter and its pipeline agree on the protocol"; + ok = + let + s = agentSettings agentOtelGrpc; + in + (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ]; + } + { + # The tier in the middle. Its OTLP receiver takes both signals on one + # port, so without this pipeline an agent's push is answered 404 on + # `/v1/logs` — and a forwarder retrying into a 404 is indistinguish- + # able from one with nothing to send. Compared against the metrics + # pipeline's exporters rather than a name, so the two signals cannot + # drift to different destinations. + name = "the hive collector forwards logs upstream, not only metrics"; + ok = + (hiveOtelPipelines ? logs) + && hiveOtelPipelines.metrics.exporters != [ ] + && hiveOtelPipelines.logs.receivers == [ "otlp" ] + && hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters; + } + { + # `deltatocumulative` is metrics-only: naming it in a logs pipeline + # kills the collector at startup rather than doing nothing. The second + # clause is the control — the metrics pipeline still names it, so a + # pass means the two processor lists differ rather than that the + # processor left the module. + name = "the hive collector keeps the metrics-only processor out of its logs pipeline"; + ok = + !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors) + && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors; + } + { + # Read against the roster the fixture declares rather than against + # names spelled here: an arm naming `h1` passes on a single-hive + # config however the mapping is written. The length clause is what + # makes the `all` mean anything — over an empty roster it holds + # vacuously. + name = "the swarm collector routes every hive's logs, not just one"; + ok = + let + p = (otelSettings otelTwoHives).service.pipelines; + hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives; + in + lib.length hives == 2 + && lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives; + } + { + # The same split as the metrics case above — defining an exporter and + # naming it are two lists — plus the half one shared list cannot have: + # the metrics store's exporter renders perfectly well inside a logs + # pipeline and posts journal records at an ingest route that is not + # for them. + name = "every logs pipeline sends to the log store and to no metrics one"; + ok = + let + s = otelSettings otelTwoHives; + logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines; + used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes)); + in + logPipes != { } + && builtins.elem "otlphttp/victorialogs" used + && !(builtins.elem "otlphttp/victoriametrics" used) + && lib.all (e: s.exporters ? ${e}) used; + } { # The store's seal is spread over six gates — the stanza, the # provisioning unit, two bind mounts, a device and an EnvironmentFile.