# `checks.module-eval-agent-otel` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, shared, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) agent carriesJournaldSeverity runGroup ; # The log path's three hops, one fixture each. Nothing carries a journal # record end to end at eval time, so what these defend is the part no tier # can check for itself: each hop's output is the next hop's input, and # every mismatch between them is silent — a push accepted and routed # nowhere, a receiver pointed at an empty directory, a pipeline that does # not exist. agentBridge = "http://10.42.0.1:4318"; agentOtel = agent { otel.enable = true; otel.endpoint = agentBridge; }; # The same agent over the other wire protocol. An exporter's NAME is what # selects it, so this is where a defined exporter and the pipeline's # reference to it can drift apart. Also an agent config still setting the # removed metric interval. agentOtelGrpc = agent { otel.enable = true; otel.endpoint = agentBridge; otel.protocol = "grpc"; otel.metricIntervalMs = 10000; }; # Sets nothing: what an agent gets on a hive, whose meta flake renders no # otel line. agentDefault = shared.agentBare; agentNoOtel = agent { otel.enable = false; }; agentSettings = machine: machine.services.opentelemetry-collector.settings; cases = [ { # The journald receiver's own default directory is the RUNTIME # journal, and a container that stores persistently leaves that # empty. At the default the forwarder validates, starts, reports # healthy and ships nothing, so this one literal is the difference # between the path working and silently not. name = "the agent forwarder reads the persistent journal, not the runtime one"; ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; } { # Wiring, not contents. A journald record arrives with its level in # `PRIORITY` and nothing downstream reads that field, so a receiver with # no severity parser ships every line to the log store labelled # `Unspecified` — healthy, complete, and unqueryable by level. # # What this asks is only whether THIS receiver carries the shared # mapping. ⚠️ Whether that mapping is RIGHT — the inverted direction, # which is the part of it a reviewer cannot check by looking — is # asserted once, in ./journald-severity.nix, against the file both tiers # import. Re-asserting the table here would test that file twice and # report one defect as two failures. # # The swarm collector's own receiver gets the sibling of this case in # ./swarm-otel-core.nix. Deliberately NOT folded together with it: they # cover different journals — this container's own, and the host's — and # one tier quietly losing its parser while the other keeps one is # precisely the half-fixed state worth catching. # # Membership rather than equality of the whole operator list, so a tier # that later grows an unrelated operator of its own still passes. name = "the agent forwarder's journald receiver carries the shared PRIORITY mapping"; ok = carriesJournaldSeverity (agentSettings agentOtel).receivers.journald; } { # The hop's two ends: what it reads, and where what it reads goes. # The endpoint is compared against the value the fixture handed the # option rather than a literal spelled here, so an exporter that # stopped reading the option fails instead of matching a constant # that travelled beside it. name = "the agent forwarder ships the journal to the endpoint its hive gave it"; ok = let s = agentSettings agentOtel; p = s.service.pipelines.logs; in p.receivers == [ "journald" ] && p.exporters != [ ] && lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters; } { # 🎯 What the meta flake no longer says: an agent that sets nothing # exports, to a name rather than an address. The name carries no swarm # domain and no hive, so the same build is correct on every hive; the # resolver of the hive it runs on answers it. `agentNoOtel` above is the # control that these lookups find nothing where the switch is off. name = "an agent with no otel config exports to the fixed hive-collector name by default"; ok = let endpoint = "http://hive-collector.hive:4318"; env = agentDefault.systemd.globalEnvironment; s = agentSettings agentDefault; in agentDefault.services.hyperhive.agent.otel.enable && env.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint && agentDefault.environment.variables.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint && lib.all (e: s.exporters.${e}.endpoint == endpoint) s.service.pipelines.logs.exporters && !(agentNoOtel.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT); } { # The hive's extra labels arrive at container start, so the build # cannot hold the whole `OTEL_RESOURCE_ATTRIBUTES`: a boot-time # generator computes it, and a unit-level `Environment=` copy would # override that. The endpoint beside it is the presence control for the # two absence lookups; `agentNoOtel` is the control for the generator. name = "an agent's OTEL resource labels come from a boot-time generator, not the build"; ok = let generator = m: m.environment.etc ? "systemd/system-environment-generators/60-hyperhive-otel-resource"; in agentDefault.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT && !(agentDefault.systemd.globalEnvironment ? OTEL_RESOURCE_ATTRIBUTES) && agentDefault.environment.variables ? OTEL_EXPORTER_OTLP_ENDPOINT && !(agentDefault.environment.variables ? OTEL_RESOURCE_ATTRIBUTES) && generator agentDefault && !(generator agentNoOtel); } { # Presence control for the two cases above: with the switch off there # is no collector in the container at all, so their passing is about # the wiring rather than about a unit that renders regardless. name = "an agent that switched telemetry off runs no collector"; ok = !agentNoOtel.services.opentelemetry-collector.enable; } { # `otlp` and `otlphttp` are different components and the protocol # option picks which one is defined. A pipeline left naming the other # is a startup failure; an exporter no pipeline names is silence. name = "the agent forwarder's exporter and its pipeline agree on the protocol"; ok = let s = agentSettings agentOtelGrpc; in (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ]; } { # Control: `agentOtel` sets no interval. The endpoint lookup shows the # environment it reads is the one the OTEL SDKs get. name = "an agent config setting the removed metric interval evaluates, warns, and exports none"; ok = let warned = lib.any ( w: lib.hasInfix "services.hyperhive.agent.otel.metricIntervalMs is removed and ignored" w ); env = machine: machine.systemd.globalEnvironment; in warned agentOtelGrpc.warnings && !(warned agentOtel.warnings) && (env agentOtelGrpc) ? OTEL_EXPORTER_OTLP_ENDPOINT && !((env agentOtelGrpc) ? OTEL_METRIC_EXPORT_INTERVAL); } ]; in runGroup "agent-otel" cases