vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

otel: evaluate the agent log forwarder in module-eval

Browse source 
Nothing in this suite evaluated nix/agent-modules/ at all: every fixture
was a host, so a typo in a rendered container config surfaced on a real
deploy and nowhere else. This adds an `agent` constructor beside `hive`,
off the same `nixosModules.agent-base` the meta flake hands a container.

It also adds the suite's first two-hive fixture. Every existing one
declares `swarm.hives.h1` alone, so a per-hive arm written against those
passes on a hardcoded literal — which is exactly what the new per-hive
logs pipeline needed covering.

Eight cases, each paired with the control that makes it mean something:
the absence arms with a presence half, the per-hive arm with a roster
length check, because `lib.all` over an empty roster holds vacuously.

Each was then shown to fail. Eight mutations across the three files —
`directory` back to the runtime default, a pipeline naming no receiver,
an exporter aimed at a loopback literal, an exporter name that stops
reading `protocol`, a missing hive-tier logs pipeline, the metrics-only
processor inside it, a per-hive pipeline hardcoded to one hive, and logs
pipelines exporting to the metrics store — all caught, none survived,
none skipped, each run's baseline green.

Part of #3940.
This commit is contained in:
atlas 2026-09-11 02:46:08 +02:00 • committed by mara
commit 07639fd364
1 changed files with 166 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

166
nix/module-eval.nix
View file

@ -58,6 +58,32 @@ let
];
}).config;
# The other half of the tree. `nix/agent-modules/` is evaluated by nothing
# else in this suite — every fixture above is a host — so a rendered
# container config was only ever read by a real deploy. Same entry point
# the meta flake hands a container, so what this evaluates is what an
# agent gets.
#
# Note `hyperhive`, not `services.hyperhive`: an agent container's options
# live at the top level.
agent =
extra:
(nixosSystem {
system = pkgs.stdenv.hostPlatform.system;
modules = [
self.nixosModules.agent-base
{
fileSystems."/" = {
device = "/dev/null";
fsType = "tmpfs";
};
boot.loader.grub.enable = false;
system.stateVersion = "25.11";
hyperhive = lib.recursiveUpdate { user.name = "a1"; } extra;
}
];
}).config;
allLocal = hive { deploy.singleHostSwarm = true; };
bare = hive { };
withCi = hive { deploy.forgejo.ci.enable = true; };
@ -373,6 +399,45 @@ let
swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret";
};
# The log path's three hops, one fixture each. Nothing carries a journal
# record end to end at eval time, so what these defend is the part no tier
# can check for itself: each hop's output is the next hop's input, and
# every mismatch between them is silent — a push accepted and routed
# nowhere, a receiver pointed at an empty directory, a pipeline that does
# not exist.
agentBridge = "http://10.42.0.1:4318";
agentOtel = agent {
otel.enable = true;
otel.endpoint = agentBridge;
};
# The same agent over the other wire protocol. An exporter's NAME is what
# selects it, so this is where a defined exporter and the pipeline's
# reference to it can drift apart.
agentOtelGrpc = agent {
otel.enable = true;
otel.endpoint = agentBridge;
otel.protocol = "grpc";
};
agentNoOtel = agent { };
agentSettings = machine: machine.services.opentelemetry-collector.settings;
# This hive's own collector, which is a HOST service — unlike the swarm
# tier's, which lives in a container and is read through `otelSettings`.
hiveOtel = hive {
otel.enable = true;
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
};
hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;
# Two hives in the roster, which no other fixture here has: every one of
# them declares `swarm.hives.h1` alone, so a per-hive arm written against
# one of those passes on a hardcoded literal.
otelTwoHives = hive {
deploy.swarm-otel.enable = true;
deploy.authelia.enable = true;
swarm.hives.h2.domain = "h2.t.local";
};
# A priority collision is a property of the *option*, not
# of the merged value's interior — nix throws the moment the value is
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
@ -1022,6 +1087,107 @@ let
in
named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named;
}
{
# The journald receiver's own default directory is the RUNTIME
# journal, and a container that stores persistently leaves that
# empty. At the default the forwarder validates, starts, reports
# healthy and ships nothing, so this one literal is the difference
# between the path working and silently not.
name = "the agent forwarder reads the persistent journal, not the runtime one";
ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal";
}
{
# The hop's two ends: what it reads, and where what it reads goes.
# The endpoint is compared against the value the fixture handed the
# option rather than a literal spelled here, so an exporter that
# stopped reading the option fails instead of matching a constant
# that travelled beside it.
name = "the agent forwarder ships the journal to the endpoint its hive gave it";
ok =
let
s = agentSettings agentOtel;
p = s.service.pipelines.logs;
in
p.receivers == [ "journald" ]
&& p.exporters != [ ]
&& lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters;
}
{
# Presence control for the two cases above: with the switch off there
# is no collector in the container at all, so their passing is about
# the wiring rather than about a unit that renders regardless.
name = "an agent that has not opted into telemetry runs no collector";
ok = !agentNoOtel.services.opentelemetry-collector.enable;
}
{
# `otlp` and `otlphttp` are different components and the protocol
# option picks which one is defined. A pipeline left naming the other
# is a startup failure; an exporter no pipeline names is silence.
name = "the agent forwarder's exporter and its pipeline agree on the protocol";
ok =
let
s = agentSettings agentOtelGrpc;
in
(s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ];
}
{
# The tier in the middle. Its OTLP receiver takes both signals on one
# port, so without this pipeline an agent's push is answered 404 on
# `/v1/logs` — and a forwarder retrying into a 404 is indistinguish-
# able from one with nothing to send. Compared against the metrics
# pipeline's exporters rather than a name, so the two signals cannot
# drift to different destinations.
name = "the hive collector forwards logs upstream, not only metrics";
ok =
(hiveOtelPipelines ? logs)
&& hiveOtelPipelines.metrics.exporters != [ ]
&& hiveOtelPipelines.logs.receivers == [ "otlp" ]
&& hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
}
{
# `deltatocumulative` is metrics-only: naming it in a logs pipeline
# kills the collector at startup rather than doing nothing. The second
# clause is the control — the metrics pipeline still names it, so a
# pass means the two processor lists differ rather than that the
# processor left the module.
name = "the hive collector keeps the metrics-only processor out of its logs pipeline";
ok =
!(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors)
&& builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors;
}
{
# Read against the roster the fixture declares rather than against
# names spelled here: an arm naming `h1` passes on a single-hive
# config however the mapping is written. The length clause is what
# makes the `all` mean anything — over an empty roster it holds
# vacuously.
name = "the swarm collector routes every hive's logs, not just one";
ok =
let
p = (otelSettings otelTwoHives).service.pipelines;
hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives;
in
lib.length hives == 2
&& lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives;
}
{
# The same split as the metrics case above — defining an exporter and
# naming it are two lists — plus the half one shared list cannot have:
# the metrics store's exporter renders perfectly well inside a logs
# pipeline and posts journal records at an ingest route that is not
# for them.
name = "every logs pipeline sends to the log store and to no metrics one";
ok =
let
s = otelSettings otelTwoHives;
logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines;
used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes));
in
logPipes != { }
&& builtins.elem "otlphttp/victorialogs" used
&& !(builtins.elem "otlphttp/victoriametrics" used)
&& lib.all (e: s.exporters ? ${e}) used;
}
{
# The store's seal is spread over six gates — the stanza, the
# provisioning unit, two bind mounts, a device and an EnvironmentFile.