hyperhive/nix/module-eval/agent-otel.nix

# `checks.module-eval-agent-otel` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    agent
    carriesJournaldSeverity
    runGroup
    ;

  # The log path's three hops, one fixture each. Nothing carries a journal
  # record end to end at eval time, so what these defend is the part no tier
  # can check for itself: each hop's output is the next hop's input, and
  # every mismatch between them is silent — a push accepted and routed
  # nowhere, a receiver pointed at an empty directory, a pipeline that does
  # not exist.
  agentBridge = "http://10.42.0.1:4318";

  agentOtel = agent {
    otel.enable = true;
    otel.endpoint = agentBridge;
  };

  # The same agent over the other wire protocol. An exporter's NAME is what
  # selects it, so this is where a defined exporter and the pipeline's
  # reference to it can drift apart. Also an agent config still setting the
  # removed metric interval.
  agentOtelGrpc = agent {
    otel.enable = true;
    otel.endpoint = agentBridge;
    otel.protocol = "grpc";
    otel.metricIntervalMs = 10000;
  };

  # Sets nothing: what an agent gets on a hive, whose meta flake renders no
  # otel line.
  agentDefault = shared.agentBare;

  agentNoOtel = agent { otel.enable = false; };

  agentSettings = machine: machine.services.opentelemetry-collector.settings;
  cases = [
    {
      # The journald receiver's own default directory is the RUNTIME
      # journal, and a container that stores persistently leaves that
      # empty. At the default the forwarder validates, starts, reports
      # healthy and ships nothing, so this one literal is the difference
      # between the path working and silently not.
      name = "the agent forwarder reads the persistent journal, not the runtime one";
      ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal";
    }
    {
      # Wiring, not contents. A journald record arrives with its level in
      # `PRIORITY` and nothing downstream reads that field, so a receiver with
      # no severity parser ships every line to the log store labelled
      # `Unspecified` — healthy, complete, and unqueryable by level.
      #
      # What this asks is only whether THIS receiver carries the shared
      # mapping. ⚠️ Whether that mapping is RIGHT — the inverted direction,
      # which is the part of it a reviewer cannot check by looking — is
      # asserted once, in ./journald-severity.nix, against the file both tiers
      # import. Re-asserting the table here would test that file twice and
      # report one defect as two failures.
      #
      # The swarm collector's own receiver gets the sibling of this case in
      # ./swarm-otel-core.nix. Deliberately NOT folded together with it: they
      # cover different journals — this container's own, and the host's — and
      # one tier quietly losing its parser while the other keeps one is
      # precisely the half-fixed state worth catching.
      #
      # Membership rather than equality of the whole operator list, so a tier
      # that later grows an unrelated operator of its own still passes.
      name = "the agent forwarder's journald receiver carries the shared PRIORITY mapping";
      ok = carriesJournaldSeverity (agentSettings agentOtel).receivers.journald;
    }
    {
      # The hop's two ends: what it reads, and where what it reads goes.
      # The endpoint is compared against the value the fixture handed the
      # option rather than a literal spelled here, so an exporter that
      # stopped reading the option fails instead of matching a constant
      # that travelled beside it.
      name = "the agent forwarder ships the journal to the endpoint its hive gave it";
      ok =
        let
          s = agentSettings agentOtel;
          p = s.service.pipelines.logs;
        in
        p.receivers == [ "journald" ]
        && p.exporters != [ ]
        && lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters;
    }
    {
      # 🎯 What the meta flake no longer says: an agent that sets nothing
      # exports, to a name rather than an address. The name carries no swarm
      # domain and no hive, so the same build is correct on every hive; the
      # resolver of the hive it runs on answers it. `agentNoOtel` above is the
      # control that these lookups find nothing where the switch is off.
      name = "an agent with no otel config exports to the fixed hive-collector name by default";
      ok =
        let
          endpoint = "http://hive-collector.hive:4318";
          env = agentDefault.systemd.globalEnvironment;
          s = agentSettings agentDefault;
        in
        agentDefault.services.hyperhive.agent.otel.enable
        && env.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint
        && agentDefault.environment.variables.OTEL_EXPORTER_OTLP_ENDPOINT == endpoint
        && lib.all (e: s.exporters.${e}.endpoint == endpoint) s.service.pipelines.logs.exporters
        && !(agentNoOtel.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT);
    }
    {
      # The hive's extra labels arrive at container start, so the build
      # cannot hold the whole `OTEL_RESOURCE_ATTRIBUTES`: a boot-time
      # generator computes it, and a unit-level `Environment=` copy would
      # override that. The endpoint beside it is the presence control for the
      # two absence lookups; `agentNoOtel` is the control for the generator.
      name = "an agent's OTEL resource labels come from a boot-time generator, not the build";
      ok =
        let
          generator =
            m: m.environment.etc ? "systemd/system-environment-generators/60-hyperhive-otel-resource";
        in
        agentDefault.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT
        && !(agentDefault.systemd.globalEnvironment ? OTEL_RESOURCE_ATTRIBUTES)
        && agentDefault.environment.variables ? OTEL_EXPORTER_OTLP_ENDPOINT
        && !(agentDefault.environment.variables ? OTEL_RESOURCE_ATTRIBUTES)
        && generator agentDefault
        && !(generator agentNoOtel);
    }
    {
      # Presence control for the two cases above: with the switch off there
      # is no collector in the container at all, so their passing is about
      # the wiring rather than about a unit that renders regardless.
      name = "an agent that switched telemetry off runs no collector";
      ok = !agentNoOtel.services.opentelemetry-collector.enable;
    }
    {
      # `otlp` and `otlphttp` are different components and the protocol
      # option picks which one is defined. A pipeline left naming the other
      # is a startup failure; an exporter no pipeline names is silence.
      name = "the agent forwarder's exporter and its pipeline agree on the protocol";
      ok =
        let
          s = agentSettings agentOtelGrpc;
        in
        (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ];
    }
    {
      # Control: `agentOtel` sets no interval. The endpoint lookup shows the
      # environment it reads is the one the OTEL SDKs get.
      name = "an agent config setting the removed metric interval evaluates, warns, and exports none";
      ok =
        let
          warned = lib.any (
            w: lib.hasInfix "services.hyperhive.agent.otel.metricIntervalMs is removed and ignored" w
          );
          env = machine: machine.systemd.globalEnvironment;
        in
        warned agentOtelGrpc.warnings
        && !(warned agentOtel.warnings)
        && (env agentOtelGrpc) ? OTEL_EXPORTER_OTLP_ENDPOINT
        && !((env agentOtelGrpc) ? OTEL_METRIC_EXPORT_INTERVAL);
    }
  ];
in
runGroup "agent-otel" cases