# `checks.module-eval-hive-otel` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup otelSettings ; # This hive's own collector, which is a HOST service — unlike the swarm # tier's, which lives in a container and is read through `otelSettings`. hiveOtel = hive { otel.enable = true; otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; }; hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines; hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings; # The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for # the swarm tier — the option is one string, what prometheus is handed is a # job — but this collector is a host service, so the path to it differs. hiveScrapeJob = job: lib.findFirst ( c: c.job_name == job ) null hiveOtelSettings.receivers.prometheus.config.scrape_configs; # Duplicated from bao-otel-collector.nix — a case here needs it too. # The store with and without a collector on the same host. `scrapeTargets` # is only ever read by a local collector, so the metrics endpoint is a # function of the pairing rather than of the store. baoWithCollector = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = true; # A second job declared as bare `host:port`, so the pair of cases below # reads one rendered scrape list: the store's entry carries a path, this # one carries none. swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; cases = [ { # The tier in the middle. Its OTLP receiver takes both signals on one # port, so without this pipeline an agent's push is answered 404 on # `/v1/logs` — and a forwarder retrying into a 404 is indistinguish- # able from one with nothing to send. Compared against the metrics # pipeline's exporters rather than a name, so the two signals cannot # drift to different destinations. name = "the hive collector forwards logs upstream, not only metrics"; ok = (hiveOtelPipelines ? logs) && hiveOtelPipelines.metrics.exporters != [ ] && hiveOtelPipelines.logs.receivers == [ "otlp" ] && hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters; } { # `deltatocumulative` is metrics-only: naming it in a logs pipeline # kills the collector at startup rather than doing nothing. The second # clause is the control — the metrics pipeline still names it, so a # pass means the two processor lists differ rather than that the # processor left the module. name = "the hive collector keeps the metrics-only processor out of its logs pipeline"; ok = !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors) && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors; } { # The counters that say telemetry is being LOST — refused, failed, # queue depth — are served on loopback and reach no store unless # something reads them. Read off the rendered job rather than the # option: only the job is what prometheus actually requests. name = "the hive collector scrapes its own telemetry endpoint"; ok = let j = hiveScrapeJob "collector"; in j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ]; } { # A `prometheus` receiver no pipeline names collects nothing while # rendering and starting perfectly, so the scrape above is inert # without this. The path is newly reachable: until the hive tier had a # target of its own, this receiver was never emitted on any hive. name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs"; ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers; } { # `metrics.address` is the spelling that looks right and is rejected by # this collector version. The first clause is the control: without it a # missing telemetry block would pass the port check vacuously. name = "the hive collector binds its telemetry port through readers, not address"; ok = let m = hiveOtelSettings.service.telemetry.metrics; in !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888; } { # Both collectors share a network namespace whenever they are # co-located, and this port appears in no config the port-collision # assertion can read — so equal defaults mean the second to start dies # at `bind()`. Pinned as a case rather than an assertion: enforcing it # belongs with the other port checks, not here. name = "the two collector tiers do not claim the same self-telemetry port"; ok = let portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port; in portOf hiveOtelSettings != portOf (otelSettings baoWithCollector); } ]; in runGroup "hive-otel" cases