# The HIVE tier of the collector pair: every hive runs one, on the host that # runs hive-c0re (`deploy.hive-controller.enable`). Agents export to it at # the fixed name `services.hyperhive.otel.collector.host`, which this hive's # resolver answers with its own bridge IP, so an agent's build carries # nothing hive-specific. # # This tier receives from this hive's agents and forwards to the swarm's # collector (./swarm-otel.nix). It holds no credential and picks no # destination: an agent's samples cross a hive boundary exactly once, and # what happens after that is the swarm's decision, not a hive's. The # upstream export is a swarm-collector deploy option, # `services.hyperhive.deploy.swarm-otel.upstream` (./swarm-otel.nix); the # old `otel.*` paths forward to it below. { pkgs, lib, config, ... }: let # This tier now reaches the swarm's collector by name through the # gateway (`swarm-otel-service.nix`'s `domain`) instead of a loopback URL, so it # needs the same trust-bundle wiring every other host consumer of an `https://` # swarm-service name needs — see `swarm-controller.nix` for the sibling # wiring this copies. # # `hostUnit`: `opentelemetry-collector` is a host systemd service, not a # container, so it reads the CA from the host path and the bundle oneshot # waits on `hive-tls-ca.service` itself. `enable`: `imports` is # unconditional at the host's top level, so without it a hive with this # tier off would still get a bundle oneshot and a phantom # `opentelemetry-collector` service holding an `SSL_CERT_FILE`. caTrust = import ./lib/hive-ca-trust.nix { inherit lib; tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; gatewayCfg = config.services.hyperhive.gateway; }; # The hive's own store identity. The collector authenticates to the swarm # as the hive, so it reads its secret as the hive: the hive leaf's policy # (`swarm-secret-client`'s `policy::render`) grants `swarm/hives//*`. runsHive = config.services.hyperhive.otel.collector.enable; hiveLeaf = config.services.hyperhive.deploy.bao.clientCertFile != null && config.services.hyperhive.deploy.bao.clientKeyFile != null; # Where `swarm-bao-hive-otel-oidc` lands the secret on this host. fetchedSecretPath = "/var/lib/hive-otel-oidc/${config.services.hyperhive.swarm.authelia.hiveClientPrefix}${toString config.services.hyperhive.hiveName}.secret"; in { imports = [ (caTrust.trustBundle { inherit pkgs; name = "hive-otel"; consumers = [ "opentelemetry-collector" ]; hostUnit = true; enable = runsHive; }) (lib.mkRenamedOptionModule [ "services" "hyperhive" "otel" "endpoint" ] [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "endpoint" ] ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "otel" "protocol" ] [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "protocol" ] ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "otel" "headersCredential" ] [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headersCredential" ] ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "otel" "collector" "upstreamHeaderName" ] [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headerName" ] ) (lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "enable" ] '' Every hive runs its own telemetry collector, and every agent exports to it by default, so there is no switch. Remove this definition. '') (lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "collector" "port" ] '' The hive collector's receiver port is fixed at 4318: agents dial that literal. Remove this definition. '') ]; options.services.hyperhive.otel = { collector.enable = lib.mkOption { type = lib.types.bool; readOnly = true; default = config.services.hyperhive.deploy.hive-controller.enable || config.services.hyperhive.deploy.allSwarmServices; defaultText = lib.literalExpression "deploy.hive-controller.enable || deploy.allSwarmServices"; description = '' Whether this host runs the hive tier's collector. Not a switch: every host that runs a hive ({option}`services.hyperhive.deploy.hive-controller.enable`) runs one, and so does a host carrying all the swarm's shared services ({option}`services.hyperhive.deploy.allSwarmServices`), whose own containers export to it. Agents export to it unauthenticated at {option}`services.hyperhive.otel.collector.host`, a name this hive's resolver answers with a bridge address only its own containers can reach. It forwards to the swarm's collector ({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds the upstream credential and writes the swarm's store, so neither an agent nor this tier ever sees the credential. ⚠️ The collector is in the path of all telemetry. It runs on the same host as the agents and restarts on failure; telemetry is not the control plane, so degraded telemetry is not degraded operation, but the export does not survive independently of anything host-side. ''; }; collector.host = lib.mkOption { type = lib.types.str; readOnly = true; default = "hive-collector.hive"; description = '' The name agents reach this hive's collector on. A literal, the same on every hive and in every swarm: each hive's own resolver answers it with its own bridge IP, so an agent's build carries neither a swarm domain nor a hive-specific address. The agent module's `services.hyperhive.agent.otel.endpoint` default spells the same name. It sits outside every domain a hive or the swarm serves: a hive's domain is `.`, so it never ends in `.hive`. ''; }; telemetryPort = lib.mkOption { type = lib.types.port; default = 8888; description = '' Port this collector serves its **own** metrics on — queue depth, refused and dropped samples, exporter failures. How you find out that telemetry is being lost. 8888 is the collector's built-in default; declaring it here makes the value *configured* rather than implied, which is the point of the option. An undeclared port is invisible to anything comparing ports, so a second collector on the same host picking the same number fails at `bind()` rather than at eval — `services.hyperhive.swarm.otel.telemetryPort` sidesteps 8888 by hand for that reason. ''; }; scrapeTargets = lib.mkOption { type = lib.types.attrsOf lib.types.str; default = { }; example = lib.literalExpression ''{ hive-forge = "127.0.0.1:3000"; }''; description = '' Prometheus exposition endpoints this hive's collector scrapes, as ` = ":"`. **A service declares its own entry, from its own module, under its own `enable`** — the same rule as the swarm tier's option of the same name, and for the same reason: an entry exists only where the service that named it runs, so the scraper and its target are co-located by construction rather than by luck. ⚠️ **This tier, not the swarm one, is where a HIVE-owned target belongs.** The two are not interchangeable: a swarm service does not belong to a hive, so its samples must never acquire a `hive` label — which is why that scraper lives one tier up. Putting a hive-owned target there would either mislabel it or leave it unattributed. Scraping is unauthenticated on purpose. This receiver is a prometheus-to-OTLP converter sitting next to what it reads, in the same trust position as the agents already pushing to this collector; the authenticated hop is the collector's own export onward, which is one hop for the whole hive rather than one per target. Not empty on a default hive: this collector's own endpoint is always in here, so the scrape receiver is emitted everywhere. The `!= { }` guards around it stay meaningful — the set can still be forced empty, and a `prometheus` receiver with nothing to scrape is a config that renders, starts and collects nothing. ''; }; clientSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = if hiveLeaf && config.services.hyperhive.hiveName != null then fetchedSecretPath else null; defaultText = lib.literalExpression ''"/var/lib/hive-otel-oidc/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when deploy.bao.clientCertFile and clientKeyFile are set, else null''; example = "/var/lib/secrets/hive-telemetry.secret"; description = '' Absolute path to this hive's OAuth2 client secret, used to authenticate to the swarm's collector as this hive. **Whether this is set is what decides whether this tier authenticates at all.** A credential is the only thing that makes authenticated egress possible, so its presence is the condition rather than a second switch that could disagree with it. Defaults, on a host holding the hive's store identity ({option}`services.hyperhive.deploy.bao.clientCertFile`), to the file `swarm-bao-hive-otel-oidc.service` fetches from the swarm secret store under that identity. Set it to deliver the secret some other way; that unit then does not exist. Read by `LoadCredential`, so it is never evaluated by nix, never copied into the store and never passed in argv. ''; }; }; # Shared by every hive's otel tier and its agents. Read-only because the agent # module's `otel.endpoint` default spells the same literal, and a host that # moved the port would leave its agents dialling the old one. options.services.hyperhive.swarm.hiveOtel = { collectorPort = lib.mkOption { type = lib.types.port; default = 4318; readOnly = true; description = '' Port of every hive collector's OTLP/HTTP receiver, at that hive's `services.hyperhive.network.bridgeIp`: the port agents dial at `services.hyperhive.otel.collector.host`. 4318 is the OTLP/HTTP default. The port is contributed to `services.hyperhive.network.exposeHostPorts`, which opens it on the bridge interface only — so it is reachable from agent containers and not from the outside world. ''; }; }; # Read by this hive's hive-c0re, which stamps it on its own export and # renders it into its agents', so it is a per-host choice. options.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes = lib.mkOption { type = lib.types.str; default = ""; example = "deployment.environment=prod"; description = '' Extra comma-separated entries appended to `OTEL_RESOURCE_ATTRIBUTES` of this hive's hive-c0re and its agents, after the built-in `service.name` / `agent` / `hive` / `swarm` labels. ''; }; config = lib.mkIf runsHive ( let otel = config.services.hyperhive.otel; autheliaCfg = config.services.hyperhive.swarm.authelia; deployCfg = config.services.hyperhive.deploy; swarmOtelCfg = config.services.hyperhive.swarm.otel; hiveOtelCfg = config.services.hyperhive.swarm.hiveOtel; hiveName = config.services.hyperhive.hiveName; listen = "${config.services.hyperhive.network.bridgeIp}:${toString hiveOtelCfg.collectorPort}"; swarmName = "otlphttp/swarm"; gatewayCfg = config.services.hyperhive.gateway; authName = "oauth2client/swarm"; # A hive always authenticates to the swarm's collector as itself, so # this is not a mode — it is a precondition, and the assertion below # is what enforces it. Kept as a name because several places have to # read "do we have what it takes", and an eval error from a null path # names this file rather than the option an operator has to set. senderAuth = otel.clientSecretFile != null && hiveName != null; # This hive's client id, and also the audience it must ASK for. Both # are `hiveClientPrefix` + the hive's name because that is the one # name the swarm already agrees on; the receiver one tier up derives # the same string. hiveClient = "${autheliaCfg.hiveClientPrefix}${toString hiveName}"; # systemd exports `CREDENTIALS_DIRECTORY` to any unit with # `LoadCredential`, and the collector expands `${env:…}` at load. So # the secret reaches the process as a PATH resolved at runtime — nix # renders neither the value nor the directory, and nothing has to # hardcode `/run/credentials/`. credName = "swarm-client.secret"; baoCfg = config.services.hyperhive.swarm.bao; fetchUnit = "swarm-bao-hive-otel-oidc"; # Only when the collector reads the file the unit writes: a secret # delivered some other way is the operator's, and a unit restarting the # collector on a file it never loads would be noise. # ./glue-bao-readers-store-order.nix restates this gate. fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath; # An operator-delivered file has no unit that restarts the collector # once it lands, so the collector waits for it instead. # ./glue-hive-otel-authelia.nix restates this gate. waits = senderAuth && !fetches; secretWaitUnit = "hive-otel-client-secret"; # Written by swarm-secret-publish in authelia's container. The nix half of # `swarm_secret_client::queue::hive_client_path`. storeSecretPath = "secret/swarm/hives/${toString hiveName}/oidc/client"; atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; storeRetry = import ./lib/store-retry.nix { }; storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in { # Reachable from agent containers and nowhere else: this opens # the port on the bridge interface only. services.hyperhive.network.exposeHostPorts = [ hiveOtelCfg.collectorPort ]; # The name agents dial, answered with the bridge IP the receiver binds. services.hyperhive.gateway.localNames = [ otel.collector.host ]; # …and the receiver binds that bridge address, so the interface has # to exist. `exposeHostPorts` is a firewall hole, not a demand. services.hyperhive.network.enable = lib.mkDefault true; # The collector scrapes itself. Its own counters — refused, failed, # queue depth — are the only signal that says telemetry is being # dropped, and they reach no store unless something reads them. # # Loopback, so this deliberately does NOT go through # `exposeHostPorts`: the endpoint is for the collector beside it, # not for agent containers. services.hyperhive.otel.scrapeTargets.collector = "127.0.0.1:${toString otel.telemetryPort}"; services.opentelemetry-collector = { enable = true; # Contrib, matching the swarm tier (./swarm-otel.nix). The upstream # default build has no auth extensions at all, and this tier has to # *present* a credential to the swarm tier — `oauth2client` lives # only in contrib, so the package choice is what makes authenticated # egress expressible rather than a preference. # # Not a build-farm cost: contrib is fetched, not compiled. # # ⚠️ Read the note directly below before adding any extension here. # It describes precisely the trap this package unlocks: naming an # extension the build lacks passes `validate` and then kills the # collector at startup. With contrib the extensions exist — but the # gap it warns about (a green build proving nothing about whether # the process starts) is exactly why this module's auth wiring is # gated by a probe that runs both collectors, not by eval. package = pkgs.opentelemetry-collector-contrib; # `validateConfigFile` defaults to `isStorePath configFile`, # and `configFile` is null on the `settings` path — so the # upstream default is OFF for exactly the way this module # configures it. Turning it on runs `otelcol validate` at # build time, which is the collector checking its own config. # ⚠️ It is a PARSER, not a wiring check, and the gap is wider # than "no sample was sent": measured 2026-08-15, `validate` # ACCEPTS a receiver naming an auth extension that is absent # from the build, and the collector then dies at startup with # `Failed to start component`. So a green build does not # prove this config STARTS, never mind that a sample arrives. validateConfigFile = true; settings = { # Every extension is defined in this one attrset: a second # `extensions` joined onto `settings` with `//` replaces this one # instead of adding to it. extensions = { # The journald receiver's cursor store, same shape as the agent # tier's own (`nix/agent-modules/otel.nix`): a restart resumes # from the last-read position instead of re-reading from the # journal's tail (silently dropping whatever landed while the # collector was down) or its head (re-sending everything). # `create_directory` matters at BUILD time: `validateConfigFile` # runs `otelcol validate` in the nix sandbox, where systemd's # `StateDirectory=` has never run and the path does not exist # yet — without it the validator refuses with "directory must # exist". file_storage = { directory = "/var/lib/opentelemetry-collector"; create_directory = true; }; } // lib.optionalAttrs senderAuth { ${authName} = { client_id = hiveClient; # A real key, measured against this collector version rather # than assumed — with a deliberate typo rejected in the same # run, so "accepted" is distinguishable from "ignores # everything". Keeps the secret out of nix entirely: the # collector opens the file itself. client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credName}"; token_url = "${toString autheliaCfg.url}/api/oidc/token"; # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. # Registering it on the client only makes it permissible; a # token minted without asking carries `aud: []` and every # receiver refuses it — with a config that reads perfectly at # both ends. Measured against authelia 4.39.20. endpoint_params.audience = hiveClient; }; }; receivers = { otlp.protocols.http.endpoint = listen; # The host itself, which nothing else measures. hive-c0re emits # per-container metrics and each service exports its own, so the # store could answer "is this container using more memory than # last week" and could not answer "is the machine under them out # of memory" — the question every one of those readings is # implicitly relative to. # # Here rather than in ./swarm-otel.nix because a host is owned by # the hive running on it: the swarm tier runs one collector for # the whole swarm and would have to reach *other* machines to see # theirs. Consequence worth knowing before reading a dashboard: # these samples carry `hive` and no `agent`, so "which host" is # answered by "which hive" — true only while a hive is one host. # # Every scraper the pinned receiver offers except `process`, and # the exception is not squeamishness about volume: `process` adds # a series set PER RUNNING PROCESS, which on a hive host means # every agent's claude and every nix build. That is unbounded and # churning, where the nine below are a fixed handful per host # forever. Coverage and cardinality are different axes and only # the second one can hurt the store. # # Names verified against THIS collector version rather than # upstream's docs, with a deliberately bogus scraper rejected in # the same run so that "accepted" is distinguishable from "the # validator ignores unknown keys". `validateConfigFile` above # keeps that honest at build time. hostmetrics = { # Stated rather than inherited: the upstream default is a # value that can change under a version bump, and the sample # interval is the one knob here with a storage bill attached. collection_interval = "60s"; scrapers = { cpu = { }; memory = { }; filesystem = { }; load = { }; disk = { }; network = { }; paging = { }; # ⚠️ `processes` is the SUMMARY -- how many are running, # blocked, sleeping. A handful of series per host. Its # singular sibling `process` is the per-process one and is # deliberately absent above; the two differ by one letter and # a grep for the shorter name matches both, so read the # trailing `s` before concluding which is configured here. processes = { }; system = { }; }; }; # This tier's own host journal, scoped to the one unit whose # crash-watch warning (`"container crash detected"`, # `crash_watch.rs`) a swarm-level grafana alert # (`hh-container-crashed` in `swarm-grafana.nix`) depends on # reaching VictoriaLogs. Unlike the swarm tier's journald # receiver (`swarm-otel.nix`) this needs no bind mount: this # collector is a HOST systemd unit, not a container, so # `/var/log/journal` is the same filesystem hive-c0re itself # logs into — no id-mapping gap to cross. # # `units`, unlike the swarm tier's unfiltered receiver: this # collector has no `filter/exclude-user-sessions` processor, # and a hive host can be someone's workstation too, so an # allowlist is load-bearing here rather than a tidiness # preference. journald = { directory = "/var/log/journal"; storage = "file_storage"; units = [ "hive-c0re.service" ]; # Severity table shared with the agent and swarm tiers' # journald receivers — see ../journald-severity.nix for why # it is its own file. operators = import ../journald-severity.nix; }; } # Only when a service has actually declared a target. An enabled # `prometheus` receiver with an empty `scrape_configs` renders, # validates and starts perfectly while reading nothing — and the # empty set is the default, so that inert shape would be what # most hives deploy. // lib.optionalAttrs (otel.scrapeTargets != { }) { prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: { job_name = job; static_configs = [ { targets = [ target ]; } ]; }) otel.scrapeTargets; } # This host's gateway nginx, from the `stub_status` server # ./hive-gateway/default.nix adds. The receiver dials `http://` # only, which is why that server is on loopback TCP. // lib.optionalAttrs gatewayCfg.enable { nginx = { endpoint = "http://127.0.0.1:${toString gatewayCfg.statusPort}/stub_status"; collection_interval = "60s"; }; }; # One destination, and it is the swarm's collector. This tier # holds no upstream credential and writes no store: it receives # from this hive's agents and forwards, which is the whole of # its job. Everything that decides where telemetry ultimately # goes lives one tier up, in ./swarm-otel.nix. exporters.${swarmName} = { # ⚠️ Plain `endpoint`, and the exporter beside this one in # ./swarm-otel.nix warns against exactly that spelling — read # both before "fixing" either. The difference is the far end, # not the exporter: `endpoint` is a BASE that otlphttp appends # `/v1/metrics` to, which is precisely the path an OTLP/HTTP # receiver serves. VictoriaMetrics is the odd one out, serving # OTLP at `/opentelemetry/api/v1/push`, and that is why the # store exporter needs `metrics_endpoint` while this one must # not have it. # # By name through the gateway, not a loopback literal: a # loopback literal is correct only while listener and caller # share a netns, an assumption that has cost this project two # outages, and it is exactly the split-host case a swarm # service name exists to make a config fact rather than a code # change. `https://` because that name resolves through the # gateway even on a co-located host — see `caTrust` above for # the trust half that makes this verify. # The hive's own path under the collector's single name. The # swarm tier gives each hive its own authenticated receiver and # routes to it by this prefix, so the path is not decoration — # it selects WHICH receiver, and therefore which hive the # samples get labelled as. endpoint = "https://${swarmOtelCfg.domain}" + lib.optionalString senderAuth "/${toString hiveName}"; } // lib.optionalAttrs senderAuth { auth.authenticator = authName; }; # ⚠️ An extension not listed here is INERT: the collector starts # clean and the exporter naming it sends nothing authenticated — # and, for `file_storage`, the journald receiver's `storage:` # above would name a component the collector never starts. service.extensions = [ "file_storage" ] ++ lib.optional senderAuth authName; # Host identity, so a sample can say WHICH machine it came from. # Without this the host's metrics carry `hive` and no host # attribute, so "which host" is answered by "which hive" — true # only while a hive is one machine, which is exactly the framing # the swarm work exists to end. # # `resourcedetection` rather than a value chosen in nix: this is # the collector's own canonical mechanism for host identity, and # picking our own would answer a question the tool already # answers, differently from every other OTel deployment. A # resource attribute is CAPTURED into the stored series, so a # private naming scheme is expensive in the one way that lasts — # every historical comparison keys on it. # # Keys verified against the collector binary rather than # upstream's docs, with two deliberate mutations rejected in the # same run — a bogus resource attribute and a bogus detector key # — so "accepted" is distinguishable from a validator that # ignores what it does not recognise. `validateConfigFile` above # keeps that true at build time against the pinned version. processors.resourcedetection = { detectors = [ "system" ]; # BOTH attributes, deliberately: # host.name — readable, and what a panel groups by # host.id — /etc/machine-id, survives a rename # A name-keyed history splits silently the moment a machine is # renamed or repurposed, with nothing linking old to new — and # hosts get renamed and repurposed precisely when services move # off one box. Neither alone is sufficient: an id nobody can # read is not a dashboard, and a name that moves is not # history. Cost is one extra label per series. system.resource_attributes = { "host.name".enabled = true; "host.id".enabled = true; }; }; # Converts any DELTA-temporality counter to CUMULATIVE before it # leaves this hive. `hive-metric` counters are delta by default # (agent ergonomics — see its own docs); the swarm's store # (VictoriaMetrics) has no `deltatocumulative` of its own and # official guidance is to feed it cumulative. Default config # (`{ }`) is a no-op passthrough for anything already cumulative # — Claude Code's own export, `hostmetrics`, and any Prometheus # scrape all stay exactly as they were. Verified against the # collector binary rather than assumed: an empty config validates # clean, an invalid key is rejected (`otelcol-contrib validate`). processors.deltatocumulative = { }; service.pipelines.metrics = { # EXTENDED, not replaced. Assigning here instead of appending # would drop `otlp` — the hive would stop receiving from its # own agents while still rendering a config the collector # starts cleanly on, so nothing would report the loss. The # `optional` matters too: a pipeline naming a receiver that no # longer exists is a startup failure, which `validateConfigFile` # below turns into a build failure rather than a broken deploy. # `hostmetrics` is unconditional, unlike `prometheus` above: it # has no per-hive configuration to be empty of, so there is no # inert shape to guard against — the receiver either collects the # host's metrics or the host has none collected at all. receivers = [ "otlp" "hostmetrics" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus" ++ lib.optional gatewayCfg.enable "nginx"; # Applies to EVERY receiver above, not just `hostmetrics` — a # processor sits in the pipeline, not on a receiver. That is # what we want here: samples an agent pushed over `otlp` and # samples scraped from a target both arrive at the store with # the identity of the machine that collected them, and a # reading is only comparable across hives once it says where # it came from. `deltatocumulative` first, so a delta counter is # already fixed up by the time it picks up its host identity. processors = [ "deltatocumulative" "resourcedetection" ]; exporters = [ swarmName ]; }; # The second signal this tier carries. Two distinct sources, for # two distinct reasons: # - `otlp`: an agent's journal is unreadable from here (the # host-side per-container journal directory is an id-mapped # bind mount journald never writes into), so each agent # container runs its own forwarder (nix/agent-modules/otel.nix) # and pushes over the `otlp` receiver above. Without this # pipeline that receiver answers 404 on `/v1/logs` — measured, # and indistinguishable from a route that was never meant to # exist. # - `journald`: this collector's own host journal, scoped to # `hive-c0re.service` above — no id-mapping gap to cross here, # since this collector IS the host, unlike an agent. # # One `otlp` receiver, two pipelines: it accepts both signals and # each pipeline takes the one it names. The exporter needs no # change either — its `endpoint` is a base, and otlphttp appends # `/v1/logs` to it exactly as it appends `/v1/metrics`. service.pipelines.logs = { receivers = [ "otlp" "journald" ]; # `resourcedetection` and NOT `deltatocumulative`: that one is # a metrics-only processor, and naming it here is a startup # failure rather than a no-op. Host identity applies to both # signals for the same reason — a line is only comparable # across hives once it says which machine collected it. processors = [ "resourcedetection" ]; exporters = [ swarmName ]; }; # Bound explicitly rather than left to the collector's built-in # default, so `telemetryPort` is a value something can compare # instead of an assumption. # # ⚠️ `metrics.address` is the spelling that looks right and is # rejected by this collector version — ./swarm-otel.nix documents # the exact error it produces. `readers` is the schema it accepts, # and the difference is a startup failure rather than a warning. service.telemetry.metrics.readers = [ { pull.exporter.prometheus = { host = "127.0.0.1"; port = otel.telemetryPort; }; } ]; }; }; systemd.services.opentelemetry-collector = lib.mkMerge [ # ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them under # `[Service]` — in `serviceConfig` they render, deploy and do # nothing. `checks.module-eval-hive-otel` pins them in `unitConfig`. # `Restart` is deliberately NOT set: nixpkgs' own module defines it # at normal priority, so a second definition is a module-system # conflict. The window must exceed `RestartSec × burst`. { startLimitBurst = 12; startLimitIntervalSec = 120; serviceConfig.RestartSec = 5; } # `LoadCredential` and not a copy-oneshot: this collector is a HOST # unit, so there is no container boundary to cross and therefore no # reason for a second on-disk copy of the secret. systemd hands it to # the process in a private tmpfs and exports the directory, which is # what the config above names. (lib.mkIf senderAuth { serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; }) # `requires`, not merely `after`: ordering without gating leaves the # collector starting anyway and failing on its own `LoadCredential`, # which is the failure the wait exists to prevent. Same reasoning # `lib/hive-ca-trust.nix` records for its bundle consumers. (lib.mkIf waits { requires = [ "${secretWaitUnit}.service" ]; after = [ "${secretWaitUnit}.service" ]; }) ]; # Wait for an operator-delivered credential rather than racing it. # # `LoadCredential` naming a missing path is FATAL AT UNIT START, and # nothing orders this host unit against whatever delivers the file. # Worse, nixpkgs # sets `Restart = "always"` with no `RestartSec`, so the failure is # instant and the unit burns systemd's 5-starts-in-10s allowance in well # under a second, landing in `start-limit-hit` where it stops retrying # entirely and needs `systemctl reset-failed` by hand. # # 🔑 `Restart = always` reads like it makes this self-healing and does # the opposite: a SLOW-failing unit retries until the secret appears, a # FAST-failing one exhausts its limit before the thing it waits for can # exist. This oneshot converts the fast failure into a slow one, which # is what that restart policy is actually good at. # # Not rendered beside the store fetch below, which restarts the # collector itself when the file lands. systemd.services.${secretWaitUnit} = lib.mkIf waits { description = "wait for this hive's telemetry client secret"; # Deliberately says nothing about the collector: the CONSUMER declares # `requires` + `after` above, which both pulls this into the # transaction and gates on it. Stating the order from both sides too # would be one more thing to keep consistent for no added guarantee. # # Where authelia runs on this host the named file may be one it mints; # ./glue-hive-otel-authelia.nix orders this unit after its container # and restates `waits`. serviceConfig = { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = secretWaitUnit; # ⚠️ Longer than the bounded wait below. `DefaultTimeoutStartSec` is # 90s, so without this systemd kills the unit at 90 — before it can # emit the message naming the file it was waiting for. TimeoutStartSec = "180s"; }; path = [ pkgs.coreutils ]; script = '' set -euo pipefail secret=${lib.escapeShellArg (toString otel.clientSecretFile)} # Bounded, then FAIL — never skip. A silent skip produces a # collector that starts and ships nothing, which is the failure # mode this whole path is trying not to have. for _ in $(seq 1 60); do [ -s "$secret" ] && exit 0 sleep 2 done echo "telemetry client secret $secret has not appeared after 120s" >&2 exit 1 ''; }; # This hive's OIDC client secret, out of the swarm secret store under the # hive's own leaf. Same shape as ./hive-matrix.nix's # `swarm-bao-matrix-oidc`: the login and the read both fail loudly, and # `storeRetry` retries until the publisher in authelia's container has # written the secret, because the collector does not start without it. # # The collector re-reads `client_secret_file` on every token fetch, but # the file it reads is `LoadCredential`'s copy, made at start: a changed # secret reaches it only by a restart, which `refresh_unit` does. systemd.services.${fetchUnit} = lib.mkIf fetches { description = "fetch this hive's telemetry OIDC client secret from the swarm secret store"; # Ordered after the store where it runs here # (./glue-bao-readers-store-order.nix, which restates `fetches`); # elsewhere the timeout below bounds the attempt. before = [ "opentelemetry-collector.service" ]; wantedBy = [ "multi-user.target" "opentelemetry-collector.service" ]; path = [ deployCfg.bao.package pkgs.coreutils pkgs.systemd ]; # ./lib/store-retry.nix. The collector waits for one attempt; a secret # a later attempt lands restarts it (below). inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { # No `RemainAfterExit`: the timer below starts this again, and # starting an active unit is a no-op. Type = "oneshot"; SyslogIdentifier = fetchUnit; TimeoutStartSec = 30; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; BAO_CLIENT_CERT = deployCfg.bao.clientCertFile; BAO_CLIENT_KEY = deployCfg.bao.clientKeyFile; } // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) { BAO_CACERT = deployCfg.bao.serverCaFile; }; script = '' set -euo pipefail ${atomicWriteSecret} ${refreshConsumer} err="$(mktemp)" trap 'rm -f "$err"' EXIT # `-token-only` answers on stdout and skips the token helper, a # `sh` this unit's `path` does not carry. if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "could not log in to swarm-bao with this hive's certificate; leaving the telemetry collector's OIDC client secret as it is." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 1 fi export BAO_TOKEN if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then echo "swarm-bao did not return ${storeSecretPath}; the telemetry collector has no OIDC client secret yet." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 1 fi if [ -z "$secret" ]; then echo "swarm-bao returned an empty ${storeSecretPath}; the telemetry collector has no OIDC client secret yet." >&2 exit 1 fi # Root-owned 0400: the collector runs under `DynamicUser`, so there is # no stable uid to hand the file to, and `LoadCredential` reads it as # root before the sandbox exists. install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf fetchedSecretPath)} if secret_differs ${lib.escapeShellArg fetchedSecretPath} "$secret"; then atomic_write_secret 0400 "" ${lib.escapeShellArg fetchedSecretPath} "$secret" fi refresh_unit opentelemetry-collector.service ${lib.escapeShellArg fetchedSecretPath} ''; }; systemd.timers.${fetchUnit} = lib.mkIf fetches ( storeRefreshTimer "re-read this hive's telemetry OIDC client secret from the swarm secret store" ); assertions = [ { # A hive authenticates to the swarm's collector as itself — there # is no unauthenticated path to fall back to, so a missing # credential is a broken deployment rather than a quieter mode. # Caught here because the alternative is a collector that starts # cleanly, retries forever, and reports nothing to anyone. assertion = senderAuth; message = '' This hive runs a telemetry collector (every hive does) but has no identity to present to the swarm's collector: services.hyperhive.otel.clientSecretFile = ${ if otel.clientSecretFile == null then "null" else otel.clientSecretFile } services.hyperhive.hiveName = ${if hiveName == null then "null" else hiveName} Every hive authenticates as itself — that is what makes the `hive` label on its metrics mean anything — so both are required. With the hive's store identity set (services.hyperhive.deploy.bao.clientCertFile and clientKeyFile), the default is the file swarm-bao-hive-otel-oidc.service fetches under it. On a hive that runs the store both are set already; elsewhere place the hive's leaf and name it there — see docs/swarm/secrets.md. ''; } { # Same guard as ./swarm-otel.nix's sibling journald receiver, and # the same reason: the receiver reads /var/log/journal, which only # exists when journald stores persistently. With "volatile" the # journal lives in /run/log/journal instead and with "none" there # is none at all — either way `nixos-container`'s analogue for a # host unit is simpler but no less fatal: the collector's # `journald` receiver fails to open the directory and the unit # never starts, with nothing here saying why. assertion = !(lib.elem config.services.journald.storage [ "volatile" "none" ]); message = '' This hive runs a telemetry collector, which runs a journald receiver reading /var/log/journal for hive-c0re's own crash-watch log line — but services.journald.storage is "${config.services.journald.storage}" on this host. Set services.journald.storage = "persistent" (the NixOS default). ''; } ]; } ); }