diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index d505a6a2..666490b5 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -27,6 +27,7 @@ ./swarm-nats.nix ./swarm-controller.nix ./swarm-grafana.nix + ./swarm-otel.nix ./swarm-snapshot-store.nix ./swarm-ui.nix ./swarm-victoriametrics.nix diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix new file mode 100644 index 00000000..23ae296e --- /dev/null +++ b/nix/host-modules/swarm-otel.nix @@ -0,0 +1,213 @@ +# The swarm's telemetry collector: one per swarm, in a `swarm-otel` +# nixos-container beside the swarm's other shared services. +# +# Two tiers, and they are separate on purpose: +# +# - `otel.nix` is the **hive** tier. It receives from this hive's agents +# on the bridge and forwards, and it holds no upstream credential. +# - this is the **swarm** tier. It is the only holder of the upstream +# credential, the only writer to the swarm's metrics store, and (once +# #3283 lands) the place that stamps `hive=` from the authenticated +# connection rather than from anything a sender can choose. +# +# On a host that runs both, both processes run. They are not collapsed: +# all-local is a statement about *where* processes run, not about what +# shape the deployment has, and a local tier boundary that disappears is +# one the local deployment stops testing. `hive=` attribution is the +# property that would differ, and it is the one #3283 depends on. +# +# A container rather than a second host unit, for the same reason every +# sibling swarm service is one — and because `services.opentelemetry-collector` +# is a singleton NixOS option, already spoken for on the host by the hive +# tier. A container gets its own evaluation and therefore its own collector. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.otel; + swarmCfg = config.services.hyperhive.swarm; + otelCfg = config.services.hyperhive.otel; + vmCfg = config.services.hyperhive.swarm.victoriametrics; +in +{ + options.services.hyperhive.swarm.otel = { + enable = lib.mkOption { + type = lib.types.bool; + default = swarmCfg.enableRequiredServices; + defaultText = lib.literalExpression "services.hyperhive.swarm.enableRequiredServices"; + description = '' + Run the swarm's telemetry collector on this host. + + Derived from `swarm.enableRequiredServices` like the swarm's other + shared services: a swarm has one of these, and it belongs wherever + the shared services live rather than on every hive. + + A hive that does not run it still runs its own hive-tier collector + (`services.hyperhive.otel.enable`) and points it here with + {option}`services.hyperhive.swarm.otel.url`. + ''; + }; + + machine = lib.mkOption { + type = lib.types.str; + default = "swarm-otel"; + description = '' + Name of the nixos-container this collector runs in — also the + `machinectl` name, so other modules may read it rather than + repeating the literal. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 4319; + description = '' + Port this collector's OTLP/HTTP receiver listens on. + + ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the + hive tier already uses it (`services.hyperhive.otel.collector.port`) + and every swarm container shares the host's network namespace. Two + listeners claiming one port on one host is not a build failure — + it is a runtime coin toss over which one gets it, with nothing in + any log saying so. The same collision cost a release when grafana + and the forge both defaulted to 3000. + ''; + }; + + url = lib.mkOption { + type = lib.types.str; + default = "http://127.0.0.1:${toString cfg.port}"; + defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString config.services.hyperhive.swarm.otel.port}"''; + description = '' + Where the **hive** tier sends what it receives — this collector's + OTLP/HTTP base URL. + + The default addresses it on loopback, which is correct while the + two tiers share a host: every swarm container runs in the host's + network namespace, so a swarm service is reachable there exactly + as the metrics store already is. + + ⚠️ That default is a *default*, not an assumption baked into the + exporter. A hive whose swarm collector runs elsewhere sets this to + that host's address, and nothing else changes — a loopback literal + written directly into the exporter would have made the split-host + case a code change instead of a config one. + ''; + }; + }; + + config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { + assertions = [ + { + # The tier exists to hold the upstream credential and to write the + # swarm's store. With neither, it is a process that receives + # samples and drops them — which looks healthy and loses data. + assertion = otelCfg.endpoint != "" || vmCfg.enable; + message = '' + services.hyperhive.swarm.otel.enable is true but this collector + has nowhere to send what it receives: + services.hyperhive.otel.endpoint is empty and + services.hyperhive.swarm.victoriametrics.enable is false. + + Set the endpoint to export upstream, or enable the swarm's + metrics store. + ''; + } + ]; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm service: the hive tier + # reaches this collector, and this collector reaches the metrics + # store, without either crossing a network boundary that would need + # its own trust material. + privateNetwork = false; + + # The upstream credential is operator-provided and lives on the host. + # Read-only, and only when one is configured — binding a path that + # does not exist makes nixos-container refuse to start the container, + # which is a stall several layers from its cause. + bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) { + ${otelCfg.headersCredential} = { + hostPath = otelCfg.headersCredential; + isReadOnly = true; + }; + }; + + config = + { ... }: + { + system.stateVersion = config.system.stateVersion; + networking.firewall.enable = false; + # Keep the host-copied /etc/resolv.conf intact — same reasoning + # as the sibling swarm containers. + networking.resolvconf.enable = lib.mkForce false; + + services.opentelemetry-collector = { + enable = true; + package = pkgs.opentelemetry-collector-contrib; + # Runs `otelcol validate` at build time. ⚠️ A parser, not a + # wiring check: it accepts a receiver naming an absent + # extension, and the collector then dies at startup. A green + # build does not prove this config starts, never mind that a + # sample arrives — which is why this module's gate pushes a + # real sample through both tiers into the store. + validateConfigFile = true; + settings = { + receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; + + exporters = + lib.optionalAttrs vmCfg.enable { + # `metrics_endpoint`, NOT `endpoint`: the latter is a + # base that otlphttp appends `/v1/metrics` to, while + # VictoriaMetrics serves OTLP at + # `/opentelemetry/api/v1/push`. With `endpoint` the + # collector answers 200 to its own clients and posts the + # samples to a path that does not exist. Measured + # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`. + "otlphttp/victoriametrics".metrics_endpoint = + "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push"; + } + // lib.optionalAttrs (otelCfg.endpoint != "") { + ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { + endpoint = otelCfg.endpoint; + } + // lib.optionalAttrs (otelCfg.headersCredential != null) { + # Interpolated by the collector from its environment at + # runtime, never by nix: `EnvironmentFile` below is what + # puts it there, so the value is not read into the store. + headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; + } + // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; + }; + + service.pipelines.metrics = { + receivers = [ "otlp" ]; + # Fan-out, not a choice: with both configured the same + # samples go upstream AND into the swarm's store. The store + # is for looking at this swarm; the upstream is for whoever + # aggregates across swarms, and neither replaces the other. + exporters = + lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") + ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; + }; + }; + }; + + # The credential file is already `NAME=value`, systemd's + # EnvironmentFile format — so the secret reaches the process as an + # environment variable without being read by nix, written to the + # store, or passed in argv. + systemd.services.opentelemetry-collector.serviceConfig = + lib.optionalAttrs (otelCfg.headersCredential != null) + { + EnvironmentFile = otelCfg.headersCredential; + }; + }; + }; + }; +}