From 2aa33f12d2e2dbc8995513d9866ff3d0eb89b12b Mon Sep 17 00:00:00 2001 From: damocles Date: Wed, 26 Aug 2026 21:36:45 +0200 Subject: [PATCH] swarm-otel: add an unauthenticated swarm-tier OTLP receiver for swarm-controller to push to --- nix/host-modules/swarm-controller.nix | 36 ++++++++++++++++- nix/host-modules/swarm-otel.nix | 56 +++++++++++++++++++++++---- 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 5ba54251..672481ff 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -149,6 +149,35 @@ let SWARM_CONTROLLER_NAME = config.services.hyperhive.swarm.name; }; + # `vcs_metrics`/`hive_jobq_metrics` (this daemon's OTLP push exporters) + # read the standard `OTEL_EXPORTER_OTLP_ENDPOINT` var and are silent, + # graceful no-ops without it — same shape as `forgeEnv`/`authBridgeEnv` + # above, genuinely optional and gated on the option resolving rather + # than assumed. + # + # Gated on THIS HOST running `swarm-otel`, not merely on it existing + # somewhere in the swarm: its receiver for a swarm-tier producer binds + # `127.0.0.1` only (see `swarm-otel.nix::producerPort`), so reaching it + # needs co-location, and this checks that directly rather than assuming + # it — the unasserted-co-location gap flagged elsewhere in this codebase + # doesn't apply here because there is nothing to assert: a controller on + # a host that doesn't run the collector simply exports nothing, the same + # graceful absence `forgeEnv` already models. + otelSwarmCfg = config.services.hyperhive.swarm.otel; + otelEnv = lib.optionalAttrs otelSwarmCfg.enable { + OTEL_EXPORTER_OTLP_ENDPOINT = "http://${otelSwarmCfg.domain}:${toString otelSwarmCfg.producerPort}"; + }; + + # Only when THIS host also runs `swarm-otel` — same reasoning and same + # shape as `hive-forge/default.nix`'s `ssoLocal`-gated entry: a raw host + # systemd unit does not get the bridge's dnsmasq resolution containers + # get, so the name that reaches a co-located collector over `otelEnv` + # above needs an explicit loopback alias here, or it resolves however + # (or however it fails to) off-host DNS says on this host. + otelHostsEntry = lib.optionalAttrs otelSwarmCfg.enable { + "127.0.0.1" = [ otelSwarmCfg.domain ]; + }; + # Wrapped rather than documented: every one of these values is derived # from an option this deployment already set, so making the operator # re-supply them on the command line would be asking them to repeat the @@ -450,6 +479,10 @@ in "swarm-controller-credential" ]; + # See `otelHostsEntry`'s own comment: only present, and only ever + # `{ "127.0.0.1" = [ ... ]; }`, when this host also runs `swarm-otel`. + networking.hosts = otelHostsEntry; + users.users.swarm-controller = { isSystemUser = true; group = "swarm-controller"; @@ -676,7 +709,8 @@ in // forgeEnv // webhookEnv // authBridgeEnv - // swarmNameEnv; + // swarmNameEnv + // otelEnv; }; # A systemd credential is a SNAPSHOT: it is materialised into `%d` once, diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 9d5ebd00..e037c448 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -249,6 +249,29 @@ in ''; }; + producerPort = lib.mkOption { + type = lib.types.port; + default = 4390; + description = '' + Port the swarm tier's own unauthenticated OTLP/HTTP receiver + listens on, at `127.0.0.1` — for a **swarm-level** producer + (`swarm-controller`'s vcs/jobq counters today) to push to, on the + same host this collector runs on. No authenticator, deliberately: + unlike the per-hive receivers, the label this receiver's samples + get (`swarm`) is the one the receiver's own *existence* + establishes — there is no hive identity to forge or attribute, + so the per-hive attribution mechanism does not apply here. + + ⚠️ Deliberately NOT derived from `port + (number of hives)`: that + range grows every time a hive is added, and a fixed offset from + it would silently start colliding once the hive count caught up. + Kept as its own reserved value instead, and the assertion below + still catches a real collision (including one hive growth + eventually causes) rather than starting a collector that quietly + drops one receiver's samples. + ''; + }; + domain = lib.mkOption { type = lib.types.str; default = "otel.${domainBase}"; @@ -715,6 +738,7 @@ in derived = lib.attrValues hivePorts; others = [ cfg.telemetryPort + cfg.producerPort otelCfg.collector.port ] ++ lib.optional vmCfg.enable vmCfg.port; @@ -846,6 +870,12 @@ in }; } ) hivePorts + # The swarm tier's OWN receiver, unauthenticated on + # purpose — see `producerPort`'s description for why no + # per-hive-shaped attribution applies here. + // { + "otlp/${swarmTierName}".protocols.http.endpoint = "127.0.0.1:${toString cfg.producerPort}"; + } # MERGED with the per-hive receivers, never assigned over # them. A plain assignment here would drop every hive's # receiver and still render a valid config that starts @@ -1044,9 +1074,17 @@ in # them would acquire the one label a swarm-level service must # not have. Keeping it out of them makes the absence # structural rather than something to remember to strip. - // lib.optionalAttrs (cfg.scrapeTargets != { }) { + // { + # `otlp/${swarmTierName}` is unconditional (see the + # receiver above), so this pipeline is too — a swarm-tier + # producer must always have somewhere to land, unlike + # `prometheus`, which only joins the receiver list once + # something has actually declared a scrape target. "metrics/${swarmTierName}" = { - receivers = [ "prometheus" ]; + receivers = [ + "otlp/${swarmTierName}" + ] + ++ lib.optional (cfg.scrapeTargets != { }) "prometheus"; processors = [ "resource/${swarmTierName}" ]; exporters = exporterNames; }; @@ -1151,12 +1189,14 @@ in # an invented one (a sentinel, the local hive's name) would be # queried as though it meant something. # - # ⚠️ Emitted for the LOG pipeline too, not just the scraped one: - # a processor a pipeline names but the config does not define is - # a collector that refuses to start, and enabling the log store - # without declaring a scrape target is a perfectly ordinary - # config. - // lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) { + # ⚠️ Unconditional, not gated on `scrapeTargets != {} || + # collectLogs` (as it once was): a processor a pipeline names + # but the config does not define is a collector that refuses + # to start, and `metrics/${swarmTierName}` now ALWAYS exists + # (its `otlp/${swarmTierName}` receiver is unconditional too, + # for a swarm-tier producer to push to) — so this processor + # has to exist unconditionally right alongside it. + // { "resource/${swarmTierName}".attributes = [ { # The metric LABEL, a different namespace from the