swarm-otel: add an unauthenticated swarm-tier OTLP receiver for swarm-controller to push to

nix/host-modules/swarm-controller.nix

@@ -149,6 +149,35 @@ let
     SWARM_CONTROLLER_NAME = config.services.hyperhive.swarm.name;
   };
 
+  # `vcs_metrics`/`hive_jobq_metrics` (this daemon's OTLP push exporters)
+  # read the standard `OTEL_EXPORTER_OTLP_ENDPOINT` var and are silent,
+  # graceful no-ops without it — same shape as `forgeEnv`/`authBridgeEnv`
+  # above, genuinely optional and gated on the option resolving rather
+  # than assumed.
+  #
+  # Gated on THIS HOST running `swarm-otel`, not merely on it existing
+  # somewhere in the swarm: its receiver for a swarm-tier producer binds
+  # `127.0.0.1` only (see `swarm-otel.nix::producerPort`), so reaching it
+  # needs co-location, and this checks that directly rather than assuming
+  # it — the unasserted-co-location gap flagged elsewhere in this codebase
+  # doesn't apply here because there is nothing to assert: a controller on
+  # a host that doesn't run the collector simply exports nothing, the same
+  # graceful absence `forgeEnv` already models.
+  otelSwarmCfg = config.services.hyperhive.swarm.otel;
+  otelEnv = lib.optionalAttrs otelSwarmCfg.enable {
+    OTEL_EXPORTER_OTLP_ENDPOINT = "http://${otelSwarmCfg.domain}:${toString otelSwarmCfg.producerPort}";
+  };
+
+  # Only when THIS host also runs `swarm-otel` — same reasoning and same
+  # shape as `hive-forge/default.nix`'s `ssoLocal`-gated entry: a raw host
+  # systemd unit does not get the bridge's dnsmasq resolution containers
+  # get, so the name that reaches a co-located collector over `otelEnv`
+  # above needs an explicit loopback alias here, or it resolves however
+  # (or however it fails to) off-host DNS says on this host.
+  otelHostsEntry = lib.optionalAttrs otelSwarmCfg.enable {
+    "127.0.0.1" = [ otelSwarmCfg.domain ];
+  };
+
   # Wrapped rather than documented: every one of these values is derived
   # from an option this deployment already set, so making the operator
   # re-supply them on the command line would be asking them to repeat the
@@ -450,6 +479,10 @@ in
       "swarm-controller-credential"
     ];
 
+    # See `otelHostsEntry`'s own comment: only present, and only ever
+    # `{ "127.0.0.1" = [ ... ]; }`, when this host also runs `swarm-otel`.
+    networking.hosts = otelHostsEntry;
+
     users.users.swarm-controller = {
       isSystemUser = true;
       group = "swarm-controller";
@@ -676,7 +709,8 @@ in
       // forgeEnv
       // webhookEnv
       // authBridgeEnv
-      // swarmNameEnv;
+      // swarmNameEnv
+      // otelEnv;
     };
 
     # A systemd credential is a SNAPSHOT: it is materialised into `%d` once,

nix/host-modules/swarm-otel.nix

@@ -249,6 +249,29 @@ in
       '';
     };
 
+    producerPort = lib.mkOption {
+      type = lib.types.port;
+      default = 4390;
+      description = ''
+        Port the swarm tier's own unauthenticated OTLP/HTTP receiver
+        listens on, at `127.0.0.1` — for a **swarm-level** producer
+        (`swarm-controller`'s vcs/jobq counters today) to push to, on the
+        same host this collector runs on. No authenticator, deliberately:
+        unlike the per-hive receivers, the label this receiver's samples
+        get (`swarm`) is the one the receiver's own *existence*
+        establishes — there is no hive identity to forge or attribute,
+        so the per-hive attribution mechanism does not apply here.
+
+        ⚠️ Deliberately NOT derived from `port + (number of hives)`: that
+        range grows every time a hive is added, and a fixed offset from
+        it would silently start colliding once the hive count caught up.
+        Kept as its own reserved value instead, and the assertion below
+        still catches a real collision (including one hive growth
+        eventually causes) rather than starting a collector that quietly
+        drops one receiver's samples.
+      '';
+    };
+
     domain = lib.mkOption {
       type = lib.types.str;
       default = "otel.${domainBase}";
@@ -715,6 +738,7 @@ in
             derived = lib.attrValues hivePorts;
             others = [
               cfg.telemetryPort
+              cfg.producerPort
               otelCfg.collector.port
             ]
             ++ lib.optional vmCfg.enable vmCfg.port;
@@ -846,6 +870,12 @@ in
                     };
                   }
                 ) hivePorts
+                # The swarm tier's OWN receiver, unauthenticated on
+                # purpose — see `producerPort`'s description for why no
+                # per-hive-shaped attribution applies here.
+                // {
+                  "otlp/${swarmTierName}".protocols.http.endpoint = "127.0.0.1:${toString cfg.producerPort}";
+                }
                 # MERGED with the per-hive receivers, never assigned over
                 # them. A plain assignment here would drop every hive's
                 # receiver and still render a valid config that starts
@@ -1044,9 +1074,17 @@ in
                 # them would acquire the one label a swarm-level service must
                 # not have. Keeping it out of them makes the absence
                 # structural rather than something to remember to strip.
-                // lib.optionalAttrs (cfg.scrapeTargets != { }) {
+                // {
+                  # `otlp/${swarmTierName}` is unconditional (see the
+                  # receiver above), so this pipeline is too — a swarm-tier
+                  # producer must always have somewhere to land, unlike
+                  # `prometheus`, which only joins the receiver list once
+                  # something has actually declared a scrape target.
                   "metrics/${swarmTierName}" = {
-                    receivers = [ "prometheus" ];
+                    receivers = [
+                      "otlp/${swarmTierName}"
+                    ]
+                    ++ lib.optional (cfg.scrapeTargets != { }) "prometheus";
                     processors = [ "resource/${swarmTierName}" ];
                     exporters = exporterNames;
                   };
@@ -1151,12 +1189,14 @@ in
                 # an invented one (a sentinel, the local hive's name) would be
                 # queried as though it meant something.
                 #
-                # ⚠️ Emitted for the LOG pipeline too, not just the scraped one:
-                # a processor a pipeline names but the config does not define is
-                # a collector that refuses to start, and enabling the log store
-                # without declaring a scrape target is a perfectly ordinary
-                # config.
-                // lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) {
+                # ⚠️ Unconditional, not gated on `scrapeTargets != {} ||
+                # collectLogs` (as it once was): a processor a pipeline names
+                # but the config does not define is a collector that refuses
+                # to start, and `metrics/${swarmTierName}` now ALWAYS exists
+                # (its `otlp/${swarmTierName}` receiver is unconditional too,
+                # for a swarm-tier producer to push to) — so this processor
+                # has to exist unconditionally right alongside it.
+                // {
                   "resource/${swarmTierName}".attributes = [
                     {
                       # The metric LABEL, a different namespace from the