glue-controller-env: the controller always exports otlp to the swarm's collector

docs/swarm/services.md

@@ -198,6 +198,10 @@ and failure series on `collector` appear once something fails.
 
 **Where the data comes from.** The swarm's OTEL collector, below.
 
+The `controller` panels come from the controller itself. It exports OTLP
+through the gateway to the swarm's collector from whatever host it runs on,
+whether or not that host runs the collector.
+
 Neither container is reachable except through the gateway: each listens
 on a unix socket only the gateway's nginx can open, and VictoriaMetrics
 runs in a network namespace of its own with no interface but loopback.

nix/host-modules/glue-controller-env.nix

@@ -1,27 +1,31 @@
-# Glue: tell the swarm controller about the authelia and the collector that run
+# Glue: tell the swarm controller where the swarm's collector is and which
-# on this host.
+# authelia runs on this host.
 #
 # ONE PAIRING PER FILE — controller ← the services whose coordinates it reads
 # from the environment. Deleting this leaves a controller that exports no
 # metrics, and a `swarmctl` that fails `user add` saying the authelia binary and
 # users file are unset.
 #
-# ⚠️ Two gates, one per service, each ANDed with the controller being here:
+# ⚠️ Everything here is gated on the controller being here:
 # `systemd.services.swarm-controller.environment` on a host without the
 # controller would define a unit with no ExecStart.
 #
-# authelia → `swarmctl`'s wrapper, not the daemon's unit. Gated on authelia
+# authelia → `swarmctl`'s wrapper, not the daemon's unit. Also gated on authelia
 # running HERE: the controller can be enabled on a host that is not the swarm's
 # SSO host, and there `swarmctl user add` must fail saying the value is unset —
 # a guessed path would resolve cleanly and write a file nothing reads. The
 # CONFIGURED authelia, not whatever is on PATH: the argon2 parameters baked into
 # a hash have to match the verifier's.
 #
-# swarm-otel → the daemon's unit. `vcs_metrics`/`hive_jobq_metrics` read the
+# collector → the daemon's unit. `vcs_metrics`/`hive_jobq_metrics` read the
-# standard `OTEL_EXPORTER_OTLP_ENDPOINT` and are silent no-ops without it. The
+# standard `OTEL_EXPORTER_OTLP_ENDPOINT` and are silent no-ops without it. No
-# endpoint is the collector's swarm-service name through the gateway, never a
+# gate beyond the controller: the controller always exports to the swarm's
+# collector, which the swarm always has (./swarm-bao.nix,
+# ./glue-swarm-otel-oidc-client.nix), and `deploy.swarm-otel.enable` only says it
+# runs HERE. The endpoint is its swarm-service name through the gateway, never a
 # loopback literal, with no trailing slash: the http exporter appends
-# `/v1/metrics` and the gateway route strips `/${producerName}`.
+# `/v1/metrics` and the gateway route strips `/${producerName}`. The port suffix
+# is elided on 443, as in ./swarm-bao.nix.
 # `SWARM_CONTROLLER_OTEL_AUDIENCE` is the audience ./glue-controller-oidc-client.nix
 # registers and ./swarm-otel.nix's `oidc/${producerName}` authenticator checks.
 {
@@ -34,6 +38,9 @@ let
   deployCfg = hyperhiveCfg.deploy;
   otelSwarmCfg = hyperhiveCfg.swarm.otel;
   queueClientId = hyperhiveCfg.swarm.controller.queueClientId;
+  gatewayPortSuffix = lib.optionalString (
+    hyperhiveCfg.gateway.httpsPort != 443
+  ) ":${toString hyperhiveCfg.gateway.httpsPort}";
 in
 {
   config = lib.mkIf deployCfg.swarm-controller.enable (
@@ -44,12 +51,12 @@ in
           SWARMCTL_AUTHELIA_USERS_FILE = deployCfg.authelia.hostUsersFile;
         };
       })
-      (lib.mkIf deployCfg.swarm-otel.enable {
+      {
         systemd.services.swarm-controller.environment = {
-          OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}";
+          OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}${gatewayPortSuffix}/${otelSwarmCfg.producerName}";
           SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId;
         };
-      })
+      }
     ]
   );
 }

nix/module-eval/core-toggle.nix

@@ -58,6 +58,14 @@ let
     otel.debug = true;
   };
 
+  # The controller behind a gateway on a non-default TLS port: the only
+  # fixture whose collector URL carries a port. Only the controller's
+  # environment is read off it.
+  controllerAltPort = hive {
+    deploy.swarm-controller.enable = true;
+    gateway.httpsPort = 8443;
+  };
+
   # On the store's host, which mints the runner's store identity.
   withCi = hive {
     deploy.bao.enable = true;
@@ -149,10 +157,11 @@ let
       ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
     }
     {
-      # ../host-modules/glue-controller-env.nix: the collector's coordinates
+      # ../host-modules/glue-controller-env.nix: the controller always exports to
-      # follow swarm-otel running HERE. `controllerAlone` is the split case — a
+      # the swarm's collector. `controllerAlone` runs no collector of its own
-      # controller whose collector runs elsewhere.
+      # (`allLocal` does, as the presence control); `centralToggleOff` has no
-      name = "the controller exports to the collector only where the collector runs here";
+      # controller, so no unit to set it on.
+      name = "the controller always exports to the swarm's collector";
       ok =
         let
           env = m: m.systemd.services.swarm-controller.environment;
@@ -169,8 +178,16 @@ let
         && client != null
         && lib.elem clientId client.audience
         && client.accessTokenSignedResponseAlg == "RS256"
-        && !(env shared.controllerAlone ? OTEL_EXPORTER_OTLP_ENDPOINT)
+        && allLocal.services.hyperhive.deploy.swarm-otel.enable
-        && !(env shared.controllerAlone ? SWARM_CONTROLLER_OTEL_AUDIENCE);
+        && !shared.controllerAlone.services.hyperhive.deploy.swarm-otel.enable
+        &&
+          (env shared.controllerAlone).OTEL_EXPORTER_OTLP_ENDPOINT or null
+          == "https://${otel.domain}/${otel.producerName}"
+        && (env shared.controllerAlone).SWARM_CONTROLLER_OTEL_AUDIENCE or null == clientId
+        &&
+          (env controllerAltPort).OTEL_EXPORTER_OTLP_ENDPOINT or null
+          == "https://${otel.domain}:8443/${otel.producerName}"
+        && !(centralToggleOff.systemd.services ? swarm-controller);
     }
     {
       # Same file: `swarmctl` is told where the configured authelia is only