From 2507f4f713042ffd574b649b628c7474796c2fa3 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 17 Oct 2026 14:50:49 +0200 Subject: [PATCH] glue-controller-env: the controller always exports otlp to the swarm's collector --- docs/swarm/services.md | 4 ++++ nix/host-modules/glue-controller-env.nix | 29 +++++++++++++++--------- nix/module-eval/core-toggle.nix | 29 +++++++++++++++++++----- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 2daee96b..0df067d3 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -198,6 +198,10 @@ and failure series on `collector` appear once something fails. **Where the data comes from.** The swarm's OTEL collector, below. +The `controller` panels come from the controller itself. It exports OTLP +through the gateway to the swarm's collector from whatever host it runs on, +whether or not that host runs the collector. + Neither container is reachable except through the gateway: each listens on a unix socket only the gateway's nginx can open, and VictoriaMetrics runs in a network namespace of its own with no interface but loopback. diff --git a/nix/host-modules/glue-controller-env.nix b/nix/host-modules/glue-controller-env.nix index 44f30f01..1f9d5d1d 100644 --- a/nix/host-modules/glue-controller-env.nix +++ b/nix/host-modules/glue-controller-env.nix @@ -1,27 +1,31 @@ -# Glue: tell the swarm controller about the authelia and the collector that run -# on this host. +# Glue: tell the swarm controller where the swarm's collector is and which +# authelia runs on this host. # # ONE PAIRING PER FILE — controller ← the services whose coordinates it reads # from the environment. Deleting this leaves a controller that exports no # metrics, and a `swarmctl` that fails `user add` saying the authelia binary and # users file are unset. # -# ⚠️ Two gates, one per service, each ANDed with the controller being here: +# ⚠️ Everything here is gated on the controller being here: # `systemd.services.swarm-controller.environment` on a host without the # controller would define a unit with no ExecStart. # -# authelia → `swarmctl`'s wrapper, not the daemon's unit. Gated on authelia +# authelia → `swarmctl`'s wrapper, not the daemon's unit. Also gated on authelia # running HERE: the controller can be enabled on a host that is not the swarm's # SSO host, and there `swarmctl user add` must fail saying the value is unset — # a guessed path would resolve cleanly and write a file nothing reads. The # CONFIGURED authelia, not whatever is on PATH: the argon2 parameters baked into # a hash have to match the verifier's. # -# swarm-otel → the daemon's unit. `vcs_metrics`/`hive_jobq_metrics` read the -# standard `OTEL_EXPORTER_OTLP_ENDPOINT` and are silent no-ops without it. The -# endpoint is the collector's swarm-service name through the gateway, never a +# collector → the daemon's unit. `vcs_metrics`/`hive_jobq_metrics` read the +# standard `OTEL_EXPORTER_OTLP_ENDPOINT` and are silent no-ops without it. No +# gate beyond the controller: the controller always exports to the swarm's +# collector, which the swarm always has (./swarm-bao.nix, +# ./glue-swarm-otel-oidc-client.nix), and `deploy.swarm-otel.enable` only says it +# runs HERE. The endpoint is its swarm-service name through the gateway, never a # loopback literal, with no trailing slash: the http exporter appends -# `/v1/metrics` and the gateway route strips `/${producerName}`. +# `/v1/metrics` and the gateway route strips `/${producerName}`. The port suffix +# is elided on 443, as in ./swarm-bao.nix. # `SWARM_CONTROLLER_OTEL_AUDIENCE` is the audience ./glue-controller-oidc-client.nix # registers and ./swarm-otel.nix's `oidc/${producerName}` authenticator checks. { @@ -34,6 +38,9 @@ let deployCfg = hyperhiveCfg.deploy; otelSwarmCfg = hyperhiveCfg.swarm.otel; queueClientId = hyperhiveCfg.swarm.controller.queueClientId; + gatewayPortSuffix = lib.optionalString ( + hyperhiveCfg.gateway.httpsPort != 443 + ) ":${toString hyperhiveCfg.gateway.httpsPort}"; in { config = lib.mkIf deployCfg.swarm-controller.enable ( @@ -44,12 +51,12 @@ in SWARMCTL_AUTHELIA_USERS_FILE = deployCfg.authelia.hostUsersFile; }; }) - (lib.mkIf deployCfg.swarm-otel.enable { + { systemd.services.swarm-controller.environment = { - OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}"; + OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}${gatewayPortSuffix}/${otelSwarmCfg.producerName}"; SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId; }; - }) + } ] ); } diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index a3d8dad1..812c3d0e 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -58,6 +58,14 @@ let otel.debug = true; }; + # The controller behind a gateway on a non-default TLS port: the only + # fixture whose collector URL carries a port. Only the controller's + # environment is read off it. + controllerAltPort = hive { + deploy.swarm-controller.enable = true; + gateway.httpsPort = 8443; + }; + # On the store's host, which mints the runner's store identity. withCi = hive { deploy.bao.enable = true; @@ -149,10 +157,11 @@ let ok = allLocal.services.hyperhive.deploy.swarm-controller.enable; } { - # ../host-modules/glue-controller-env.nix: the collector's coordinates - # follow swarm-otel running HERE. `controllerAlone` is the split case — a - # controller whose collector runs elsewhere. - name = "the controller exports to the collector only where the collector runs here"; + # ../host-modules/glue-controller-env.nix: the controller always exports to + # the swarm's collector. `controllerAlone` runs no collector of its own + # (`allLocal` does, as the presence control); `centralToggleOff` has no + # controller, so no unit to set it on. + name = "the controller always exports to the swarm's collector"; ok = let env = m: m.systemd.services.swarm-controller.environment; @@ -169,8 +178,16 @@ let && client != null && lib.elem clientId client.audience && client.accessTokenSignedResponseAlg == "RS256" - && !(env shared.controllerAlone ? OTEL_EXPORTER_OTLP_ENDPOINT) - && !(env shared.controllerAlone ? SWARM_CONTROLLER_OTEL_AUDIENCE); + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !shared.controllerAlone.services.hyperhive.deploy.swarm-otel.enable + && + (env shared.controllerAlone).OTEL_EXPORTER_OTLP_ENDPOINT or null + == "https://${otel.domain}/${otel.producerName}" + && (env shared.controllerAlone).SWARM_CONTROLLER_OTEL_AUDIENCE or null == clientId + && + (env controllerAltPort).OTEL_EXPORTER_OTLP_ENDPOINT or null + == "https://${otel.domain}:8443/${otel.producerName}" + && !(centralToggleOff.systemd.services ? swarm-controller); } { # Same file: `swarmctl` is told where the configured authelia is only