From c06aa78b6384eb32eba971463dc1ebca8f3120e8 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 13 Oct 2026 08:33:28 +0200 Subject: [PATCH] glue: otel scrape targets for authelia, grafana and nats move out of the services Part of #3742. --- nix/host-modules/default.nix | 1 + nix/host-modules/glue-otel-scrape-targets.nix | 38 +++++++++++++++++++ nix/host-modules/swarm-authelia.nix | 28 -------------- nix/host-modules/swarm-grafana.nix | 8 ---- nix/host-modules/swarm-nats.nix | 27 ------------- nix/host-modules/swarm-otel-service.nix | 6 ++- nix/module-eval/forge-placement.nix | 11 ++++++ nix/module-eval/grafana.nix | 6 +++ nix/module-eval/nats-authelia.nix | 7 ++++ 9 files changed, 68 insertions(+), 64 deletions(-) create mode 100644 nix/host-modules/glue-otel-scrape-targets.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index aedbd81c..ff894b82 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -42,6 +42,7 @@ ./glue-nats-auth-bao-identity.nix ./glue-nats-bao-identity.nix ./glue-nats-oidc-client.nix + ./glue-otel-scrape-targets.nix ./glue-queue-agent-credential.nix ./glue-review-classifier-oidc-client.nix ./glue-secret-publisher-bao-identity.nix diff --git a/nix/host-modules/glue-otel-scrape-targets.nix b/nix/host-modules/glue-otel-scrape-targets.nix new file mode 100644 index 00000000..a105a279 --- /dev/null +++ b/nix/host-modules/glue-otel-scrape-targets.nix @@ -0,0 +1,38 @@ +# Glue: declare the loopback scrape targets of authelia, grafana and nats on the +# swarm's OTEL collector, wherever one of them shares a host with it. +# +# ONE FILE, THREE PAIRINGS — each service ← the collector, with the same shape. +# Deleting this leaves the collector scraping none of them: no error, no +# metrics. +# +# Each target is gated on the service AND the collector being enabled here. The +# loopback address is only honest when both run on this host; a target nobody +# can read asserts a collection that is not happening. Both services default +# from `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, +# so co-location is a property of the auto-deployed topology, not the modules. +# +# ⚠️ Silent when split: separate hosts are separate evaluations with no shared +# context, so a host running a service and no collector declares nothing and +# the collector's host never learns of the service. No error, no warning — a +# healthy exporter and an empty dashboard. Not guardable from here. +# +# ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only +# this one reads `scrapeTargets`. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + swarmCfg = hyperhiveCfg.swarm; + withCollector = service: lib.mkIf (deployCfg.${service}.enable && deployCfg.swarm-otel.enable); +in +{ + config.services.hyperhive.swarm.otel.scrapeTargets = { + authelia = withCollector "authelia" "127.0.0.1:${toString swarmCfg.authelia.metricsPort}"; + grafana = withCollector "grafana" "127.0.0.1:${toString swarmCfg.grafana.metricsPort}"; + nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}"; + }; +} diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 014bc10f..a4338cd7 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -713,34 +713,6 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; - # Declared here rather than in the collector's module, per the option's - # own rule: an entry exists only where the service that named it runs. - # - # Gated on the collector's `enable` as well, and that second condition is - # what makes the loopback address honest. Both services default from - # `deploy.allSwarmServices` — but `mkDefault` is an invitation to - # override, not a guarantee, so "they are on the same host" is a property - # of the auto-deployed topology rather than of the module. Without this - # gate, a host running authelia and no collector would declare a target - # nothing can read, and the absence would be silent: no error, no metrics, - # nothing to notice. - # - # It does not make authelia scrapeable from ANOTHER host — that needs the - # endpoint published under a name with a cert and an audience, which is a - # different piece of work. This only stops the config asserting a - # collection that is not happening. - # ⚠️ `swarm.otel` and not `hyperhive.otel` — two different collectors one - # word apart. This one is the swarm's; `hyperhive.otel` is the per-hive - # tier that ships telemetry upstream and never reads `scrapeTargets`. - # Written out in full rather than through a `let` binding so the gate and - # the option it gates are visibly the same path: gating the wrong one is - # not a build error, it is a target that is always declared or never is. - services.hyperhive.swarm.otel.scrapeTargets = - lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable - { - authelia = "127.0.0.1:${toString cfg.metricsPort}"; - }; - # `server_name = authelia.domain`, all of `/` → authelia. # # ⚠️ The server name must be exactly `cfg.domain`, not a near-miss: diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index d4743f25..36fef4b0 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -503,14 +503,6 @@ in locations."= /metrics".proxyPass = "http://unix:${socketPath}:/metrics"; }; - # Declared here rather than in the collector's module, per that option's - # rule: an entry exists where the service that named it runs. - services.hyperhive.swarm.otel.scrapeTargets = - lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable - { - grafana = "127.0.0.1:${toString cfg.metricsPort}"; - }; - # Order the container after the host CA service so the bind source below # exists before nspawn sets the mount up. systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index c2ff6938..81972782 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -459,33 +459,6 @@ in } ]; - # Declared here rather than in the collector's module: an entry then - # exists only where the service that named it runs. Gated on a - # collector, because a target nobody reads asserts a collection that is - # not happening. - # - # ⚠️ KNOWN LIMITATION, and it is SILENT. That rule constrains the - # target, not the scraper — nothing places the collector on this host. - # `swarm-required-services.nix` derives `nats.enable` and `otel.enable` - # from one `lib.mkDefault`, so they are co-located by *default*, and an - # operator may split them. Split, NATS is never scraped: this host - # declares an entry no local collector reads, the collector's host never - # enabled this module. No error, no warning — a healthy exporter and an - # empty dashboard. - # - # Not guardable: separate hosts are separate evaluations with no shared - # context, so this one cannot see what that one runs. Saying so is the - # only mechanism there is. - # - # ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, - # and only this one reads `scrapeTargets`. Written in full so the gate - # and the option it gates are visibly the same path. - services.hyperhive.swarm.otel.scrapeTargets = - lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable - { - nats = "127.0.0.1:${toString cfg.metricsPort}"; - }; - # Open the client port on the bridge, so agent containers can reach # the queue. `privateNetwork = false` below means the server binds in # the host netns, which is the precondition this option names — but diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index a28e107a..82e9ef28 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -237,7 +237,11 @@ in **A service declares its own entry, from its own module, under its own `enable`.** Do not assemble the list here: an entry then exists only where the service that named it runs, so a target is never - declared on a host that does not serve it. + declared on a host that does not serve it. An entry that also needs + the collector on the same host (the loopback address is only honest + then) is declared from a glue module instead, which reads both + switches: authelia, grafana and nats are declared by + `glue-otel-scrape-targets.nix`. ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference is a silent gap.** Nothing here puts the collector on the same host. diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index 3ab329f7..cf0fc743 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -137,6 +137,17 @@ let && !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames) && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile"); } + { + name = "a host running the collector and authelia, grafana and nats scrapes all three on loopback"; + ok = + let + t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets; + port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort; + in + t.authelia == "127.0.0.1:${port "authelia"}" + && t.grafana == "127.0.0.1:${port "grafana"}" + && t.nats == "127.0.0.1:${port "nats"}"; + } { name = "hosting the swarm's shared services runs the forge"; ok = runsForge servicesHere; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index c1319f1b..61520d06 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -109,6 +109,12 @@ let && lib.hasInfix option a.message ) m.assertions; cases = [ + { + name = "grafana declares no scrape target on a host with no collector"; + ok = + !(grafanaOldPath.services.hyperhive.deploy.swarm-otel.enable) + && !(grafanaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? grafana); + } { # The socket's directory is its access control, so it must stay dedicated # to this one socket. `readOnly` is what makes that inexpressible: a diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index df227e53..deade9ee 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -99,6 +99,13 @@ let autheliaColocated = shared.autheliaOnly; cases = [ + { + name = "authelia and nats declare no scrape target on a host with no collector"; + ok = + !(autheliaOldPath.services.hyperhive.deploy.swarm-otel.enable) + && !(autheliaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? authelia) + && !(autheliaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? nats); + } { # The xkey and the responder's OIDC client secret are read only from # the store, so a queue with no responder leaf cannot start. Refused at