glue: otel scrape targets for authelia, grafana and nats move out of the services

Part of #3742.

nix/host-modules/default.nix

@@ -42,6 +42,7 @@
     ./glue-nats-auth-bao-identity.nix
     ./glue-nats-bao-identity.nix
     ./glue-nats-oidc-client.nix
+    ./glue-otel-scrape-targets.nix
     ./glue-queue-agent-credential.nix
     ./glue-review-classifier-oidc-client.nix
     ./glue-secret-publisher-bao-identity.nix

nix/host-modules/glue-otel-scrape-targets.nix

@@ -0,0 +1,38 @@
+# Glue: declare the loopback scrape targets of authelia, grafana and nats on the
+# swarm's OTEL collector, wherever one of them shares a host with it.
+#
+# ONE FILE, THREE PAIRINGS — each service ← the collector, with the same shape.
+# Deleting this leaves the collector scraping none of them: no error, no
+# metrics.
+#
+# Each target is gated on the service AND the collector being enabled here. The
+# loopback address is only honest when both run on this host; a target nobody
+# can read asserts a collection that is not happening. Both services default
+# from `deploy.allSwarmServices`, but `mkDefault` is an invitation to override,
+# so co-location is a property of the auto-deployed topology, not the modules.
+#
+# ⚠️ Silent when split: separate hosts are separate evaluations with no shared
+# context, so a host running a service and no collector declares nothing and
+# the collector's host never learns of the service. No error, no warning — a
+# healthy exporter and an empty dashboard. Not guardable from here.
+#
+# ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only
+# this one reads `scrapeTargets`.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  hyperhiveCfg = config.services.hyperhive;
+  deployCfg = hyperhiveCfg.deploy;
+  swarmCfg = hyperhiveCfg.swarm;
+  withCollector = service: lib.mkIf (deployCfg.${service}.enable && deployCfg.swarm-otel.enable);
+in
+{
+  config.services.hyperhive.swarm.otel.scrapeTargets = {
+    authelia = withCollector "authelia" "127.0.0.1:${toString swarmCfg.authelia.metricsPort}";
+    grafana = withCollector "grafana" "127.0.0.1:${toString swarmCfg.grafana.metricsPort}";
+    nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}";
+  };
+}

nix/host-modules/swarm-authelia.nix

@@ -713,34 +713,6 @@ in
     services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
-    # Declared here rather than in the collector's module, per the option's
-    # own rule: an entry exists only where the service that named it runs.
-    #
-    # Gated on the collector's `enable` as well, and that second condition is
-    # what makes the loopback address honest. Both services default from
-    # `deploy.allSwarmServices` — but `mkDefault` is an invitation to
-    # override, not a guarantee, so "they are on the same host" is a property
-    # of the auto-deployed topology rather than of the module. Without this
-    # gate, a host running authelia and no collector would declare a target
-    # nothing can read, and the absence would be silent: no error, no metrics,
-    # nothing to notice.
-    #
-    # It does not make authelia scrapeable from ANOTHER host — that needs the
-    # endpoint published under a name with a cert and an audience, which is a
-    # different piece of work. This only stops the config asserting a
-    # collection that is not happening.
-    # ⚠️ `swarm.otel` and not `hyperhive.otel` — two different collectors one
-    # word apart. This one is the swarm's; `hyperhive.otel` is the per-hive
-    # tier that ships telemetry upstream and never reads `scrapeTargets`.
-    # Written out in full rather than through a `let` binding so the gate and
-    # the option it gates are visibly the same path: gating the wrong one is
-    # not a build error, it is a target that is always declared or never is.
-    services.hyperhive.swarm.otel.scrapeTargets =
-      lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable
-        {
-          authelia = "127.0.0.1:${toString cfg.metricsPort}";
-        };
-
     # `server_name = authelia.domain`, all of `/` → authelia.
     #
     # ⚠️ The server name must be exactly `cfg.domain`, not a near-miss:

nix/host-modules/swarm-grafana.nix

@@ -503,14 +503,6 @@ in
       locations."= /metrics".proxyPass = "http://unix:${socketPath}:/metrics";
     };
 
-    # Declared here rather than in the collector's module, per that option's
-    # rule: an entry exists where the service that named it runs.
-    services.hyperhive.swarm.otel.scrapeTargets =
-      lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable
-        {
-          grafana = "127.0.0.1:${toString cfg.metricsPort}";
-        };
-
     # Order the container after the host CA service so the bind source below
     # exists before nspawn sets the mount up.
     systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;

nix/host-modules/swarm-nats.nix

@@ -459,33 +459,6 @@ in
       }
     ];
 
-    # Declared here rather than in the collector's module: an entry then
-    # exists only where the service that named it runs. Gated on a
-    # collector, because a target nobody reads asserts a collection that is
-    # not happening.
-    #
-    # ⚠️ KNOWN LIMITATION, and it is SILENT. That rule constrains the
-    # target, not the scraper — nothing places the collector on this host.
-    # `swarm-required-services.nix` derives `nats.enable` and `otel.enable`
-    # from one `lib.mkDefault`, so they are co-located by *default*, and an
-    # operator may split them. Split, NATS is never scraped: this host
-    # declares an entry no local collector reads, the collector's host never
-    # enabled this module. No error, no warning — a healthy exporter and an
-    # empty dashboard.
-    #
-    # Not guardable: separate hosts are separate evaluations with no shared
-    # context, so this one cannot see what that one runs. Saying so is the
-    # only mechanism there is.
-    #
-    # ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart,
-    # and only this one reads `scrapeTargets`. Written in full so the gate
-    # and the option it gates are visibly the same path.
-    services.hyperhive.swarm.otel.scrapeTargets =
-      lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable
-        {
-          nats = "127.0.0.1:${toString cfg.metricsPort}";
-        };
-
     # Open the client port on the bridge, so agent containers can reach
     # the queue. `privateNetwork = false` below means the server binds in
     # the host netns, which is the precondition this option names — but

nix/host-modules/swarm-otel-service.nix

@@ -237,7 +237,11 @@ in
         **A service declares its own entry, from its own module, under its
         own `enable`.** Do not assemble the list here: an entry then exists
         only where the service that named it runs, so a target is never
-        declared on a host that does not serve it.
+        declared on a host that does not serve it. An entry that also needs
+        the collector on the same host (the loopback address is only honest
+        then) is declared from a glue module instead, which reads both
+        switches: authelia, grafana and nats are declared by
+        `glue-otel-scrape-targets.nix`.
 
         ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference
         is a silent gap.** Nothing here puts the collector on the same host.

nix/module-eval/forge-placement.nix

@@ -137,6 +137,17 @@ let
         && !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames)
         && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
     }
+    {
+      name = "a host running the collector and authelia, grafana and nats scrapes all three on loopback";
+      ok =
+        let
+          t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets;
+          port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort;
+        in
+        t.authelia == "127.0.0.1:${port "authelia"}"
+        && t.grafana == "127.0.0.1:${port "grafana"}"
+        && t.nats == "127.0.0.1:${port "nats"}";
+    }
     {
       name = "hosting the swarm's shared services runs the forge";
       ok = runsForge servicesHere;

nix/module-eval/grafana.nix

@@ -109,6 +109,12 @@ let
       && lib.hasInfix option a.message
     ) m.assertions;
   cases = [
+    {
+      name = "grafana declares no scrape target on a host with no collector";
+      ok =
+        !(grafanaOldPath.services.hyperhive.deploy.swarm-otel.enable)
+        && !(grafanaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? grafana);
+    }
     {
       # The socket's directory is its access control, so it must stay dedicated
       # to this one socket. `readOnly` is what makes that inexpressible: a

nix/module-eval/nats-authelia.nix

@@ -99,6 +99,13 @@ let
   autheliaColocated = shared.autheliaOnly;
 
   cases = [
+    {
+      name = "authelia and nats declare no scrape target on a host with no collector";
+      ok =
+        !(autheliaOldPath.services.hyperhive.deploy.swarm-otel.enable)
+        && !(autheliaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? authelia)
+        && !(autheliaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? nats);
+    }
     {
       # The xkey and the responder's OIDC client secret are read only from
       # the store, so a queue with no responder leaf cannot start. Refused at