From 6c1ab87f68b18fa027af852737d07d728976e369 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 13 Oct 2026 10:24:17 +0200 Subject: [PATCH] swarm-otel: declare forgejo's published scrape target on every host The forgejo entry in `swarm.otel.publishedScrapeTargets` was declared under `deploy.forgejo.enable`, so it existed only in the forge host's evaluation. Its two readers evaluate elsewhere: the collector renders its prometheus jobs from its own host's list, and authelia registers the collector client's audience from its own host's list. With the forge, the collector and authelia on different hosts, the collector had no forgejo job and authelia would have refused its token. Declare the entry unconditionally in swarm-otel-service.nix, keyed on `swarm.forge.domain`, which every host evaluates identically. A swarm with no forge scrapes a name that answers nothing and shows a failing job, the same trade the collector and grafana OIDC clients already make. Single-host render is unchanged: the entry already existed there. Part of #5205 --- docs/swarm/secrets.md | 9 ++--- nix/host-modules/hive-forge/default.nix | 15 +++----- nix/host-modules/swarm-otel-service.nix | 35 +++++++++++++++---- nix/module-eval/swarm-otel-identity.nix | 46 ++++++++++++++++++++++++- 4 files changed, 82 insertions(+), 23 deletions(-) diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 260bd2d7..b4dacaef 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -86,14 +86,15 @@ out of the store on every host that runs the collector and holds a store identity, whether authelia is a network away or in the container next door. The same call `glue-grafana-oidc-client.nix` makes for Grafana's registers the client it reads for unconditionally: authelia refuses a -bearer-authz client with no audience, and the push audiences supply one even -for a collector with nothing published to scrape. +bearer-authz client with no audience, and every host publishes at least +forgejo's `/metrics` as a scrape target, so the audience is never empty. ⚠️ **A collector with no store identity isn't refused.** Unlike Grafana, where an absent OIDC block is a container with no way in at all, a collector with nothing to authenticate with still receives every hive's telemetry — -only its own pushes to the stores go out unauthenticated and get refused -there. `swarm-bao-otel-oidc.service` therefore doesn't render at all without +its pushes to the stores go out unauthenticated and get refused there, and its +scrape jobs for the published targets read a credential file nothing +loads, so they can't get a token. `swarm-bao-otel-oidc.service` therefore doesn't render at all without `deploy.bao.otelOidcClientCertFile` / `otelOidcClientKeyFile`, the same choice `glue-queue-agent-credential.nix` makes for its own reader, rather than the hard assertion `swarm-grafana.nix` uses for its non-optional one. The operator can still deliver the secret by diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 1c0c7d5c..f207c627 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -290,17 +290,6 @@ in # The forge container resolves the rest of the hive through dnsmasq. services.hyperhive.gateway.dns.enable = lib.mkDefault true; - # The metrics endpoint, declared once: the collector both scrapes this - # URL and derives from it the audience its token is minted for. - # - # ⚠️ Written as the exact URL a collector requests, because that is - # what authelia compares against — this string agreeing with the - # `location` block above it is the whole mechanism. A near miss is a - # correctly minted token refused at the target. - services.hyperhive.swarm.otel.publishedScrapeTargets = { - forgejo = "https://${cfg.domain}/metrics"; - }; - # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB # clones). SSH stays direct on `forge.sshPort`. See @@ -327,6 +316,10 @@ in # endpoint would ride that catch-all to anyone. `= /metrics` # outranks the `/` prefix in nginx, so this location — and its # auth — is the one that runs. + # + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.forgejo` in ../swarm-otel-service.nix; + # the two must agree, since that string is also the token's audience. "= /metrics" = { proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/metrics"; extraConfig = '' diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 6b250ad8..a28e107a 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -1,7 +1,8 @@ # The swarm's telemetry collector as every hive sees it: the name it is -# reached on, its receiver ports, the client it is registered as and the -# audiences that client may present, identical on every host. What the host -# running it decides, and the container itself, are in ./swarm-otel.nix. +# reached on, its receiver ports, the published targets it scrapes, the client +# it is registered as and the audiences that client may present, identical on +# every host. What the host running it decides, and the container itself, are +# in ./swarm-otel.nix. { lib, config, @@ -274,10 +275,18 @@ in Prometheus endpoints this collector scrapes **by name, with a credential**, as ` = ""`. - A service module declares its own entry, from its own module, the - same way it does for `scrapeTargets` — and the collector's OAuth2 - client derives its permitted audiences from these URLs, so a - target and the authorisation to reach it are **one declaration**. + **Entries are declared swarm-wide**: from this module's `config`, + unconditionally, keyed on the service's `swarm..domain`, so + every host evaluates the same set. Unlike `scrapeTargets`, an entry + is not declared under the service's own `enable`: the collector + reads this list on its host and authelia's registration reads it on + authelia's host, and neither need be the service's host. A swarm + that does not run the service anywhere scrapes a name that answers + nothing, which shows as a failing job rather than a missing one. + + The collector's OAuth2 client derives its permitted audiences from + these URLs, so a target and the authorisation to reach it are **one + declaration**. Two lists that must agree would be a drift to maintain, and its failure mode is the bad one: a target whose audience was forgotten authenticates against nothing and reads as a scrape failure rather @@ -331,4 +340,16 @@ in ''; }; }; + + # Not under any `enable`: see `publishedScrapeTargets` for why every host + # declares every entry. + # + # ⚠️ Each URL must be exactly what the collector requests, because authelia + # checks the token's audience against it: it has to agree with the gated + # `= /metrics` location on the service's vhost. A near miss is a correctly + # minted token refused at the target. + config.services.hyperhive.swarm.otel.publishedScrapeTargets = { + # The vhost location is in ./hive-forge/default.nix. + forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics"; + }; } diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index d23f0074..814d053b 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -77,7 +77,7 @@ let }; # A collector whose ONLY scrape work is published: loopback targets forced - # empty, one published job declared. Unreachable in a real deploy today — + # empty, published jobs only. Unreachable in a real deploy today — # the module seeds `scrapeTargets.collector` under its own `enable`, so the # loopback set is never empty on its own — which is exactly why the arm # below needs a fixture that takes that seeding away. `mkForce` is what @@ -97,6 +97,11 @@ let deploy.authelia.enable = true; swarm.hives.h2.domain = "h2.t.local"; }; + + # authelia with neither the forge nor the collector beside it. + inherit (shared) autheliaOnly; + + forgeMetricsUrl = "https://forge.t.local/metrics"; cases = [ { # The collector's non-assertion, the deliberate mirror of Grafana's @@ -152,6 +157,45 @@ let && (s.receivers ? prometheus) && builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers; } + { + # The collector reads its own host's target list, so a forge entry + # declared only where the forge runs is a job no split deploy renders. + # `otelRemoteAuthelia` runs neither the forge nor authelia. + name = "a collector on a host without the forge scrapes the forge by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst (j: j.job_name == "forgejo") null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.forgejo.enable + && job != null + && job.static_configs == [ { targets = [ "forge.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == forgeMetricsUrl + && builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers; + } + { + # The other half: authelia refuses a token for an audience its client + # was not registered for, and registers it from its OWN host's list. + name = "authelia on a host without the forge or the collector admits the collector's forge scrape"; + ok = + let + h = autheliaOnly.services.hyperhive; + client = lib.findFirst (c: c.id == h.swarm.otel.clientId) null h.swarm.authelia.oidc.clients; + rules = + autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; + in + !h.deploy.forgejo.enable + && !h.deploy.swarm-otel.enable + && client != null + && builtins.elem forgeMetricsUrl client.audience + && builtins.elem { + domain = "forge.t.local"; + resources = [ "^/metrics$" ]; + policy = "one_factor"; + subject = [ "oauth2:client:${h.swarm.otel.clientId}" ]; + } rules; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive