From 393e84afd0fdf60513610b0cdf821279eb4c72a8 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 13 Oct 2026 10:36:15 +0200 Subject: [PATCH] swarm-otel: scrape authelia through its published name Authelia's metrics were scraped over loopback, declared only on a host running both authelia and the collector, so a split deploy collected none. Its target is now declared on every host as `https:///metrics`, next to forgejo's, and authelia's vhost serves `= /metrics` behind the same collector auth subrequest as the forge's. The loopback entry is gone, so a single host still scrapes authelia once, now by name. The forge-only `metricsRule` becomes `metricsRules`, derived from every `publishedScrapeTargets` entry. Each target gets the collector-scoped `one_factor` rule followed by an unconditional `deny`. The `deny` is needed because authelia 4.39 skips a rule whose `subject` does not match and moves on. With nothing after the collector rule, any other authenticated session reached `default_policy = one_factor` and was admitted. That was already true of the forge's `/metrics`, and this change closes it there too. Part of #5205 --- nix/host-modules/swarm-authelia-service.nix | 19 ++-- nix/host-modules/swarm-authelia.nix | 114 +++++++++++++++----- nix/host-modules/swarm-otel-service.nix | 7 +- nix/module-eval/swarm-otel-identity.nix | 96 +++++++++++++++++ 4 files changed, 194 insertions(+), 42 deletions(-) diff --git a/nix/host-modules/swarm-authelia-service.nix b/nix/host-modules/swarm-authelia-service.nix index eb9b1006..9b527bcd 100644 --- a/nix/host-modules/swarm-authelia-service.nix +++ b/nix/host-modules/swarm-authelia-service.nix @@ -43,24 +43,17 @@ in authelia's documentation finds what they expect. A separate port from {option}`port` because it is a separate - listener with a different audience: the main one is proxied by - the gateway and reachable from the swarm, this one is scraped by - the collector on this host and by nothing else. + listener with a different audience: the main one serves logins and + is reachable from the swarm, this one carries no authentication of + its own. The swarm collector scrapes it, from whichever host it + runs on, as `https://''${domain}/metrics`: the gateway's + `= /metrics` location proxies to this port only after authelia + has accepted the collector's token for that URL. ⚠️ Every swarm container shares the host network namespace, so two services defaulting to the same port do not conflict at build time — one simply loses at runtime, with nothing in any log. Check a new value against the others before changing this. - - ::: {.note} - Loopback means this endpoint is only reachable by a collector on - the *same host*, so the scrape target is declared only when one is - enabled here. Run the swarm's collector elsewhere and authelia's - metrics are simply not collected — no error, and nothing in a log - to say so. Making them reachable across hosts is a different piece - of work: the endpoint would have to be published under a name, - with a certificate and an audience. - ::: ''; }; diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a4338cd7..c514460a 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -61,27 +61,44 @@ let collectorClientId = config.services.hyperhive.swarm.otel.clientId; collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients; - # The forge's metrics endpoint: deny until a collector exists to allow. + # Every published metrics endpoint: the collector's client, and nobody else. # - # `deny` is not a placeholder, it is the protection. The endpoint is - # always served (a swarm-integrated forge always has metrics) and - # `default_policy` is `one_factor`, which means *any* authenticated - # subject — every operator today, every agent once they hold authelia - # accounts. Nor does the audience stand between a browser session and - # this data: `authn_strategies` on the authz endpoint also accepts - # `CookieSession`, and a cookie carries no audience at all. - metricsRule = { - domain = hyperhiveCfg.swarm.forge.domain; - resources = [ "^/metrics$" ]; - } - // ( - if collectorRegistered then - { - policy = "one_factor"; - subject = [ "oauth2:client:${collectorClientId}" ]; - } - else - { policy = "deny"; } + # The `deny` is the protection, and it has to be a rule of its own. A rule + # with a `subject` does not match a subject outside it: authelia skips it and + # moves on, and with nothing after it lands on `default_policy`, which is + # `one_factor` — *any* authenticated subject, every operator today, every + # agent once they hold authelia accounts. Nor does the audience stand + # between a browser session and this data: `authn_strategies` on the authz + # endpoint also accepts `CookieSession`, and a cookie carries no audience. + # Until a collector is registered, the `deny` is the only rule. + # + # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same + # URL the collector requests and its token is minted for, so a target + # cannot be published without the rules that confine it to the collector. + metricsRules = lib.concatLists ( + lib.mapAttrsToList ( + job: url: + let + # The split `parsePublished` in ./swarm-otel.nix makes for the scrape + # job; the two must agree on what the URL's host and path are. + parts = builtins.match "https://([^/]+)(/.*)" url; + target = { + domain = builtins.elemAt parts 0; + resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}$" ]; + }; + in + if parts == null then + throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived." + else + lib.optional collectorRegistered ( + target + // { + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; + } + ) + ++ [ (target // { policy = "deny"; }) ] + ) config.services.hyperhive.swarm.otel.publishedScrapeTargets ); # Upstream's `services.authelia.instances.` derives the unit, @@ -779,6 +796,48 @@ in locations."/bridge/" = { proxyPass = "http://127.0.0.1:${toString cfg.bridgePort}/"; }; + + # Authelia's own metrics, for the swarm collector wherever it runs. + # The listener (`metricsPort`) is loopback-only and unauthenticated; + # this location is the only way in from off the machine. `= /metrics` + # outranks the `/` prefix, so its auth is the one that runs. + # + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.authelia` in ./swarm-otel-service.nix; the two + # must agree, since that string is also the token's audience. + locations."= /metrics" = { + proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics"; + extraConfig = '' + auth_request /__authelia_metrics_authz; + ''; + }; + + # The subrequest, same shape as the forge's `= /__forge_metrics_authz` + # (./hive-forge/default.nix), and machine-facing for the same reason: no + # `error_page 401 =302`, so a scraper gets the 401 rather than a login + # page parsed as metrics. + # + # Authelia authorising a request for itself does not loop: the + # subrequest lands on `/api/` above, which carries no `auth_request`. + # An authelia that is down answers the subrequest 502, which + # `auth_request` turns into a 500 for the scrape — closed, not open. + locations."= /__authelia_metrics_authz" = { + proxyPass = "https://${cfg.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which would + # override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo cfg.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; }; # Order the container after the host CA generator, so the bind source @@ -1038,7 +1097,8 @@ in # Loopback only, like the main listener above and for a # stronger reason: this endpoint has no authentication of its # own, and it reports request volumes and outcomes for every - # SSO login on the swarm. + # SSO login on the swarm. The collector reaches it through the + # vhost's gated `= /metrics` location. telemetry.metrics = { enabled = true; address = "tcp://127.0.0.1:${toString cfg.metricsPort}"; @@ -1070,13 +1130,13 @@ in access_control = { default_policy = "one_factor"; # ⚠️ ORDER MATTERS — authelia takes the FIRST matching rule. - # The metrics rule is listed first so it cannot be shadowed - # by a broader domain rule added later. + # The metrics rules are listed first so they cannot be + # shadowed by a broader domain rule added later. rules = - # Denied or client-scoped depending on whether a - # collector is registered — see `metricsRule` above, - # which is where the reasoning for both halves lives. - [ metricsRule ] + # The collector's client where one is registered, then + # `deny` — see `metricsRules` above, which is where the + # reasoning for both halves lives. + metricsRules # Also guarded on the domain being set: without it a null # apex would render a rule matching the string "null". ++ lib.optional (deployCfg.swarm-controller.enable && swarmDomain != null) { diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 7bb04080..086e72d8 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -289,8 +289,9 @@ in nothing, which shows as a failing job rather than a missing one. The collector's OAuth2 client derives its permitted audiences from - these URLs, so a target and the authorisation to reach it are **one - declaration**. + these URLs, and authelia derives from each one the access rule that + admits only that client to it, so a target and the authorisation to + reach it are **one declaration**. Two lists that must agree would be a drift to maintain, and its failure mode is the bad one: a target whose audience was forgotten authenticates against nothing and reads as a scrape failure rather @@ -355,5 +356,7 @@ in config.services.hyperhive.swarm.otel.publishedScrapeTargets = { # The vhost location is in ./hive-forge/default.nix. forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics"; + # The vhost location is in ./swarm-authelia.nix. + authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics"; }; } diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 814d053b..6d22d213 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -101,7 +101,30 @@ let # authelia with neither the forge nor the collector beside it. inherit (shared) autheliaOnly; + # Every swarm service on one host, the collector and authelia included. + inherit (shared) allLocal; + forgeMetricsUrl = "https://forge.t.local/metrics"; + autheliaMetricsUrl = "https://auth.t.local/metrics"; + + # The policy authelia applies to `subject` requesting `https://`: + # the first rule whose domain, resources and subject all match, else + # `default_policy`. ⚠️ A rule with a `subject` list does NOT match a subject + # outside it — authelia moves on to the next rule — which is the semantics a + # check of the rule's own fields cannot see. + autheliaPolicy = + m: subject: domain: path: + let + ac = m.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control; + matches = + r: + r.domain == domain + && lib.any (re: builtins.match re path != null) (r.resources or [ ".*" ]) + && (!(r ? subject) || builtins.elem subject r.subject); + hit = lib.findFirst matches null ac.rules; + in + if hit == null then ac.default_policy else hit.policy; + cases = [ { # The collector's non-assertion, the deliberate mirror of Grafana's @@ -196,6 +219,79 @@ let subject = [ "oauth2:client:${h.swarm.otel.clientId}" ]; } rules; } + { + # Authelia's half of the same move: its target is published by name, so + # a collector anywhere in the swarm has a job for it. + name = "a collector on a host without authelia scrapes authelia by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst (j: j.job_name == "authelia") null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.authelia.enable + && job != null + && job.static_configs == [ { targets = [ "auth.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == autheliaMetricsUrl; + } + { + # The location the collector's job requests: gated, and proxying to the + # loopback listener that has no authentication of its own. + name = "authelia's vhost serves /metrics only behind the collector's auth subrequest"; + ok = + let + h = autheliaOnly.services.hyperhive; + locs = autheliaOnly.services.nginx.virtualHosts.${h.swarm.authelia.domain}.locations; + authz = locs."= /__authelia_metrics_authz"; + in + locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.authelia.metricsPort}/metrics" + && lib.hasInfix "auth_request /__authelia_metrics_authz;" locs."= /metrics".extraConfig + && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request" + && lib.hasInfix "internal;" authz.extraConfig; + } + { + # Every published target admits the collector's client and refuses any + # other authenticated subject. Read through authelia's first-match + # semantics: a client-scoped rule alone lets an operator's session fall + # through to `default_policy = one_factor`. + name = "every published metrics URL admits the collector and refuses a plain session"; + ok = + let + h = autheliaOnly.services.hyperhive; + collector = "oauth2:client:${h.swarm.otel.clientId}"; + targets = lib.mapAttrsToList ( + _: url: builtins.match "https://([^/]+)(/.*)" url + ) h.swarm.otel.publishedScrapeTargets; + policy = + subject: t: autheliaPolicy autheliaOnly subject (builtins.elemAt t 0) (builtins.elemAt t 1); + in + lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ + autheliaMetricsUrl + forgeMetricsUrl + ] + && lib.all (t: policy collector t == "one_factor") targets + && lib.all (t: policy "user:someone" t == "deny") targets + && lib.all (t: policy "group:admins" t == "deny") targets; + } + { + # Single host: authelia is scraped once, by its published name. A + # loopback job beside it would scrape the same listener twice — and share + # its job name, which ./swarm-otel.nix refuses. + name = "a host running authelia and the collector scrapes authelia once, by name"; + ok = + let + o = allLocal.services.hyperhive.swarm.otel; + jobs = + lib.filter (j: j.job_name == "authelia") + (otelSettings allLocal).receivers.prometheus.config.scrape_configs; + in + allLocal.services.hyperhive.deploy.authelia.enable + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !(o.scrapeTargets ? authelia) + && (o.publishedScrapeTargets.authelia or null) == autheliaMetricsUrl + && lib.length jobs == 1 + && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ]; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive