From 393e84afd0fdf60513610b0cdf821279eb4c72a8 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 13 Oct 2026 10:36:15 +0200 Subject: [PATCH 1/3] swarm-otel: scrape authelia through its published name Authelia's metrics were scraped over loopback, declared only on a host running both authelia and the collector, so a split deploy collected none. Its target is now declared on every host as `https:///metrics`, next to forgejo's, and authelia's vhost serves `= /metrics` behind the same collector auth subrequest as the forge's. The loopback entry is gone, so a single host still scrapes authelia once, now by name. The forge-only `metricsRule` becomes `metricsRules`, derived from every `publishedScrapeTargets` entry. Each target gets the collector-scoped `one_factor` rule followed by an unconditional `deny`. The `deny` is needed because authelia 4.39 skips a rule whose `subject` does not match and moves on. With nothing after the collector rule, any other authenticated session reached `default_policy = one_factor` and was admitted. That was already true of the forge's `/metrics`, and this change closes it there too. Part of #5205 --- nix/host-modules/swarm-authelia-service.nix | 19 ++-- nix/host-modules/swarm-authelia.nix | 114 +++++++++++++++----- nix/host-modules/swarm-otel-service.nix | 7 +- nix/module-eval/swarm-otel-identity.nix | 96 +++++++++++++++++ 4 files changed, 194 insertions(+), 42 deletions(-) diff --git a/nix/host-modules/swarm-authelia-service.nix b/nix/host-modules/swarm-authelia-service.nix index eb9b1006..9b527bcd 100644 --- a/nix/host-modules/swarm-authelia-service.nix +++ b/nix/host-modules/swarm-authelia-service.nix @@ -43,24 +43,17 @@ in authelia's documentation finds what they expect. A separate port from {option}`port` because it is a separate - listener with a different audience: the main one is proxied by - the gateway and reachable from the swarm, this one is scraped by - the collector on this host and by nothing else. + listener with a different audience: the main one serves logins and + is reachable from the swarm, this one carries no authentication of + its own. The swarm collector scrapes it, from whichever host it + runs on, as `https://''${domain}/metrics`: the gateway's + `= /metrics` location proxies to this port only after authelia + has accepted the collector's token for that URL. ⚠️ Every swarm container shares the host network namespace, so two services defaulting to the same port do not conflict at build time — one simply loses at runtime, with nothing in any log. Check a new value against the others before changing this. - - ::: {.note} - Loopback means this endpoint is only reachable by a collector on - the *same host*, so the scrape target is declared only when one is - enabled here. Run the swarm's collector elsewhere and authelia's - metrics are simply not collected — no error, and nothing in a log - to say so. Making them reachable across hosts is a different piece - of work: the endpoint would have to be published under a name, - with a certificate and an audience. - ::: ''; }; diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a4338cd7..c514460a 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -61,27 +61,44 @@ let collectorClientId = config.services.hyperhive.swarm.otel.clientId; collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients; - # The forge's metrics endpoint: deny until a collector exists to allow. + # Every published metrics endpoint: the collector's client, and nobody else. # - # `deny` is not a placeholder, it is the protection. The endpoint is - # always served (a swarm-integrated forge always has metrics) and - # `default_policy` is `one_factor`, which means *any* authenticated - # subject — every operator today, every agent once they hold authelia - # accounts. Nor does the audience stand between a browser session and - # this data: `authn_strategies` on the authz endpoint also accepts - # `CookieSession`, and a cookie carries no audience at all. - metricsRule = { - domain = hyperhiveCfg.swarm.forge.domain; - resources = [ "^/metrics$" ]; - } - // ( - if collectorRegistered then - { - policy = "one_factor"; - subject = [ "oauth2:client:${collectorClientId}" ]; - } - else - { policy = "deny"; } + # The `deny` is the protection, and it has to be a rule of its own. A rule + # with a `subject` does not match a subject outside it: authelia skips it and + # moves on, and with nothing after it lands on `default_policy`, which is + # `one_factor` — *any* authenticated subject, every operator today, every + # agent once they hold authelia accounts. Nor does the audience stand + # between a browser session and this data: `authn_strategies` on the authz + # endpoint also accepts `CookieSession`, and a cookie carries no audience. + # Until a collector is registered, the `deny` is the only rule. + # + # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same + # URL the collector requests and its token is minted for, so a target + # cannot be published without the rules that confine it to the collector. + metricsRules = lib.concatLists ( + lib.mapAttrsToList ( + job: url: + let + # The split `parsePublished` in ./swarm-otel.nix makes for the scrape + # job; the two must agree on what the URL's host and path are. + parts = builtins.match "https://([^/]+)(/.*)" url; + target = { + domain = builtins.elemAt parts 0; + resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}$" ]; + }; + in + if parts == null then + throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived." + else + lib.optional collectorRegistered ( + target + // { + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; + } + ) + ++ [ (target // { policy = "deny"; }) ] + ) config.services.hyperhive.swarm.otel.publishedScrapeTargets ); # Upstream's `services.authelia.instances.` derives the unit, @@ -779,6 +796,48 @@ in locations."/bridge/" = { proxyPass = "http://127.0.0.1:${toString cfg.bridgePort}/"; }; + + # Authelia's own metrics, for the swarm collector wherever it runs. + # The listener (`metricsPort`) is loopback-only and unauthenticated; + # this location is the only way in from off the machine. `= /metrics` + # outranks the `/` prefix, so its auth is the one that runs. + # + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.authelia` in ./swarm-otel-service.nix; the two + # must agree, since that string is also the token's audience. + locations."= /metrics" = { + proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics"; + extraConfig = '' + auth_request /__authelia_metrics_authz; + ''; + }; + + # The subrequest, same shape as the forge's `= /__forge_metrics_authz` + # (./hive-forge/default.nix), and machine-facing for the same reason: no + # `error_page 401 =302`, so a scraper gets the 401 rather than a login + # page parsed as metrics. + # + # Authelia authorising a request for itself does not loop: the + # subrequest lands on `/api/` above, which carries no `auth_request`. + # An authelia that is down answers the subrequest 502, which + # `auth_request` turns into a 500 for the scrape — closed, not open. + locations."= /__authelia_metrics_authz" = { + proxyPass = "https://${cfg.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which would + # override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo cfg.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; }; # Order the container after the host CA generator, so the bind source @@ -1038,7 +1097,8 @@ in # Loopback only, like the main listener above and for a # stronger reason: this endpoint has no authentication of its # own, and it reports request volumes and outcomes for every - # SSO login on the swarm. + # SSO login on the swarm. The collector reaches it through the + # vhost's gated `= /metrics` location. telemetry.metrics = { enabled = true; address = "tcp://127.0.0.1:${toString cfg.metricsPort}"; @@ -1070,13 +1130,13 @@ in access_control = { default_policy = "one_factor"; # ⚠️ ORDER MATTERS — authelia takes the FIRST matching rule. - # The metrics rule is listed first so it cannot be shadowed - # by a broader domain rule added later. + # The metrics rules are listed first so they cannot be + # shadowed by a broader domain rule added later. rules = - # Denied or client-scoped depending on whether a - # collector is registered — see `metricsRule` above, - # which is where the reasoning for both halves lives. - [ metricsRule ] + # The collector's client where one is registered, then + # `deny` — see `metricsRules` above, which is where the + # reasoning for both halves lives. + metricsRules # Also guarded on the domain being set: without it a null # apex would render a rule matching the string "null". ++ lib.optional (deployCfg.swarm-controller.enable && swarmDomain != null) { diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 7bb04080..086e72d8 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -289,8 +289,9 @@ in nothing, which shows as a failing job rather than a missing one. The collector's OAuth2 client derives its permitted audiences from - these URLs, so a target and the authorisation to reach it are **one - declaration**. + these URLs, and authelia derives from each one the access rule that + admits only that client to it, so a target and the authorisation to + reach it are **one declaration**. Two lists that must agree would be a drift to maintain, and its failure mode is the bad one: a target whose audience was forgotten authenticates against nothing and reads as a scrape failure rather @@ -355,5 +356,7 @@ in config.services.hyperhive.swarm.otel.publishedScrapeTargets = { # The vhost location is in ./hive-forge/default.nix. forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics"; + # The vhost location is in ./swarm-authelia.nix. + authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics"; }; } diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 814d053b..6d22d213 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -101,7 +101,30 @@ let # authelia with neither the forge nor the collector beside it. inherit (shared) autheliaOnly; + # Every swarm service on one host, the collector and authelia included. + inherit (shared) allLocal; + forgeMetricsUrl = "https://forge.t.local/metrics"; + autheliaMetricsUrl = "https://auth.t.local/metrics"; + + # The policy authelia applies to `subject` requesting `https://`: + # the first rule whose domain, resources and subject all match, else + # `default_policy`. ⚠️ A rule with a `subject` list does NOT match a subject + # outside it — authelia moves on to the next rule — which is the semantics a + # check of the rule's own fields cannot see. + autheliaPolicy = + m: subject: domain: path: + let + ac = m.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control; + matches = + r: + r.domain == domain + && lib.any (re: builtins.match re path != null) (r.resources or [ ".*" ]) + && (!(r ? subject) || builtins.elem subject r.subject); + hit = lib.findFirst matches null ac.rules; + in + if hit == null then ac.default_policy else hit.policy; + cases = [ { # The collector's non-assertion, the deliberate mirror of Grafana's @@ -196,6 +219,79 @@ let subject = [ "oauth2:client:${h.swarm.otel.clientId}" ]; } rules; } + { + # Authelia's half of the same move: its target is published by name, so + # a collector anywhere in the swarm has a job for it. + name = "a collector on a host without authelia scrapes authelia by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst (j: j.job_name == "authelia") null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.authelia.enable + && job != null + && job.static_configs == [ { targets = [ "auth.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == autheliaMetricsUrl; + } + { + # The location the collector's job requests: gated, and proxying to the + # loopback listener that has no authentication of its own. + name = "authelia's vhost serves /metrics only behind the collector's auth subrequest"; + ok = + let + h = autheliaOnly.services.hyperhive; + locs = autheliaOnly.services.nginx.virtualHosts.${h.swarm.authelia.domain}.locations; + authz = locs."= /__authelia_metrics_authz"; + in + locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.authelia.metricsPort}/metrics" + && lib.hasInfix "auth_request /__authelia_metrics_authz;" locs."= /metrics".extraConfig + && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request" + && lib.hasInfix "internal;" authz.extraConfig; + } + { + # Every published target admits the collector's client and refuses any + # other authenticated subject. Read through authelia's first-match + # semantics: a client-scoped rule alone lets an operator's session fall + # through to `default_policy = one_factor`. + name = "every published metrics URL admits the collector and refuses a plain session"; + ok = + let + h = autheliaOnly.services.hyperhive; + collector = "oauth2:client:${h.swarm.otel.clientId}"; + targets = lib.mapAttrsToList ( + _: url: builtins.match "https://([^/]+)(/.*)" url + ) h.swarm.otel.publishedScrapeTargets; + policy = + subject: t: autheliaPolicy autheliaOnly subject (builtins.elemAt t 0) (builtins.elemAt t 1); + in + lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ + autheliaMetricsUrl + forgeMetricsUrl + ] + && lib.all (t: policy collector t == "one_factor") targets + && lib.all (t: policy "user:someone" t == "deny") targets + && lib.all (t: policy "group:admins" t == "deny") targets; + } + { + # Single host: authelia is scraped once, by its published name. A + # loopback job beside it would scrape the same listener twice — and share + # its job name, which ./swarm-otel.nix refuses. + name = "a host running authelia and the collector scrapes authelia once, by name"; + ok = + let + o = allLocal.services.hyperhive.swarm.otel; + jobs = + lib.filter (j: j.job_name == "authelia") + (otelSettings allLocal).receivers.prometheus.config.scrape_configs; + in + allLocal.services.hyperhive.deploy.authelia.enable + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !(o.scrapeTargets ? authelia) + && (o.publishedScrapeTargets.authelia or null) == autheliaMetricsUrl + && lib.length jobs == 1 + && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ]; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive From 8eb11a91da5b0645dacf8325a74aedd491225943 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 13 Oct 2026 10:52:51 +0200 Subject: [PATCH 2/3] swarm-authelia: match the metrics rules with or without a query string Authelia 4.39.20 matches `resources` against the cleaned path plus `?` (authorization/types.go:94 NewObject, utils/url.go:11-34 URLPathFullClean), while nginx's `= /metrics` serves the request whatever its query. `^/metrics$` therefore missed `/metrics?x=1`: both the collector rule and the trailing `deny` were skipped and a plain session reached `default_policy = one_factor`. The pattern is now `^([?].*)?$` for every published target. Forgejo strips trailing slashes before routing (routers/common/ middleware.go stripSlashesMiddleware) and serves metrics without a token when none is configured, so `forge./metrics/` reached its metrics through the ungated `/` location. The forge vhost now answers `= /metrics/` with 404; nginx merges slashes and decodes `%2F` before matching, so that also covers `/metrics//` and `/metrics%2F`. Part of #5205 --- nix/host-modules/hive-forge/default.nix | 7 ++ nix/host-modules/swarm-authelia.nix | 6 +- nix/module-eval/swarm-otel-identity.nix | 93 ++++++++++++++++++++++--- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index f207c627..abe84206 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -327,6 +327,13 @@ in ''; }; + # ⚠️ Forgejo strips trailing slashes before routing, so `/metrics/` + # reaches the same unauthenticated handler — and the exact match + # above does not take it, leaving it to the `/` catch-all. nginx + # merges slashes and decodes `%2F` before matching, so this one + # location also takes `/metrics//` and `/metrics%2F`. + "= /metrics/".return = "404"; + # The subrequest. Same implementation and header set as # `swarm-ui.nix` uses, for the same reason: `X-Original-URL` # and `X-Original-Method` are what authelia's `auth-request` diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index c514460a..a5c5d760 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -84,7 +84,11 @@ let parts = builtins.match "https://([^/]+)(/.*)" url; target = { domain = builtins.elemAt parts 0; - resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}$" ]; + # ⚠️ Authelia matches `resources` against the cleaned path PLUS the + # query string, while nginx's `= ` location ignores the query. + # A bare `^$` therefore misses `?x`, which the location + # still serves, and that request falls through to `default_policy`. + resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}([?].*)?$" ]; }; in if parts == null then diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 6d22d213..bf666f9f 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -107,8 +107,10 @@ let forgeMetricsUrl = "https://forge.t.local/metrics"; autheliaMetricsUrl = "https://auth.t.local/metrics"; - # The policy authelia applies to `subject` requesting `https://`: - # the first rule whose domain, resources and subject all match, else + # The policy authelia applies to `subject` requesting `https://`, + # where `path` is what authelia matches `resources` against: the cleaned path + # plus `?` when there is one. + # It is the first rule whose domain, resources and subject all match, else # `default_policy`. ⚠️ A rule with a `subject` list does NOT match a subject # outside it — authelia moves on to the next rule — which is the semantics a # check of the rule's own fields cannot see. @@ -214,7 +216,7 @@ let && builtins.elem forgeMetricsUrl client.audience && builtins.elem { domain = "forge.t.local"; - resources = [ "^/metrics$" ]; + resources = [ "^/metrics([?].*)?$" ]; policy = "one_factor"; subject = [ "oauth2:client:${h.swarm.otel.clientId}" ]; } rules; @@ -251,17 +253,30 @@ let } { # Every published target admits the collector's client and refuses any - # other authenticated subject. Read through authelia's first-match - # semantics: a client-scoped rule alone lets an operator's session fall - # through to `default_policy = one_factor`. + # other authenticated subject, with or without a query string. Read + # through authelia's first-match semantics: a client-scoped rule alone + # lets an operator's session fall through to `default_policy = one_factor`, + # and so does a request the rule's regex does not match. name = "every published metrics URL admits the collector and refuses a plain session"; ok = let h = autheliaOnly.services.hyperhive; collector = "oauth2:client:${h.swarm.otel.clientId}"; - targets = lib.mapAttrsToList ( - _: url: builtins.match "https://([^/]+)(/.*)" url - ) h.swarm.otel.publishedScrapeTargets; + targets = lib.concatMap ( + url: + let + t = builtins.match "https://([^/]+)(/.*)" url; + in + map + (q: [ + (builtins.elemAt t 0) + "${builtins.elemAt t 1}${q}" + ]) + [ + "" + "?x=1" + ] + ) (lib.attrValues h.swarm.otel.publishedScrapeTargets); policy = subject: t: autheliaPolicy autheliaOnly subject (builtins.elemAt t 0) (builtins.elemAt t 1); in @@ -269,10 +284,70 @@ let autheliaMetricsUrl forgeMetricsUrl ] + && lib.length targets == 4 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; } + { + # Which request forms reach `= /metrics`, and whether the rule's regex + # matches the path authelia derives from each. `nginx` records the + # location each form lands in (measured against a stand-alone nginx); + # `path` is authelia's cleaned path plus query for that form. The + # invariant: every form the gated location serves is matched by the rule, + # so none of them reaches `default_policy`. + name = "the metrics rule matches every request form the gated location serves"; + ok = + let + rules = + autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; + metricsRules = lib.filter (r: r ? resources) rules; + re = lib.head (lib.head metricsRules).resources; + table = [ + { + request = "/metrics"; + nginx = "= /metrics"; + path = "/metrics"; + } + { + request = "/metrics?x=1"; + nginx = "= /metrics"; + path = "/metrics?x=1"; + } + { + request = "//metrics"; + nginx = "= /metrics"; + path = "/metrics"; + } + { + request = "/metrics/"; + nginx = "= /metrics/ (forge: 404) or /"; + path = "/metrics/"; + } + { + request = "/metricsx"; + nginx = "/"; + path = "/metricsx"; + } + ]; + matched = row: builtins.match re row.path != null; + in + lib.length metricsRules == 4 + && lib.all (r: r.resources == [ re ]) metricsRules + && re == "^/metrics([?].*)?$" + && lib.all (row: matched row == (row.nginx == "= /metrics")) table; + } + { + # Forgejo strips trailing slashes before routing, so `/metrics/` reaches + # its metrics handler unless nginx refuses it first. + name = "the forge refuses /metrics/ rather than proxying it to forgejo's metrics"; + ok = + let + locs = + allLocal.services.nginx.virtualHosts.${allLocal.services.hyperhive.swarm.forge.domain}.locations; + in + (locs."= /metrics/".return or null) == "404"; + } { # Single host: authelia is scraped once, by its published name. A # loopback job beside it would scrape the same listener twice — and share From e7bf33762441f7e3b9d05acf399b7c1ddb570179 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 14 Oct 2026 01:13:33 +0200 Subject: [PATCH 3/3] docs: swarm services publish authelia's /metrics to the collector --- docs/swarm/services.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 94e020f5..26628c8b 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -122,6 +122,11 @@ small-deployment choices, and the scope is the justification: redis buys shared session state across replicas and there is one instance; SMTP exists to mail humans, and provisioning here is programmatic. +The swarm collector scrapes authelia's own metrics at +`https:///metrics`, whichever host runs the +container. The gateway admits only the collector's OAuth2 client there; +any other session gets `deny`. + See [`sso.md`](sso.md) for bootstrapping the first user and the OIDC relying-party flow, and [`secrets.md`](secrets.md) for where authelia generates and reads each of its keys.