diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 94e020f5..26628c8b 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -122,6 +122,11 @@ small-deployment choices, and the scope is the justification: redis buys shared session state across replicas and there is one instance; SMTP exists to mail humans, and provisioning here is programmatic. +The swarm collector scrapes authelia's own metrics at +`https:///metrics`, whichever host runs the +container. The gateway admits only the collector's OAuth2 client there; +any other session gets `deny`. + See [`sso.md`](sso.md) for bootstrapping the first user and the OIDC relying-party flow, and [`secrets.md`](secrets.md) for where authelia generates and reads each of its keys. diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index f207c627..abe84206 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -327,6 +327,13 @@ in ''; }; + # ⚠️ Forgejo strips trailing slashes before routing, so `/metrics/` + # reaches the same unauthenticated handler — and the exact match + # above does not take it, leaving it to the `/` catch-all. nginx + # merges slashes and decodes `%2F` before matching, so this one + # location also takes `/metrics//` and `/metrics%2F`. + "= /metrics/".return = "404"; + # The subrequest. Same implementation and header set as # `swarm-ui.nix` uses, for the same reason: `X-Original-URL` # and `X-Original-Method` are what authelia's `auth-request` diff --git a/nix/host-modules/swarm-authelia-service.nix b/nix/host-modules/swarm-authelia-service.nix index eb9b1006..9b527bcd 100644 --- a/nix/host-modules/swarm-authelia-service.nix +++ b/nix/host-modules/swarm-authelia-service.nix @@ -43,24 +43,17 @@ in authelia's documentation finds what they expect. A separate port from {option}`port` because it is a separate - listener with a different audience: the main one is proxied by - the gateway and reachable from the swarm, this one is scraped by - the collector on this host and by nothing else. + listener with a different audience: the main one serves logins and + is reachable from the swarm, this one carries no authentication of + its own. The swarm collector scrapes it, from whichever host it + runs on, as `https://''${domain}/metrics`: the gateway's + `= /metrics` location proxies to this port only after authelia + has accepted the collector's token for that URL. ⚠️ Every swarm container shares the host network namespace, so two services defaulting to the same port do not conflict at build time — one simply loses at runtime, with nothing in any log. Check a new value against the others before changing this. - - ::: {.note} - Loopback means this endpoint is only reachable by a collector on - the *same host*, so the scrape target is declared only when one is - enabled here. Run the swarm's collector elsewhere and authelia's - metrics are simply not collected — no error, and nothing in a log - to say so. Making them reachable across hosts is a different piece - of work: the endpoint would have to be published under a name, - with a certificate and an audience. - ::: ''; }; diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a4338cd7..a5c5d760 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -61,27 +61,48 @@ let collectorClientId = config.services.hyperhive.swarm.otel.clientId; collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients; - # The forge's metrics endpoint: deny until a collector exists to allow. + # Every published metrics endpoint: the collector's client, and nobody else. # - # `deny` is not a placeholder, it is the protection. The endpoint is - # always served (a swarm-integrated forge always has metrics) and - # `default_policy` is `one_factor`, which means *any* authenticated - # subject — every operator today, every agent once they hold authelia - # accounts. Nor does the audience stand between a browser session and - # this data: `authn_strategies` on the authz endpoint also accepts - # `CookieSession`, and a cookie carries no audience at all. - metricsRule = { - domain = hyperhiveCfg.swarm.forge.domain; - resources = [ "^/metrics$" ]; - } - // ( - if collectorRegistered then - { - policy = "one_factor"; - subject = [ "oauth2:client:${collectorClientId}" ]; - } - else - { policy = "deny"; } + # The `deny` is the protection, and it has to be a rule of its own. A rule + # with a `subject` does not match a subject outside it: authelia skips it and + # moves on, and with nothing after it lands on `default_policy`, which is + # `one_factor` — *any* authenticated subject, every operator today, every + # agent once they hold authelia accounts. Nor does the audience stand + # between a browser session and this data: `authn_strategies` on the authz + # endpoint also accepts `CookieSession`, and a cookie carries no audience. + # Until a collector is registered, the `deny` is the only rule. + # + # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same + # URL the collector requests and its token is minted for, so a target + # cannot be published without the rules that confine it to the collector. + metricsRules = lib.concatLists ( + lib.mapAttrsToList ( + job: url: + let + # The split `parsePublished` in ./swarm-otel.nix makes for the scrape + # job; the two must agree on what the URL's host and path are. + parts = builtins.match "https://([^/]+)(/.*)" url; + target = { + domain = builtins.elemAt parts 0; + # ⚠️ Authelia matches `resources` against the cleaned path PLUS the + # query string, while nginx's `= ` location ignores the query. + # A bare `^$` therefore misses `?x`, which the location + # still serves, and that request falls through to `default_policy`. + resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}([?].*)?$" ]; + }; + in + if parts == null then + throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived." + else + lib.optional collectorRegistered ( + target + // { + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; + } + ) + ++ [ (target // { policy = "deny"; }) ] + ) config.services.hyperhive.swarm.otel.publishedScrapeTargets ); # Upstream's `services.authelia.instances.` derives the unit, @@ -779,6 +800,48 @@ in locations."/bridge/" = { proxyPass = "http://127.0.0.1:${toString cfg.bridgePort}/"; }; + + # Authelia's own metrics, for the swarm collector wherever it runs. + # The listener (`metricsPort`) is loopback-only and unauthenticated; + # this location is the only way in from off the machine. `= /metrics` + # outranks the `/` prefix, so its auth is the one that runs. + # + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.authelia` in ./swarm-otel-service.nix; the two + # must agree, since that string is also the token's audience. + locations."= /metrics" = { + proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics"; + extraConfig = '' + auth_request /__authelia_metrics_authz; + ''; + }; + + # The subrequest, same shape as the forge's `= /__forge_metrics_authz` + # (./hive-forge/default.nix), and machine-facing for the same reason: no + # `error_page 401 =302`, so a scraper gets the 401 rather than a login + # page parsed as metrics. + # + # Authelia authorising a request for itself does not loop: the + # subrequest lands on `/api/` above, which carries no `auth_request`. + # An authelia that is down answers the subrequest 502, which + # `auth_request` turns into a 500 for the scrape — closed, not open. + locations."= /__authelia_metrics_authz" = { + proxyPass = "https://${cfg.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which would + # override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo cfg.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; }; # Order the container after the host CA generator, so the bind source @@ -1038,7 +1101,8 @@ in # Loopback only, like the main listener above and for a # stronger reason: this endpoint has no authentication of its # own, and it reports request volumes and outcomes for every - # SSO login on the swarm. + # SSO login on the swarm. The collector reaches it through the + # vhost's gated `= /metrics` location. telemetry.metrics = { enabled = true; address = "tcp://127.0.0.1:${toString cfg.metricsPort}"; @@ -1070,13 +1134,13 @@ in access_control = { default_policy = "one_factor"; # ⚠️ ORDER MATTERS — authelia takes the FIRST matching rule. - # The metrics rule is listed first so it cannot be shadowed - # by a broader domain rule added later. + # The metrics rules are listed first so they cannot be + # shadowed by a broader domain rule added later. rules = - # Denied or client-scoped depending on whether a - # collector is registered — see `metricsRule` above, - # which is where the reasoning for both halves lives. - [ metricsRule ] + # The collector's client where one is registered, then + # `deny` — see `metricsRules` above, which is where the + # reasoning for both halves lives. + metricsRules # Also guarded on the domain being set: without it a null # apex would render a rule matching the string "null". ++ lib.optional (deployCfg.swarm-controller.enable && swarmDomain != null) { diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 7bb04080..086e72d8 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -289,8 +289,9 @@ in nothing, which shows as a failing job rather than a missing one. The collector's OAuth2 client derives its permitted audiences from - these URLs, so a target and the authorisation to reach it are **one - declaration**. + these URLs, and authelia derives from each one the access rule that + admits only that client to it, so a target and the authorisation to + reach it are **one declaration**. Two lists that must agree would be a drift to maintain, and its failure mode is the bad one: a target whose audience was forgotten authenticates against nothing and reads as a scrape failure rather @@ -355,5 +356,7 @@ in config.services.hyperhive.swarm.otel.publishedScrapeTargets = { # The vhost location is in ./hive-forge/default.nix. forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics"; + # The vhost location is in ./swarm-authelia.nix. + authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics"; }; } diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 814d053b..bf666f9f 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -101,7 +101,32 @@ let # authelia with neither the forge nor the collector beside it. inherit (shared) autheliaOnly; + # Every swarm service on one host, the collector and authelia included. + inherit (shared) allLocal; + forgeMetricsUrl = "https://forge.t.local/metrics"; + autheliaMetricsUrl = "https://auth.t.local/metrics"; + + # The policy authelia applies to `subject` requesting `https://`, + # where `path` is what authelia matches `resources` against: the cleaned path + # plus `?` when there is one. + # It is the first rule whose domain, resources and subject all match, else + # `default_policy`. ⚠️ A rule with a `subject` list does NOT match a subject + # outside it — authelia moves on to the next rule — which is the semantics a + # check of the rule's own fields cannot see. + autheliaPolicy = + m: subject: domain: path: + let + ac = m.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control; + matches = + r: + r.domain == domain + && lib.any (re: builtins.match re path != null) (r.resources or [ ".*" ]) + && (!(r ? subject) || builtins.elem subject r.subject); + hit = lib.findFirst matches null ac.rules; + in + if hit == null then ac.default_policy else hit.policy; + cases = [ { # The collector's non-assertion, the deliberate mirror of Grafana's @@ -191,11 +216,157 @@ let && builtins.elem forgeMetricsUrl client.audience && builtins.elem { domain = "forge.t.local"; - resources = [ "^/metrics$" ]; + resources = [ "^/metrics([?].*)?$" ]; policy = "one_factor"; subject = [ "oauth2:client:${h.swarm.otel.clientId}" ]; } rules; } + { + # Authelia's half of the same move: its target is published by name, so + # a collector anywhere in the swarm has a job for it. + name = "a collector on a host without authelia scrapes authelia by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst (j: j.job_name == "authelia") null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.authelia.enable + && job != null + && job.static_configs == [ { targets = [ "auth.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == autheliaMetricsUrl; + } + { + # The location the collector's job requests: gated, and proxying to the + # loopback listener that has no authentication of its own. + name = "authelia's vhost serves /metrics only behind the collector's auth subrequest"; + ok = + let + h = autheliaOnly.services.hyperhive; + locs = autheliaOnly.services.nginx.virtualHosts.${h.swarm.authelia.domain}.locations; + authz = locs."= /__authelia_metrics_authz"; + in + locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.authelia.metricsPort}/metrics" + && lib.hasInfix "auth_request /__authelia_metrics_authz;" locs."= /metrics".extraConfig + && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request" + && lib.hasInfix "internal;" authz.extraConfig; + } + { + # Every published target admits the collector's client and refuses any + # other authenticated subject, with or without a query string. Read + # through authelia's first-match semantics: a client-scoped rule alone + # lets an operator's session fall through to `default_policy = one_factor`, + # and so does a request the rule's regex does not match. + name = "every published metrics URL admits the collector and refuses a plain session"; + ok = + let + h = autheliaOnly.services.hyperhive; + collector = "oauth2:client:${h.swarm.otel.clientId}"; + targets = lib.concatMap ( + url: + let + t = builtins.match "https://([^/]+)(/.*)" url; + in + map + (q: [ + (builtins.elemAt t 0) + "${builtins.elemAt t 1}${q}" + ]) + [ + "" + "?x=1" + ] + ) (lib.attrValues h.swarm.otel.publishedScrapeTargets); + policy = + subject: t: autheliaPolicy autheliaOnly subject (builtins.elemAt t 0) (builtins.elemAt t 1); + in + lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ + autheliaMetricsUrl + forgeMetricsUrl + ] + && lib.length targets == 4 + && lib.all (t: policy collector t == "one_factor") targets + && lib.all (t: policy "user:someone" t == "deny") targets + && lib.all (t: policy "group:admins" t == "deny") targets; + } + { + # Which request forms reach `= /metrics`, and whether the rule's regex + # matches the path authelia derives from each. `nginx` records the + # location each form lands in (measured against a stand-alone nginx); + # `path` is authelia's cleaned path plus query for that form. The + # invariant: every form the gated location serves is matched by the rule, + # so none of them reaches `default_policy`. + name = "the metrics rule matches every request form the gated location serves"; + ok = + let + rules = + autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; + metricsRules = lib.filter (r: r ? resources) rules; + re = lib.head (lib.head metricsRules).resources; + table = [ + { + request = "/metrics"; + nginx = "= /metrics"; + path = "/metrics"; + } + { + request = "/metrics?x=1"; + nginx = "= /metrics"; + path = "/metrics?x=1"; + } + { + request = "//metrics"; + nginx = "= /metrics"; + path = "/metrics"; + } + { + request = "/metrics/"; + nginx = "= /metrics/ (forge: 404) or /"; + path = "/metrics/"; + } + { + request = "/metricsx"; + nginx = "/"; + path = "/metricsx"; + } + ]; + matched = row: builtins.match re row.path != null; + in + lib.length metricsRules == 4 + && lib.all (r: r.resources == [ re ]) metricsRules + && re == "^/metrics([?].*)?$" + && lib.all (row: matched row == (row.nginx == "= /metrics")) table; + } + { + # Forgejo strips trailing slashes before routing, so `/metrics/` reaches + # its metrics handler unless nginx refuses it first. + name = "the forge refuses /metrics/ rather than proxying it to forgejo's metrics"; + ok = + let + locs = + allLocal.services.nginx.virtualHosts.${allLocal.services.hyperhive.swarm.forge.domain}.locations; + in + (locs."= /metrics/".return or null) == "404"; + } + { + # Single host: authelia is scraped once, by its published name. A + # loopback job beside it would scrape the same listener twice — and share + # its job name, which ./swarm-otel.nix refuses. + name = "a host running authelia and the collector scrapes authelia once, by name"; + ok = + let + o = allLocal.services.hyperhive.swarm.otel; + jobs = + lib.filter (j: j.job_name == "authelia") + (otelSettings allLocal).receivers.prometheus.config.scrape_configs; + in + allLocal.services.hyperhive.deploy.authelia.enable + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !(o.scrapeTargets ? authelia) + && (o.publishedScrapeTargets.authelia or null) == autheliaMetricsUrl + && lib.length jobs == 1 + && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ]; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive