diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 26628c8b..94e020f5 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -122,11 +122,6 @@ small-deployment choices, and the scope is the justification: redis buys shared session state across replicas and there is one instance; SMTP exists to mail humans, and provisioning here is programmatic. -The swarm collector scrapes authelia's own metrics at -`https:///metrics`, whichever host runs the -container. The gateway admits only the collector's OAuth2 client there; -any other session gets `deny`. - See [`sso.md`](sso.md) for bootstrapping the first user and the OIDC relying-party flow, and [`secrets.md`](secrets.md) for where authelia generates and reads each of its keys. diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index abe84206..f207c627 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -327,13 +327,6 @@ in ''; }; - # ⚠️ Forgejo strips trailing slashes before routing, so `/metrics/` - # reaches the same unauthenticated handler — and the exact match - # above does not take it, leaving it to the `/` catch-all. nginx - # merges slashes and decodes `%2F` before matching, so this one - # location also takes `/metrics//` and `/metrics%2F`. - "= /metrics/".return = "404"; - # The subrequest. Same implementation and header set as # `swarm-ui.nix` uses, for the same reason: `X-Original-URL` # and `X-Original-Method` are what authelia's `auth-request` diff --git a/nix/host-modules/swarm-authelia-service.nix b/nix/host-modules/swarm-authelia-service.nix index 9b527bcd..eb9b1006 100644 --- a/nix/host-modules/swarm-authelia-service.nix +++ b/nix/host-modules/swarm-authelia-service.nix @@ -43,17 +43,24 @@ in authelia's documentation finds what they expect. A separate port from {option}`port` because it is a separate - listener with a different audience: the main one serves logins and - is reachable from the swarm, this one carries no authentication of - its own. The swarm collector scrapes it, from whichever host it - runs on, as `https://''${domain}/metrics`: the gateway's - `= /metrics` location proxies to this port only after authelia - has accepted the collector's token for that URL. + listener with a different audience: the main one is proxied by + the gateway and reachable from the swarm, this one is scraped by + the collector on this host and by nothing else. ⚠️ Every swarm container shares the host network namespace, so two services defaulting to the same port do not conflict at build time — one simply loses at runtime, with nothing in any log. Check a new value against the others before changing this. + + ::: {.note} + Loopback means this endpoint is only reachable by a collector on + the *same host*, so the scrape target is declared only when one is + enabled here. Run the swarm's collector elsewhere and authelia's + metrics are simply not collected — no error, and nothing in a log + to say so. Making them reachable across hosts is a different piece + of work: the endpoint would have to be published under a name, + with a certificate and an audience. + ::: ''; }; diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a5c5d760..a4338cd7 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -61,48 +61,27 @@ let collectorClientId = config.services.hyperhive.swarm.otel.clientId; collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients; - # Every published metrics endpoint: the collector's client, and nobody else. + # The forge's metrics endpoint: deny until a collector exists to allow. # - # The `deny` is the protection, and it has to be a rule of its own. A rule - # with a `subject` does not match a subject outside it: authelia skips it and - # moves on, and with nothing after it lands on `default_policy`, which is - # `one_factor` — *any* authenticated subject, every operator today, every - # agent once they hold authelia accounts. Nor does the audience stand - # between a browser session and this data: `authn_strategies` on the authz - # endpoint also accepts `CookieSession`, and a cookie carries no audience. - # Until a collector is registered, the `deny` is the only rule. - # - # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same - # URL the collector requests and its token is minted for, so a target - # cannot be published without the rules that confine it to the collector. - metricsRules = lib.concatLists ( - lib.mapAttrsToList ( - job: url: - let - # The split `parsePublished` in ./swarm-otel.nix makes for the scrape - # job; the two must agree on what the URL's host and path are. - parts = builtins.match "https://([^/]+)(/.*)" url; - target = { - domain = builtins.elemAt parts 0; - # ⚠️ Authelia matches `resources` against the cleaned path PLUS the - # query string, while nginx's `= ` location ignores the query. - # A bare `^$` therefore misses `?x`, which the location - # still serves, and that request falls through to `default_policy`. - resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}([?].*)?$" ]; - }; - in - if parts == null then - throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived." - else - lib.optional collectorRegistered ( - target - // { - policy = "one_factor"; - subject = [ "oauth2:client:${collectorClientId}" ]; - } - ) - ++ [ (target // { policy = "deny"; }) ] - ) config.services.hyperhive.swarm.otel.publishedScrapeTargets + # `deny` is not a placeholder, it is the protection. The endpoint is + # always served (a swarm-integrated forge always has metrics) and + # `default_policy` is `one_factor`, which means *any* authenticated + # subject — every operator today, every agent once they hold authelia + # accounts. Nor does the audience stand between a browser session and + # this data: `authn_strategies` on the authz endpoint also accepts + # `CookieSession`, and a cookie carries no audience at all. + metricsRule = { + domain = hyperhiveCfg.swarm.forge.domain; + resources = [ "^/metrics$" ]; + } + // ( + if collectorRegistered then + { + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; + } + else + { policy = "deny"; } ); # Upstream's `services.authelia.instances.` derives the unit, @@ -800,48 +779,6 @@ in locations."/bridge/" = { proxyPass = "http://127.0.0.1:${toString cfg.bridgePort}/"; }; - - # Authelia's own metrics, for the swarm collector wherever it runs. - # The listener (`metricsPort`) is loopback-only and unauthenticated; - # this location is the only way in from off the machine. `= /metrics` - # outranks the `/` prefix, so its auth is the one that runs. - # - # The collector requests exactly this URL, declared as - # `publishedScrapeTargets.authelia` in ./swarm-otel-service.nix; the two - # must agree, since that string is also the token's audience. - locations."= /metrics" = { - proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics"; - extraConfig = '' - auth_request /__authelia_metrics_authz; - ''; - }; - - # The subrequest, same shape as the forge's `= /__forge_metrics_authz` - # (./hive-forge/default.nix), and machine-facing for the same reason: no - # `error_page 401 =302`, so a scraper gets the 401 rather than a login - # page parsed as metrics. - # - # Authelia authorising a request for itself does not loop: the - # subrequest lands on `/api/` above, which carries no `auth_request`. - # An authelia that is down answers the subrequest 502, which - # `auth_request` turns into a 500 for the scrape — closed, not open. - locations."= /__authelia_metrics_authz" = { - proxyPass = "https://${cfg.domain}/api/authz/auth-request"; - # nixpkgs appends its OWN `Host $host` after extraConfig, which would - # override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. - recommendedProxySettings = false; - extraConfig = '' - internal; - ${gatewayCfg.lib.verifiedProxyTo cfg.domain} - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; }; # Order the container after the host CA generator, so the bind source @@ -1101,8 +1038,7 @@ in # Loopback only, like the main listener above and for a # stronger reason: this endpoint has no authentication of its # own, and it reports request volumes and outcomes for every - # SSO login on the swarm. The collector reaches it through the - # vhost's gated `= /metrics` location. + # SSO login on the swarm. telemetry.metrics = { enabled = true; address = "tcp://127.0.0.1:${toString cfg.metricsPort}"; @@ -1134,13 +1070,13 @@ in access_control = { default_policy = "one_factor"; # ⚠️ ORDER MATTERS — authelia takes the FIRST matching rule. - # The metrics rules are listed first so they cannot be - # shadowed by a broader domain rule added later. + # The metrics rule is listed first so it cannot be shadowed + # by a broader domain rule added later. rules = - # The collector's client where one is registered, then - # `deny` — see `metricsRules` above, which is where the - # reasoning for both halves lives. - metricsRules + # Denied or client-scoped depending on whether a + # collector is registered — see `metricsRule` above, + # which is where the reasoning for both halves lives. + [ metricsRule ] # Also guarded on the domain being set: without it a null # apex would render a rule matching the string "null". ++ lib.optional (deployCfg.swarm-controller.enable && swarmDomain != null) { diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 086e72d8..7bb04080 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -289,9 +289,8 @@ in nothing, which shows as a failing job rather than a missing one. The collector's OAuth2 client derives its permitted audiences from - these URLs, and authelia derives from each one the access rule that - admits only that client to it, so a target and the authorisation to - reach it are **one declaration**. + these URLs, so a target and the authorisation to reach it are **one + declaration**. Two lists that must agree would be a drift to maintain, and its failure mode is the bad one: a target whose audience was forgotten authenticates against nothing and reads as a scrape failure rather @@ -356,7 +355,5 @@ in config.services.hyperhive.swarm.otel.publishedScrapeTargets = { # The vhost location is in ./hive-forge/default.nix. forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics"; - # The vhost location is in ./swarm-authelia.nix. - authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics"; }; } diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index bf666f9f..814d053b 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -101,32 +101,7 @@ let # authelia with neither the forge nor the collector beside it. inherit (shared) autheliaOnly; - # Every swarm service on one host, the collector and authelia included. - inherit (shared) allLocal; - forgeMetricsUrl = "https://forge.t.local/metrics"; - autheliaMetricsUrl = "https://auth.t.local/metrics"; - - # The policy authelia applies to `subject` requesting `https://`, - # where `path` is what authelia matches `resources` against: the cleaned path - # plus `?` when there is one. - # It is the first rule whose domain, resources and subject all match, else - # `default_policy`. ⚠️ A rule with a `subject` list does NOT match a subject - # outside it — authelia moves on to the next rule — which is the semantics a - # check of the rule's own fields cannot see. - autheliaPolicy = - m: subject: domain: path: - let - ac = m.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control; - matches = - r: - r.domain == domain - && lib.any (re: builtins.match re path != null) (r.resources or [ ".*" ]) - && (!(r ? subject) || builtins.elem subject r.subject); - hit = lib.findFirst matches null ac.rules; - in - if hit == null then ac.default_policy else hit.policy; - cases = [ { # The collector's non-assertion, the deliberate mirror of Grafana's @@ -216,157 +191,11 @@ let && builtins.elem forgeMetricsUrl client.audience && builtins.elem { domain = "forge.t.local"; - resources = [ "^/metrics([?].*)?$" ]; + resources = [ "^/metrics$" ]; policy = "one_factor"; subject = [ "oauth2:client:${h.swarm.otel.clientId}" ]; } rules; } - { - # Authelia's half of the same move: its target is published by name, so - # a collector anywhere in the swarm has a job for it. - name = "a collector on a host without authelia scrapes authelia by name"; - ok = - let - s = otelSettings otelRemoteAuthelia; - job = lib.findFirst (j: j.job_name == "authelia") null s.receivers.prometheus.config.scrape_configs; - in - !otelRemoteAuthelia.services.hyperhive.deploy.authelia.enable - && job != null - && job.static_configs == [ { targets = [ "auth.t.local" ]; } ] - && job.metrics_path == "/metrics" - && job.oauth2.endpoint_params.audience == autheliaMetricsUrl; - } - { - # The location the collector's job requests: gated, and proxying to the - # loopback listener that has no authentication of its own. - name = "authelia's vhost serves /metrics only behind the collector's auth subrequest"; - ok = - let - h = autheliaOnly.services.hyperhive; - locs = autheliaOnly.services.nginx.virtualHosts.${h.swarm.authelia.domain}.locations; - authz = locs."= /__authelia_metrics_authz"; - in - locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.authelia.metricsPort}/metrics" - && lib.hasInfix "auth_request /__authelia_metrics_authz;" locs."= /metrics".extraConfig - && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request" - && lib.hasInfix "internal;" authz.extraConfig; - } - { - # Every published target admits the collector's client and refuses any - # other authenticated subject, with or without a query string. Read - # through authelia's first-match semantics: a client-scoped rule alone - # lets an operator's session fall through to `default_policy = one_factor`, - # and so does a request the rule's regex does not match. - name = "every published metrics URL admits the collector and refuses a plain session"; - ok = - let - h = autheliaOnly.services.hyperhive; - collector = "oauth2:client:${h.swarm.otel.clientId}"; - targets = lib.concatMap ( - url: - let - t = builtins.match "https://([^/]+)(/.*)" url; - in - map - (q: [ - (builtins.elemAt t 0) - "${builtins.elemAt t 1}${q}" - ]) - [ - "" - "?x=1" - ] - ) (lib.attrValues h.swarm.otel.publishedScrapeTargets); - policy = - subject: t: autheliaPolicy autheliaOnly subject (builtins.elemAt t 0) (builtins.elemAt t 1); - in - lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ - autheliaMetricsUrl - forgeMetricsUrl - ] - && lib.length targets == 4 - && lib.all (t: policy collector t == "one_factor") targets - && lib.all (t: policy "user:someone" t == "deny") targets - && lib.all (t: policy "group:admins" t == "deny") targets; - } - { - # Which request forms reach `= /metrics`, and whether the rule's regex - # matches the path authelia derives from each. `nginx` records the - # location each form lands in (measured against a stand-alone nginx); - # `path` is authelia's cleaned path plus query for that form. The - # invariant: every form the gated location serves is matched by the rule, - # so none of them reaches `default_policy`. - name = "the metrics rule matches every request form the gated location serves"; - ok = - let - rules = - autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; - metricsRules = lib.filter (r: r ? resources) rules; - re = lib.head (lib.head metricsRules).resources; - table = [ - { - request = "/metrics"; - nginx = "= /metrics"; - path = "/metrics"; - } - { - request = "/metrics?x=1"; - nginx = "= /metrics"; - path = "/metrics?x=1"; - } - { - request = "//metrics"; - nginx = "= /metrics"; - path = "/metrics"; - } - { - request = "/metrics/"; - nginx = "= /metrics/ (forge: 404) or /"; - path = "/metrics/"; - } - { - request = "/metricsx"; - nginx = "/"; - path = "/metricsx"; - } - ]; - matched = row: builtins.match re row.path != null; - in - lib.length metricsRules == 4 - && lib.all (r: r.resources == [ re ]) metricsRules - && re == "^/metrics([?].*)?$" - && lib.all (row: matched row == (row.nginx == "= /metrics")) table; - } - { - # Forgejo strips trailing slashes before routing, so `/metrics/` reaches - # its metrics handler unless nginx refuses it first. - name = "the forge refuses /metrics/ rather than proxying it to forgejo's metrics"; - ok = - let - locs = - allLocal.services.nginx.virtualHosts.${allLocal.services.hyperhive.swarm.forge.domain}.locations; - in - (locs."= /metrics/".return or null) == "404"; - } - { - # Single host: authelia is scraped once, by its published name. A - # loopback job beside it would scrape the same listener twice — and share - # its job name, which ./swarm-otel.nix refuses. - name = "a host running authelia and the collector scrapes authelia once, by name"; - ok = - let - o = allLocal.services.hyperhive.swarm.otel; - jobs = - lib.filter (j: j.job_name == "authelia") - (otelSettings allLocal).receivers.prometheus.config.scrape_configs; - in - allLocal.services.hyperhive.deploy.authelia.enable - && allLocal.services.hyperhive.deploy.swarm-otel.enable - && !(o.scrapeTargets ? authelia) - && (o.publishedScrapeTargets.authelia or null) == autheliaMetricsUrl - && lib.length jobs == 1 - && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ]; - } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive