swarm-otel: scrape authelia through its published name

Authelia's metrics were scraped over loopback, declared only on a host
running both authelia and the collector, so a split deploy collected none.
Its target is now declared on every host as
`https://<swarm.authelia.domain>/metrics`, next to forgejo's, and authelia's
vhost serves `= /metrics` behind the same collector auth subrequest as the
forge's. The loopback entry is gone, so a single host still scrapes
authelia once, now by name.

The forge-only `metricsRule` becomes `metricsRules`, derived from every
`publishedScrapeTargets` entry. Each target gets the collector-scoped
`one_factor` rule followed by an unconditional `deny`. The `deny` is
needed because authelia 4.39 skips a rule whose `subject` does not
match and moves on. With nothing after the collector rule, any other
authenticated session reached `default_policy = one_factor` and was
admitted. That was already true of the forge's `/metrics`, and this
change closes it there too.

Part of #5205

nix/host-modules/swarm-authelia-service.nix

@@ -43,24 +43,17 @@ in
         authelia's documentation finds what they expect.
 
         A separate port from {option}`port` because it is a separate
-        listener with a different audience: the main one is proxied by
+        listener with a different audience: the main one serves logins and
-        the gateway and reachable from the swarm, this one is scraped by
+        is reachable from the swarm, this one carries no authentication of
-        the collector on this host and by nothing else.
+        its own. The swarm collector scrapes it, from whichever host it
+        runs on, as `https://''${domain}/metrics`: the gateway's
+        `= /metrics` location proxies to this port only after authelia
+        has accepted the collector's token for that URL.
 
         ⚠️ Every swarm container shares the host network namespace, so
         two services defaulting to the same port do not conflict at build
         time — one simply loses at runtime, with nothing in any log. Check
         a new value against the others before changing this.
-
-        ::: {.note}
-        Loopback means this endpoint is only reachable by a collector on
-        the *same host*, so the scrape target is declared only when one is
-        enabled here. Run the swarm's collector elsewhere and authelia's
-        metrics are simply not collected — no error, and nothing in a log
-        to say so. Making them reachable across hosts is a different piece
-        of work: the endpoint would have to be published under a name,
-        with a certificate and an audience.
-        :::
       '';
     };
 

nix/host-modules/swarm-authelia.nix

@@ -61,27 +61,44 @@ let
   collectorClientId = config.services.hyperhive.swarm.otel.clientId;
   collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients;
 
-  # The forge's metrics endpoint: deny until a collector exists to allow.
+  # Every published metrics endpoint: the collector's client, and nobody else.
   #
-  # `deny` is not a placeholder, it is the protection. The endpoint is
+  # The `deny` is the protection, and it has to be a rule of its own. A rule
-  # always served (a swarm-integrated forge always has metrics) and
+  # with a `subject` does not match a subject outside it: authelia skips it and
-  # `default_policy` is `one_factor`, which means *any* authenticated
+  # moves on, and with nothing after it lands on `default_policy`, which is
-  # subject — every operator today, every agent once they hold authelia
+  # `one_factor` — *any* authenticated subject, every operator today, every
-  # accounts. Nor does the audience stand between a browser session and
+  # agent once they hold authelia accounts. Nor does the audience stand
-  # this data: `authn_strategies` on the authz endpoint also accepts
+  # between a browser session and this data: `authn_strategies` on the authz
-  # `CookieSession`, and a cookie carries no audience at all.
+  # endpoint also accepts `CookieSession`, and a cookie carries no audience.
-  metricsRule = {
+  # Until a collector is registered, the `deny` is the only rule.
-    domain = hyperhiveCfg.swarm.forge.domain;
+  #
-    resources = [ "^/metrics$" ];
+  # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same
-  }
+  # URL the collector requests and its token is minted for, so a target
-  // (
+  # cannot be published without the rules that confine it to the collector.
-    if collectorRegistered then
+  metricsRules = lib.concatLists (
-      {
+    lib.mapAttrsToList (
+      job: url:
+      let
+        # The split `parsePublished` in ./swarm-otel.nix makes for the scrape
+        # job; the two must agree on what the URL's host and path are.
+        parts = builtins.match "https://([^/]+)(/.*)" url;
+        target = {
+          domain = builtins.elemAt parts 0;
+          resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}$" ];
+        };
+      in
+      if parts == null then
+        throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived."
+      else
+        lib.optional collectorRegistered (
+          target
+          // {
             policy = "one_factor";
             subject = [ "oauth2:client:${collectorClientId}" ];
           }
-    else
+        )
-      { policy = "deny"; }
+        ++ [ (target // { policy = "deny"; }) ]
+    ) config.services.hyperhive.swarm.otel.publishedScrapeTargets
   );
 
   # Upstream's `services.authelia.instances.<name>` derives the unit,
@@ -779,6 +796,48 @@ in
       locations."/bridge/" = {
         proxyPass = "http://127.0.0.1:${toString cfg.bridgePort}/";
       };
+
+      # Authelia's own metrics, for the swarm collector wherever it runs.
+      # The listener (`metricsPort`) is loopback-only and unauthenticated;
+      # this location is the only way in from off the machine. `= /metrics`
+      # outranks the `/` prefix, so its auth is the one that runs.
+      #
+      # The collector requests exactly this URL, declared as
+      # `publishedScrapeTargets.authelia` in ./swarm-otel-service.nix; the two
+      # must agree, since that string is also the token's audience.
+      locations."= /metrics" = {
+        proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
+        extraConfig = ''
+          auth_request /__authelia_metrics_authz;
+        '';
+      };
+
+      # The subrequest, same shape as the forge's `= /__forge_metrics_authz`
+      # (./hive-forge/default.nix), and machine-facing for the same reason: no
+      # `error_page 401 =302`, so a scraper gets the 401 rather than a login
+      # page parsed as metrics.
+      #
+      # Authelia authorising a request for itself does not loop: the
+      # subrequest lands on `/api/` above, which carries no `auth_request`.
+      # An authelia that is down answers the subrequest 502, which
+      # `auth_request` turns into a 500 for the scrape — closed, not open.
+      locations."= /__authelia_metrics_authz" = {
+        proxyPass = "https://${cfg.domain}/api/authz/auth-request";
+        # nixpkgs appends its OWN `Host $host` after extraConfig, which would
+        # override verifiedProxyTo's — see hive-gateway/vhost-lib.nix.
+        recommendedProxySettings = false;
+        extraConfig = ''
+          internal;
+          ${gatewayCfg.lib.verifiedProxyTo cfg.domain}
+          proxy_pass_request_body off;
+          proxy_set_header Content-Length "";
+          proxy_set_header X-Original-Method $request_method;
+          proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
+          proxy_set_header X-Forwarded-Proto $scheme;
+          proxy_set_header X-Forwarded-Host $http_host;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        '';
+      };
     };
 
     # Order the container after the host CA generator, so the bind source
@@ -1038,7 +1097,8 @@ in
               # Loopback only, like the main listener above and for a
               # stronger reason: this endpoint has no authentication of its
               # own, and it reports request volumes and outcomes for every
-              # SSO login on the swarm.
+              # SSO login on the swarm. The collector reaches it through the
+              # vhost's gated `= /metrics` location.
               telemetry.metrics = {
                 enabled = true;
                 address = "tcp://127.0.0.1:${toString cfg.metricsPort}";
@@ -1070,13 +1130,13 @@ in
               access_control = {
                 default_policy = "one_factor";
                 # ⚠️ ORDER MATTERS — authelia takes the FIRST matching rule.
-                # The metrics rule is listed first so it cannot be shadowed
+                # The metrics rules are listed first so they cannot be
-                # by a broader domain rule added later.
+                # shadowed by a broader domain rule added later.
                 rules =
-                  # Denied or client-scoped depending on whether a
+                  # The collector's client where one is registered, then
-                  # collector is registered — see `metricsRule` above,
+                  # `deny` — see `metricsRules` above, which is where the
-                  # which is where the reasoning for both halves lives.
+                  # reasoning for both halves lives.
-                  [ metricsRule ]
+                  metricsRules
                   # Also guarded on the domain being set: without it a null
                   # apex would render a rule matching the string "null".
                   ++ lib.optional (deployCfg.swarm-controller.enable && swarmDomain != null) {

nix/host-modules/swarm-otel-service.nix

@@ -289,8 +289,9 @@ in
         nothing, which shows as a failing job rather than a missing one.
 
         The collector's OAuth2 client derives its permitted audiences from
-        these URLs, so a target and the authorisation to reach it are **one
+        these URLs, and authelia derives from each one the access rule that
-        declaration**.
+        admits only that client to it, so a target and the authorisation to
+        reach it are **one declaration**.
         Two lists that must agree would be a drift to maintain, and its
         failure mode is the bad one: a target whose audience was forgotten
         authenticates against nothing and reads as a scrape failure rather
@@ -355,5 +356,7 @@ in
   config.services.hyperhive.swarm.otel.publishedScrapeTargets = {
     # The vhost location is in ./hive-forge/default.nix.
     forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics";
+    # The vhost location is in ./swarm-authelia.nix.
+    authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics";
   };
 }

nix/module-eval/swarm-otel-identity.nix

@@ -101,7 +101,30 @@ let
   # authelia with neither the forge nor the collector beside it.
   inherit (shared) autheliaOnly;
 
+  # Every swarm service on one host, the collector and authelia included.
+  inherit (shared) allLocal;
+
   forgeMetricsUrl = "https://forge.t.local/metrics";
+  autheliaMetricsUrl = "https://auth.t.local/metrics";
+
+  # The policy authelia applies to `subject` requesting `https://<domain><path>`:
+  # the first rule whose domain, resources and subject all match, else
+  # `default_policy`. ⚠️ A rule with a `subject` list does NOT match a subject
+  # outside it — authelia moves on to the next rule — which is the semantics a
+  # check of the rule's own fields cannot see.
+  autheliaPolicy =
+    m: subject: domain: path:
+    let
+      ac = m.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control;
+      matches =
+        r:
+        r.domain == domain
+        && lib.any (re: builtins.match re path != null) (r.resources or [ ".*" ])
+        && (!(r ? subject) || builtins.elem subject r.subject);
+      hit = lib.findFirst matches null ac.rules;
+    in
+    if hit == null then ac.default_policy else hit.policy;
+
   cases = [
     {
       # The collector's non-assertion, the deliberate mirror of Grafana's
@@ -196,6 +219,79 @@ let
           subject = [ "oauth2:client:${h.swarm.otel.clientId}" ];
         } rules;
     }
+    {
+      # Authelia's half of the same move: its target is published by name, so
+      # a collector anywhere in the swarm has a job for it.
+      name = "a collector on a host without authelia scrapes authelia by name";
+      ok =
+        let
+          s = otelSettings otelRemoteAuthelia;
+          job = lib.findFirst (j: j.job_name == "authelia") null s.receivers.prometheus.config.scrape_configs;
+        in
+        !otelRemoteAuthelia.services.hyperhive.deploy.authelia.enable
+        && job != null
+        && job.static_configs == [ { targets = [ "auth.t.local" ]; } ]
+        && job.metrics_path == "/metrics"
+        && job.oauth2.endpoint_params.audience == autheliaMetricsUrl;
+    }
+    {
+      # The location the collector's job requests: gated, and proxying to the
+      # loopback listener that has no authentication of its own.
+      name = "authelia's vhost serves /metrics only behind the collector's auth subrequest";
+      ok =
+        let
+          h = autheliaOnly.services.hyperhive;
+          locs = autheliaOnly.services.nginx.virtualHosts.${h.swarm.authelia.domain}.locations;
+          authz = locs."= /__authelia_metrics_authz";
+        in
+        locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.authelia.metricsPort}/metrics"
+        && lib.hasInfix "auth_request /__authelia_metrics_authz;" locs."= /metrics".extraConfig
+        && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request"
+        && lib.hasInfix "internal;" authz.extraConfig;
+    }
+    {
+      # Every published target admits the collector's client and refuses any
+      # other authenticated subject. Read through authelia's first-match
+      # semantics: a client-scoped rule alone lets an operator's session fall
+      # through to `default_policy = one_factor`.
+      name = "every published metrics URL admits the collector and refuses a plain session";
+      ok =
+        let
+          h = autheliaOnly.services.hyperhive;
+          collector = "oauth2:client:${h.swarm.otel.clientId}";
+          targets = lib.mapAttrsToList (
+            _: url: builtins.match "https://([^/]+)(/.*)" url
+          ) h.swarm.otel.publishedScrapeTargets;
+          policy =
+            subject: t: autheliaPolicy autheliaOnly subject (builtins.elemAt t 0) (builtins.elemAt t 1);
+        in
+        lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
+          autheliaMetricsUrl
+          forgeMetricsUrl
+        ]
+        && lib.all (t: policy collector t == "one_factor") targets
+        && lib.all (t: policy "user:someone" t == "deny") targets
+        && lib.all (t: policy "group:admins" t == "deny") targets;
+    }
+    {
+      # Single host: authelia is scraped once, by its published name. A
+      # loopback job beside it would scrape the same listener twice — and share
+      # its job name, which ./swarm-otel.nix refuses.
+      name = "a host running authelia and the collector scrapes authelia once, by name";
+      ok =
+        let
+          o = allLocal.services.hyperhive.swarm.otel;
+          jobs =
+            lib.filter (j: j.job_name == "authelia")
+              (otelSettings allLocal).receivers.prometheus.config.scrape_configs;
+        in
+        allLocal.services.hyperhive.deploy.authelia.enable
+        && allLocal.services.hyperhive.deploy.swarm-otel.enable
+        && !(o.scrapeTargets ? authelia)
+        && (o.publishedScrapeTargets.authelia or null) == autheliaMetricsUrl
+        && lib.length jobs == 1
+        && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ];
+    }
     {
       # Read against the roster the fixture declares rather than against
       # names spelled here: an arm naming `h1` passes on a single-hive