From da88d450dd53a8a8073474876358d5f38f5b4af8 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Mon, 24 Aug 2026 12:05:01 +0200
Subject: [PATCH 1/6] feat(#3517): a named capability for authelia's
 bearer-authz scope
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A scraper reaches a service published behind the gateway by presenting an
access token to authelia's authz endpoint, which requires the client to
carry authelia.bearer.authz. Nothing could express that: scopes are
derived from kind, and a machine client rendered an empty list.

A named capability rather than a free-form scopes list, for the reason
the derivation exists — authelia refuses some scope/grant combinations
outright, openid with client_credentials among them, and a list makes
those expressible again.

The two assertions carry their weight: authelia checks the same
obligations, but in its preStart validator, so a violation builds and
deploys cleanly and then fails to restart with swarm SSO attached to it.
---
 nix/host-modules/swarm-authelia.nix | 78 ++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 1 deletion(-)

diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix
index 7ed2e398..2ad3785c 100644
--- a/nix/host-modules/swarm-authelia.nix
+++ b/nix/host-modules/swarm-authelia.nix
@@ -217,7 +217,7 @@ let
       if c.kind == "machine" then
         ''
           printf -- '        grant_types: ["client_credentials"]\n'
-          printf -- '        scopes: []\n'
+          printf -- '        scopes: [${lib.optionalString c.bearerAuthz "authelia.bearer.authz"}]\n'
         ''
       else
         ''
@@ -575,6 +575,34 @@ in
               '';
             };
 
+            bearerAuthz = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              example = true;
+              description = ''
+                Grant this client the `authelia.bearer.authz` scope, so it
+                may present its access token to authelia's authz endpoint
+                and be authorised by an `access_control` rule — how a
+                scraper reaches a service published behind the gateway.
+
+                A named capability rather than a free-form `scopes` list,
+                for the same reason `kind` derives the rest: authelia
+                refuses some scope/grant combinations outright (`openid`
+                with `client_credentials` among them), and a list would
+                make those combinations expressible again. This admits the
+                one value that is legal here and nothing else.
+
+                ::: {.note}
+                Setting this obliges two other options, and the assertions
+                below enforce it. Authelia checks the same thing, but only
+                in its `preStart` validator — which means a violation
+                builds and deploys cleanly and then fails to restart,
+                taking swarm SSO down. The assertions move that to
+                evaluation, where a wrong value costs nothing.
+                :::
+              '';
+            };
+
             accessTokenSignedResponseAlg = lib.mkOption {
               type = lib.types.nullOr (
                 lib.types.enum [
@@ -820,6 +848,54 @@ in
             + "services.hyperhive.swarm.hives; rename the hive or the "
             + "colliding client.";
         }
+        # The two obligations `authelia.bearer.authz` carries. Authelia
+        # enforces both itself — but in its `preStart` validator, so a
+        # violation produces a green `nixos-rebuild switch` and an authelia
+        # that then refuses to come back up, taking swarm SSO with it.
+        # Asserting here moves the same failure to evaluation, where a
+        # wrong value costs a build and nothing else.
+        {
+          assertion = lib.all (c: !c.bearerAuthz || c.audience != [ ]) cfg.oidc.clients;
+          message =
+            "services.hyperhive.swarm.authelia.oidc.clients: "
+            + lib.concatStringsSep ", " (
+              map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.audience == [ ]) cfg.oidc.clients)
+            )
+            + " sets bearerAuthz but declares no audience. The audience is "
+            + "what authorises a bearer token at a given URL, so without "
+            + "one the scope grants access to nothing and authelia refuses "
+            + "the configuration outright.";
+        }
+        {
+          assertion = lib.all (
+            c:
+            !c.bearerAuthz
+            || lib.elem c.tokenEndpointAuthMethod [
+              "client_secret_basic"
+              "client_secret_jwt"
+              "private_key_jwt"
+            ]
+          ) cfg.oidc.clients;
+          message =
+            "services.hyperhive.swarm.authelia.oidc.clients: "
+            + lib.concatStringsSep ", " (
+              map (c: "client '${c.id}' (tokenEndpointAuthMethod = ${toString c.tokenEndpointAuthMethod})") (
+                lib.filter (
+                  c:
+                  c.bearerAuthz
+                  && !lib.elem c.tokenEndpointAuthMethod [
+                    "client_secret_basic"
+                    "client_secret_jwt"
+                    "private_key_jwt"
+                  ]
+                ) cfg.oidc.clients
+              )
+            )
+            + ". A confidential client carrying authelia.bearer.authz must "
+            + "authenticate with client_secret_basic, client_secret_jwt or "
+            + "private_key_jwt. Notably client_secret_post is refused, and "
+            + "it is what an OAuth2 client library may reach for first.";
+        }
       ];
 
     # Authelia's own gateway surface: the vhost that fronts it and the

From 3895a1e21d5875787598c9d5fcbe019802586d1b Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Mon, 24 Aug 2026 12:53:23 +0200
Subject: [PATCH 2/6] feat(#3517): register the swarm collector as an audienced
 oauth2 client
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The forge's `/metrics` is published behind the gateway and denied to
everyone, waiting on a client to allow. This is that client.

An audience is a URL: authelia validates a bearer token against the
address being requested, and a client may only request an audience it is
registered for, so registration is the authorisation. The URLs are owned
by the services that publish them while audiences attach to one client,
so services contribute to a list and this module builds the single entry
— the `gateway.localNames` split, forced here by client definitions
concatenating rather than merging into a shared entry.

The access-control rule asks the client list whether the collector is
registered rather than re-deriving the conditions that register it. The
two drifting is not a build failure: authelia refuses a rule naming an
unknown client in its startup validator, so SSO fails to restart.
---
 nix/host-modules/hive-forge/default.nix | 12 ++++
 nix/host-modules/swarm-authelia.nix     | 71 ++++++++++++--------
 nix/host-modules/swarm-otel.nix         | 88 +++++++++++++++++++++++++
 3 files changed, 145 insertions(+), 26 deletions(-)

diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix
index 6775ee23..e319da0c 100644
--- a/nix/host-modules/hive-forge/default.nix
+++ b/nix/host-modules/hive-forge/default.nix
@@ -442,6 +442,18 @@ in
       url = "https://${cfg.domain}/";
     };
 
+    # The address a scraper has to hold a token for. Same `behindGateway`
+    # guard, and for a stronger reason than the two above: with it off
+    # there is no `= /metrics` location and no `auth_request` in front of
+    # it, so the URL this names does not exist to be authorised.
+    #
+    # ⚠️ Written as the exact URL a collector requests, because that is
+    # what authelia compares against — this string agreeing with the
+    # `location` block above it is the whole mechanism. A near miss is a
+    # correctly minted token refused at the target.
+    services.hyperhive.swarm.otel.collectorAudiences =
+      lib.optional cfg.behindGateway "https://${cfg.domain}/metrics";
+
     # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for
     # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB
     # clones). SSH stays direct on `forge.sshPort`. See
diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix
index 2ad3785c..9b6c9584 100644
--- a/nix/host-modules/swarm-authelia.nix
+++ b/nix/host-modules/swarm-authelia.nix
@@ -57,6 +57,47 @@ let
   # failing the check it was supposed to pass.
   operatorGroup = "admins";
 
+  # Whether the swarm collector's client is actually registered here.
+  #
+  # Asked of the client list rather than re-derived from the conditions
+  # that produce it (`otel.enable`, a non-empty audience set, authelia
+  # being co-located). A second copy of that predicate is a second thing
+  # to keep in step, and the two drifting is not a build failure: a rule
+  # naming an unregistered client is refused by authelia's *startup*
+  # validator, so the whole SSO service fails to restart.
+  #
+  # Reading the registration itself also makes the rule correct for a
+  # registrar this module has never heard of — an operator registering
+  # the collector by hand against a provider that is not co-located with
+  # it gets the same rule, from the same expression.
+  # ⚠️ `swarm.otel` named in full, not through a let binding: there are two
+  # otel options one word apart, and only this one is the swarm's collector.
+  collectorClientId = config.services.hyperhive.swarm.otel.clientId;
+  collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients;
+
+  # The forge's metrics endpoint: deny until a collector exists to allow.
+  #
+  # `deny` is not a placeholder, it is the protection. The endpoint is
+  # always served (a swarm-integrated forge always has metrics) and
+  # `default_policy` is `one_factor`, which means *any* authenticated
+  # subject — every operator today, every agent once they hold authelia
+  # accounts. Nor does the audience stand between a browser session and
+  # this data: `authn_strategies` on the authz endpoint also accepts
+  # `CookieSession`, and a cookie carries no audience at all.
+  metricsRule = {
+    domain = forgeCfg.domain;
+    resources = [ "^/metrics$" ];
+  }
+  // (
+    if collectorRegistered then
+      {
+        policy = "one_factor";
+        subject = [ "oauth2:client:${collectorClientId}" ];
+      }
+    else
+      { policy = "deny"; }
+  );
+
   # Upstream's `services.authelia.instances.<name>` derives the unit,
   # user, group and StateDirectory from the instance name
   # (`authelia` + `-<name>`). Naming them here rather than repeating the
@@ -1281,32 +1322,10 @@ in
                 # The metrics rule is listed first so it cannot be shadowed
                 # by a broader domain rule added later.
                 rules =
-                  # The forge's metrics endpoint. `deny` is deliberate and
-                  # is the whole protection right now: the endpoint is
-                  # always served (a swarm-integrated forge always has
-                  # metrics), and `default_policy` is `one_factor`, which
-                  # means *any* authenticated subject — every operator
-                  # today, every agent once they hold authelia accounts.
-                  #
-                  # Being reachable by a Bearer token is not sufficient on
-                  # its own: `authn_strategies` on this endpoint also
-                  # accepts `CookieSession`, and a cookie carries no
-                  # audience, so the audience is not what stands between a
-                  # browser session and this data.
-                  #
-                  # The collector gets in by REPLACING this with a
-                  # client-scoped allow (`subject = ["oauth2:client:<id>"]`)
-                  # once such a client is registered. Denying until then is
-                  # what makes publishing the endpoint safe on its own —
-                  # authelia refuses a subject naming a client that is not
-                  # registered, and it does so in a `preStart` validator,
-                  # so naming one early takes the whole SSO service down on
-                  # the next restart rather than failing the build.
-                  lib.optional forgeCfg.behindGateway {
-                    domain = forgeCfg.domain;
-                    resources = [ "^/metrics$" ];
-                    policy = "deny";
-                  }
+                  # Denied or client-scoped depending on whether a
+                  # collector is registered — see `metricsRule` above,
+                  # which is where the reasoning for both halves lives.
+                  lib.optional forgeCfg.behindGateway metricsRule
                   ++ lib.optional uiCfg.enable {
                     domain = uiCfg.domain;
                     subject = [ "group:${operatorGroup}" ];
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index cb67b155..c02563c4 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -223,6 +223,57 @@ in
         scrape is the inert configuration this option exists to avoid.
       '';
     };
+
+    collectorAudiences = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = lib.literalExpression ''[ "https://forge.example.com/metrics" ]'';
+      description = ''
+        URLs this collector may mint an access token for, contributed by
+        the modules that publish them.
+
+        A service module says **which URL it exposes**; this module turns
+        the set into the collector's single OAuth2 client — the same split
+        as `services.hyperhive.gateway.localNames`, and for the same
+        reason. The collector has no business knowing another service's
+        routes, and a service cannot reach into the client list to add
+        itself: client definitions concatenate rather than merging into a
+        shared entry, so the entry has to be built in one place from a
+        list many modules can write.
+
+        ⚠️ An audience is a **URL**, not a label. Authelia validates a
+        bearer token against the address being requested, so a token
+        minted for one target is refused at another and a client may only
+        request an audience it is registered for — registration is the
+        authorisation. A near miss (a trailing slash, `http` for `https`)
+        presents as a valid token rejected at the target, several layers
+        from its cause.
+
+        ⚠️ Deliberately not part of `scrapeTargets`. An entry there is
+        trusted because the scraper and the target share a host; an entry
+        here is trusted because it presents a credential. Folding the two
+        together would leave a reader unable to tell which of those a
+        given target relies on.
+      '';
+    };
+
+    clientId = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = "swarm-collector";
+      description = ''
+        OAuth2 client id this collector authenticates as. Published so
+        authelia's `access_control` rules can name it without carrying a
+        second copy of the string, exactly as
+        `services.hyperhive.swarm.authelia.hiveClientPrefix` is published
+        for the queue's responder.
+
+        Two spellings drifting apart is not a build failure: authelia
+        refuses a rule naming an unregistered client in its startup
+        validator, so the swarm's SSO service fails to *restart* — long
+        after the change that caused it evaluated cleanly.
+      '';
+    };
   };
 
   config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
@@ -267,6 +318,43 @@ in
     # telemetry.
     services.hyperhive.swarm.authelia.oidc.hiveIdentities = true;
 
+    # The collector's own identity, for the other direction: the hive
+    # identities above are how this collector authenticates its *callers*,
+    # this is how it authenticates *itself* to a service published behind
+    # the gateway.
+    #
+    # Only where authelia is co-located. A client is a row in this host's
+    # provider config, so declaring one against a remote provider would
+    # render nothing while reading as done; a swarm whose authelia lives
+    # elsewhere registers it there.
+    #
+    # ⚠️ Also conditional on the audience set being non-empty, and that is
+    # the shipped case rather than an edge — nothing contributes a URL
+    # until some service publishes one behind the gateway. Authelia
+    # refuses a bearer-authz client with no audience, so an unconditional
+    # declaration would break every hive that runs a collector and
+    # publishes nothing.
+    services.hyperhive.swarm.authelia.oidc.clients =
+      lib.mkIf (autheliaCfg.enable && cfg.collectorAudiences != [ ])
+        [
+          {
+            id = cfg.clientId;
+            description = "HyperHive swarm collector";
+            kind = "machine";
+            # Grants `authelia.bearer.authz`, without which the authz
+            # endpoint refuses an otherwise valid token and blames the
+            # token rather than the missing grant.
+            bearerAuthz = true;
+            audience = cfg.collectorAudiences;
+            # Stated rather than left on authelia's default, because the
+            # two agreeing today is not the same as this being the
+            # required value: authelia permits only basic / JWT methods
+            # for a confidential client holding that scope, and enforces
+            # it in the startup validator.
+            tokenEndpointAuthMethod = "client_secret_basic";
+          }
+        ];
+
     # The CA bind source is written at runtime by a host unit, so the
     # container has to start after it — otherwise nspawn sets up a mount
     # over a file that does not exist yet.

From 94204ac82b71a3e2a2ca0ac7b689632b15b1b7c4 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Mon, 24 Aug 2026 13:14:23 +0200
Subject: [PATCH 3/6] fix(swarm-authelia): guard bearerAuthz against a silent
 no-op on kind
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review found that nothing stopped `bearerAuthz = true` on an interactive
client. `renderClient` reads the flag only in the machine branch, so the
scope is never emitted: the client authenticates, is authorised for
nothing, and the build is green. `kind` defaults to `interactive`, so it
is reached by forgetting a field rather than by writing a wrong one —
this module's own failure mode one level up.

The other half of the review asked to relax the method assertion to
accept `null`, on the strength of the option's doc calling `null`
authelia's default. Measured instead: under `authelia.bearer.authz`
authelia refuses the omission outright, so the assertion was right and
the DOC was wrong. The doc now carries the exception, and the assertion
message says null is refused rather than leaving a reader to infer it.
---
 nix/host-modules/swarm-authelia.nix | 34 ++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix
index 9b6c9584..5fff0a71 100644
--- a/nix/host-modules/swarm-authelia.nix
+++ b/nix/host-modules/swarm-authelia.nix
@@ -587,6 +587,15 @@ in
                 a successful consent** — the login looks like it worked
                 right up to the last hop, and neither the redirect nor
                 the secret is at fault.
+
+                ⚠️ `null` is NOT accepted on a client with `bearerAuthz`.
+                Measured against authelia 4.39.20: under that scope the
+                method must be *stated*: omitting it is refused with
+                `must be configured as 'client_secret_basic', … but it's
+                configured as` an empty string. The sentence above is
+                true of an ordinary client and false of that one, which
+                is exactly how a reviewer reads this option and concludes
+                the assertion below is wrong.
               '';
             };
 
@@ -935,7 +944,30 @@ in
             + ". A confidential client carrying authelia.bearer.authz must "
             + "authenticate with client_secret_basic, client_secret_jwt or "
             + "private_key_jwt. Notably client_secret_post is refused, and "
-            + "it is what an OAuth2 client library may reach for first.";
+            + "it is what an OAuth2 client library may reach for first. "
+            + "Leaving it null is refused too: authelia requires the method "
+            + "to be STATED under this scope rather than defaulted.";
+        }
+        # Nothing else stops `bearerAuthz` on an interactive client, and it
+        # would silently do nothing: `renderClient` reads the flag only in
+        # the `machine` branch, so the scope is simply never emitted and the
+        # client authenticates fine while being authorised for nothing.
+        #
+        # That is this module's own failure mode one level up — a green
+        # build and a grant that does not exist — and `kind` defaults to
+        # `interactive`, so it is reached by FORGETTING a field rather than
+        # by writing a wrong one.
+        {
+          assertion = lib.all (c: !c.bearerAuthz || c.kind == "machine") cfg.oidc.clients;
+          message =
+            "services.hyperhive.swarm.authelia.oidc.clients: "
+            + lib.concatStringsSep ", " (
+              map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.kind != "machine") cfg.oidc.clients)
+            )
+            + " sets bearerAuthz with kind != \"machine\". The scope is only "
+            + "emitted for machine clients, so this grants nothing while "
+            + "evaluating and deploying cleanly. A browser client has a user "
+            + "to authorise and does not need it.";
         }
       ];
 

From 94ef8428e03d2aaf57ea3bb622cd62e528819715 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Mon, 24 Aug 2026 13:36:32 +0200
Subject: [PATCH 4/6] refactor(swarm-otel): one declaration for a published
 scrape target
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`collectorAudiences` was a list of URLs services contributed so the
collector's client could be registered for them. Slice C needs the same
URLs as scrape jobs, and a job needs a name the audience list has no
room for — so services would have contributed to two options that must
agree.

They now contribute `publishedScrapeTargets` once, as `<job> = "<url>"`,
and the client's audiences derive from it. The drift that would have
needed maintaining is gone, and its failure mode was the quiet one: a
target whose audience was forgotten authenticates against nothing and
reads as a broken scrape rather than a missing registration.

Adds an assertion for the one collision the module system cannot catch.
Two definitions of the same key within one option are already refused
(measured); across the two scrape options nothing arbitrates, and both
entries would render into a single scrape_configs list under one
job_name.
---
 nix/host-modules/hive-forge/default.nix |  15 ++--
 nix/host-modules/swarm-otel.nix         | 100 ++++++++++++++++--------
 2 files changed, 77 insertions(+), 38 deletions(-)

diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix
index e319da0c..4367fd28 100644
--- a/nix/host-modules/hive-forge/default.nix
+++ b/nix/host-modules/hive-forge/default.nix
@@ -442,17 +442,20 @@ in
       url = "https://${cfg.domain}/";
     };
 
-    # The address a scraper has to hold a token for. Same `behindGateway`
-    # guard, and for a stronger reason than the two above: with it off
-    # there is no `= /metrics` location and no `auth_request` in front of
-    # it, so the URL this names does not exist to be authorised.
+    # The metrics endpoint, declared once: the collector both scrapes this
+    # URL and derives from it the audience its token is minted for. Same
+    # `behindGateway` guard, and for a stronger reason than the two above:
+    # with it off there is no `= /metrics` location and no `auth_request`
+    # in front of it, so the URL this names does not exist to be scraped
+    # or authorised.
     #
     # ⚠️ Written as the exact URL a collector requests, because that is
     # what authelia compares against — this string agreeing with the
     # `location` block above it is the whole mechanism. A near miss is a
     # correctly minted token refused at the target.
-    services.hyperhive.swarm.otel.collectorAudiences =
-      lib.optional cfg.behindGateway "https://${cfg.domain}/metrics";
+    services.hyperhive.swarm.otel.publishedScrapeTargets = lib.optionalAttrs cfg.behindGateway {
+      forgejo = "https://${cfg.domain}/metrics";
+    };
 
     # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for
     # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index c02563c4..c1b7fea2 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -224,36 +224,35 @@ in
       '';
     };
 
-    collectorAudiences = lib.mkOption {
-      type = lib.types.listOf lib.types.str;
-      default = [ ];
-      example = lib.literalExpression ''[ "https://forge.example.com/metrics" ]'';
+    publishedScrapeTargets = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      example = lib.literalExpression ''{ forge = "https://forge.example.com/metrics"; }'';
       description = ''
-        URLs this collector may mint an access token for, contributed by
-        the modules that publish them.
+        Prometheus endpoints this collector scrapes **by name, with a
+        credential**, as `<job name> = "<url>"`.
 
-        A service module says **which URL it exposes**; this module turns
-        the set into the collector's single OAuth2 client — the same split
-        as `services.hyperhive.gateway.localNames`, and for the same
-        reason. The collector has no business knowing another service's
-        routes, and a service cannot reach into the client list to add
-        itself: client definitions concatenate rather than merging into a
-        shared entry, so the entry has to be built in one place from a
-        list many modules can write.
+        A service module declares its own entry, from its own module, the
+        same way it does for `scrapeTargets` — and the collector's OAuth2
+        client derives its permitted audiences from these URLs, so a
+        target and the authorisation to reach it are **one declaration**.
+        Two lists that must agree would be a drift to maintain, and its
+        failure mode is the bad one: a target whose audience was forgotten
+        authenticates against nothing and reads as a scrape failure rather
+        than a config mistake.
 
-        ⚠️ An audience is a **URL**, not a label. Authelia validates a
-        bearer token against the address being requested, so a token
-        minted for one target is refused at another and a client may only
-        request an audience it is registered for — registration is the
-        authorisation. A near miss (a trailing slash, `http` for `https`)
-        presents as a valid token rejected at the target, several layers
-        from its cause.
+        ⚠️ A **full URL**, not `host:port`. Authelia validates a bearer
+        token against the address being requested, so the string here is
+        also the audience the token is minted for; a near miss (a trailing
+        slash, `http` for `https`) presents as a valid token rejected at
+        the target, several layers from its cause.
 
-        ⚠️ Deliberately not part of `scrapeTargets`. An entry there is
-        trusted because the scraper and the target share a host; an entry
-        here is trusted because it presents a credential. Folding the two
-        together would leave a reader unable to tell which of those a
-        given target relies on.
+        ⚠️ Deliberately separate from `scrapeTargets`. An entry there is
+        trusted because the scraper and the target share a host — that
+        option is loopback-and-unauthenticated by contract. An entry here
+        is trusted because it presents a credential. One shape for both
+        would leave a reader unable to tell which of those a given target
+        relies on.
       '';
     };
 
@@ -328,14 +327,14 @@ in
     # render nothing while reading as done; a swarm whose authelia lives
     # elsewhere registers it there.
     #
-    # ⚠️ Also conditional on the audience set being non-empty, and that is
-    # the shipped case rather than an edge — nothing contributes a URL
-    # until some service publishes one behind the gateway. Authelia
-    # refuses a bearer-authz client with no audience, so an unconditional
+    # ⚠️ Also conditional on there being a published target, and that is
+    # the shipped case rather than an edge — nothing declares one until
+    # some service publishes behind the gateway. Authelia refuses a
+    # bearer-authz client with no audience, so an unconditional
     # declaration would break every hive that runs a collector and
     # publishes nothing.
     services.hyperhive.swarm.authelia.oidc.clients =
-      lib.mkIf (autheliaCfg.enable && cfg.collectorAudiences != [ ])
+      lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { })
         [
           {
             id = cfg.clientId;
@@ -345,7 +344,13 @@ in
             # endpoint refuses an otherwise valid token and blames the
             # token rather than the missing grant.
             bearerAuthz = true;
-            audience = cfg.collectorAudiences;
+            # DERIVED from the targets rather than contributed alongside
+            # them. A service declares a URL once and this is the
+            # permission to reach it; two lists that had to agree would be
+            # a drift to maintain, and the failure mode is the quiet one —
+            # a target whose audience was forgotten authenticates against
+            # nothing and looks like a broken scrape.
+            audience = lib.attrValues cfg.publishedScrapeTargets;
             # Stated rather than left on authelia's default, because the
             # two agreeing today is not the same as this being the
             # required value: authelia permits only basic / JWT methods
@@ -416,6 +421,37 @@ in
           Rename the hive. Reserved: ${lib.concatMapStringsSep ", " (n: "'${n}'") reservedOwners}.
         '';
       }
+      {
+        # A job name used by BOTH scrape options. Within one option this
+        # cannot happen — the module system refuses two definitions of the
+        # same key with different values, measured — but the two options
+        # are separate, so nothing arbitrates between them, and they
+        # render into a single `scrape_configs` LIST where nothing
+        # overwrites anything: both entries ship under one `job_name`.
+        #
+        # The message names which side each collision came from, which is
+        # the part a rendered-config error could not tell an operator.
+        assertion =
+          lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets)
+          == [ ];
+        message = ''
+          services.hyperhive.swarm.otel: ${
+            lib.concatMapStringsSep ", " (j: "'${j}'") (
+              lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets)
+            )
+          } is declared as both a loopback
+          scrapeTargets job and a publishedScrapeTargets job.
+
+          They render into one prometheus scrape_configs list, so both
+          entries would ship under the same job_name — nothing overwrites
+          anything, and the two are scraped by different rules with
+          different trust.
+
+          Rename one. A target is either reachable on loopback because it
+          shares this host, or published and reached with a credential; it
+          should not be described as both.
+        '';
+      }
       {
         # A hive proves who it is with a token this provider mints, so
         # there is no version of this collector that runs without one.

From 97b104b54f49190a322cd25a6bf7ea0c18018756 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Mon, 24 Aug 2026 13:42:14 +0200
Subject: [PATCH 5/6] feat(swarm-otel): scrape a published target with an
 authelia bearer token
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The published targets now render as prometheus scrape jobs, so the
option's name is true: the collector reaches them by name over https,
using prometheus-native oauth2 with the audience set to the target's own
url. Registered is not the same as requested — a client that does not
ask for an audience gets a token with an empty one however complete its
registration looks.

The url is split into scheme, target and metrics_path rather than asked
for three times: two spellings of one address is a mismatch waiting to
happen, and the failure it produces is a valid token refused at the
target. A malformed or non-https url is an assertion rather than a null
dereference from inside the renderer.
---
 nix/host-modules/swarm-otel.nix | 87 +++++++++++++++++++++++++++++++--
 1 file changed, 82 insertions(+), 5 deletions(-)

diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index c1b7fea2..232e7459 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -58,6 +58,24 @@ let
   # swarm-tier pipeline would be added here and inherit the check for free.
   reservedOwners = [ swarmTierName ];
 
+  # A published target is declared as ONE url, because that url is also the
+  # audience its token is minted for — but prometheus wants the same fact in
+  # three fields. Split it here rather than asking a service to state it
+  # twice: two spellings of one address is a mismatch waiting to happen, and
+  # the failure is a valid token refused at the target.
+  #
+  # `null` when the shape is wrong, which the assertion below reports by name.
+  # ⚠️ The pattern requires `https`. A published endpoint reached over plain
+  # http would ship a bearer token in clear text, and that is worth an eval
+  # error rather than a warning nobody reads.
+  parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
+
+  # Where the collector reads its own client secret. Under /var/lib and not
+  # /run for the same reason the forge and matrix secrets are: the collector
+  # may start before the delivery unit on a later boot, and a secret that
+  # evaporates on reboot turns a working scrape into an intermittent one.
+  collectorSecretPath = "/var/lib/swarm-otel/${cfg.clientId}.secret";
+
   # `attrNames` is sorted, so this is a function of the hive SET and not of
   # the order anyone wrote it in.
   #
@@ -421,6 +439,28 @@ in
           Rename the hive. Reserved: ${lib.concatMapStringsSep ", " (n: "'${n}'") reservedOwners}.
         '';
       }
+      {
+        # A published target that is not an `https://host/path` url. Without
+        # this the split returns null and the failure surfaces as
+        # `attempt to call elemAt on null` from inside the renderer, naming
+        # neither the option nor the offending value.
+        #
+        # It also enforces the scheme: a published endpoint scraped over plain
+        # http would put a bearer token on the wire in clear text.
+        assertion = lib.all (u: parsePublished u != null) (lib.attrValues cfg.publishedScrapeTargets);
+        message = ''
+          services.hyperhive.swarm.otel.publishedScrapeTargets has ${
+            lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") (
+              lib.filter (kv: parsePublished kv.value == null) (lib.attrsToList cfg.publishedScrapeTargets)
+            )
+          }, which is not of the form https://<host>/<path>.
+
+          The value is both the address scraped and the audience the
+          collector's token is minted for, so it has to be the exact url a
+          request goes to. https is required: this hop carries a bearer
+          token, and plain http would put it on the wire in clear text.
+        '';
+      }
       {
         # A job name used by BOTH scrape options. Within one option this
         # cannot happen — the module system refuses two definitions of the
@@ -600,11 +640,48 @@ in
                 # target: a `prometheus` receiver with nothing to scrape is
                 # the shape this whole issue is about, a config that renders
                 # and deploys perfectly while adding no data.
-                // lib.optionalAttrs (cfg.scrapeTargets != { }) {
-                  prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: {
-                    job_name = job;
-                    static_configs = [ { targets = [ target ]; } ];
-                  }) cfg.scrapeTargets;
+                // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) {
+                  prometheus.config.scrape_configs =
+                    lib.mapAttrsToList (job: target: {
+                      job_name = job;
+                      static_configs = [ { targets = [ target ]; } ];
+                    }) cfg.scrapeTargets
+                    # ⚠️ `++`, so the two kinds of target land in ONE list —
+                    # which is exactly why a job name may not appear in both
+                    # options. A list concatenation does not resolve a
+                    # collision the way an attrset would: both entries ship
+                    # under the same `job_name`. The assertion above is what
+                    # stands between that and a deploy.
+                    ++ lib.mapAttrsToList (
+                      job: url:
+                      let
+                        parts = parsePublished url;
+                      in
+                      {
+                        job_name = job;
+                        # Split from the declared url — see `parsePublished`.
+                        scheme = "https";
+                        static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ];
+                        metrics_path = lib.elemAt parts 1;
+                        # Prometheus-native oauth2, not the collector's
+                        # `oauth2client` extension: the prometheus receiver
+                        # takes upstream's scrape config verbatim and does not
+                        # accept an `auth` block naming a collector extension.
+                        oauth2 = {
+                          client_id = cfg.clientId;
+                          # A path, never a value — nothing here may read the
+                          # secret, or it lands in the store world-readable.
+                          client_secret_file = collectorSecretPath;
+                          token_url = "${autheliaCfg.url}/api/oidc/token";
+                          # The audience is the target's own url, and authelia
+                          # checks it against the address being requested.
+                          # Registered ≠ requested: a client that does not ASK
+                          # for an audience gets a token with `aud: []` however
+                          # complete its registration looks.
+                          endpoint_params.audience = url;
+                        };
+                      }
+                    ) cfg.publishedScrapeTargets;
                 };
 
               exporters =

From 2e06957b3241b99d503e64651a31512285b5c494 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Mon, 24 Aug 2026 13:46:48 +0200
Subject: [PATCH 6/6] feat(swarm-otel): deliver the collector's client secret
 from authelia
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The scrape config names a client_secret_file; this is what puts a file
there. A host oneshot copies authelia's minted secret between the two
container trees — a copy and not a bind mount, because the secret does
not exist until authelia's first boot and nixos-container refuses to
start on a missing bind source, which on a fresh swarm is a permanent
stall presenting as broken metrics.

The collector runs under DynamicUser and the prometheus receiver opens
client_secret_file itself, as that user, so there is no uid to hand the
file to. LoadCredential reads it as root before the sandbox exists and
re-exposes it under a path that does not depend on which uid the unit
got; the scrape config points there. Both spellings derive from one
binding, since a mismatch is a file the collector cannot open and
nothing but a runtime 401 would say so.
---
 nix/host-modules/swarm-otel.nix | 104 +++++++++++++++++++++++++++++---
 1 file changed, 95 insertions(+), 9 deletions(-)

diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index 232e7459..06e9ec9d 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -70,11 +70,30 @@ let
   # error rather than a warning nobody reads.
   parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
 
-  # Where the collector reads its own client secret. Under /var/lib and not
-  # /run for the same reason the forge and matrix secrets are: the collector
-  # may start before the delivery unit on a later boot, and a secret that
-  # evaporates on reboot turns a working scrape into an intermittent one.
-  collectorSecretPath = "/var/lib/swarm-otel/${cfg.clientId}.secret";
+  # The client secret takes three names, and the reason is `DynamicUser`.
+  #
+  # Upstream's collector unit runs with `DynamicUser = true`, and the
+  # prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that
+  # user — so there is no stable uid to hand a file to, and the root-owned
+  # 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable.
+  # (Matrix gets away with it because `LoadCredential` reads the file as root
+  # before the sandbox exists, and tuwunel never opens that path itself.)
+  #
+  # `LoadCredential` solves both halves: systemd reads the file as root and
+  # re-exposes it to the dynamic user under a path that does not depend on
+  # which uid it turned out to be.
+  #
+  # At rest in the container's tree — written by the host oneshot below.
+  # Under /var/lib and not /run because the collector may start before the
+  # delivery unit on a later boot, and a secret that evaporates on reboot
+  # turns a working scrape into an intermittent one.
+  collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret";
+  collectorCredentialId = "oidc-client-secret";
+  # What the scrape config points at. ⚠️ This path and the `LoadCredential`
+  # id below are one fact spelled twice by systemd's design — both derive from
+  # `collectorCredentialId` so they cannot drift; a mismatch is a file the
+  # collector cannot open, discovered at runtime and nowhere else.
+  collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
 
   # `attrNames` is sorted, so this is a function of the hive SET and not of
   # the order anyone wrote it in.
@@ -378,6 +397,63 @@ in
           }
         ];
 
+    # Deliver the collector's client secret from authelia's container into
+    # this one. On the HOST because that is the only place both container
+    # trees are addressable: they share this host's network namespace, which
+    # makes them feel co-located, but their filesystem roots are separate.
+    #
+    # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container
+    # refuses to start when a bind source is missing, and this secret does not
+    # exist until authelia's first boot has minted it — so binding it would
+    # make the collector wait on a file that waits on a container that starts
+    # after it. On a fresh swarm that is a permanent stall presenting as
+    # "metrics are broken", several layers from its cause.
+    systemd.services.swarm-otel-oidc-secret =
+      lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { })
+        {
+          description = "deliver the swarm collector's OIDC client secret from authelia";
+          after = [ "container@${autheliaCfg.machine}.service" ];
+          requires = [ "container@${autheliaCfg.machine}.service" ];
+          before = [ "container@${cfg.machine}.service" ];
+          wantedBy = [ "container@${cfg.machine}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            SyslogIdentifier = "swarm-otel-oidc-secret";
+            # Longer than the bounded wait below, and that is the point:
+            # `DefaultTimeoutStartSec` is 90s, so without this systemd kills
+            # the unit before it can emit the message naming the file it was
+            # waiting for — the failure then reads as a timeout with no cause.
+            TimeoutStartSec = "180s";
+          };
+          path = [ pkgs.coreutils ];
+          script = ''
+            set -euo pipefail
+
+            src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"}
+            dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"}
+
+            # authelia's container is up, but its first-boot generator may
+            # still be minting. Bounded wait, then fail: skipping silently
+            # produces a collector whose scrape gets a 401 forever, which is
+            # the failure this whole design exists to make impossible.
+            for _ in $(seq 1 60); do
+              [ -s "$src" ] && break
+              sleep 2
+            done
+            if [ ! -s "$src" ]; then
+              echo "authelia has not minted $src after 120s" >&2
+              exit 1
+            fi
+
+            # root-owned 0400. The collector runs under `DynamicUser`, so
+            # there is no uid to give it to — `LoadCredential` reads this as
+            # root before the sandbox exists and re-exposes it to whichever
+            # uid the unit got.
+            install -D -m 0400 -o root -g root "$src" "$dst"
+          '';
+        };
+
     # The CA bind source is written at runtime by a host unit, so the
     # container has to start after it — otherwise nspawn sets up a mount
     # over a file that does not exist yet.
@@ -863,10 +939,20 @@ in
           # environment variable without being read by nix, written to the
           # store, or passed in argv.
           systemd.services.opentelemetry-collector.serviceConfig =
-            lib.optionalAttrs (otelCfg.headersCredential != null)
-              {
-                EnvironmentFile = otelCfg.headersCredential;
-              };
+            lib.optionalAttrs (otelCfg.headersCredential != null) {
+              EnvironmentFile = otelCfg.headersCredential;
+            }
+            # The other credential, and the other direction: the one above
+            # authenticates this collector's export onward, this one
+            # authenticates it to a service it scrapes.
+            #
+            # ⚠️ Only where a published target exists. `LoadCredential` on a
+            # missing source is a unit that refuses to start, so declaring it
+            # unconditionally would take the collector down on every hive that
+            # scrapes nothing published — the empty case is the shipped one.
+            // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) {
+              LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ];
+            };
         };
     };
   };