feat(swarm-otel): deliver the collector's client secret from authelia

The scrape config names a client_secret_file; this is what puts a file
there. A host oneshot copies authelia's minted secret between the two
container trees — a copy and not a bind mount, because the secret does
not exist until authelia's first boot and nixos-container refuses to
start on a missing bind source, which on a fresh swarm is a permanent
stall presenting as broken metrics.

The collector runs under DynamicUser and the prometheus receiver opens
client_secret_file itself, as that user, so there is no uid to hand the
file to. LoadCredential reads it as root before the sandbox exists and
re-exposes it under a path that does not depend on which uid the unit
got; the scrape config points there. Both spellings derive from one
binding, since a mismatch is a file the collector cannot open and
nothing but a runtime 401 would say so.

nix/host-modules/hive-forge/default.nix

@@ -442,6 +442,21 @@ in
       url = "https://${cfg.domain}/";
     };
 
+    # The metrics endpoint, declared once: the collector both scrapes this
+    # URL and derives from it the audience its token is minted for. Same
+    # `behindGateway` guard, and for a stronger reason than the two above:
+    # with it off there is no `= /metrics` location and no `auth_request`
+    # in front of it, so the URL this names does not exist to be scraped
+    # or authorised.
+    #
+    # ⚠️ Written as the exact URL a collector requests, because that is
+    # what authelia compares against — this string agreeing with the
+    # `location` block above it is the whole mechanism. A near miss is a
+    # correctly minted token refused at the target.
+    services.hyperhive.swarm.otel.publishedScrapeTargets = lib.optionalAttrs cfg.behindGateway {
+      forgejo = "https://${cfg.domain}/metrics";
+    };
+
     # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for
     # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB
     # clones). SSH stays direct on `forge.sshPort`. See

nix/host-modules/swarm-authelia.nix

@@ -57,6 +57,47 @@ let
   # failing the check it was supposed to pass.
   operatorGroup = "admins";
 
+  # Whether the swarm collector's client is actually registered here.
+  #
+  # Asked of the client list rather than re-derived from the conditions
+  # that produce it (`otel.enable`, a non-empty audience set, authelia
+  # being co-located). A second copy of that predicate is a second thing
+  # to keep in step, and the two drifting is not a build failure: a rule
+  # naming an unregistered client is refused by authelia's *startup*
+  # validator, so the whole SSO service fails to restart.
+  #
+  # Reading the registration itself also makes the rule correct for a
+  # registrar this module has never heard of — an operator registering
+  # the collector by hand against a provider that is not co-located with
+  # it gets the same rule, from the same expression.
+  # ⚠️ `swarm.otel` named in full, not through a let binding: there are two
+  # otel options one word apart, and only this one is the swarm's collector.
+  collectorClientId = config.services.hyperhive.swarm.otel.clientId;
+  collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients;
+
+  # The forge's metrics endpoint: deny until a collector exists to allow.
+  #
+  # `deny` is not a placeholder, it is the protection. The endpoint is
+  # always served (a swarm-integrated forge always has metrics) and
+  # `default_policy` is `one_factor`, which means *any* authenticated
+  # subject — every operator today, every agent once they hold authelia
+  # accounts. Nor does the audience stand between a browser session and
+  # this data: `authn_strategies` on the authz endpoint also accepts
+  # `CookieSession`, and a cookie carries no audience at all.
+  metricsRule = {
+    domain = forgeCfg.domain;
+    resources = [ "^/metrics$" ];
+  }
+  // (
+    if collectorRegistered then
+      {
+        policy = "one_factor";
+        subject = [ "oauth2:client:${collectorClientId}" ];
+      }
+    else
+      { policy = "deny"; }
+  );
+
   # Upstream's `services.authelia.instances.<name>` derives the unit,
   # user, group and StateDirectory from the instance name
   # (`authelia` + `-<name>`). Naming them here rather than repeating the
@@ -217,7 +258,7 @@ let
       if c.kind == "machine" then
         ''
           printf -- '        grant_types: ["client_credentials"]\n'
-          printf -- '        scopes: []\n'
+          printf -- '        scopes: [${lib.optionalString c.bearerAuthz "authelia.bearer.authz"}]\n'
         ''
       else
         ''
@@ -546,6 +587,15 @@ in
                 a successful consent** — the login looks like it worked
                 right up to the last hop, and neither the redirect nor
                 the secret is at fault.
+
+                ⚠️ `null` is NOT accepted on a client with `bearerAuthz`.
+                Measured against authelia 4.39.20: under that scope the
+                method must be *stated*: omitting it is refused with
+                `must be configured as 'client_secret_basic', … but it's
+                configured as` an empty string. The sentence above is
+                true of an ordinary client and false of that one, which
+                is exactly how a reviewer reads this option and concludes
+                the assertion below is wrong.
               '';
             };
 
@@ -575,6 +625,34 @@ in
               '';
             };
 
+            bearerAuthz = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              example = true;
+              description = ''
+                Grant this client the `authelia.bearer.authz` scope, so it
+                may present its access token to authelia's authz endpoint
+                and be authorised by an `access_control` rule — how a
+                scraper reaches a service published behind the gateway.
+
+                A named capability rather than a free-form `scopes` list,
+                for the same reason `kind` derives the rest: authelia
+                refuses some scope/grant combinations outright (`openid`
+                with `client_credentials` among them), and a list would
+                make those combinations expressible again. This admits the
+                one value that is legal here and nothing else.
+
+                ::: {.note}
+                Setting this obliges two other options, and the assertions
+                below enforce it. Authelia checks the same thing, but only
+                in its `preStart` validator — which means a violation
+                builds and deploys cleanly and then fails to restart,
+                taking swarm SSO down. The assertions move that to
+                evaluation, where a wrong value costs nothing.
+                :::
+              '';
+            };
+
             accessTokenSignedResponseAlg = lib.mkOption {
               type = lib.types.nullOr (
                 lib.types.enum [
@@ -820,6 +898,77 @@ in
             + "services.hyperhive.swarm.hives; rename the hive or the "
             + "colliding client.";
         }
+        # The two obligations `authelia.bearer.authz` carries. Authelia
+        # enforces both itself — but in its `preStart` validator, so a
+        # violation produces a green `nixos-rebuild switch` and an authelia
+        # that then refuses to come back up, taking swarm SSO with it.
+        # Asserting here moves the same failure to evaluation, where a
+        # wrong value costs a build and nothing else.
+        {
+          assertion = lib.all (c: !c.bearerAuthz || c.audience != [ ]) cfg.oidc.clients;
+          message =
+            "services.hyperhive.swarm.authelia.oidc.clients: "
+            + lib.concatStringsSep ", " (
+              map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.audience == [ ]) cfg.oidc.clients)
+            )
+            + " sets bearerAuthz but declares no audience. The audience is "
+            + "what authorises a bearer token at a given URL, so without "
+            + "one the scope grants access to nothing and authelia refuses "
+            + "the configuration outright.";
+        }
+        {
+          assertion = lib.all (
+            c:
+            !c.bearerAuthz
+            || lib.elem c.tokenEndpointAuthMethod [
+              "client_secret_basic"
+              "client_secret_jwt"
+              "private_key_jwt"
+            ]
+          ) cfg.oidc.clients;
+          message =
+            "services.hyperhive.swarm.authelia.oidc.clients: "
+            + lib.concatStringsSep ", " (
+              map (c: "client '${c.id}' (tokenEndpointAuthMethod = ${toString c.tokenEndpointAuthMethod})") (
+                lib.filter (
+                  c:
+                  c.bearerAuthz
+                  && !lib.elem c.tokenEndpointAuthMethod [
+                    "client_secret_basic"
+                    "client_secret_jwt"
+                    "private_key_jwt"
+                  ]
+                ) cfg.oidc.clients
+              )
+            )
+            + ". A confidential client carrying authelia.bearer.authz must "
+            + "authenticate with client_secret_basic, client_secret_jwt or "
+            + "private_key_jwt. Notably client_secret_post is refused, and "
+            + "it is what an OAuth2 client library may reach for first. "
+            + "Leaving it null is refused too: authelia requires the method "
+            + "to be STATED under this scope rather than defaulted.";
+        }
+        # Nothing else stops `bearerAuthz` on an interactive client, and it
+        # would silently do nothing: `renderClient` reads the flag only in
+        # the `machine` branch, so the scope is simply never emitted and the
+        # client authenticates fine while being authorised for nothing.
+        #
+        # That is this module's own failure mode one level up — a green
+        # build and a grant that does not exist — and `kind` defaults to
+        # `interactive`, so it is reached by FORGETTING a field rather than
+        # by writing a wrong one.
+        {
+          assertion = lib.all (c: !c.bearerAuthz || c.kind == "machine") cfg.oidc.clients;
+          message =
+            "services.hyperhive.swarm.authelia.oidc.clients: "
+            + lib.concatStringsSep ", " (
+              map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.kind != "machine") cfg.oidc.clients)
+            )
+            + " sets bearerAuthz with kind != \"machine\". The scope is only "
+            + "emitted for machine clients, so this grants nothing while "
+            + "evaluating and deploying cleanly. A browser client has a user "
+            + "to authorise and does not need it.";
+        }
       ];
 
     # Authelia's own gateway surface: the vhost that fronts it and the
@@ -1205,32 +1354,10 @@ in
                 # The metrics rule is listed first so it cannot be shadowed
                 # by a broader domain rule added later.
                 rules =
-                  # The forge's metrics endpoint. `deny` is deliberate and
-                  # is the whole protection right now: the endpoint is
-                  # always served (a swarm-integrated forge always has
-                  # metrics), and `default_policy` is `one_factor`, which
-                  # means *any* authenticated subject — every operator
-                  # today, every agent once they hold authelia accounts.
-                  #
-                  # Being reachable by a Bearer token is not sufficient on
-                  # its own: `authn_strategies` on this endpoint also
-                  # accepts `CookieSession`, and a cookie carries no
-                  # audience, so the audience is not what stands between a
-                  # browser session and this data.
-                  #
-                  # The collector gets in by REPLACING this with a
-                  # client-scoped allow (`subject = ["oauth2:client:<id>"]`)
-                  # once such a client is registered. Denying until then is
-                  # what makes publishing the endpoint safe on its own —
-                  # authelia refuses a subject naming a client that is not
-                  # registered, and it does so in a `preStart` validator,
-                  # so naming one early takes the whole SSO service down on
-                  # the next restart rather than failing the build.
-                  lib.optional forgeCfg.behindGateway {
-                    domain = forgeCfg.domain;
-                    resources = [ "^/metrics$" ];
-                    policy = "deny";
-                  }
+                  # Denied or client-scoped depending on whether a
+                  # collector is registered — see `metricsRule` above,
+                  # which is where the reasoning for both halves lives.
+                  lib.optional forgeCfg.behindGateway metricsRule
                   ++ lib.optional uiCfg.enable {
                     domain = uiCfg.domain;
                     subject = [ "group:${operatorGroup}" ];

nix/host-modules/swarm-otel.nix

@@ -58,6 +58,43 @@ let
   # swarm-tier pipeline would be added here and inherit the check for free.
   reservedOwners = [ swarmTierName ];
 
+  # A published target is declared as ONE url, because that url is also the
+  # audience its token is minted for — but prometheus wants the same fact in
+  # three fields. Split it here rather than asking a service to state it
+  # twice: two spellings of one address is a mismatch waiting to happen, and
+  # the failure is a valid token refused at the target.
+  #
+  # `null` when the shape is wrong, which the assertion below reports by name.
+  # ⚠️ The pattern requires `https`. A published endpoint reached over plain
+  # http would ship a bearer token in clear text, and that is worth an eval
+  # error rather than a warning nobody reads.
+  parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
+
+  # The client secret takes three names, and the reason is `DynamicUser`.
+  #
+  # Upstream's collector unit runs with `DynamicUser = true`, and the
+  # prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that
+  # user — so there is no stable uid to hand a file to, and the root-owned
+  # 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable.
+  # (Matrix gets away with it because `LoadCredential` reads the file as root
+  # before the sandbox exists, and tuwunel never opens that path itself.)
+  #
+  # `LoadCredential` solves both halves: systemd reads the file as root and
+  # re-exposes it to the dynamic user under a path that does not depend on
+  # which uid it turned out to be.
+  #
+  # At rest in the container's tree — written by the host oneshot below.
+  # Under /var/lib and not /run because the collector may start before the
+  # delivery unit on a later boot, and a secret that evaporates on reboot
+  # turns a working scrape into an intermittent one.
+  collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret";
+  collectorCredentialId = "oidc-client-secret";
+  # What the scrape config points at. ⚠️ This path and the `LoadCredential`
+  # id below are one fact spelled twice by systemd's design — both derive from
+  # `collectorCredentialId` so they cannot drift; a mismatch is a file the
+  # collector cannot open, discovered at runtime and nowhere else.
+  collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
+
   # `attrNames` is sorted, so this is a function of the hive SET and not of
   # the order anyone wrote it in.
   #
@@ -223,6 +260,56 @@ in
         scrape is the inert configuration this option exists to avoid.
       '';
     };
+
+    publishedScrapeTargets = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      example = lib.literalExpression ''{ forge = "https://forge.example.com/metrics"; }'';
+      description = ''
+        Prometheus endpoints this collector scrapes **by name, with a
+        credential**, as `<job name> = "<url>"`.
+
+        A service module declares its own entry, from its own module, the
+        same way it does for `scrapeTargets` — and the collector's OAuth2
+        client derives its permitted audiences from these URLs, so a
+        target and the authorisation to reach it are **one declaration**.
+        Two lists that must agree would be a drift to maintain, and its
+        failure mode is the bad one: a target whose audience was forgotten
+        authenticates against nothing and reads as a scrape failure rather
+        than a config mistake.
+
+        ⚠️ A **full URL**, not `host:port`. Authelia validates a bearer
+        token against the address being requested, so the string here is
+        also the audience the token is minted for; a near miss (a trailing
+        slash, `http` for `https`) presents as a valid token rejected at
+        the target, several layers from its cause.
+
+        ⚠️ Deliberately separate from `scrapeTargets`. An entry there is
+        trusted because the scraper and the target share a host — that
+        option is loopback-and-unauthenticated by contract. An entry here
+        is trusted because it presents a credential. One shape for both
+        would leave a reader unable to tell which of those a given target
+        relies on.
+      '';
+    };
+
+    clientId = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = "swarm-collector";
+      description = ''
+        OAuth2 client id this collector authenticates as. Published so
+        authelia's `access_control` rules can name it without carrying a
+        second copy of the string, exactly as
+        `services.hyperhive.swarm.authelia.hiveClientPrefix` is published
+        for the queue's responder.
+
+        Two spellings drifting apart is not a build failure: authelia
+        refuses a rule naming an unregistered client in its startup
+        validator, so the swarm's SSO service fails to *restart* — long
+        after the change that caused it evaluated cleanly.
+      '';
+    };
   };
 
   config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
@@ -267,6 +354,106 @@ in
     # telemetry.
     services.hyperhive.swarm.authelia.oidc.hiveIdentities = true;
 
+    # The collector's own identity, for the other direction: the hive
+    # identities above are how this collector authenticates its *callers*,
+    # this is how it authenticates *itself* to a service published behind
+    # the gateway.
+    #
+    # Only where authelia is co-located. A client is a row in this host's
+    # provider config, so declaring one against a remote provider would
+    # render nothing while reading as done; a swarm whose authelia lives
+    # elsewhere registers it there.
+    #
+    # ⚠️ Also conditional on there being a published target, and that is
+    # the shipped case rather than an edge — nothing declares one until
+    # some service publishes behind the gateway. Authelia refuses a
+    # bearer-authz client with no audience, so an unconditional
+    # declaration would break every hive that runs a collector and
+    # publishes nothing.
+    services.hyperhive.swarm.authelia.oidc.clients =
+      lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { })
+        [
+          {
+            id = cfg.clientId;
+            description = "HyperHive swarm collector";
+            kind = "machine";
+            # Grants `authelia.bearer.authz`, without which the authz
+            # endpoint refuses an otherwise valid token and blames the
+            # token rather than the missing grant.
+            bearerAuthz = true;
+            # DERIVED from the targets rather than contributed alongside
+            # them. A service declares a URL once and this is the
+            # permission to reach it; two lists that had to agree would be
+            # a drift to maintain, and the failure mode is the quiet one —
+            # a target whose audience was forgotten authenticates against
+            # nothing and looks like a broken scrape.
+            audience = lib.attrValues cfg.publishedScrapeTargets;
+            # Stated rather than left on authelia's default, because the
+            # two agreeing today is not the same as this being the
+            # required value: authelia permits only basic / JWT methods
+            # for a confidential client holding that scope, and enforces
+            # it in the startup validator.
+            tokenEndpointAuthMethod = "client_secret_basic";
+          }
+        ];
+
+    # Deliver the collector's client secret from authelia's container into
+    # this one. On the HOST because that is the only place both container
+    # trees are addressable: they share this host's network namespace, which
+    # makes them feel co-located, but their filesystem roots are separate.
+    #
+    # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container
+    # refuses to start when a bind source is missing, and this secret does not
+    # exist until authelia's first boot has minted it — so binding it would
+    # make the collector wait on a file that waits on a container that starts
+    # after it. On a fresh swarm that is a permanent stall presenting as
+    # "metrics are broken", several layers from its cause.
+    systemd.services.swarm-otel-oidc-secret =
+      lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { })
+        {
+          description = "deliver the swarm collector's OIDC client secret from authelia";
+          after = [ "container@${autheliaCfg.machine}.service" ];
+          requires = [ "container@${autheliaCfg.machine}.service" ];
+          before = [ "container@${cfg.machine}.service" ];
+          wantedBy = [ "container@${cfg.machine}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            SyslogIdentifier = "swarm-otel-oidc-secret";
+            # Longer than the bounded wait below, and that is the point:
+            # `DefaultTimeoutStartSec` is 90s, so without this systemd kills
+            # the unit before it can emit the message naming the file it was
+            # waiting for — the failure then reads as a timeout with no cause.
+            TimeoutStartSec = "180s";
+          };
+          path = [ pkgs.coreutils ];
+          script = ''
+            set -euo pipefail
+
+            src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"}
+            dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"}
+
+            # authelia's container is up, but its first-boot generator may
+            # still be minting. Bounded wait, then fail: skipping silently
+            # produces a collector whose scrape gets a 401 forever, which is
+            # the failure this whole design exists to make impossible.
+            for _ in $(seq 1 60); do
+              [ -s "$src" ] && break
+              sleep 2
+            done
+            if [ ! -s "$src" ]; then
+              echo "authelia has not minted $src after 120s" >&2
+              exit 1
+            fi
+
+            # root-owned 0400. The collector runs under `DynamicUser`, so
+            # there is no uid to give it to — `LoadCredential` reads this as
+            # root before the sandbox exists and re-exposes it to whichever
+            # uid the unit got.
+            install -D -m 0400 -o root -g root "$src" "$dst"
+          '';
+        };
+
     # The CA bind source is written at runtime by a host unit, so the
     # container has to start after it — otherwise nspawn sets up a mount
     # over a file that does not exist yet.
@@ -328,6 +515,59 @@ in
           Rename the hive. Reserved: ${lib.concatMapStringsSep ", " (n: "'${n}'") reservedOwners}.
         '';
       }
+      {
+        # A published target that is not an `https://host/path` url. Without
+        # this the split returns null and the failure surfaces as
+        # `attempt to call elemAt on null` from inside the renderer, naming
+        # neither the option nor the offending value.
+        #
+        # It also enforces the scheme: a published endpoint scraped over plain
+        # http would put a bearer token on the wire in clear text.
+        assertion = lib.all (u: parsePublished u != null) (lib.attrValues cfg.publishedScrapeTargets);
+        message = ''
+          services.hyperhive.swarm.otel.publishedScrapeTargets has ${
+            lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") (
+              lib.filter (kv: parsePublished kv.value == null) (lib.attrsToList cfg.publishedScrapeTargets)
+            )
+          }, which is not of the form https://<host>/<path>.
+
+          The value is both the address scraped and the audience the
+          collector's token is minted for, so it has to be the exact url a
+          request goes to. https is required: this hop carries a bearer
+          token, and plain http would put it on the wire in clear text.
+        '';
+      }
+      {
+        # A job name used by BOTH scrape options. Within one option this
+        # cannot happen — the module system refuses two definitions of the
+        # same key with different values, measured — but the two options
+        # are separate, so nothing arbitrates between them, and they
+        # render into a single `scrape_configs` LIST where nothing
+        # overwrites anything: both entries ship under one `job_name`.
+        #
+        # The message names which side each collision came from, which is
+        # the part a rendered-config error could not tell an operator.
+        assertion =
+          lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets)
+          == [ ];
+        message = ''
+          services.hyperhive.swarm.otel: ${
+            lib.concatMapStringsSep ", " (j: "'${j}'") (
+              lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets)
+            )
+          } is declared as both a loopback
+          scrapeTargets job and a publishedScrapeTargets job.
+
+          They render into one prometheus scrape_configs list, so both
+          entries would ship under the same job_name — nothing overwrites
+          anything, and the two are scraped by different rules with
+          different trust.
+
+          Rename one. A target is either reachable on loopback because it
+          shares this host, or published and reached with a credential; it
+          should not be described as both.
+        '';
+      }
       {
         # A hive proves who it is with a token this provider mints, so
         # there is no version of this collector that runs without one.
@@ -476,11 +716,48 @@ in
                 # target: a `prometheus` receiver with nothing to scrape is
                 # the shape this whole issue is about, a config that renders
                 # and deploys perfectly while adding no data.
-                // lib.optionalAttrs (cfg.scrapeTargets != { }) {
-                  prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: {
-                    job_name = job;
-                    static_configs = [ { targets = [ target ]; } ];
-                  }) cfg.scrapeTargets;
+                // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) {
+                  prometheus.config.scrape_configs =
+                    lib.mapAttrsToList (job: target: {
+                      job_name = job;
+                      static_configs = [ { targets = [ target ]; } ];
+                    }) cfg.scrapeTargets
+                    # ⚠️ `++`, so the two kinds of target land in ONE list —
+                    # which is exactly why a job name may not appear in both
+                    # options. A list concatenation does not resolve a
+                    # collision the way an attrset would: both entries ship
+                    # under the same `job_name`. The assertion above is what
+                    # stands between that and a deploy.
+                    ++ lib.mapAttrsToList (
+                      job: url:
+                      let
+                        parts = parsePublished url;
+                      in
+                      {
+                        job_name = job;
+                        # Split from the declared url — see `parsePublished`.
+                        scheme = "https";
+                        static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ];
+                        metrics_path = lib.elemAt parts 1;
+                        # Prometheus-native oauth2, not the collector's
+                        # `oauth2client` extension: the prometheus receiver
+                        # takes upstream's scrape config verbatim and does not
+                        # accept an `auth` block naming a collector extension.
+                        oauth2 = {
+                          client_id = cfg.clientId;
+                          # A path, never a value — nothing here may read the
+                          # secret, or it lands in the store world-readable.
+                          client_secret_file = collectorSecretPath;
+                          token_url = "${autheliaCfg.url}/api/oidc/token";
+                          # The audience is the target's own url, and authelia
+                          # checks it against the address being requested.
+                          # Registered ≠ requested: a client that does not ASK
+                          # for an audience gets a token with `aud: []` however
+                          # complete its registration looks.
+                          endpoint_params.audience = url;
+                        };
+                      }
+                    ) cfg.publishedScrapeTargets;
                 };
 
               exporters =
@@ -662,10 +939,20 @@ in
           # environment variable without being read by nix, written to the
           # store, or passed in argv.
           systemd.services.opentelemetry-collector.serviceConfig =
-            lib.optionalAttrs (otelCfg.headersCredential != null)
-              {
-                EnvironmentFile = otelCfg.headersCredential;
-              };
+            lib.optionalAttrs (otelCfg.headersCredential != null) {
+              EnvironmentFile = otelCfg.headersCredential;
+            }
+            # The other credential, and the other direction: the one above
+            # authenticates this collector's export onward, this one
+            # authenticates it to a service it scrapes.
+            #
+            # ⚠️ Only where a published target exists. `LoadCredential` on a
+            # missing source is a unit that refuses to start, so declaring it
+            # unconditionally would take the collector down on every hive that
+            # scrapes nothing published — the empty case is the shipped one.
+            // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) {
+              LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ];
+            };
         };
     };
   };