feat(swarm-otel): deliver the collector's client secret from authelia

The scrape config names a client_secret_file; this is what puts a file
there. A host oneshot copies authelia's minted secret between the two
container trees — a copy and not a bind mount, because the secret does
not exist until authelia's first boot and nixos-container refuses to
start on a missing bind source, which on a fresh swarm is a permanent
stall presenting as broken metrics.

The collector runs under DynamicUser and the prometheus receiver opens
client_secret_file itself, as that user, so there is no uid to hand the
file to. LoadCredential reads it as root before the sandbox exists and
re-exposes it under a path that does not depend on which uid the unit
got; the scrape config points there. Both spellings derive from one
binding, since a mismatch is a file the collector cannot open and
nothing but a runtime 401 would say so.

nix/host-modules/swarm-otel.nix

@@ -70,11 +70,30 @@ let
   # error rather than a warning nobody reads.
   parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
 
-  # Where the collector reads its own client secret. Under /var/lib and not
+  # The client secret takes three names, and the reason is `DynamicUser`.
-  # /run for the same reason the forge and matrix secrets are: the collector
+  #
-  # may start before the delivery unit on a later boot, and a secret that
+  # Upstream's collector unit runs with `DynamicUser = true`, and the
-  # evaporates on reboot turns a working scrape into an intermittent one.
+  # prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that
-  collectorSecretPath = "/var/lib/swarm-otel/${cfg.clientId}.secret";
+  # user — so there is no stable uid to hand a file to, and the root-owned
+  # 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable.
+  # (Matrix gets away with it because `LoadCredential` reads the file as root
+  # before the sandbox exists, and tuwunel never opens that path itself.)
+  #
+  # `LoadCredential` solves both halves: systemd reads the file as root and
+  # re-exposes it to the dynamic user under a path that does not depend on
+  # which uid it turned out to be.
+  #
+  # At rest in the container's tree — written by the host oneshot below.
+  # Under /var/lib and not /run because the collector may start before the
+  # delivery unit on a later boot, and a secret that evaporates on reboot
+  # turns a working scrape into an intermittent one.
+  collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret";
+  collectorCredentialId = "oidc-client-secret";
+  # What the scrape config points at. ⚠️ This path and the `LoadCredential`
+  # id below are one fact spelled twice by systemd's design — both derive from
+  # `collectorCredentialId` so they cannot drift; a mismatch is a file the
+  # collector cannot open, discovered at runtime and nowhere else.
+  collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
 
   # `attrNames` is sorted, so this is a function of the hive SET and not of
   # the order anyone wrote it in.
@@ -378,6 +397,63 @@ in
           }
         ];
 
+    # Deliver the collector's client secret from authelia's container into
+    # this one. On the HOST because that is the only place both container
+    # trees are addressable: they share this host's network namespace, which
+    # makes them feel co-located, but their filesystem roots are separate.
+    #
+    # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container
+    # refuses to start when a bind source is missing, and this secret does not
+    # exist until authelia's first boot has minted it — so binding it would
+    # make the collector wait on a file that waits on a container that starts
+    # after it. On a fresh swarm that is a permanent stall presenting as
+    # "metrics are broken", several layers from its cause.
+    systemd.services.swarm-otel-oidc-secret =
+      lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { })
+        {
+          description = "deliver the swarm collector's OIDC client secret from authelia";
+          after = [ "container@${autheliaCfg.machine}.service" ];
+          requires = [ "container@${autheliaCfg.machine}.service" ];
+          before = [ "container@${cfg.machine}.service" ];
+          wantedBy = [ "container@${cfg.machine}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            SyslogIdentifier = "swarm-otel-oidc-secret";
+            # Longer than the bounded wait below, and that is the point:
+            # `DefaultTimeoutStartSec` is 90s, so without this systemd kills
+            # the unit before it can emit the message naming the file it was
+            # waiting for — the failure then reads as a timeout with no cause.
+            TimeoutStartSec = "180s";
+          };
+          path = [ pkgs.coreutils ];
+          script = ''
+            set -euo pipefail
+
+            src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"}
+            dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"}
+
+            # authelia's container is up, but its first-boot generator may
+            # still be minting. Bounded wait, then fail: skipping silently
+            # produces a collector whose scrape gets a 401 forever, which is
+            # the failure this whole design exists to make impossible.
+            for _ in $(seq 1 60); do
+              [ -s "$src" ] && break
+              sleep 2
+            done
+            if [ ! -s "$src" ]; then
+              echo "authelia has not minted $src after 120s" >&2
+              exit 1
+            fi
+
+            # root-owned 0400. The collector runs under `DynamicUser`, so
+            # there is no uid to give it to — `LoadCredential` reads this as
+            # root before the sandbox exists and re-exposes it to whichever
+            # uid the unit got.
+            install -D -m 0400 -o root -g root "$src" "$dst"
+          '';
+        };
+
     # The CA bind source is written at runtime by a host unit, so the
     # container has to start after it — otherwise nspawn sets up a mount
     # over a file that does not exist yet.
@@ -863,10 +939,20 @@ in
           # environment variable without being read by nix, written to the
           # store, or passed in argv.
           systemd.services.opentelemetry-collector.serviceConfig =
-            lib.optionalAttrs (otelCfg.headersCredential != null)
+            lib.optionalAttrs (otelCfg.headersCredential != null) {
-              {
+              EnvironmentFile = otelCfg.headersCredential;
-                EnvironmentFile = otelCfg.headersCredential;
+            }
-              };
+            # The other credential, and the other direction: the one above
+            # authenticates this collector's export onward, this one
+            # authenticates it to a service it scrapes.
+            #
+            # ⚠️ Only where a published target exists. `LoadCredential` on a
+            # missing source is a unit that refuses to start, so declaring it
+            # unconditionally would take the collector down on every hive that
+            # scrapes nothing published — the empty case is the shipped one.
+            // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) {
+              LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ];
+            };
         };
     };
   };