diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 232e7459..06e9ec9d 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -70,11 +70,30 @@ let # error rather than a warning nobody reads. parsePublished = url: builtins.match "https://([^/]+)(/.*)" url; - # Where the collector reads its own client secret. Under /var/lib and not - # /run for the same reason the forge and matrix secrets are: the collector - # may start before the delivery unit on a later boot, and a secret that - # evaporates on reboot turns a working scrape into an intermittent one. - collectorSecretPath = "/var/lib/swarm-otel/${cfg.clientId}.secret"; + # The client secret takes three names, and the reason is `DynamicUser`. + # + # Upstream's collector unit runs with `DynamicUser = true`, and the + # prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that + # user — so there is no stable uid to hand a file to, and the root-owned + # 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable. + # (Matrix gets away with it because `LoadCredential` reads the file as root + # before the sandbox exists, and tuwunel never opens that path itself.) + # + # `LoadCredential` solves both halves: systemd reads the file as root and + # re-exposes it to the dynamic user under a path that does not depend on + # which uid it turned out to be. + # + # At rest in the container's tree — written by the host oneshot below. + # Under /var/lib and not /run because the collector may start before the + # delivery unit on a later boot, and a secret that evaporates on reboot + # turns a working scrape into an intermittent one. + collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret"; + collectorCredentialId = "oidc-client-secret"; + # What the scrape config points at. ⚠️ This path and the `LoadCredential` + # id below are one fact spelled twice by systemd's design — both derive from + # `collectorCredentialId` so they cannot drift; a mismatch is a file the + # collector cannot open, discovered at runtime and nowhere else. + collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}"; # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. @@ -378,6 +397,63 @@ in } ]; + # Deliver the collector's client secret from authelia's container into + # this one. On the HOST because that is the only place both container + # trees are addressable: they share this host's network namespace, which + # makes them feel co-located, but their filesystem roots are separate. + # + # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container + # refuses to start when a bind source is missing, and this secret does not + # exist until authelia's first boot has minted it — so binding it would + # make the collector wait on a file that waits on a container that starts + # after it. On a fresh swarm that is a permanent stall presenting as + # "metrics are broken", several layers from its cause. + systemd.services.swarm-otel-oidc-secret = + lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { }) + { + description = "deliver the swarm collector's OIDC client secret from authelia"; + after = [ "container@${autheliaCfg.machine}.service" ]; + requires = [ "container@${autheliaCfg.machine}.service" ]; + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ "container@${cfg.machine}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-otel-oidc-secret"; + # Longer than the bounded wait below, and that is the point: + # `DefaultTimeoutStartSec` is 90s, so without this systemd kills + # the unit before it can emit the message naming the file it was + # waiting for — the failure then reads as a timeout with no cause. + TimeoutStartSec = "180s"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + + src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"} + dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"} + + # authelia's container is up, but its first-boot generator may + # still be minting. Bounded wait, then fail: skipping silently + # produces a collector whose scrape gets a 401 forever, which is + # the failure this whole design exists to make impossible. + for _ in $(seq 1 60); do + [ -s "$src" ] && break + sleep 2 + done + if [ ! -s "$src" ]; then + echo "authelia has not minted $src after 120s" >&2 + exit 1 + fi + + # root-owned 0400. The collector runs under `DynamicUser`, so + # there is no uid to give it to — `LoadCredential` reads this as + # root before the sandbox exists and re-exposes it to whichever + # uid the unit got. + install -D -m 0400 -o root -g root "$src" "$dst" + ''; + }; + # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. @@ -863,10 +939,20 @@ in # environment variable without being read by nix, written to the # store, or passed in argv. systemd.services.opentelemetry-collector.serviceConfig = - lib.optionalAttrs (otelCfg.headersCredential != null) - { - EnvironmentFile = otelCfg.headersCredential; - }; + lib.optionalAttrs (otelCfg.headersCredential != null) { + EnvironmentFile = otelCfg.headersCredential; + } + # The other credential, and the other direction: the one above + # authenticates this collector's export onward, this one + # authenticates it to a service it scrapes. + # + # ⚠️ Only where a published target exists. `LoadCredential` on a + # missing source is a unit that refuses to start, so declaring it + # unconditionally would take the collector down on every hive that + # scrapes nothing published — the empty case is the shipped one. + // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { + LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; + }; }; }; };