diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 2800717c..b8e95529 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -274,6 +274,7 @@ in # renders neither the value nor the directory, and nothing has to # hardcode `/run/credentials/`. credName = "swarm-client.secret"; + secretWaitUnit = "hive-otel-client-secret"; in { # Reachable from agent containers and nowhere else: this opens @@ -380,8 +381,71 @@ in # reason for a second on-disk copy of the secret. systemd hands it to # the process in a private tmpfs and exports the directory, which is # what the config above names. - systemd.services.opentelemetry-collector.serviceConfig = lib.optionalAttrs senderAuth { - LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; + systemd.services.opentelemetry-collector = lib.optionalAttrs senderAuth { + serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; + + # `requires`, not merely `after`: ordering without gating leaves the + # collector starting anyway and failing on its own `LoadCredential`, + # which is the failure the wait exists to prevent. Same reasoning + # `lib/hive-ca-trust.nix` records for its bundle consumers. + requires = [ "${secretWaitUnit}.service" ]; + after = [ "${secretWaitUnit}.service" ]; + }; + + # Wait for the credential rather than racing it. + # + # `LoadCredential` naming a missing path is FATAL AT UNIT START, and the + # secret is minted by authelia's first-boot generator inside its own + # container — nothing orders a host unit against that. Worse, nixpkgs + # sets `Restart = "always"` with no `RestartSec`, so the failure is + # instant and the unit burns systemd's 5-starts-in-10s allowance in well + # under a second, landing in `start-limit-hit` where it stops retrying + # entirely and needs `systemctl reset-failed` by hand. + # + # 🔑 `Restart = always` reads like it makes this self-healing and does + # the opposite: a SLOW-failing unit retries until the secret appears, a + # FAST-failing one exhausts its limit before the thing it waits for can + # exist. This oneshot converts the fast failure into a slow one, which + # is what that restart policy is actually good at. + # + # Copied from `hive-forge-oidc-secret.service`, which solves exactly + # this for the forge — see `docs/swarm/secrets.md`. + systemd.services.${secretWaitUnit} = lib.mkIf senderAuth { + description = "wait for this hive's telemetry client secret"; + # Deliberately says nothing about the collector: the CONSUMER declares + # `requires` + `after` above, which both pulls this into the + # transaction and gates on it. Stating the order from both sides too + # would be one more thing to keep consistent for no added guarantee. + # + # Only when the minting container is on THIS host. Elsewhere the file + # is operator-provided and there is no local unit to order against — + # naming one that does not exist orders nothing, silently. + after = lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service"; + requires = lib.optional autheliaCfg.enable "container@${autheliaCfg.machine}.service"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = secretWaitUnit; + # ⚠️ Longer than the bounded wait below. `DefaultTimeoutStartSec` is + # 90s, so without this systemd kills the unit at 90 — before it can + # emit the message naming the file it was waiting for. + TimeoutStartSec = "180s"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + secret=${lib.escapeShellArg (toString otel.clientSecretFile)} + + # Bounded, then FAIL — never skip. A silent skip produces a + # collector that starts and ships nothing, which is the failure + # mode this whole path is trying not to have. + for _ in $(seq 1 60); do + [ -s "$secret" ] && exit 0 + sleep 2 + done + echo "telemetry client secret $secret has not appeared after 120s" >&2 + exit 1 + ''; }; assertions = [