From 9ef4f5511345a8ec9f2b127b99802cae1948f474 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 10 Oct 2026 01:16:04 +0200 Subject: [PATCH] hive telemetry collector: keep the secret wait for an operator-named file The previous commit dropped `hive-otel-client-secret` and the collector's `requires`/`after` on it for every hive. Only the store fetch restarts the collector when its file lands; an operator-named `otel.clientSecretFile` has nothing doing that, so a late file failed `LoadCredential` straight into the collector's start limit with no way back. The wait unit and the ordering are back for `senderAuth && !fetches`, and absent only where `swarm-bao-hive-otel-oidc` delivers the file. The module-eval case for a hand-named secret now also pins the wait unit and the collector's `requires`/`after` on it, on both of its fixtures. Refs #5015 --- docs/scheduler/observability.md | 4 +- nix/host-modules/otel.nix | 72 +++++++++++++++++++++++++++++++++ nix/module-eval/hive-otel.nix | 21 ++++++++-- 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index e3f47fe8..0fdd7c61 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -139,8 +139,8 @@ so a secret rotated later arrives the next time the unit runs. On a hive that runs the store the leaf is already there; elsewhere it's the credential you place — see [secret placement](../swarm/secrets.md). -To deliver the secret some other way, name the file, and the fetch unit isn't -rendered: +To deliver the secret some other way, name the file. The fetch unit isn't +rendered, and the collector waits up to 120s for the file to appear: ```nix services.hyperhive.otel.clientSecretFile = "/run/secrets/hive-telemetry.secret"; diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 52ac5b9a..f8dfd75b 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -338,6 +338,10 @@ in # delivered some other way is the operator's, and a unit restarting the # collector on a file it never loads would be noise. fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath; + # An operator-delivered file has no unit that restarts the collector + # once it lands, so the collector waits for it instead. + waits = senderAuth && !fetches; + secretWaitUnit = "hive-otel-client-secret"; # Written by swarm-secret-publish on authelia's host. The nix half of # `swarm_secret_client::queue::hive_client_path`. storeSecretPath = "secret/swarm/hives/${toString hiveName}/oidc/client"; @@ -661,8 +665,76 @@ in (lib.mkIf senderAuth { serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; }) + + # `requires`, not merely `after`: ordering without gating leaves the + # collector starting anyway and failing on its own `LoadCredential`, + # which is the failure the wait exists to prevent. Same reasoning + # `lib/hive-ca-trust.nix` records for its bundle consumers. + (lib.mkIf waits { + requires = [ "${secretWaitUnit}.service" ]; + after = [ "${secretWaitUnit}.service" ]; + }) ]; + # Wait for an operator-delivered credential rather than racing it. + # + # `LoadCredential` naming a missing path is FATAL AT UNIT START, and + # nothing orders this host unit against whatever delivers the file. + # Worse, nixpkgs + # sets `Restart = "always"` with no `RestartSec`, so the failure is + # instant and the unit burns systemd's 5-starts-in-10s allowance in well + # under a second, landing in `start-limit-hit` where it stops retrying + # entirely and needs `systemctl reset-failed` by hand. + # + # 🔑 `Restart = always` reads like it makes this self-healing and does + # the opposite: a SLOW-failing unit retries until the secret appears, a + # FAST-failing one exhausts its limit before the thing it waits for can + # exist. This oneshot converts the fast failure into a slow one, which + # is what that restart policy is actually good at. + # + # Copied from `hive-forge-oidc-secret.service`, which solves exactly + # this for the forge — see `docs/swarm/secrets.md`. + # + # Not rendered beside the store fetch below, which restarts the + # collector itself when the file lands. + systemd.services.${secretWaitUnit} = lib.mkIf waits { + description = "wait for this hive's telemetry client secret"; + # Deliberately says nothing about the collector: the CONSUMER declares + # `requires` + `after` above, which both pulls this into the + # transaction and gates on it. Stating the order from both sides too + # would be one more thing to keep consistent for no added guarantee. + # + # When authelia runs on this host the named file may be one it mints. + # Elsewhere there is no local unit to order against — naming one that + # does not exist orders nothing, silently. + after = lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"; + requires = lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = secretWaitUnit; + # ⚠️ Longer than the bounded wait below. `DefaultTimeoutStartSec` is + # 90s, so without this systemd kills the unit at 90 — before it can + # emit the message naming the file it was waiting for. + TimeoutStartSec = "180s"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + secret=${lib.escapeShellArg (toString otel.clientSecretFile)} + + # Bounded, then FAIL — never skip. A silent skip produces a + # collector that starts and ships nothing, which is the failure + # mode this whole path is trying not to have. + for _ in $(seq 1 60); do + [ -s "$secret" ] && exit 0 + sleep 2 + done + echo "telemetry client secret $secret has not appeared after 120s" >&2 + exit 1 + ''; + }; + # This hive's OIDC client secret, out of the swarm secret store under the # hive's own leaf. Same shape as ./hive-matrix.nix's # `swarm-bao-matrix-oidc`: the login and the read both fail loudly, and diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index ff45247c..42941877 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -261,12 +261,27 @@ let } { # A secret delivered some other way is the operator's: no unit fetches - # into a file the collector does not load, or restarts it on one. - name = "a hand-named collector secret renders no store fetch"; + # into a file the collector does not load, or restarts it on one. Nothing + # restarts the collector when that file lands, so it waits for the file + # instead of failing `LoadCredential` into its start limit. + name = "a hand-named collector secret renders no store fetch, and the collector waits for the file"; ok = + let + waitsFor = + machine: + let + c = machine.systemd.services.opentelemetry-collector; + w = machine.systemd.services.hive-otel-client-secret; + in + builtins.elem "hive-otel-client-secret.service" c.requires + && builtins.elem "hive-otel-client-secret.service" c.after + && lib.hasInfix "/var/lib/hive-otel-oidc/client.secret" w.script; + in !(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc) && !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc) - && builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; + && builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential + && waitsFor hiveOtel + && waitsFor baoWithHiveOtel; } ]; in