hive telemetry collector: keep the secret wait for an operator-named file

The previous commit dropped `hive-otel-client-secret` and the collector's
`requires`/`after` on it for every hive. Only the store fetch restarts the
collector when its file lands; an operator-named `otel.clientSecretFile`
has nothing doing that, so a late file failed `LoadCredential` straight into
the collector's start limit with no way back.

The wait unit and the ordering are back for `senderAuth && !fetches`, and
absent only where `swarm-bao-hive-otel-oidc` delivers the file. The
module-eval case for a hand-named secret now also pins the wait unit and
the collector's `requires`/`after` on it, on both of its fixtures.

Refs #5015

docs/scheduler/observability.md

@@ -139,8 +139,8 @@ so a secret rotated later arrives the next time the unit runs. On a hive that
 runs the store the leaf is already there; elsewhere it's the credential you
 place — see [secret placement](../swarm/secrets.md).
 
-To deliver the secret some other way, name the file, and the fetch unit isn't
+To deliver the secret some other way, name the file. The fetch unit isn't
-rendered:
+rendered, and the collector waits up to 120s for the file to appear:
 
 ```nix
 services.hyperhive.otel.clientSecretFile = "/run/secrets/hive-telemetry.secret";

nix/host-modules/otel.nix

@@ -338,6 +338,10 @@ in
       # delivered some other way is the operator's, and a unit restarting the
       # collector on a file it never loads would be noise.
       fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath;
+      # An operator-delivered file has no unit that restarts the collector
+      # once it lands, so the collector waits for it instead.
+      waits = senderAuth && !fetches;
+      secretWaitUnit = "hive-otel-client-secret";
       # Written by swarm-secret-publish on authelia's host. The nix half of
       # `swarm_secret_client::queue::hive_client_path`.
       storeSecretPath = "secret/swarm/hives/${toString hiveName}/oidc/client";
@@ -661,8 +665,76 @@ in
         (lib.mkIf senderAuth {
           serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ];
         })
+
+        # `requires`, not merely `after`: ordering without gating leaves the
+        # collector starting anyway and failing on its own `LoadCredential`,
+        # which is the failure the wait exists to prevent. Same reasoning
+        # `lib/hive-ca-trust.nix` records for its bundle consumers.
+        (lib.mkIf waits {
+          requires = [ "${secretWaitUnit}.service" ];
+          after = [ "${secretWaitUnit}.service" ];
+        })
       ];
 
+      # Wait for an operator-delivered credential rather than racing it.
+      #
+      # `LoadCredential` naming a missing path is FATAL AT UNIT START, and
+      # nothing orders this host unit against whatever delivers the file.
+      # Worse, nixpkgs
+      # sets `Restart = "always"` with no `RestartSec`, so the failure is
+      # instant and the unit burns systemd's 5-starts-in-10s allowance in well
+      # under a second, landing in `start-limit-hit` where it stops retrying
+      # entirely and needs `systemctl reset-failed` by hand.
+      #
+      # 🔑 `Restart = always` reads like it makes this self-healing and does
+      # the opposite: a SLOW-failing unit retries until the secret appears, a
+      # FAST-failing one exhausts its limit before the thing it waits for can
+      # exist. This oneshot converts the fast failure into a slow one, which
+      # is what that restart policy is actually good at.
+      #
+      # Copied from `hive-forge-oidc-secret.service`, which solves exactly
+      # this for the forge — see `docs/swarm/secrets.md`.
+      #
+      # Not rendered beside the store fetch below, which restarts the
+      # collector itself when the file lands.
+      systemd.services.${secretWaitUnit} = lib.mkIf waits {
+        description = "wait for this hive's telemetry client secret";
+        # Deliberately says nothing about the collector: the CONSUMER declares
+        # `requires` + `after` above, which both pulls this into the
+        # transaction and gates on it. Stating the order from both sides too
+        # would be one more thing to keep consistent for no added guarantee.
+        #
+        # When authelia runs on this host the named file may be one it mints.
+        # Elsewhere there is no local unit to order against — naming one that
+        # does not exist orders nothing, silently.
+        after = lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service";
+        requires = lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service";
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          SyslogIdentifier = secretWaitUnit;
+          # ⚠️ Longer than the bounded wait below. `DefaultTimeoutStartSec` is
+          # 90s, so without this systemd kills the unit at 90 — before it can
+          # emit the message naming the file it was waiting for.
+          TimeoutStartSec = "180s";
+        };
+        path = [ pkgs.coreutils ];
+        script = ''
+          set -euo pipefail
+          secret=${lib.escapeShellArg (toString otel.clientSecretFile)}
+
+          # Bounded, then FAIL — never skip. A silent skip produces a
+          # collector that starts and ships nothing, which is the failure
+          # mode this whole path is trying not to have.
+          for _ in $(seq 1 60); do
+            [ -s "$secret" ] && exit 0
+            sleep 2
+          done
+          echo "telemetry client secret $secret has not appeared after 120s" >&2
+          exit 1
+        '';
+      };
+
       # This hive's OIDC client secret, out of the swarm secret store under the
       # hive's own leaf. Same shape as ./hive-matrix.nix's
       # `swarm-bao-matrix-oidc`: the login and the read both fail loudly, and

nix/module-eval/hive-otel.nix

@@ -261,12 +261,27 @@ let
     }
     {
       # A secret delivered some other way is the operator's: no unit fetches
-      # into a file the collector does not load, or restarts it on one.
+      # into a file the collector does not load, or restarts it on one. Nothing
-      name = "a hand-named collector secret renders no store fetch";
+      # restarts the collector when that file lands, so it waits for the file
+      # instead of failing `LoadCredential` into its start limit.
+      name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
       ok =
+        let
+          waitsFor =
+            machine:
+            let
+              c = machine.systemd.services.opentelemetry-collector;
+              w = machine.systemd.services.hive-otel-client-secret;
+            in
+            builtins.elem "hive-otel-client-secret.service" c.requires
+            && builtins.elem "hive-otel-client-secret.service" c.after
+            && lib.hasInfix "/var/lib/hive-otel-oidc/client.secret" w.script;
+        in
         !(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
         && !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
-        && builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
+        && builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
+        && waitsFor hiveOtel
+        && waitsFor baoWithHiveOtel;
     }
   ];
 in