collectors: retry a failed start forever with a growing delay instead of hitting start-limit

nix/module-eval/hive-otel.nix

@@ -190,25 +190,28 @@ let
         !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
     }
     {
-      # A bind that loses the race with the outgoing process is the whole
-      # failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
-      # without this the unit spends its five default attempts inside two
-      # seconds and lands in `start-limit-hit`, where it stops retrying. The
-      # third clause is the one that has to hold — `StartLimit*` are `[Unit]`
-      # settings that systemd ignores under `[Service]`, so a bound written
-      # into `serviceConfig` renders, deploys and does nothing. Asserted
-      # where nixpkgs puts it rather than where it was written, same as
-      # ./bao-grants.nix.
-      name = "the hive collector backs off a failed bind from [Unit], not [Service]";
+      # A failed start (a bind that loses the race with the outgoing process,
+      # a credential not yet delivered) must keep retrying: any start limit
+      # leaves the collector failed until someone starts it by hand.
+      # `StartLimit*` are `[Unit]` settings that systemd ignores under
+      # `[Service]`, so a limit written into `serviceConfig` renders, deploys
+      # and does nothing. Asserted where nixpkgs puts it rather than where it
+      # was written, same as ./bao-grants.nix. `Restart = "always"` is
+      # nixpkgs' own setting: it shows the lookup reaches the real unit, so a
+      # misspelt attribute path cannot pass the other clauses.
+      name = "the hive collector retries a failed start forever, with a growing delay";
       ok =
         let
           u = hiveOtel.systemd.services.opentelemetry-collector;
+          sc = u.serviceConfig;
         in
-        u.serviceConfig.RestartSec or 0 > 0
-        && toString u.unitConfig.StartLimitBurst == "12"
-        && !(u.serviceConfig ? StartLimitBurst)
-        # The window has to outlast every attempt, or the burst is unreachable.
-        && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
+        sc.Restart == "always"
+        && toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0"
+        && !(sc ? StartLimitIntervalSec)
+        && !(sc ? StartLimitBurst)
+        && sc.RestartSec > 0
+        && sc.RestartSteps > 0
+        && sc.RestartMaxDelaySec > sc.RestartSec;
     }
     {
       # Collectors share a network namespace whenever they are co-located,
@@ -319,14 +322,15 @@ let
         && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
     }
     {
-      # On a first boot the file does not exist until the fetch succeeds, so
-      # the collector fails `LoadCredential` into its start limit, i.e. the
-      # `failed` state. Only `reset-failed` clears that limit, so the fetch
-      # has to reset before it starts; `try-restart` acts on no failed unit.
+      # On a first boot the file does not exist until the fetch succeeds, and
+      # the collector retries `LoadCredential` with a growing delay. A
+      # collector that is `failed` for another reason is not retried by
+      # systemd, and `try-restart` acts on no failed unit, so the fetch has to
+      # `reset-failed` before it starts.
       # A never-active collector has an empty `ActiveEnterTimestamp`, read as
       # 0, so the file always counts as newer. The collector only `wants` the
       # fetch, so an unreachable store does not hold it back.
-      name = "the hive collector's fetch revives a collector that hit its start limit without the file";
+      name = "the hive collector's fetch revives a collector that is failed without the file";
       ok =
         let
           s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
@@ -396,7 +400,7 @@ let
       # A secret delivered some other way is the operator's: no unit fetches
       # into a file the collector does not load, or restarts it on one. Nothing
       # restarts the collector when that file lands, so it waits for the file
-      # instead of failing `LoadCredential` into its start limit.
+      # instead of looping on a failing `LoadCredential`.
       name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
       ok =
         let