collectors: retry a failed start forever with a growing delay instead of hitting start-limit

nix/host-modules/otel.nix

@@ -768,16 +768,26 @@ in
       };
 
       systemd.services.opentelemetry-collector = lib.mkMerge [
-        # ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them under
-        # `[Service]` — in `serviceConfig` they render, deploy and do
-        # nothing. `checks.module-eval-hive-otel` pins them in `unitConfig`.
-        # `Restart` is deliberately NOT set: nixpkgs' own module defines it
-        # at normal priority, so a second definition is a module-system
-        # conflict. The window must exceed `RestartSec × burst`.
+        # A failed start (a bind that loses the race with the outgoing
+        # process, a credential not yet delivered) must keep retrying: any
+        # start limit leaves the collector failed, and the hive's telemetry
+        # queues on disk, until someone starts it by hand.
+        #
+        # ⚠️ `StartLimitIntervalSec` is a `[Unit]` setting; systemd IGNORES it
+        # under `[Service]`, so it is the unit-level option here and
+        # `checks.module-eval-hive-otel` pins it in `unitConfig`. `Restart` is
+        # deliberately NOT set: nixpkgs' own module defines it as `"always"` at
+        # normal priority, so a second definition is a module-system conflict.
+        #
+        # The delay grows from `RestartSec` to `RestartMaxDelaySec` over
+        # `RestartSteps` restarts (systemd.service(5)).
         {
-          startLimitBurst = 12;
-          startLimitIntervalSec = 120;
-          serviceConfig.RestartSec = 5;
+          startLimitIntervalSec = 0;
+          serviceConfig = {
+            RestartSec = 2;
+            RestartSteps = 6;
+            RestartMaxDelaySec = 60;
+          };
         }
 
         # `LoadCredential` and not a copy-oneshot: this collector is a HOST
@@ -803,17 +813,12 @@ in
       #
       # `LoadCredential` naming a missing path is FATAL AT UNIT START, and
       # nothing orders this host unit against whatever delivers the file.
-      # Worse, nixpkgs
-      # sets `Restart = "always"` with no `RestartSec`, so the failure is
-      # instant and the unit burns systemd's 5-starts-in-10s allowance in well
-      # under a second, landing in `start-limit-hit` where it stops retrying
-      # entirely and needs `systemctl reset-failed` by hand.
+      # The collector's restart policy retries a failed start with no limit,
+      # so without this wait the unit loops on the missing file, one start a
+      # minute at worst, until the file lands.
       #
-      # 🔑 `Restart = always` reads like it makes this self-healing and does
-      # the opposite: a SLOW-failing unit retries until the secret appears, a
-      # FAST-failing one exhausts its limit before the thing it waits for can
-      # exist. This oneshot converts the fast failure into a slow one, which
-      # is what that restart policy is actually good at.
+      # This oneshot holds the collector back until the file exists, so the
+      # first start succeeds instead of failing and backing off.
       #
       # Not rendered beside the store fetch below, which restarts the
       # collector itself when the file lands.