swarm-bao, swarm-nats: bound each attempt of the hand-copied store-retry units

The granting units (granterPolicyUnit's 17, swarm-bao-granter-role,
swarm-bao-controller-policy, swarm-bao-agent-pki) and swarm-bao-nats-tls
copy store-retry's restart budget by hand, are Type=oneshot, and set no
TimeoutStartSec, so systemd gives each attempt no start timeout. A store
that accepts a connection and never answers leaves the attempt running
forever: Restart=on-failure never fires, units ordered after it wait, and
swarm-bao-nats-tls's daily timer starts a unit that is still activating.

Each now sets TimeoutStartSec: 60 for the granting units and the
granter's role, 2min for controller-policy (it may generate the services
root), 5min for agent-pki and nats-tls (each may wait on a blocking
restart of a server in a container). module-eval-bao-grants and
module-eval-nats-tls assert the bound is finite.

The restart budget itself is unchanged; that is #5305.

Closes #5306

nix/host-modules/swarm-bao.nix

@@ -1125,6 +1125,7 @@ let
         RemainAfterExit = true;
         Restart = "on-failure";
         RestartSec = 30;
+        TimeoutStartSec = 60;
       };
       script = ''
         set -euo pipefail
@@ -2935,6 +2936,7 @@ in
           RemainAfterExit = true;
           Restart = "on-failure";
           RestartSec = 30;
+          TimeoutStartSec = 60;
           # Runs only once the script above exits 0, which is when the
           # granter may first configure `auth/oidc`; the viewer unit exits 0
           # without doing so before then, and nothing else re-runs it. A
@@ -3055,6 +3057,9 @@ in
           RemainAfterExit = true;
           Restart = "on-failure";
           RestartSec = 30;
+          # Above the granting units' 60s: an attempt may generate the
+          # services root, a 4096-bit RSA key.
+          TimeoutStartSec = "2min";
         };
         script = ''
           set -euo pipefail
@@ -3462,6 +3467,10 @@ in
           RemainAfterExit = true;
           Restart = "on-failure";
           RestartSec = 30;
+          # An attempt may generate a 4096-bit RSA root and then wait on a
+          # blocking restart of openbao: its stop and its start, 90s each by
+          # systemd's defaults.
+          TimeoutStartSec = "5min";
         };
         script = ''
           set -euo pipefail

nix/host-modules/swarm-nats.nix

@@ -1043,6 +1043,10 @@ in
         SyslogIdentifier = "swarm-bao-nats-tls";
         Restart = "on-failure";
         RestartSec = 30;
+        # An attempt that rotates the leaf waits on a blocking restart of the
+        # server in its container: up to 150s to stop (nixpkgs' nats module)
+        # and 90s to start.
+        TimeoutStartSec = "5min";
       };
       environment = {
         BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";

nix/module-eval/bao-grants.nix

@@ -1058,6 +1058,21 @@ let
         && u.serviceConfig.Restart == "on-failure"
       ) grantingUnitNames;
     }
+    {
+      # A oneshot has no start timeout by default, so an attempt that never
+      # ends never fails into a retry.
+      name = "the granter's role and every granting unit bound their attempt";
+      ok = lib.all (
+        unit:
+        !(lib.elem
+          (toString (baoGrantHere.systemd.services.${unit}.serviceConfig.TimeoutStartSec or "infinity"))
+          [
+            "infinity"
+            "0"
+          ]
+        )
+      ) ([ "swarm-bao-granter-role" ] ++ grantingUnitNames);
+    }
     {
       # A fetch from the store outlives a store or gateway that is down or
       # sealed for hours: `start-limit-hit` does not clear when the store comes

nix/module-eval/nats-tls.nix

@@ -297,6 +297,16 @@ let
         !(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
         && !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
     }
+    {
+      # A oneshot has no start timeout by default, so an attempt that never
+      # ends never fails into a retry, and the daily timer's start is a no-op.
+      name = "the leaf unit bounds its attempt";
+      ok =
+        !(lib.elem (toString (leafUnit.serviceConfig.TimeoutStartSec or "infinity")) [
+          "infinity"
+          "0"
+        ]);
+    }
     {
       # Every hive dials the queue by the same name, so a hive away from it
       # needs no URL of its own. Control and property in one: the scan must