hive-tls: bound each store attempt so a silent store fails into a retry

swarm-services-cert, swarm-services-cert-renew and hive-tls-resign are
oneshots, which systemd gives no start timeout, and the root-only
`ca/pem` curl set no connect or total timeout. A store that accepted the
connection and never answered held the attempt forever: no failure, so
no store-retry restart, and a hung renew unit swallowed every later
timer tick.

The curl now gives up after 10s to connect and 30s overall. Each unit
gets a TimeoutStartSec: 20min for the services-leaf pair (the script
waits up to 10min on its lock), 10min for the gateway leaf. The
`--no-block` comment no longer cites a TimeoutStartSec that did not
exist.

module-eval-hive-tls gains a case pinning both.

Closes #5303

nix/host-modules/hive-tls.nix

@@ -229,6 +229,10 @@ let
     # The root-only fetch on a host fronting no swarm name.
     pkgs.curl
   ];
+  # One attempt of the issuance: up to 10 minutes on the lock at the top of
+  # the script, then the store calls. A oneshot has no start timeout of its
+  # own, and an attempt that never ends never fails into a retry.
+  servicesCertTimeoutStartSec = "20min";
   servicesCertEnvironment = {
     BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
   }
@@ -455,6 +459,9 @@ in
         Type = "oneshot";
         UMask = "0077";
         SyslogIdentifier = "hive-tls-resign";
+        # One attempt. A oneshot has no start timeout of its own, and an
+        # attempt that never ends never fails into a retry.
+        TimeoutStartSec = "10min";
       };
       script = ''
         set -euo pipefail
@@ -619,6 +626,7 @@ in
         RemainAfterExit = true;
         UMask = "0077";
         SyslogIdentifier = "swarm-services-cert";
+        TimeoutStartSec = servicesCertTimeoutStartSec;
       };
       environment = servicesCertEnvironment;
       script = ''
@@ -654,6 +662,7 @@ in
               raw="$(mktemp "$d/swarm-services-root.XXXXXX")"
               trap 'rm -f "$err" "$raw"' EXIT
               if ! curl --fail --silent --show-error \
+                  --connect-timeout 10 --max-time 30 \
                   ${storeRootCurlTls} --output "$raw" \
                   "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then
                 echo "could not read the store's ${baoServicesPkiMount} root." >&2
@@ -824,10 +833,10 @@ in
         # `Before=` both of these, so a blocking `systemctl restart` or
         # `start` enqueues a job that systemd will not start until this
         # unit is active — and this unit is not active until its ExecStart
-        # returns, which is waiting on that job. A deadlock, held until
-        # the 24h retry window's `TimeoutStartSec` fires. Queueing the
-        # job and letting it run once we exit is the only ordering that
-        # terminates.
+        # returns, which is waiting on that job. A deadlock, broken only
+        # when `TimeoutStartSec` fails the attempt, and entered again by
+        # every retry. Queueing the job and letting it run once we exit is
+        # the only ordering that terminates.
         #
         # The cost is real and is the reason `hive-tls-resign` does NOT
         # do this: a propagation that fails after we return does not fail
@@ -885,6 +894,7 @@ in
         Type = "oneshot";
         UMask = "0077";
         SyslogIdentifier = "swarm-services-cert-renew";
+        TimeoutStartSec = servicesCertTimeoutStartSec;
       };
     };
 

nix/module-eval/hive-tls.nix

@@ -684,6 +684,18 @@ let
         && (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service"
         && !(remainsAfterExit leafUnit);
     }
+    {
+      # A oneshot has no start timeout by default, so an attempt that never
+      # ends never fails into a retry.
+      name = "every unit that calls the store bounds its attempt, and the root fetch bounds its curl";
+      ok =
+        lib.all (u: (u.serviceConfig.TimeoutStartSec or "infinity") != "infinity") [
+          boot
+          triggered
+          leafUnit
+        ]
+        && lib.hasInfix "--connect-timeout 10 --max-time 30" (noSvcBranch noSvcFetch);
+    }
   ];
 in
 runGroup "hive-tls" cases