agent trust bundle: retry the root fetch inside its start job so a dns blip at boot queues the harness instead of failing it

nix/agent-modules/hive-ca.nix

@@ -14,7 +14,10 @@
 #
 # A process reads its trust when it starts, so a rotated root reaches the
 # consumers below at their next start. They `Requires=` this unit, which keeps
-# one from running with a store that lacks the root.
+# one from running with a store that lacks the root. The fetch retries inside
+# the start job (../host-modules/lib/retry-in-job.nix), so a consumer queues
+# behind a store that does not answer yet rather than failing with
+# `dependency`.
 {
   pkgs,
   lib,
@@ -40,6 +43,12 @@ let
   configured = cfg.addr != null;
 
   storeRetry = import ../host-modules/lib/store-retry.nix { };
+  retryInJob = import ../host-modules/lib/retry-in-job.nix { };
+
+  # How long the fetch retries inside one start job. The store's name does not
+  # resolve for minutes after a host switch, and the consumers below wait in
+  # the job queue for this long instead of failing with `dependency`.
+  fetchBudgetSec = 840;
 
   # Units that dial a swarm service over TLS. The first group is defined
   # whenever the agent module is; the rest exist under the same condition as
@@ -99,7 +108,7 @@ in
         serviceConfig = storeRetry.serviceConfig // {
           Type = "oneshot";
           RemainAfterExit = false;
-          TimeoutStartSec = 60;
+          TimeoutStartSec = fetchBudgetSec + 60;
           User = agentName;
           Group = agentName;
           RuntimeDirectory = runtimeDir;
@@ -121,6 +130,8 @@ in
         script = ''
           set -euo pipefail
 
+          ${retryInJob}
+
           for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
             if [ ! -s "$f" ]; then
               echo "this agent has no store identity at $f, so it cannot fetch the swarm root certificate." >&2
@@ -145,12 +156,15 @@ in
 
           # The listener demands a client certificate and the endpoint takes no
           # token, so no login precedes this.
-          if ! curl --fail --silent --show-error \
+          fetch_root() {
+            curl --fail --silent --show-error \
               --connect-timeout 10 --max-time 30 \
               ''${cacert:+--cacert "$cacert"} \
               --cert "$BAO_CLIENT_CERT" --key "$BAO_CLIENT_KEY" \
               --output "$raw" \
-              ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"; then
+              ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"
+          }
+          if ! retry_in_job ${toString fetchBudgetSec} fetch_root; then
             echo "could not read the store's ${cfg.servicesPkiMount} root at $BAO_ADDR." >&2
             cat "$err" >&2
             exit 1