agent trust bundle: retry the root fetch inside its start job so a dns blip at boot queues the harness instead of failing it

docs/swarm/ca.md

@@ -263,7 +263,7 @@ naming, because it's subtle and it bites at a distance:
 
 Two consumers, both reading the root at runtime:
 
-- **Agents fetch it themselves.** `hive-agent-trust-bundle.service` (`nix/agent-modules/hive-ca.nix`) reads `<mount>/ca/pem` from the store with the agent's own store certificate, then writes `/run/hive-agent-trust-bundle/bundle.pem` (system CAs plus the root) and `root.pem` (the root alone). The agent's units point `SSL_CERT_FILE` at the first (Rust, Go and OpenSSL clients) and `NODE_EXTRA_CA_CERTS` at the second (Node and Claude Code), and order after the unit with `Requires=`. The meta flake names no CA file, and a timer re-reads the root hourly. A consumer reads it at start, so a rotated root reaches an agent unit at its next start.
+- **Agents fetch it themselves.** `hive-agent-trust-bundle.service` (`nix/agent-modules/hive-ca.nix`) reads `<mount>/ca/pem` from the store with the agent's own store certificate, then writes `/run/hive-agent-trust-bundle/bundle.pem` (system CAs plus the root) and `root.pem` (the root alone). The agent's units point `SSL_CERT_FILE` at the first (Rust, Go and OpenSSL clients) and `NODE_EXTRA_CA_CERTS` at the second (Node and Claude Code), and order after the unit with `Requires=`. The fetch retries inside its start job for up to 14 minutes, with a wait that doubles from 2 to 60 seconds, so a store name that doesn't resolve yet delays those units instead of failing them; after that the job fails and they fail with it. The meta flake names no CA file, and a timer re-reads the root hourly. A consumer reads it at start, so a rotated root reaches an agent unit at its next start.
 - **The Matrix container gets a bind mount**, via the shared
   `lib/hive-ca-trust.nix` helper that `hive-forge` and `swarm-ci-runner`
   use: the bundle is bind-mounted read-only into the container, and a

nix/agent-modules/hive-ca.nix

@@ -14,7 +14,10 @@
 #
 # A process reads its trust when it starts, so a rotated root reaches the
 # consumers below at their next start. They `Requires=` this unit, which keeps
-# one from running with a store that lacks the root.
+# one from running with a store that lacks the root. The fetch retries inside
+# the start job (../host-modules/lib/retry-in-job.nix), so a consumer queues
+# behind a store that does not answer yet rather than failing with
+# `dependency`.
 {
   pkgs,
   lib,
@@ -40,6 +43,12 @@ let
   configured = cfg.addr != null;
 
   storeRetry = import ../host-modules/lib/store-retry.nix { };
+  retryInJob = import ../host-modules/lib/retry-in-job.nix { };
+
+  # How long the fetch retries inside one start job. The store's name does not
+  # resolve for minutes after a host switch, and the consumers below wait in
+  # the job queue for this long instead of failing with `dependency`.
+  fetchBudgetSec = 840;
 
   # Units that dial a swarm service over TLS. The first group is defined
   # whenever the agent module is; the rest exist under the same condition as
@@ -99,7 +108,7 @@ in
         serviceConfig = storeRetry.serviceConfig // {
           Type = "oneshot";
           RemainAfterExit = false;
-          TimeoutStartSec = 60;
+          TimeoutStartSec = fetchBudgetSec + 60;
           User = agentName;
           Group = agentName;
           RuntimeDirectory = runtimeDir;
@@ -121,6 +130,8 @@ in
         script = ''
           set -euo pipefail
 
+          ${retryInJob}
+
           for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
             if [ ! -s "$f" ]; then
               echo "this agent has no store identity at $f, so it cannot fetch the swarm root certificate." >&2
@@ -145,12 +156,15 @@ in
 
           # The listener demands a client certificate and the endpoint takes no
           # token, so no login precedes this.
-          if ! curl --fail --silent --show-error \
+          fetch_root() {
+            curl --fail --silent --show-error \
               --connect-timeout 10 --max-time 30 \
               ''${cacert:+--cacert "$cacert"} \
               --cert "$BAO_CLIENT_CERT" --key "$BAO_CLIENT_KEY" \
               --output "$raw" \
-              ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"; then
+              ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"
+          }
+          if ! retry_in_job ${toString fetchBudgetSec} fetch_root; then
             echo "could not read the store's ${cfg.servicesPkiMount} root at $BAO_ADDR." >&2
             cat "$err" >&2
             exit 1

nix/checks.nix

@@ -77,6 +77,10 @@ in
   # Runs ./host-modules/lib/refresh-consumer.nix's restart decision under a
   # comma-decimal locale, against a stub `systemctl`.
   script-test-refresh-consumer = import ./script-tests/refresh-consumer.nix { inherit pkgs; };
+  # Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent
+  # trust-bundle fetch retries through inside its start job: success after two
+  # failures, the wait schedule, and a return of 1 once the budget is spent.
+  script-test-retry-in-job = import ./script-tests/retry-in-job.nix { inherit pkgs; };
   # Runs ./host-modules/lib/attic-init.nix's two scripts, which create the swarm
   # cache's key, cache and tokens, twice over against stub tools: the second
   # run must mint nothing.

nix/host-modules/lib/retry-in-job.nix

@@ -0,0 +1,38 @@
+# Shared shell step for a oneshot that fetches over the network and that other
+# units `Requires=`. `retry_in_job` repeats the fetch inside the unit's own
+# start job, so a unit ordered `After=` it waits for the fetch to succeed. A
+# unit that exits non-zero ends its job as `failed` and fails every `Requires=`
+# dependent with `dependency`; a dependent whose start job failed is not
+# started again when the retry succeeds, so ./store-retry.nix cannot cover
+# this case.
+#
+# The retry stops at the budget and returns non-zero, so the unit still fails
+# closed. Set the unit's `TimeoutStartSec` above the budget, or systemd kills
+# the job before the helper gives up.
+#
+# Pure function — NOT a NixOS module. Call it from a unit's `script`:
+#
+#   retryInJob = import ../host-modules/lib/retry-in-job.nix { };
+#   script = ''
+#     ${retryInJob}
+#     if ! retry_in_job 840 fetch; then exit 1; fi
+#   '';
+{ }:
+''
+  # retry_in_job <budget-seconds> <command…>: run the command until it
+  # succeeds, waiting 2s, doubling to at most 60s between attempts. Returns 1
+  # once another wait would pass the budget.
+  retry_in_job() {
+    local rc delay=2 deadline=$((SECONDS + $1))
+    shift
+    until "$@"; do
+      rc=$?
+      if [ $((SECONDS + delay)) -gt "$deadline" ]; then
+        return 1
+      fi
+      echo "the attempt failed with exit status $rc; retrying in $delay seconds." >&2
+      sleep "$delay"
+      delay=$((delay * 2 > 60 ? 60 : delay * 2))
+    done
+  }
+''

nix/module-eval/agent-forge-bao.nix

@@ -97,6 +97,19 @@ let
         && !(agentForgeNoBao.systemd.globalEnvironment ? SSL_CERT_FILE)
         && !(lib.elem unit (agentForgeNoBao.systemd.services.hive-agent.after or [ ]));
     }
+    {
+      # ../agent-modules/hive-ca.nix. The fetch retries inside the start job
+      # and the job's timeout outlasts the retry budget, so a consumer that
+      # `Requires=` the unit waits instead of failing with `dependency`.
+      name = "the swarm root fetch retries inside its start job and the timeout outlasts the retries";
+      ok =
+        let
+          unit = agentForgeBao.systemd.services.hive-agent-trust-bundle;
+          budget = 840;
+        in
+        lib.hasInfix "retry_in_job ${toString budget} fetch_root" unit.script
+        && unit.serviceConfig.TimeoutStartSec > budget;
+    }
     {
       # The whole switch is the store address, as for the queue credential.
       name = "an agent with a store address fetches its forge token";

nix/script-tests/retry-in-job.nix

@@ -0,0 +1,18 @@
+# `checks.script-test-retry-in-job` — runs ../host-modules/lib/retry-in-job.nix's
+# `retry_in_job` against a command that fails a set number of times. The cases
+# are in ./retry-in-job.sh.
+#
+# What runs is the rendered helper, sourced as the trust-bundle unit's script
+# includes it.
+{
+  pkgs,
+  ...
+}:
+pkgs.runCommand "hyperhive-script-test-retry-in-job"
+  {
+    HELPER = pkgs.writeText "retry-in-job.sh" (import ../host-modules/lib/retry-in-job.nix { });
+  }
+  ''
+    ${pkgs.bash}/bin/bash ${./retry-in-job.sh}
+    touch "$out"
+  ''

nix/script-tests/retry-in-job.sh

@@ -0,0 +1,66 @@
+# Cases for ./retry-in-job.nix. `sleep` is a function that advances `SECONDS`
+# and records the wait, so a case takes no wall time and the budget is
+# measured on the clock the helper reads.
+set -uo pipefail
+
+failures=0
+count=0
+
+fail() {
+  echo "FAILED: $case: $*" >&2
+  failures=$((failures + 1))
+}
+
+errlog=$(mktemp)
+waits=
+sleep() {
+  waits="$waits $1"
+  SECONDS=$((SECONDS + $1))
+}
+
+# shellcheck source=/dev/null
+. "$HELPER"
+
+attempts=0
+# Fails the first $FAIL_FIRST calls, then succeeds.
+flaky() {
+  attempts=$((attempts + 1))
+  [ "$attempts" -gt "$FAIL_FIRST" ] || return 6
+}
+
+# run <description> <fail-first> <budget>
+run() {
+  case=$1
+  FAIL_FIRST=$2
+  attempts=0
+  waits=
+  SECONDS=0
+  count=$((count + 1))
+  rc=0
+  retry_in_job "$3" flaky 2>"$errlog" || rc=$?
+}
+
+run "a command that succeeds at once is not waited for" 0 100
+[ "$rc" = 0 ] || fail "returned $rc"
+[ "$attempts" = 1 ] || fail "ran $attempts times, wanted 1"
+[ -z "$waits" ] || fail "waited:$waits"
+
+run "fail, fail, then succeed returns success after waiting 2s and 4s" 2 100
+[ "$rc" = 0 ] || fail "returned $rc"
+[ "$attempts" = 3 ] || fail "ran $attempts times, wanted 3"
+[ "$waits" = " 2 4" ] || fail "waited:$waits, wanted: 2 4"
+grep -q "failed with exit status 6; retrying in 2 seconds" "$errlog" || fail "the cause of a failed attempt is not logged: $(<"$errlog")"
+
+run "the wait doubles and stops growing at 60s" 8 1000
+[ "$rc" = 0 ] || fail "returned $rc"
+[ "$waits" = " 2 4 8 16 32 60 60 60" ] || fail "waited:$waits"
+
+run "a command that always fails returns 1 once the budget is spent" 1000000 100
+[ "$rc" = 1 ] || fail "returned $rc, wanted 1"
+[ "$waits" = " 2 4 8 16 32" ] || fail "waited:$waits, wanted: 2 4 8 16 32 (the next 60s wait passes the budget)"
+
+if [ "$failures" -gt 0 ]; then
+  echo "script-test-retry-in-job: $failures failed assertions over $count runs" >&2
+  exit 1
+fi
+echo "script-test-retry-in-job: $count runs, all assertions hold"