agent trust bundle: retry the root fetch inside its start job so a dns blip at boot queues the harness instead of failing it

nix/script-tests/retry-in-job.nix

@@ -0,0 +1,18 @@
+# `checks.script-test-retry-in-job` — runs ../host-modules/lib/retry-in-job.nix's
+# `retry_in_job` against a command that fails a set number of times. The cases
+# are in ./retry-in-job.sh.
+#
+# What runs is the rendered helper, sourced as the trust-bundle unit's script
+# includes it.
+{
+  pkgs,
+  ...
+}:
+pkgs.runCommand "hyperhive-script-test-retry-in-job"
+  {
+    HELPER = pkgs.writeText "retry-in-job.sh" (import ../host-modules/lib/retry-in-job.nix { });
+  }
+  ''
+    ${pkgs.bash}/bin/bash ${./retry-in-job.sh}
+    touch "$out"
+  ''

nix/script-tests/retry-in-job.sh

@@ -0,0 +1,66 @@
+# Cases for ./retry-in-job.nix. `sleep` is a function that advances `SECONDS`
+# and records the wait, so a case takes no wall time and the budget is
+# measured on the clock the helper reads.
+set -uo pipefail
+
+failures=0
+count=0
+
+fail() {
+  echo "FAILED: $case: $*" >&2
+  failures=$((failures + 1))
+}
+
+errlog=$(mktemp)
+waits=
+sleep() {
+  waits="$waits $1"
+  SECONDS=$((SECONDS + $1))
+}
+
+# shellcheck source=/dev/null
+. "$HELPER"
+
+attempts=0
+# Fails the first $FAIL_FIRST calls, then succeeds.
+flaky() {
+  attempts=$((attempts + 1))
+  [ "$attempts" -gt "$FAIL_FIRST" ] || return 6
+}
+
+# run <description> <fail-first> <budget>
+run() {
+  case=$1
+  FAIL_FIRST=$2
+  attempts=0
+  waits=
+  SECONDS=0
+  count=$((count + 1))
+  rc=0
+  retry_in_job "$3" flaky 2>"$errlog" || rc=$?
+}
+
+run "a command that succeeds at once is not waited for" 0 100
+[ "$rc" = 0 ] || fail "returned $rc"
+[ "$attempts" = 1 ] || fail "ran $attempts times, wanted 1"
+[ -z "$waits" ] || fail "waited:$waits"
+
+run "fail, fail, then succeed returns success after waiting 2s and 4s" 2 100
+[ "$rc" = 0 ] || fail "returned $rc"
+[ "$attempts" = 3 ] || fail "ran $attempts times, wanted 3"
+[ "$waits" = " 2 4" ] || fail "waited:$waits, wanted: 2 4"
+grep -q "failed with exit status 6; retrying in 2 seconds" "$errlog" || fail "the cause of a failed attempt is not logged: $(<"$errlog")"
+
+run "the wait doubles and stops growing at 60s" 8 1000
+[ "$rc" = 0 ] || fail "returned $rc"
+[ "$waits" = " 2 4 8 16 32 60 60 60" ] || fail "waited:$waits"
+
+run "a command that always fails returns 1 once the budget is spent" 1000000 100
+[ "$rc" = 1 ] || fail "returned $rc, wanted 1"
+[ "$waits" = " 2 4 8 16 32" ] || fail "waited:$waits, wanted: 2 4 8 16 32 (the next 60s wait passes the budget)"
+
+if [ "$failures" -gt 0 ]; then
+  echo "script-test-retry-in-job: $failures failed assertions over $count runs" >&2
+  exit 1
+fi
+echo "script-test-retry-in-job: $count runs, all assertions hold"