diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 696094f0..06883b6b 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -263,7 +263,7 @@ naming, because it's subtle and it bites at a distance: Two consumers, both reading the root at runtime: -- **Agents fetch it themselves.** `hive-agent-trust-bundle.service` (`nix/agent-modules/hive-ca.nix`) reads `/ca/pem` from the store with the agent's own store certificate, then writes `/run/hive-agent-trust-bundle/bundle.pem` (system CAs plus the root) and `root.pem` (the root alone). The agent's units point `SSL_CERT_FILE` at the first (Rust, Go and OpenSSL clients) and `NODE_EXTRA_CA_CERTS` at the second (Node and Claude Code), and order after the unit with `Requires=`. The meta flake names no CA file, and a timer re-reads the root hourly. A consumer reads it at start, so a rotated root reaches an agent unit at its next start. +- **Agents fetch it themselves.** `hive-agent-trust-bundle.service` (`nix/agent-modules/hive-ca.nix`) reads `/ca/pem` from the store with the agent's own store certificate, then writes `/run/hive-agent-trust-bundle/bundle.pem` (system CAs plus the root) and `root.pem` (the root alone). The agent's units point `SSL_CERT_FILE` at the first (Rust, Go and OpenSSL clients) and `NODE_EXTRA_CA_CERTS` at the second (Node and Claude Code), and order after the unit with `Requires=`. The fetch retries inside its start job for up to 14 minutes, with a wait that doubles from 2 to 60 seconds, so a store name that doesn't resolve yet delays those units instead of failing them; after that the job fails and they fail with it. The meta flake names no CA file, and a timer re-reads the root hourly. A consumer reads it at start, so a rotated root reaches an agent unit at its next start. - **The Matrix container gets a bind mount**, via the shared `lib/hive-ca-trust.nix` helper that `hive-forge` and `swarm-ci-runner` use: the bundle is bind-mounted read-only into the container, and a diff --git a/nix/agent-modules/hive-ca.nix b/nix/agent-modules/hive-ca.nix index 56db8980..aea35c27 100644 --- a/nix/agent-modules/hive-ca.nix +++ b/nix/agent-modules/hive-ca.nix @@ -14,7 +14,10 @@ # # A process reads its trust when it starts, so a rotated root reaches the # consumers below at their next start. They `Requires=` this unit, which keeps -# one from running with a store that lacks the root. +# one from running with a store that lacks the root. The fetch retries inside +# the start job (../host-modules/lib/retry-in-job.nix), so a consumer queues +# behind a store that does not answer yet rather than failing with +# `dependency`. { pkgs, lib, @@ -40,6 +43,12 @@ let configured = cfg.addr != null; storeRetry = import ../host-modules/lib/store-retry.nix { }; + retryInJob = import ../host-modules/lib/retry-in-job.nix { }; + + # How long the fetch retries inside one start job. The store's name does not + # resolve for minutes after a host switch, and the consumers below wait in + # the job queue for this long instead of failing with `dependency`. + fetchBudgetSec = 840; # Units that dial a swarm service over TLS. The first group is defined # whenever the agent module is; the rest exist under the same condition as @@ -99,7 +108,7 @@ in serviceConfig = storeRetry.serviceConfig // { Type = "oneshot"; RemainAfterExit = false; - TimeoutStartSec = 60; + TimeoutStartSec = fetchBudgetSec + 60; User = agentName; Group = agentName; RuntimeDirectory = runtimeDir; @@ -121,6 +130,8 @@ in script = '' set -euo pipefail + ${retryInJob} + for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do if [ ! -s "$f" ]; then echo "this agent has no store identity at $f, so it cannot fetch the swarm root certificate." >&2 @@ -145,12 +156,15 @@ in # The listener demands a client certificate and the endpoint takes no # token, so no login precedes this. - if ! curl --fail --silent --show-error \ + fetch_root() { + curl --fail --silent --show-error \ --connect-timeout 10 --max-time 30 \ ''${cacert:+--cacert "$cacert"} \ --cert "$BAO_CLIENT_CERT" --key "$BAO_CLIENT_KEY" \ --output "$raw" \ - ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"; then + ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err" + } + if ! retry_in_job ${toString fetchBudgetSec} fetch_root; then echo "could not read the store's ${cfg.servicesPkiMount} root at $BAO_ADDR." >&2 cat "$err" >&2 exit 1 diff --git a/nix/checks.nix b/nix/checks.nix index 9ff80ff8..a749da24 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -77,6 +77,10 @@ in # Runs ./host-modules/lib/refresh-consumer.nix's restart decision under a # comma-decimal locale, against a stub `systemctl`. script-test-refresh-consumer = import ./script-tests/refresh-consumer.nix { inherit pkgs; }; + # Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent + # trust-bundle fetch retries through inside its start job: success after two + # failures, the wait schedule, and a return of 1 once the budget is spent. + script-test-retry-in-job = import ./script-tests/retry-in-job.nix { inherit pkgs; }; # Runs ./host-modules/lib/attic-init.nix's two scripts, which create the swarm # cache's key, cache and tokens, twice over against stub tools: the second # run must mint nothing. diff --git a/nix/host-modules/lib/retry-in-job.nix b/nix/host-modules/lib/retry-in-job.nix new file mode 100644 index 00000000..afc166a0 --- /dev/null +++ b/nix/host-modules/lib/retry-in-job.nix @@ -0,0 +1,38 @@ +# Shared shell step for a oneshot that fetches over the network and that other +# units `Requires=`. `retry_in_job` repeats the fetch inside the unit's own +# start job, so a unit ordered `After=` it waits for the fetch to succeed. A +# unit that exits non-zero ends its job as `failed` and fails every `Requires=` +# dependent with `dependency`; a dependent whose start job failed is not +# started again when the retry succeeds, so ./store-retry.nix cannot cover +# this case. +# +# The retry stops at the budget and returns non-zero, so the unit still fails +# closed. Set the unit's `TimeoutStartSec` above the budget, or systemd kills +# the job before the helper gives up. +# +# Pure function — NOT a NixOS module. Call it from a unit's `script`: +# +# retryInJob = import ../host-modules/lib/retry-in-job.nix { }; +# script = '' +# ${retryInJob} +# if ! retry_in_job 840 fetch; then exit 1; fi +# ''; +{ }: +'' + # retry_in_job : run the command until it + # succeeds, waiting 2s, doubling to at most 60s between attempts. Returns 1 + # once another wait would pass the budget. + retry_in_job() { + local rc delay=2 deadline=$((SECONDS + $1)) + shift + until "$@"; do + rc=$? + if [ $((SECONDS + delay)) -gt "$deadline" ]; then + return 1 + fi + echo "the attempt failed with exit status $rc; retrying in $delay seconds." >&2 + sleep "$delay" + delay=$((delay * 2 > 60 ? 60 : delay * 2)) + done + } +'' diff --git a/nix/module-eval/agent-forge-bao.nix b/nix/module-eval/agent-forge-bao.nix index 04fe2cb0..bd320c65 100644 --- a/nix/module-eval/agent-forge-bao.nix +++ b/nix/module-eval/agent-forge-bao.nix @@ -97,6 +97,19 @@ let && !(agentForgeNoBao.systemd.globalEnvironment ? SSL_CERT_FILE) && !(lib.elem unit (agentForgeNoBao.systemd.services.hive-agent.after or [ ])); } + { + # ../agent-modules/hive-ca.nix. The fetch retries inside the start job + # and the job's timeout outlasts the retry budget, so a consumer that + # `Requires=` the unit waits instead of failing with `dependency`. + name = "the swarm root fetch retries inside its start job and the timeout outlasts the retries"; + ok = + let + unit = agentForgeBao.systemd.services.hive-agent-trust-bundle; + budget = 840; + in + lib.hasInfix "retry_in_job ${toString budget} fetch_root" unit.script + && unit.serviceConfig.TimeoutStartSec > budget; + } { # The whole switch is the store address, as for the queue credential. name = "an agent with a store address fetches its forge token"; diff --git a/nix/script-tests/retry-in-job.nix b/nix/script-tests/retry-in-job.nix new file mode 100644 index 00000000..de12c3bd --- /dev/null +++ b/nix/script-tests/retry-in-job.nix @@ -0,0 +1,18 @@ +# `checks.script-test-retry-in-job` — runs ../host-modules/lib/retry-in-job.nix's +# `retry_in_job` against a command that fails a set number of times. The cases +# are in ./retry-in-job.sh. +# +# What runs is the rendered helper, sourced as the trust-bundle unit's script +# includes it. +{ + pkgs, + ... +}: +pkgs.runCommand "hyperhive-script-test-retry-in-job" + { + HELPER = pkgs.writeText "retry-in-job.sh" (import ../host-modules/lib/retry-in-job.nix { }); + } + '' + ${pkgs.bash}/bin/bash ${./retry-in-job.sh} + touch "$out" + '' diff --git a/nix/script-tests/retry-in-job.sh b/nix/script-tests/retry-in-job.sh new file mode 100644 index 00000000..b849f882 --- /dev/null +++ b/nix/script-tests/retry-in-job.sh @@ -0,0 +1,66 @@ +# Cases for ./retry-in-job.nix. `sleep` is a function that advances `SECONDS` +# and records the wait, so a case takes no wall time and the budget is +# measured on the clock the helper reads. +set -uo pipefail + +failures=0 +count=0 + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +errlog=$(mktemp) +waits= +sleep() { + waits="$waits $1" + SECONDS=$((SECONDS + $1)) +} + +# shellcheck source=/dev/null +. "$HELPER" + +attempts=0 +# Fails the first $FAIL_FIRST calls, then succeeds. +flaky() { + attempts=$((attempts + 1)) + [ "$attempts" -gt "$FAIL_FIRST" ] || return 6 +} + +# run +run() { + case=$1 + FAIL_FIRST=$2 + attempts=0 + waits= + SECONDS=0 + count=$((count + 1)) + rc=0 + retry_in_job "$3" flaky 2>"$errlog" || rc=$? +} + +run "a command that succeeds at once is not waited for" 0 100 +[ "$rc" = 0 ] || fail "returned $rc" +[ "$attempts" = 1 ] || fail "ran $attempts times, wanted 1" +[ -z "$waits" ] || fail "waited:$waits" + +run "fail, fail, then succeed returns success after waiting 2s and 4s" 2 100 +[ "$rc" = 0 ] || fail "returned $rc" +[ "$attempts" = 3 ] || fail "ran $attempts times, wanted 3" +[ "$waits" = " 2 4" ] || fail "waited:$waits, wanted: 2 4" +grep -q "failed with exit status 6; retrying in 2 seconds" "$errlog" || fail "the cause of a failed attempt is not logged: $(<"$errlog")" + +run "the wait doubles and stops growing at 60s" 8 1000 +[ "$rc" = 0 ] || fail "returned $rc" +[ "$waits" = " 2 4 8 16 32 60 60 60" ] || fail "waited:$waits" + +run "a command that always fails returns 1 once the budget is spent" 1000000 100 +[ "$rc" = 1 ] || fail "returned $rc, wanted 1" +[ "$waits" = " 2 4 8 16 32" ] || fail "waited:$waits, wanted: 2 4 8 16 32 (the next 60s wait passes the budget)" + +if [ "$failures" -gt 0 ]; then + echo "script-test-retry-in-job: $failures failed assertions over $count runs" >&2 + exit 1 +fi +echo "script-test-retry-in-job: $count runs, all assertions hold"