From 97cf8a1b2b07aa5cb0064b43e9d4ebbb214fce99 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 26 Sep 2026 19:24:32 +0200 Subject: [PATCH] agent-modules: write bao's stderr where UMask=0377 lets it, name what failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hive-agent-forge-token and hive-agent-queue-credential both run with UMask=0377. Their scripts captured bao's stderr in `err="$(mktemp)"`, which under that umask is created 0400; the very next `2>"$err"` on the `bao login` line cannot reopen it for writing, so bash fails the redirect with "Permission denied" before bao ever runs. The `if !` around the login then took the only error branch it had and printed "this agent's certificate was refused by the swarm secret store" — the store was never contacted. No agent has fetched either credential. The stderr file now lives in each unit's own 0700 RuntimeDirectory and is removed before every redirect into it, so the redirect creates it — the idiom forge-token.nix already used for its staging file. The login's error branch now says which of these happened, then quotes bao's output: - `$err` could not be created, so bao never ran; - the store answered with HTTP 4xx (refusal) or another status; - the store sent a TLS alert rejecting the certificate; - no answer at all (network, DNS, or local TLS). Unreadable cert/key credentials are reported before bao runs. bao.nix has the same fetch shape but no UMask=, so its mktemp file is 0600 and writable; it is untouched. Closes #4735 --- nix/agent-modules/forge-token.nix | 31 +++++++++++++++++++++++++--- nix/agent-modules/queue-identity.nix | 30 +++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/nix/agent-modules/forge-token.nix b/nix/agent-modules/forge-token.nix index 58ca6a94..5f081f91 100644 --- a/nix/agent-modules/forge-token.nix +++ b/nix/agent-modules/forge-token.nix @@ -44,6 +44,8 @@ let tokenFile = "/run/${runtimeDir}/token"; # Beside the token, so the rename that replaces it stays in one directory. stagingFile = "/run/${runtimeDir}/token.new"; + # bao's stderr, in the unit's own `0700` directory rather than `/tmp`. + errFile = "/run/${runtimeDir}/bao.err"; # The store's address is the whole switch, as in ./bao.nix and # ./queue-identity.nix. @@ -126,17 +128,40 @@ in echo "this agent has no store identity, so it cannot fetch its own forge token." >&2 exit 0 fi + if [ ! -r "$CREDENTIALS_DIRECTORY/$id" ]; then + echo "cannot read $CREDENTIALS_DIRECTORY/$id, so this agent cannot present its store identity." >&2 + exit 1 + fi done if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}" fi - err="$(mktemp)" + # `UMask=0377` makes every file this script creates `0400`, so only + # the redirect that creates a file can write to it: `$err` is removed + # before each redirect into it. + err=${lib.escapeShellArg errFile} trap 'rm -f "$err" ${lib.escapeShellArg stagingFile}' EXIT + rm -f "$err" if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2 + # The redirect creates `$err` before bao starts, so no file means + # bao never ran. bao prints `Code: ` only for an HTTP + # answer, and `remote error: tls:` only for an alert the store sent. + re='Code: ([0-9]{3})' + if [ ! -e "$err" ]; then + echo "could not create $err, so bao never ran and the store at $BAO_ADDR was not asked." >&2 + elif [[ "$(<"$err")" =~ $re ]]; then + case "''${BASH_REMATCH[1]}" in + 4*) echo "the swarm secret store at $BAO_ADDR refused this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;; + *) echo "the swarm secret store at $BAO_ADDR failed this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;; + esac + elif [[ "$(<"$err")" == *"remote error: tls:"* ]]; then + echo "the swarm secret store at $BAO_ADDR refused this agent's certificate in the TLS handshake:" >&2 + else + echo "bao got no answer from the swarm secret store at $BAO_ADDR (network, DNS, or TLS on this side):" >&2 + fi if [ -s "$err" ]; then cat "$err" >&2; fi exit 1 fi @@ -152,7 +177,7 @@ in # staging file is removed first so the redirect creates it — a file # `UMask=0377` left behind is `0400` and could not be reopened for # writing. - rm -f ${lib.escapeShellArg stagingFile} + rm -f "$err" ${lib.escapeShellArg stagingFile} if ! bao kv get -field=value ${lib.escapeShellArg tokenPath} > ${lib.escapeShellArg stagingFile} 2>"$err"; then echo "no forge token at ${tokenPath} yet; consumers keep using the state-dir token if there is one." >&2 if [ -s "$err" ]; then cat "$err" >&2; fi diff --git a/nix/agent-modules/queue-identity.nix b/nix/agent-modules/queue-identity.nix index 7aad8897..01f03812 100644 --- a/nix/agent-modules/queue-identity.nix +++ b/nix/agent-modules/queue-identity.nix @@ -52,6 +52,8 @@ let # surviving one. runtimeDir = "${unitName}"; secretFile = "/run/${runtimeDir}/secret"; + # bao's stderr, in the unit's own `0700` directory rather than `/tmp`. + errFile = "/run/${runtimeDir}/bao.err"; # The store's address is the whole switch, exactly as in ./bao.nix — and # deliberately *not* the queue coordinates in ./queue.nix. Those are the @@ -144,6 +146,10 @@ in echo "this agent has no store identity, so it cannot fetch its own queue credential." >&2 exit 0 fi + if [ ! -r "$CREDENTIALS_DIRECTORY/$id" ]; then + echo "cannot read $CREDENTIALS_DIRECTORY/$id, so this agent cannot present its store identity." >&2 + exit 1 + fi done # Only when one was delivered — absent means verify the store's @@ -153,15 +159,34 @@ in export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}" fi - err="$(mktemp)" + # `UMask=0377` makes every file this script creates `0400`, so only + # the redirect that creates a file can write to it: `$err` is removed + # before each redirect into it. + err=${lib.escapeShellArg errFile} trap 'rm -f "$err"' EXIT # Cert auth is a login, not a transport setting: the `BAO_CLIENT_*` # variables above only pick the certificate the handshake presents. # `-token-only` answers on stdout and skips the token helper, which # is a `sh` this unit's `path` does not carry. + rm -f "$err" if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2 + # The redirect creates `$err` before bao starts, so no file means + # bao never ran. bao prints `Code: ` only for an HTTP + # answer, and `remote error: tls:` only for an alert the store sent. + re='Code: ([0-9]{3})' + if [ ! -e "$err" ]; then + echo "could not create $err, so bao never ran and the store at $BAO_ADDR was not asked." >&2 + elif [[ "$(<"$err")" =~ $re ]]; then + case "''${BASH_REMATCH[1]}" in + 4*) echo "the swarm secret store at $BAO_ADDR refused this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;; + *) echo "the swarm secret store at $BAO_ADDR failed this agent's certificate login with HTTP ''${BASH_REMATCH[1]}:" >&2 ;; + esac + elif [[ "$(<"$err")" == *"remote error: tls:"* ]]; then + echo "the swarm secret store at $BAO_ADDR refused this agent's certificate in the TLS handshake:" >&2 + else + echo "bao got no answer from the swarm secret store at $BAO_ADDR (network, DNS, or TLS on this side):" >&2 + fi if [ -s "$err" ]; then cat "$err" >&2; fi exit 1 fi @@ -180,6 +205,7 @@ in # # ⚠️ Written by redirect into the runtime directory, never echoed: # the field is the secret itself. + rm -f "$err" if ! bao kv get -field=value ${lib.escapeShellArg queuePath} > ${lib.escapeShellArg secretFile} 2>"$err"; then rm -f ${lib.escapeShellArg secretFile} echo "no per-agent queue credential at ${queuePath} yet; this agent falls back to its hive's shared one." >&2