From ed53e9abccf912b81a257ed9f76077c162daa8c9 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Sat, 26 Sep 2026 15:16:42 +0200
Subject: [PATCH] host-modules: write credential files atomically;
 agent-modules: retry a failed .claude migration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Four host glue units fetched a secret from swarm-bao and rendered it
with `> path; chmod`: a reader racing the write could see a truncated
file, and briefly one at the wrong mode before the chmod landed.
glue-matrix-bao-token.nix, glue-queue-agent-credential.nix (both
files), swarm-grafana.nix and swarm-otel.nix now write to a same-
directory temp file, set its final mode/owner, then `mv -f` it over
the target — a shared `atomic_write_secret` helper
(nix/host-modules/lib/atomic-write-secret.nix) so the five call sites
share one implementation.

The first-boot `/root/.claude` migration in nix/agent-modules/user.nix
wrote its done-marker unconditionally, so a failed `cp` (disk full,
permission error) left the marker behind and no boot ever retried the
copy. The marker is now written only when there was nothing to
migrate or the copy succeeded; `cp -an`'s no-clobber semantics already
make a retry after a partial copy safe.

Refs #4723
---
 docs/agent-lifecycle/persistence.md           |  7 +++-
 nix/agent-modules/user.nix                    | 27 ++++++++++----
 nix/host-modules/glue-matrix-bao-token.nix    |  8 ++--
 .../glue-queue-agent-credential.nix           | 11 +++---
 nix/host-modules/lib/atomic-write-secret.nix  | 37 +++++++++++++++++++
 nix/host-modules/swarm-grafana.nix            |  9 +++--
 nix/host-modules/swarm-otel.nix               |  9 +++--
 7 files changed, 83 insertions(+), 25 deletions(-)
 create mode 100644 nix/host-modules/lib/atomic-write-secret.nix

diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md
index 6f697e73..ac536424 100644
--- a/docs/agent-lifecycle/persistence.md
+++ b/docs/agent-lifecycle/persistence.md
@@ -535,9 +535,12 @@ container lifetime:
 2. **Migrate any leftover `/root/.claude` content into
    `${homeDir}/.claude`** — legacy `claude` wrote to root's
    empty home; the bind mount didn't exist yet. Marker
-   (`/var/lib/hive-agent-user-migrated`) guards single-shot.
+   (`/var/lib/hive-agent-user-migrated`) guards single-shot; it is
+   only written once there is nothing left to migrate, so a `cp`
+   failure leaves it absent and the next boot retries.
    `cp -an` (no-clobber) so any pre-existing files at the new
-   location win — never blow over data already there.
+   location win — never blow over data already there, and so a
+   retry after a partial copy is as safe as the first attempt.
 3. **Chown the bind-mounted state dir** (`/agents/*/state`)
    recursively so the agent user can read/write it. Wildcard
    matches the single agent that container sees; `-h` skips
diff --git a/nix/agent-modules/user.nix b/nix/agent-modules/user.nix
index 52424e00..8d620cce 100644
--- a/nix/agent-modules/user.nix
+++ b/nix/agent-modules/user.nix
@@ -172,16 +172,29 @@ in
       userName=${lib.escapeShellArg userName}
       mkdir -p "$homeDir"
       chown "$userName:$userName" "$homeDir"
+      # The marker is only written once nothing is left to migrate — either
+      # there was nothing under /root/.claude, or `cp` copied it all. A
+      # failed `cp` (disk full, permission error) leaves the marker absent,
+      # so the next boot's activation retries; `-an` never clobbers a file
+      # this attempt already placed, so a retry after a partial copy is
+      # exactly as safe as the first attempt.
       marker=/var/lib/hive-agent-user-migrated
-      if [ ! -e "$marker" ] && [ -d /root/.claude ] && [ "$(ls -A /root/.claude 2>/dev/null)" ]; then
-        mkdir -p "$homeDir/.claude"
-        if cp -an /root/.claude/. "$homeDir/.claude/" 2>/dev/null; then
-          rm -rf /root/.claude
-          echo "hive-agent-user-migrate: moved /root/.claude → $homeDir/.claude"
+      if [ ! -e "$marker" ]; then
+        if [ -d /root/.claude ] && [ "$(ls -A /root/.claude 2>/dev/null)" ]; then
+          mkdir -p "$homeDir/.claude"
+          if cp -an /root/.claude/. "$homeDir/.claude/" 2>/dev/null; then
+            rm -rf /root/.claude
+            echo "hive-agent-user-migrate: moved /root/.claude → $homeDir/.claude"
+            mkdir -p "$(dirname "$marker")"
+            : > "$marker"
+          else
+            echo "hive-agent-user-migrate: copying /root/.claude to $homeDir/.claude failed; will retry next boot" >&2
+          fi
+        else
+          mkdir -p "$(dirname "$marker")"
+          : > "$marker"
         fi
       fi
-      mkdir -p "$(dirname "$marker")"
-      : > "$marker"
       # Scope state + harness chowns to THIS container's own dirs only.
       # The glob `/agents/*/state` also matches other agents' state dirs
       # bind-mounted into a `ManageRootAgent` holder's container, which
diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix
index 3c1bdb62..d1e36de0 100644
--- a/nix/host-modules/glue-matrix-bao-token.nix
+++ b/nix/host-modules/glue-matrix-bao-token.nix
@@ -80,6 +80,8 @@ let
   # resolution, so this is the kind of mistake only reading the target module
   # catches.
   matrixMachine = "hive-matrix";
+
+  atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
 in
 {
   config = lib.mkMerge [
@@ -187,6 +189,8 @@ in
         script = ''
           set -euo pipefail
 
+          ${atomicWriteSecret}
+
           # A sealed or uninitialised store answers on the port and never
           # answers the read, so "the store is up" is not the same as "the
           # store can answer". `TimeoutStartSec` above is the bound; the
@@ -237,9 +241,7 @@ in
             exit 0
           fi
 
-          umask 077
-          printf '%s\n' "$token" > ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
-          chmod 0600 ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
+          printf '%s\n' "$token" | atomic_write_secret 0600 "" ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
 
           # Re-stamp the registration file from the token just written. The
           # token is half an agreement — the registration the homeserver loads
diff --git a/nix/host-modules/glue-queue-agent-credential.nix b/nix/host-modules/glue-queue-agent-credential.nix
index 21900692..33efb086 100644
--- a/nix/host-modules/glue-queue-agent-credential.nix
+++ b/nix/host-modules/glue-queue-agent-credential.nix
@@ -77,6 +77,8 @@ let
   # its own use of it: it is asserted set wherever
   # `deploy.hive-controller.enable` is.
   credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent";
+
+  atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
 in
 {
   options.services.hyperhive.deploy.hive-controller.queue = {
@@ -217,6 +219,8 @@ in
         script = ''
           set -euo pipefail
 
+          ${atomicWriteSecret}
+
           # `bao`'s own message is the only thing separating a missing value from
           # a refused identity from an unreachable host. This unit's degraded
           # mode is correct for all three, so it reports which one rather than
@@ -279,14 +283,11 @@ in
 
           install -d -m 0755 ${lib.escapeShellArg credentialDir}
 
-          umask 077
-          printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile}
-          chmod 0600 ${lib.escapeShellArg secretFile}
+          printf '%s\n' "$secret" | atomic_write_secret 0600 "" ${lib.escapeShellArg secretFile}
 
           # `0644` on purpose: an OIDC client id is presented to the token
           # endpoint on every connection and is public by construction.
-          printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile}
-          chmod 0644 ${lib.escapeShellArg clientIdFile}
+          printf '%s\n' "$client_id" | atomic_write_secret 0644 "" ${lib.escapeShellArg clientIdFile}
         '';
       };
     })
diff --git a/nix/host-modules/lib/atomic-write-secret.nix b/nix/host-modules/lib/atomic-write-secret.nix
new file mode 100644
index 00000000..dccf7e9d
--- /dev/null
+++ b/nix/host-modules/lib/atomic-write-secret.nix
@@ -0,0 +1,37 @@
+# Shared shell step for a systemd oneshot that lands a freshly-fetched
+# secret on disk: never `> path` the live file directly, because a reader
+# racing the write can open it between the truncate and the write, or
+# between the write and a `chmod` that follows it, and see an empty file
+# or one at the wrong mode. `mktemp` always creates its file at 0600
+# regardless of umask, so the temp file is private for its whole life; only
+# the `chmod`/`chown`/`mv -f` sequence below ever makes the target mode and
+# owner visible, and only once the content is already final.
+#
+# Pure function — NOT a NixOS module. Call it from a module's `let`:
+#
+#   atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
+#   ...
+#   script = ''
+#     ${atomicWriteSecret}
+#     printf '%s\n' "$secret" | atomic_write_secret 0600 "" "$path"
+#     printf '%s\n' "$secret" | atomic_write_secret 0400 "grafana:0" "$path"
+#   '';
+#
+# Third argument to `atomic_write_secret` is the target path; the second is
+# an owner for `chown` (`user:group` or a bare uid), or "" to leave the
+# mktemp-created root:root ownership as it is. Reads its content from
+# stdin. Requires `coreutils` on the caller's `path`.
+{ }:
+''
+  atomic_write_secret() {
+    local mode="$1" owner="$2" target="$3" tmp
+    tmp="$(mktemp "$(dirname -- "$target")/.$(basename -- "$target").XXXXXX")"
+    trap 'rm -f "$tmp"' RETURN
+    cat > "$tmp"
+    chmod "$mode" "$tmp"
+    if [ -n "$owner" ]; then
+      chown "$owner" "$tmp"
+    fi
+    mv -f "$tmp" "$target"
+  }
+''
diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index b447427a..355e8c6f 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -168,6 +168,8 @@ let
   nginxGid = config.ids.gids.nginx;
   grafanaUid = config.ids.uids.grafana;
 
+  atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
+
 in
 {
   # `enable` moved to `services.hyperhive.deploy.grafana.enable` — see
@@ -633,6 +635,8 @@ in
       script = ''
         set -euo pipefail
 
+        ${atomicWriteSecret}
+
         # `bao`'s own message is the only thing separating a missing value from
         # a refused identity from an unreachable host. This unit's degraded
         # mode is correct for all three, so it reports which one rather than
@@ -689,10 +693,7 @@ in
         # grants the group nothing; if this mode ever widens, the gid has to be
         # discovered at runtime rather than assumed.
         install -d -m 0755 ${lib.escapeShellArg hostSecretDir}
-        umask 077
-        printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath}
-        chown ${toString config.ids.uids.grafana}:0 ${lib.escapeShellArg hostSecretPath}
-        chmod 0400 ${lib.escapeShellArg hostSecretPath}
+        printf '%s\n' "$secret" | atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostSecretPath}
       '';
     };
 
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index edb36ff5..3b1f2e38 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -166,6 +166,8 @@ let
   # collector cannot open, discovered at runtime and nowhere else.
   collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
 
+  atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
+
   # `attrNames` is sorted, so this is a function of the hive SET and not of
   # the order anyone wrote it in.
   #
@@ -817,6 +819,8 @@ in
       script = ''
         set -euo pipefail
 
+        ${atomicWriteSecret}
+
         # `bao`'s own message is the only thing separating a missing value
         # from a refused identity from an unreachable host. This unit's
         # degraded mode is correct for all three, so it reports which one
@@ -862,10 +866,7 @@ in
         # no uid to give this to — `LoadCredential` reads it as root before
         # the sandbox exists and re-exposes it to whichever uid the unit got.
         install -d -m 0755 ${lib.escapeShellArg collectorHostSecretDir}
-        umask 077
-        printf '%s\n' "$secret" > ${lib.escapeShellArg collectorHostSecretPath}
-        chown root:root ${lib.escapeShellArg collectorHostSecretPath}
-        chmod 0400 ${lib.escapeShellArg collectorHostSecretPath}
+        printf '%s\n' "$secret" | atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorHostSecretPath}
       '';
     };