diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 6f697e73..ac536424 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -535,9 +535,12 @@ container lifetime: 2. **Migrate any leftover `/root/.claude` content into `${homeDir}/.claude`** — legacy `claude` wrote to root's empty home; the bind mount didn't exist yet. Marker - (`/var/lib/hive-agent-user-migrated`) guards single-shot. + (`/var/lib/hive-agent-user-migrated`) guards single-shot; it is + only written once there is nothing left to migrate, so a `cp` + failure leaves it absent and the next boot retries. `cp -an` (no-clobber) so any pre-existing files at the new - location win — never blow over data already there. + location win — never blow over data already there, and so a + retry after a partial copy is as safe as the first attempt. 3. **Chown the bind-mounted state dir** (`/agents/*/state`) recursively so the agent user can read/write it. Wildcard matches the single agent that container sees; `-h` skips diff --git a/nix/agent-modules/user.nix b/nix/agent-modules/user.nix index 52424e00..8d620cce 100644 --- a/nix/agent-modules/user.nix +++ b/nix/agent-modules/user.nix @@ -172,16 +172,29 @@ in userName=${lib.escapeShellArg userName} mkdir -p "$homeDir" chown "$userName:$userName" "$homeDir" + # The marker is only written once nothing is left to migrate — either + # there was nothing under /root/.claude, or `cp` copied it all. A + # failed `cp` (disk full, permission error) leaves the marker absent, + # so the next boot's activation retries; `-an` never clobbers a file + # this attempt already placed, so a retry after a partial copy is + # exactly as safe as the first attempt. marker=/var/lib/hive-agent-user-migrated - if [ ! -e "$marker" ] && [ -d /root/.claude ] && [ "$(ls -A /root/.claude 2>/dev/null)" ]; then - mkdir -p "$homeDir/.claude" - if cp -an /root/.claude/. "$homeDir/.claude/" 2>/dev/null; then - rm -rf /root/.claude - echo "hive-agent-user-migrate: moved /root/.claude → $homeDir/.claude" + if [ ! -e "$marker" ]; then + if [ -d /root/.claude ] && [ "$(ls -A /root/.claude 2>/dev/null)" ]; then + mkdir -p "$homeDir/.claude" + if cp -an /root/.claude/. "$homeDir/.claude/" 2>/dev/null; then + rm -rf /root/.claude + echo "hive-agent-user-migrate: moved /root/.claude → $homeDir/.claude" + mkdir -p "$(dirname "$marker")" + : > "$marker" + else + echo "hive-agent-user-migrate: copying /root/.claude to $homeDir/.claude failed; will retry next boot" >&2 + fi + else + mkdir -p "$(dirname "$marker")" + : > "$marker" fi fi - mkdir -p "$(dirname "$marker")" - : > "$marker" # Scope state + harness chowns to THIS container's own dirs only. # The glob `/agents/*/state` also matches other agents' state dirs # bind-mounted into a `ManageRootAgent` holder's container, which diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 3c1bdb62..d1e36de0 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -80,6 +80,8 @@ let # resolution, so this is the kind of mistake only reading the target module # catches. matrixMachine = "hive-matrix"; + + atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; in { config = lib.mkMerge [ @@ -187,6 +189,8 @@ in script = '' set -euo pipefail + ${atomicWriteSecret} + # A sealed or uninitialised store answers on the port and never # answers the read, so "the store is up" is not the same as "the # store can answer". `TimeoutStartSec` above is the bound; the @@ -237,9 +241,7 @@ in exit 0 fi - umask 077 - printf '%s\n' "$token" > ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)} - chmod 0600 ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)} + printf '%s\n' "$token" | atomic_write_secret 0600 "" ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)} # Re-stamp the registration file from the token just written. The # token is half an agreement — the registration the homeserver loads diff --git a/nix/host-modules/glue-queue-agent-credential.nix b/nix/host-modules/glue-queue-agent-credential.nix index 21900692..33efb086 100644 --- a/nix/host-modules/glue-queue-agent-credential.nix +++ b/nix/host-modules/glue-queue-agent-credential.nix @@ -77,6 +77,8 @@ let # its own use of it: it is asserted set wherever # `deploy.hive-controller.enable` is. credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent"; + + atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; in { options.services.hyperhive.deploy.hive-controller.queue = { @@ -217,6 +219,8 @@ in script = '' set -euo pipefail + ${atomicWriteSecret} + # `bao`'s own message is the only thing separating a missing value from # a refused identity from an unreachable host. This unit's degraded # mode is correct for all three, so it reports which one rather than @@ -279,14 +283,11 @@ in install -d -m 0755 ${lib.escapeShellArg credentialDir} - umask 077 - printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile} - chmod 0600 ${lib.escapeShellArg secretFile} + printf '%s\n' "$secret" | atomic_write_secret 0600 "" ${lib.escapeShellArg secretFile} # `0644` on purpose: an OIDC client id is presented to the token # endpoint on every connection and is public by construction. - printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile} - chmod 0644 ${lib.escapeShellArg clientIdFile} + printf '%s\n' "$client_id" | atomic_write_secret 0644 "" ${lib.escapeShellArg clientIdFile} ''; }; }) diff --git a/nix/host-modules/lib/atomic-write-secret.nix b/nix/host-modules/lib/atomic-write-secret.nix new file mode 100644 index 00000000..dccf7e9d --- /dev/null +++ b/nix/host-modules/lib/atomic-write-secret.nix @@ -0,0 +1,37 @@ +# Shared shell step for a systemd oneshot that lands a freshly-fetched +# secret on disk: never `> path` the live file directly, because a reader +# racing the write can open it between the truncate and the write, or +# between the write and a `chmod` that follows it, and see an empty file +# or one at the wrong mode. `mktemp` always creates its file at 0600 +# regardless of umask, so the temp file is private for its whole life; only +# the `chmod`/`chown`/`mv -f` sequence below ever makes the target mode and +# owner visible, and only once the content is already final. +# +# Pure function — NOT a NixOS module. Call it from a module's `let`: +# +# atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; +# ... +# script = '' +# ${atomicWriteSecret} +# printf '%s\n' "$secret" | atomic_write_secret 0600 "" "$path" +# printf '%s\n' "$secret" | atomic_write_secret 0400 "grafana:0" "$path" +# ''; +# +# Third argument to `atomic_write_secret` is the target path; the second is +# an owner for `chown` (`user:group` or a bare uid), or "" to leave the +# mktemp-created root:root ownership as it is. Reads its content from +# stdin. Requires `coreutils` on the caller's `path`. +{ }: +'' + atomic_write_secret() { + local mode="$1" owner="$2" target="$3" tmp + tmp="$(mktemp "$(dirname -- "$target")/.$(basename -- "$target").XXXXXX")" + trap 'rm -f "$tmp"' RETURN + cat > "$tmp" + chmod "$mode" "$tmp" + if [ -n "$owner" ]; then + chown "$owner" "$tmp" + fi + mv -f "$tmp" "$target" + } +'' diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index b447427a..355e8c6f 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -168,6 +168,8 @@ let nginxGid = config.ids.gids.nginx; grafanaUid = config.ids.uids.grafana; + atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; + in { # `enable` moved to `services.hyperhive.deploy.grafana.enable` — see @@ -633,6 +635,8 @@ in script = '' set -euo pipefail + ${atomicWriteSecret} + # `bao`'s own message is the only thing separating a missing value from # a refused identity from an unreachable host. This unit's degraded # mode is correct for all three, so it reports which one rather than @@ -689,10 +693,7 @@ in # grants the group nothing; if this mode ever widens, the gid has to be # discovered at runtime rather than assumed. install -d -m 0755 ${lib.escapeShellArg hostSecretDir} - umask 077 - printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath} - chown ${toString config.ids.uids.grafana}:0 ${lib.escapeShellArg hostSecretPath} - chmod 0400 ${lib.escapeShellArg hostSecretPath} + printf '%s\n' "$secret" | atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostSecretPath} ''; }; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index edb36ff5..3b1f2e38 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -166,6 +166,8 @@ let # collector cannot open, discovered at runtime and nowhere else. collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}"; + atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; + # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # @@ -817,6 +819,8 @@ in script = '' set -euo pipefail + ${atomicWriteSecret} + # `bao`'s own message is the only thing separating a missing value # from a refused identity from an unreachable host. This unit's # degraded mode is correct for all three, so it reports which one @@ -862,10 +866,7 @@ in # no uid to give this to — `LoadCredential` reads it as root before # the sandbox exists and re-exposes it to whichever uid the unit got. install -d -m 0755 ${lib.escapeShellArg collectorHostSecretDir} - umask 077 - printf '%s\n' "$secret" > ${lib.escapeShellArg collectorHostSecretPath} - chown root:root ${lib.escapeShellArg collectorHostSecretPath} - chmod 0400 ${lib.escapeShellArg collectorHostSecretPath} + printf '%s\n' "$secret" | atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorHostSecretPath} ''; };