host-modules: write credential files atomically; agent-modules: retry a failed .claude migration

Four host glue units fetched a secret from swarm-bao and rendered it
with `> path; chmod`: a reader racing the write could see a truncated
file, and briefly one at the wrong mode before the chmod landed.
glue-matrix-bao-token.nix, glue-queue-agent-credential.nix (both
files), swarm-grafana.nix and swarm-otel.nix now write to a same-
directory temp file, set its final mode/owner, then `mv -f` it over
the target — a shared `atomic_write_secret` helper
(nix/host-modules/lib/atomic-write-secret.nix) so the five call sites
share one implementation.

The first-boot `/root/.claude` migration in nix/agent-modules/user.nix
wrote its done-marker unconditionally, so a failed `cp` (disk full,
permission error) left the marker behind and no boot ever retried the
copy. The marker is now written only when there was nothing to
migrate or the copy succeeded; `cp -an`'s no-clobber semantics already
make a retry after a partial copy safe.

Refs #4723
This commit is contained in:
atlas 2026-09-26 15:16:42 +02:00 • committed by mara
commit ed53e9abcc
7 changed files with 83 additions and 25 deletions

View file

@ -80,6 +80,8 @@ let
# resolution, so this is the kind of mistake only reading the target module
# catches.
matrixMachine = "hive-matrix";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
in
{
config = lib.mkMerge [
@ -187,6 +189,8 @@ in
script = ''
set -euo pipefail
${atomicWriteSecret}
# A sealed or uninitialised store answers on the port and never
# answers the read, so "the store is up" is not the same as "the
# store can answer". `TimeoutStartSec` above is the bound; the
@ -237,9 +241,7 @@ in
exit 0
fi
umask 077
printf '%s\n' "$token" > ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
chmod 0600 ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
printf '%s\n' "$token" | atomic_write_secret 0600 "" ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
# Re-stamp the registration file from the token just written. The
# token is half an agreement — the registration the homeserver loads

View file

@ -77,6 +77,8 @@ let
# its own use of it: it is asserted set wherever
# `deploy.hive-controller.enable` is.
credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
in
{
options.services.hyperhive.deploy.hive-controller.queue = {
@ -217,6 +219,8 @@ in
script = ''
set -euo pipefail
${atomicWriteSecret}
# `bao`'s own message is the only thing separating a missing value from
# a refused identity from an unreachable host. This unit's degraded
# mode is correct for all three, so it reports which one rather than
@ -279,14 +283,11 @@ in
install -d -m 0755 ${lib.escapeShellArg credentialDir}
umask 077
printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile}
chmod 0600 ${lib.escapeShellArg secretFile}
printf '%s\n' "$secret" | atomic_write_secret 0600 "" ${lib.escapeShellArg secretFile}
# `0644` on purpose: an OIDC client id is presented to the token
# endpoint on every connection and is public by construction.
printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile}
chmod 0644 ${lib.escapeShellArg clientIdFile}
printf '%s\n' "$client_id" | atomic_write_secret 0644 "" ${lib.escapeShellArg clientIdFile}
'';
};
})

View file

@ -0,0 +1,37 @@
# Shared shell step for a systemd oneshot that lands a freshly-fetched
# secret on disk: never `> path` the live file directly, because a reader
# racing the write can open it between the truncate and the write, or
# between the write and a `chmod` that follows it, and see an empty file
# or one at the wrong mode. `mktemp` always creates its file at 0600
# regardless of umask, so the temp file is private for its whole life; only
# the `chmod`/`chown`/`mv -f` sequence below ever makes the target mode and
# owner visible, and only once the content is already final.
#
# Pure function — NOT a NixOS module. Call it from a module's `let`:
#
# atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
# ...
# script = ''
# ${atomicWriteSecret}
# printf '%s\n' "$secret" | atomic_write_secret 0600 "" "$path"
# printf '%s\n' "$secret" | atomic_write_secret 0400 "grafana:0" "$path"
# '';
#
# Third argument to `atomic_write_secret` is the target path; the second is
# an owner for `chown` (`user:group` or a bare uid), or "" to leave the
# mktemp-created root:root ownership as it is. Reads its content from
# stdin. Requires `coreutils` on the caller's `path`.
{ }:
''
atomic_write_secret() {
local mode="$1" owner="$2" target="$3" tmp
tmp="$(mktemp "$(dirname -- "$target")/.$(basename -- "$target").XXXXXX")"
trap 'rm -f "$tmp"' RETURN
cat > "$tmp"
chmod "$mode" "$tmp"
if [ -n "$owner" ]; then
chown "$owner" "$tmp"
fi
mv -f "$tmp" "$target"
}
''

View file

@ -168,6 +168,8 @@ let
nginxGid = config.ids.gids.nginx;
grafanaUid = config.ids.uids.grafana;
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
in
{
# `enable` moved to `services.hyperhive.deploy.grafana.enable` — see
@ -633,6 +635,8 @@ in
script = ''
set -euo pipefail
${atomicWriteSecret}
# `bao`'s own message is the only thing separating a missing value from
# a refused identity from an unreachable host. This unit's degraded
# mode is correct for all three, so it reports which one rather than
@ -689,10 +693,7 @@ in
# grants the group nothing; if this mode ever widens, the gid has to be
# discovered at runtime rather than assumed.
install -d -m 0755 ${lib.escapeShellArg hostSecretDir}
umask 077
printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath}
chown ${toString config.ids.uids.grafana}:0 ${lib.escapeShellArg hostSecretPath}
chmod 0400 ${lib.escapeShellArg hostSecretPath}
printf '%s\n' "$secret" | atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostSecretPath}
'';
};

View file

@ -166,6 +166,8 @@ let
# collector cannot open, discovered at runtime and nowhere else.
collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
# `attrNames` is sorted, so this is a function of the hive SET and not of
# the order anyone wrote it in.
#
@ -817,6 +819,8 @@ in
script = ''
set -euo pipefail
${atomicWriteSecret}
# `bao`'s own message is the only thing separating a missing value
# from a refused identity from an unreachable host. This unit's
# degraded mode is correct for all three, so it reports which one
@ -862,10 +866,7 @@ in
# no uid to give this to — `LoadCredential` reads it as root before
# the sandbox exists and re-exposes it to whichever uid the unit got.
install -d -m 0755 ${lib.escapeShellArg collectorHostSecretDir}
umask 077
printf '%s\n' "$secret" > ${lib.escapeShellArg collectorHostSecretPath}
chown root:root ${lib.escapeShellArg collectorHostSecretPath}
chmod 0400 ${lib.escapeShellArg collectorHostSecretPath}
printf '%s\n' "$secret" | atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorHostSecretPath}
'';
};