Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6a778f666 | ||
|
|
770f68c272 | ||
|
|
7a9fadc21a | ||
|
|
5466cec066 | ||
|
|
ed53e9abcc |
8 changed files with 120 additions and 31 deletions
|
|
@ -535,9 +535,12 @@ container lifetime:
|
||||||
2. **Migrate any leftover `/root/.claude` content into
|
2. **Migrate any leftover `/root/.claude` content into
|
||||||
`${homeDir}/.claude`** — legacy `claude` wrote to root's
|
`${homeDir}/.claude`** — legacy `claude` wrote to root's
|
||||||
empty home; the bind mount didn't exist yet. Marker
|
empty home; the bind mount didn't exist yet. Marker
|
||||||
(`/var/lib/hive-agent-user-migrated`) guards single-shot.
|
(`/var/lib/hive-agent-user-migrated`) guards single-shot; it's
|
||||||
|
only written once there's nothing left to migrate, so a `cp`
|
||||||
|
failure leaves it absent and the next boot retries.
|
||||||
`cp -an` (no-clobber) so any pre-existing files at the new
|
`cp -an` (no-clobber) so any pre-existing files at the new
|
||||||
location win — never blow over data already there.
|
location win — never blow over data already there, and so a
|
||||||
|
retry after a partial copy is as safe as the first attempt.
|
||||||
3. **Chown the bind-mounted state dir** (`/agents/*/state`)
|
3. **Chown the bind-mounted state dir** (`/agents/*/state`)
|
||||||
recursively so the agent user can read/write it. Wildcard
|
recursively so the agent user can read/write it. Wildcard
|
||||||
matches the single agent that container sees; `-h` skips
|
matches the single agent that container sees; `-h` skips
|
||||||
|
|
|
||||||
|
|
@ -172,16 +172,29 @@ in
|
||||||
userName=${lib.escapeShellArg userName}
|
userName=${lib.escapeShellArg userName}
|
||||||
mkdir -p "$homeDir"
|
mkdir -p "$homeDir"
|
||||||
chown "$userName:$userName" "$homeDir"
|
chown "$userName:$userName" "$homeDir"
|
||||||
|
# The marker is only written once nothing is left to migrate — either
|
||||||
|
# there was nothing under /root/.claude, or `cp` copied it all. A
|
||||||
|
# failed `cp` (disk full, permission error) leaves the marker absent,
|
||||||
|
# so the next boot's activation retries; `-an` never clobbers a file
|
||||||
|
# this attempt already placed, so a retry after a partial copy is
|
||||||
|
# exactly as safe as the first attempt.
|
||||||
marker=/var/lib/hive-agent-user-migrated
|
marker=/var/lib/hive-agent-user-migrated
|
||||||
if [ ! -e "$marker" ] && [ -d /root/.claude ] && [ "$(ls -A /root/.claude 2>/dev/null)" ]; then
|
if [ ! -e "$marker" ]; then
|
||||||
mkdir -p "$homeDir/.claude"
|
if [ -d /root/.claude ] && [ "$(ls -A /root/.claude 2>/dev/null)" ]; then
|
||||||
if cp -an /root/.claude/. "$homeDir/.claude/" 2>/dev/null; then
|
mkdir -p "$homeDir/.claude"
|
||||||
rm -rf /root/.claude
|
if cp -an /root/.claude/. "$homeDir/.claude/" 2>/dev/null; then
|
||||||
echo "hive-agent-user-migrate: moved /root/.claude → $homeDir/.claude"
|
rm -rf /root/.claude
|
||||||
|
echo "hive-agent-user-migrate: moved /root/.claude → $homeDir/.claude"
|
||||||
|
mkdir -p "$(dirname "$marker")"
|
||||||
|
: > "$marker"
|
||||||
|
else
|
||||||
|
echo "hive-agent-user-migrate: copying /root/.claude to $homeDir/.claude failed; will retry next boot" >&2
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
mkdir -p "$(dirname "$marker")"
|
||||||
|
: > "$marker"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
mkdir -p "$(dirname "$marker")"
|
|
||||||
: > "$marker"
|
|
||||||
# Scope state + harness chowns to THIS container's own dirs only.
|
# Scope state + harness chowns to THIS container's own dirs only.
|
||||||
# The glob `/agents/*/state` also matches other agents' state dirs
|
# The glob `/agents/*/state` also matches other agents' state dirs
|
||||||
# bind-mounted into a `ManageRootAgent` holder's container, which
|
# bind-mounted into a `ManageRootAgent` holder's container, which
|
||||||
|
|
|
||||||
|
|
@ -80,6 +80,8 @@ let
|
||||||
# resolution, so this is the kind of mistake only reading the target module
|
# resolution, so this is the kind of mistake only reading the target module
|
||||||
# catches.
|
# catches.
|
||||||
matrixMachine = "hive-matrix";
|
matrixMachine = "hive-matrix";
|
||||||
|
|
||||||
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
config = lib.mkMerge [
|
config = lib.mkMerge [
|
||||||
|
|
@ -187,6 +189,8 @@ in
|
||||||
script = ''
|
script = ''
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
${atomicWriteSecret}
|
||||||
|
|
||||||
# A sealed or uninitialised store answers on the port and never
|
# A sealed or uninitialised store answers on the port and never
|
||||||
# answers the read, so "the store is up" is not the same as "the
|
# answers the read, so "the store is up" is not the same as "the
|
||||||
# store can answer". `TimeoutStartSec` above is the bound; the
|
# store can answer". `TimeoutStartSec` above is the bound; the
|
||||||
|
|
@ -237,9 +241,7 @@ in
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
umask 077
|
atomic_write_secret 0600 "" ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)} "$token"
|
||||||
printf '%s\n' "$token" > ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
|
|
||||||
chmod 0600 ${lib.escapeShellArg (toString deployCfg.matrix.appserviceTokenFile)}
|
|
||||||
|
|
||||||
# Re-stamp the registration file from the token just written. The
|
# Re-stamp the registration file from the token just written. The
|
||||||
# token is half an agreement — the registration the homeserver loads
|
# token is half an agreement — the registration the homeserver loads
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,8 @@ let
|
||||||
# its own use of it: it is asserted set wherever
|
# its own use of it: it is asserted set wherever
|
||||||
# `deploy.hive-controller.enable` is.
|
# `deploy.hive-controller.enable` is.
|
||||||
credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent";
|
credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent";
|
||||||
|
|
||||||
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.hyperhive.deploy.hive-controller.queue = {
|
options.services.hyperhive.deploy.hive-controller.queue = {
|
||||||
|
|
@ -217,6 +219,8 @@ in
|
||||||
script = ''
|
script = ''
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
${atomicWriteSecret}
|
||||||
|
|
||||||
# `bao`'s own message is the only thing separating a missing value from
|
# `bao`'s own message is the only thing separating a missing value from
|
||||||
# a refused identity from an unreachable host. This unit's degraded
|
# a refused identity from an unreachable host. This unit's degraded
|
||||||
# mode is correct for all three, so it reports which one rather than
|
# mode is correct for all three, so it reports which one rather than
|
||||||
|
|
@ -279,14 +283,11 @@ in
|
||||||
|
|
||||||
install -d -m 0755 ${lib.escapeShellArg credentialDir}
|
install -d -m 0755 ${lib.escapeShellArg credentialDir}
|
||||||
|
|
||||||
umask 077
|
atomic_write_secret 0600 "" ${lib.escapeShellArg secretFile} "$secret"
|
||||||
printf '%s\n' "$secret" > ${lib.escapeShellArg secretFile}
|
|
||||||
chmod 0600 ${lib.escapeShellArg secretFile}
|
|
||||||
|
|
||||||
# `0644` on purpose: an OIDC client id is presented to the token
|
# `0644` on purpose: an OIDC client id is presented to the token
|
||||||
# endpoint on every connection and is public by construction.
|
# endpoint on every connection and is public by construction.
|
||||||
printf '%s\n' "$client_id" > ${lib.escapeShellArg clientIdFile}
|
atomic_write_secret 0644 "" ${lib.escapeShellArg clientIdFile} "$client_id"
|
||||||
chmod 0644 ${lib.escapeShellArg clientIdFile}
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
|
|
|
||||||
60
nix/host-modules/lib/atomic-write-secret.nix
Normal file
60
nix/host-modules/lib/atomic-write-secret.nix
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
# Shared shell step for a systemd oneshot that lands a freshly-fetched
|
||||||
|
# secret on disk: never `> path` the live file directly, because a reader
|
||||||
|
# racing the write can open it between the truncate and the write, or
|
||||||
|
# between the write and a `chmod` that follows it, and see an empty file
|
||||||
|
# or one at the wrong mode. `mktemp` always creates its file at 0600
|
||||||
|
# regardless of umask, so the temp file is private for its whole life; only
|
||||||
|
# the `chmod`/`chown`/`mv -f` sequence below ever makes the target mode and
|
||||||
|
# owner visible, and only once the content is already final.
|
||||||
|
#
|
||||||
|
# The value is a function ARGUMENT, not piped in on stdin: a producer that
|
||||||
|
# exits non-zero partway through a pipe still leaves `cat` a clean EOF, so
|
||||||
|
# a truncated read is indistinguishable from an intentionally short one,
|
||||||
|
# and the partial content would still land at the live path via `mv`. A
|
||||||
|
# caller that computes the value with a command captures it into a
|
||||||
|
# variable first (`value=$(cmd)`, which fails under `set -e` before this
|
||||||
|
# function is ever called) and passes the finished value in — never handed
|
||||||
|
# to an external program, so it never appears in a process's own
|
||||||
|
# `/proc/<pid>/cmdline`, and written with `printf`, a shell builtin.
|
||||||
|
#
|
||||||
|
# Pure function — NOT a NixOS module. Call it from a module's `let`:
|
||||||
|
#
|
||||||
|
# atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||||
|
# script = ''
|
||||||
|
# ${atomicWriteSecret}
|
||||||
|
# atomic_write_secret 0600 "" "$path" "$secret"
|
||||||
|
# '';
|
||||||
|
#
|
||||||
|
# Args: mode, an owner for `chown` ("" to keep mktemp's root:root), target
|
||||||
|
# path, value — written with a trailing newline like every call site's
|
||||||
|
# prior `printf '%s\n'`. Requires `coreutils` on the caller's `path`.
|
||||||
|
{ }:
|
||||||
|
''
|
||||||
|
atomic_write_secret() {
|
||||||
|
local mode="$1" owner="$2" target="$3" value="$4" tmp
|
||||||
|
tmp="$(mktemp "$(dirname -- "$target")/.$(basename -- "$target").XXXXXX")"
|
||||||
|
# The write happens in a subshell so its own EXIT trap cleans up `$tmp`
|
||||||
|
# on failure (a `RETURN` trap does not fire when `set -e` aborts the
|
||||||
|
# function mid-body) without touching an EXIT trap the calling script's
|
||||||
|
# own `script` may already have — subshell traps are local to the
|
||||||
|
# subshell. A named handler, not an inline trap string, so `local rc=$?`
|
||||||
|
# is a normal function-local assignment shellcheck can see: it only
|
||||||
|
# removes `$tmp` when `$?` is nonzero — on the ordinary path the
|
||||||
|
# subshell also exits successfully, and `$tmp` has to survive that to
|
||||||
|
# reach the `mv` below.
|
||||||
|
(
|
||||||
|
_atomic_write_secret_cleanup() {
|
||||||
|
local rc=$?
|
||||||
|
[ "$rc" -eq 0 ] || rm -f "$tmp"
|
||||||
|
exit "$rc"
|
||||||
|
}
|
||||||
|
trap _atomic_write_secret_cleanup EXIT
|
||||||
|
printf '%s\n' "$value" > "$tmp"
|
||||||
|
chmod "$mode" "$tmp"
|
||||||
|
if [ -n "$owner" ]; then
|
||||||
|
chown "$owner" "$tmp"
|
||||||
|
fi
|
||||||
|
)
|
||||||
|
mv -f "$tmp" "$target"
|
||||||
|
}
|
||||||
|
''
|
||||||
|
|
@ -764,6 +764,8 @@ let
|
||||||
forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
|
forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
|
||||||
forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}";
|
forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}";
|
||||||
forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath;
|
forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath;
|
||||||
|
|
||||||
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||||
# The collector runs under `DynamicUser` and opens `client_secret_file`
|
# The collector runs under `DynamicUser` and opens `client_secret_file`
|
||||||
# itself, so there is no uid to hand a 0400 file to. `LoadCredential` reads
|
# itself, so there is no uid to hand a 0400 file to. `LoadCredential` reads
|
||||||
# it as root before the sandbox exists and re-exposes it under a path that
|
# it as root before the sandbox exists and re-exposes it under a path that
|
||||||
|
|
@ -1797,6 +1799,8 @@ in
|
||||||
script = ''
|
script = ''
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
${atomicWriteSecret}
|
||||||
|
|
||||||
# `bao`'s own message is the only thing separating a missing value
|
# `bao`'s own message is the only thing separating a missing value
|
||||||
# from a refused identity from an unreachable store, and this unit
|
# from a refused identity from an unreachable store, and this unit
|
||||||
# retries on all three — so it reports which one rather than
|
# retries on all three — so it reports which one rather than
|
||||||
|
|
@ -1837,10 +1841,7 @@ in
|
||||||
# an argument in /proc the way `install <<<"$secret"` or an `echo`
|
# an argument in /proc the way `install <<<"$secret"` or an `echo`
|
||||||
# from `path` would.
|
# from `path` would.
|
||||||
install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir}
|
install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir}
|
||||||
umask 077
|
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderHostSecretPath} "$secret"
|
||||||
printf '%s\n' "$secret" > ${lib.escapeShellArg forwarderHostSecretPath}
|
|
||||||
chown root:root ${lib.escapeShellArg forwarderHostSecretPath}
|
|
||||||
chmod 0400 ${lib.escapeShellArg forwarderHostSecretPath}
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
|
|
@ -2594,6 +2595,9 @@ in
|
||||||
};
|
};
|
||||||
script = ''
|
script = ''
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
${atomicWriteSecret}
|
||||||
|
|
||||||
install -d -m 0770 -g ${tokenGroup} ${tokenStoreDir}
|
install -d -m 0770 -g ${tokenGroup} ${tokenStoreDir}
|
||||||
# Required whenever the store is not at its default location, or the
|
# Required whenever the store is not at its default location, or the
|
||||||
# library cannot find the token the seal asks for.
|
# library cannot find the token the seal asks for.
|
||||||
|
|
@ -2643,8 +2647,12 @@ in
|
||||||
chgrp ${tokenGroup} ${tokenStoreDir}/tpm2_pkcs11.sqlite3
|
chgrp ${tokenGroup} ${tokenStoreDir}/tpm2_pkcs11.sqlite3
|
||||||
chmod 0660 ${tokenStoreDir}/tpm2_pkcs11.sqlite3
|
chmod 0660 ${tokenStoreDir}/tpm2_pkcs11.sqlite3
|
||||||
|
|
||||||
( umask 077; printf 'BAO_HSM_PIN=%s\n' "$(cat ${tokenStoreDir}/user-pin)" > ${pinEnvFile} )
|
# Captured first, not piped: a `$(cmd)` that fails aborts here
|
||||||
chmod 0400 ${pinEnvFile}
|
# under `set -e`, before the helper — which cannot see a
|
||||||
|
# producer's exit code once its output is inside the pipe —
|
||||||
|
# is ever called.
|
||||||
|
pin_env_value="BAO_HSM_PIN=$(cat ${tokenStoreDir}/user-pin)"
|
||||||
|
atomic_write_secret 0400 "" ${pinEnvFile} "$pin_env_value"
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -168,6 +168,8 @@ let
|
||||||
nginxGid = config.ids.gids.nginx;
|
nginxGid = config.ids.gids.nginx;
|
||||||
grafanaUid = config.ids.uids.grafana;
|
grafanaUid = config.ids.uids.grafana;
|
||||||
|
|
||||||
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||||
|
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
# `enable` moved to `services.hyperhive.deploy.grafana.enable` — see
|
# `enable` moved to `services.hyperhive.deploy.grafana.enable` — see
|
||||||
|
|
@ -633,6 +635,8 @@ in
|
||||||
script = ''
|
script = ''
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
${atomicWriteSecret}
|
||||||
|
|
||||||
# `bao`'s own message is the only thing separating a missing value from
|
# `bao`'s own message is the only thing separating a missing value from
|
||||||
# a refused identity from an unreachable host. This unit's degraded
|
# a refused identity from an unreachable host. This unit's degraded
|
||||||
# mode is correct for all three, so it reports which one rather than
|
# mode is correct for all three, so it reports which one rather than
|
||||||
|
|
@ -689,10 +693,7 @@ in
|
||||||
# grants the group nothing; if this mode ever widens, the gid has to be
|
# grants the group nothing; if this mode ever widens, the gid has to be
|
||||||
# discovered at runtime rather than assumed.
|
# discovered at runtime rather than assumed.
|
||||||
install -d -m 0755 ${lib.escapeShellArg hostSecretDir}
|
install -d -m 0755 ${lib.escapeShellArg hostSecretDir}
|
||||||
umask 077
|
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostSecretPath} "$secret"
|
||||||
printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath}
|
|
||||||
chown ${toString config.ids.uids.grafana}:0 ${lib.escapeShellArg hostSecretPath}
|
|
||||||
chmod 0400 ${lib.escapeShellArg hostSecretPath}
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -166,6 +166,8 @@ let
|
||||||
# collector cannot open, discovered at runtime and nowhere else.
|
# collector cannot open, discovered at runtime and nowhere else.
|
||||||
collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
|
collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
|
||||||
|
|
||||||
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||||
|
|
||||||
# `attrNames` is sorted, so this is a function of the hive SET and not of
|
# `attrNames` is sorted, so this is a function of the hive SET and not of
|
||||||
# the order anyone wrote it in.
|
# the order anyone wrote it in.
|
||||||
#
|
#
|
||||||
|
|
@ -817,6 +819,8 @@ in
|
||||||
script = ''
|
script = ''
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
${atomicWriteSecret}
|
||||||
|
|
||||||
# `bao`'s own message is the only thing separating a missing value
|
# `bao`'s own message is the only thing separating a missing value
|
||||||
# from a refused identity from an unreachable host. This unit's
|
# from a refused identity from an unreachable host. This unit's
|
||||||
# degraded mode is correct for all three, so it reports which one
|
# degraded mode is correct for all three, so it reports which one
|
||||||
|
|
@ -862,10 +866,7 @@ in
|
||||||
# no uid to give this to — `LoadCredential` reads it as root before
|
# no uid to give this to — `LoadCredential` reads it as root before
|
||||||
# the sandbox exists and re-exposes it to whichever uid the unit got.
|
# the sandbox exists and re-exposes it to whichever uid the unit got.
|
||||||
install -d -m 0755 ${lib.escapeShellArg collectorHostSecretDir}
|
install -d -m 0755 ${lib.escapeShellArg collectorHostSecretDir}
|
||||||
umask 077
|
atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorHostSecretPath} "$secret"
|
||||||
printf '%s\n' "$secret" > ${lib.escapeShellArg collectorHostSecretPath}
|
|
||||||
chown root:root ${lib.escapeShellArg collectorHostSecretPath}
|
|
||||||
chmod 0400 ${lib.escapeShellArg collectorHostSecretPath}
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue