From 309879dba0465fb4cf007aae9fc6fac48ade0967 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 31 May 2026 14:39:18 +0200 Subject: [PATCH] docs: extract 3 substantive harness-base.nix prose blocks (#718, first pass) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit iris's #718 scope: move substantive design context from `#` comment blocks in `nix/` to corresponding `docs/` files, leave short references in code. iris handed it back to me on #10114 since nix/ is my lane + #775 established the pattern. First pass — three highest-density blocks in harness-base.nix: 1. **First-boot agent-user migration** (~70 lines → `~20 lines code + short ref` in the activation script). Substantive prose moves to new `docs/persistence.md::First-boot agent-user migration (post-#658)` section explaining the 4 steps the script performs + the eventual removability of the marker-guarded body. 2. **nix-daemon `sandbox-fallback = true`** (10-line block → 5-line ref). New `docs/gotchas.md::Containerized nix-daemon needs sandbox-fallback = true` section covers the user-namespaces rationale + nixpkgs-default override. 3. **Matrix daemon + token-arrival trigger** (~50 lines across two systemd units → ~10 lines code + short refs). New `docs/persistence.md::Matrix per-agent daemon + token-arrival trigger` covers the socket-path rationale, the runtime-dir ownership story, and the first-boot ordering pattern. Net: harness-base.nix -84 lines, docs +74 lines. Substantive design context moves to durable docs; in-code refs follow iris's pattern from her #712 batches (`see docs/::
`). Follow-ups: hive-c0re.nix, hive-forge.nix, hive-matrix.nix (already trimmed via #775 but a couple of remaining blocks could go), and the smaller files in #718's scope table. Shipping this first to get the pattern reviewed before larger batches. Verified: `nix eval` on agent-base toplevel still resolves. --- docs/gotchas.md | 11 ++++ docs/persistence.md | 63 ++++++++++++++++++++ nix/templates/harness-base.nix | 106 +++++++-------------------------- 3 files changed, 96 insertions(+), 84 deletions(-) diff --git a/docs/gotchas.md b/docs/gotchas.md index 5cf01a20..85fb1b85 100644 --- a/docs/gotchas.md +++ b/docs/gotchas.md @@ -227,3 +227,14 @@ hive-forge lint assignments # per-assignee open item count Credentials come from `$HYPERHIVE_STATE_DIR/forge-token`; default repo from `$HIVE_FORGE_REPO`, overridden per-invocation by the global `-r/--repo` flag. + +## Containerized nix-daemon needs `sandbox-fallback = true` + +Agent containers bind-mount the host's nix-daemon socket. nspawn +containers don't get user-namespaces by default, so `nix build` +invocations *inside* the container can't set up the build sandbox +and fail outright if the host daemon's +`nix.settings.sandbox-fallback` is `false` (nixpkgs default). +`nix/templates/harness-base.nix` does `lib.mkForce true` so builds +fall back to unsandboxed local builds rather than failing. Security +implications: `docs/security.md`. diff --git a/docs/persistence.md b/docs/persistence.md index dd9b3f52..03bf8982 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -198,3 +198,66 @@ On startup, `Coordinator::register_agent` drops any prior socket task before rebinding — idempotent so a hive-c0re restart followed by `rebuild alice` recreates the agent's socket without a clean reinstall. + +## First-boot agent-user migration (post-#658) + +Pre-#658 the harness ran as root inside the container. #658 dropped +to a per-agent unix user (`hyperhive.user.name`, defaults to the +agent's logical label so each container has a uniquely-named user). +The transition needs a one-time data shuffle so existing operators +who deployed pre-#658 don't lose their claude session. + +`system.activationScripts.hive-agent-user-migrate` (in +`nix/templates/harness-base.nix`) runs on every activation, +marker-guarded so the substantive moves only happen once per +container lifetime: + +1. **`${homeDir}` exists with the right ownership** — covers the + very first boot before `useradd`'s `createHome` has had a + chance to chown. Also re-applies on every rebuild in case the + meta-flake's per-agent name evolves (rare). +2. **Migrate any leftover `/root/.claude` content into + `${homeDir}/.claude`** — pre-#658 `claude` wrote to root's + empty home; the bind mount didn't exist yet. Marker + (`/var/lib/hive-agent-user-migrated`) guards single-shot. + `cp -an` (no-clobber) so any pre-existing files at the new + location win — never blow over data already there. +3. **Chown the bind-mounted state dir** (`/agents/*/state`) + recursively so the new agent user can read/write it. Wildcard + matches the single agent that container sees; `-h` skips + symlinks the agent might have planted. +4. **Chown the `~/.claude/` bind-mount** recursively. Pre-#658 + `claude` wrote `.credentials.json` 0600 root:root; post-#658 + the harness reads `~/.claude/` as the agent user to decide + Online vs NeedsLogin in `login::has_session`. Without the + chown the existing credentials get silently treated as "no + session" and the operator re-prompts every boot. + +The activation script will eventually become unnecessary once no +operators have pre-#658 state dirs left to migrate; drop the body ++ marker check at that point. + +## Matrix per-agent daemon + token-arrival trigger + +`hive-matrix-daemon` is a long-running matrix-sdk Client + sync +process per agent. Holds the unix socket the stdio +`hive-matrix-mcp` bridge talks to, emits hyperhive wake signals +on incoming room events via `/run/hive/mcp.sock`. Conditional on +`hyperhive.matrix.enable` (which both the daemon AND the +auto-injected `extraMcpServers.matrix` entry read). + +Socket path lives inside the systemd-managed runtime dir +(`RuntimeDirectory = "hive-matrix"` → `/run/hive-matrix/`, owned by +the agent user) so the daemon can bind without needing root over +`/run/` itself. Both daemon + bridge agree on the path via the +`HIVE_MATRIX_SOCKET` env var. + +**First-boot ordering**: hive-c0re provisions the matrix token AFTER +agent containers come up. Without the path-trigger sibling +(`systemd.paths.hive-matrix-daemon`, `PathExistsGlob = +/agents/*/state/matrix-token`), the daemon would exit 0 quietly the +first time it ran and the MCP would have no backend until the next +restart. The `.path` unit makes the appearance of the token re-fire +the service so the daemon comes alive in the same boot cycle as +provisioning. `matrix-avatar-sync.path` uses the same pattern for +the icon-upload oneshot (#571). diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index 0c433b59..ae65bc3b 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -656,46 +656,20 @@ in } ]; - # First-boot migration from the legacy root-run shape (#658). - # Runs on every activation; marker-guarded so the move only - # happens once. The bind mount that hive-c0re sets up has - # already moved from `/root/.claude` to `${homeDir}/.claude` - # by the time we get here (per `lifecycle::CONTAINER_CLAUDE_MOUNT` - # — the host-side path stays the same, the container-side - # mount target shifts), so the bulk of the data is already at - # the new location. This script just: - # - # - ensures `${homeDir}` exists with correct ownership (covers - # the very first boot before useradd's `createHome` has - # anything to chown); - # - migrates any leftover `/root/.claude` content that an - # operator might have populated before #658 deployed (the - # bind mount didn't exist in that lifecycle, so claude - # would have written into the root user's empty home — - # nothing important typically, but safer to move than to - # strand); - # - chowns the bind-mounted state dir (`/agents/*/state`) so - # the agent user can read/write it. + # Post-#658 first-boot migration to the per-agent unix user — + # creates the home dir, chowns the bind-mounted state + + # `~/.claude/`, and (marker-guarded) moves any leftover + # `/root/.claude` content from the pre-#658 root-run shape. See + # `docs/persistence.md::First-boot agent-user migration` for the + # step-by-step rationale; this script implements it. system.activationScripts.hive-agent-user-migrate = lib.stringAfter [ "users" "specialfs" ] '' homeDir=${lib.escapeShellArg homeDir} userName=${lib.escapeShellArg userName} - # Always ensure the home dir exists with the right ownership — - # useradd's createHome handles the very first creation but - # doesn't re-chown if a rebuild changes the user name (rare - # but possible if the meta-flake's per-agent name evolves). mkdir -p "$homeDir" chown "$userName:$userName" "$homeDir" - # One-time migration of pre-#658 /root/.claude content into the - # new home. Marker-guarded so the move only runs once per - # container lifetime — subsequent activations skip the legacy - # path even if claude were to repopulate /root/.claude for any - # reason. marker=/var/lib/hive-agent-user-migrated if [ ! -e "$marker" ] && [ -d /root/.claude ] && [ "$(ls -A /root/.claude 2>/dev/null)" ]; then mkdir -p "$homeDir/.claude" - # `mv -n` (no-clobber) so any pre-existing files at the - # destination (e.g. from the bind mount) win — we never - # blow over data already at the new location. if cp -an /root/.claude/. "$homeDir/.claude/" 2>/dev/null; then rm -rf /root/.claude echo "hive-agent-user-migrate: moved /root/.claude → $homeDir/.claude" @@ -703,25 +677,10 @@ in fi mkdir -p "$(dirname "$marker")" : > "$marker" - # Chown the bind-mounted state dir so the agent user can - # read/write it. `/agents/*/state` is the canonical mount - # point set by hive-c0re's `set_nspawn_flags`. Wildcard - # because each container only sees its own - # `/agents//state` (one match); -h to avoid following - # any symlinks the agent might have planted in there. for stateDir in /agents/*/state; do [ -d "$stateDir" ] || continue chown -hR "$userName:$userName" "$stateDir" 2>/dev/null || true done - # Same treatment for the bind-mounted `~/.claude/` dir. Pre-#658 - # the harness ran as root and `claude` wrote `.credentials.json` - # there 0600 root:root; post-#658 the harness reads - # `~/.claude/` as the agent user to decide Online vs - # NeedsLogin (`login::has_session`), and the host-side bind - # source is still root-owned 0700 from those legacy writes. - # Chown recursively so the existing credentials are readable - # under the new identity instead of getting silently treated - # as "no session" and re-prompting login every boot. if [ -d "$homeDir/.claude" ]; then chown -hR "$userName:$userName" "$homeDir/.claude" 2>/dev/null || true fi @@ -829,15 +788,11 @@ in "flakes" ]; - # Containers bind-mount the host's nix-daemon socket. The host daemon - # may be configured with remote builders or strict sandbox settings - # (sandbox-fallback = false) that make local `nix build` invocations - # fail inside the container. Enable sandbox-fallback so builds that - # can't set up the sandbox (no user-namespaces in nspawn) fall back - # to unsandboxed local builds rather than failing outright. - # mkForce overrides the nixpkgs nix module which sets this to false - # at normal priority -- without it agents get a conflicting definition - # error on rebuild. Security implications: see docs/security.md. + # `lib.mkForce` overrides nixpkgs's normal-priority `false` so + # in-container `nix build` invocations fall back to unsandboxed + # local builds rather than failing on the missing user-namespace. + # See `docs/gotchas.md::Containerized nix-daemon needs + # sandbox-fallback = true` + `docs/security.md` for the rationale. nix.settings.sandbox-fallback = lib.mkForce true; # `claude-code` is unfree. Each per-agent container's nixosConfiguration @@ -1005,14 +960,12 @@ in ''; }; - # Long-running matrix-sdk Client + sync per agent (#548 phase 3). - # Holds the unix socket the stdio `hive-matrix-mcp` bridge talks - # to, and emits hyperhive wake signals on incoming room events - # via `/run/hive/mcp.sock`. Conditional on `hyperhive.matrix.enable` - # AND token-file presence (the daemon binary itself exits 0 on - # missing token, but the path watcher below restarts it the - # moment the token lands — same first-boot-ordering pattern as - # matrix-avatar-sync.path / #571). + # Long-running matrix-sdk client + sync per agent. Holds the unix + # socket the stdio `hive-matrix-mcp` bridge connects to + emits + # hyperhive wake signals on incoming room events via + # `/run/hive/mcp.sock`. See + # `docs/persistence.md::Matrix per-agent daemon + token-arrival + # trigger` for the socket-path / first-boot-ordering rationale. systemd.services.hive-matrix-daemon = lib.mkIf config.hyperhive.matrix.enable { description = "long-running matrix-sdk Client + MCP daemon socket"; wantedBy = [ "multi-user.target" ]; @@ -1020,12 +973,6 @@ in wants = [ "network-online.target" ]; environment = { HIVE_MATRIX_URL = config.hyperhive.matrix.url; - # Socket path lives inside the systemd-managed runtime dir - # (`RuntimeDirectory = "hive-matrix"` → `/run/hive-matrix/`, - # owned by the agent user) so the daemon can bind it without - # needing root over `/run/` itself (#658). The stdio bridge - # picks up the same path via its own `HIVE_MATRIX_SOCKET` env - # in `extraMcpServers.matrix` below. HIVE_MATRIX_SOCKET = "/run/hive-matrix/socket"; RUST_LOG = "info"; }; @@ -1033,26 +980,17 @@ in ExecStart = "${pkgs.hyperhive}/bin/hive-matrix-daemon"; Restart = "on-failure"; RestartSec = 5; - # Run as the per-agent unix user (#658). The runtime dir - # (`/run/hive-matrix/`) is owned by that user via - # `RuntimeDirectory`; claude (also as that user) can - # connect to the socket inside it when the stdio bridge - # spawns per turn. User = userName; Group = userName; RuntimeDirectory = "hive-matrix"; }; }; - # Path-trigger sibling so hive-matrix-daemon fires the moment - # `/matrix-token` appears (#548 phase 3, mirrors the - # matrix-avatar-sync.path pattern from #571). On clean boot - # hive-c0re provisions the token AFTER agent containers come up; - # without the trigger the daemon would exit 0 quietly and the - # MCP would have no backend until next restart. With the watcher - # the daemon comes alive in the same boot cycle as provisioning. - # The glob matches every agent (manager sees its own state at - # `/agents/hm1nd/state/` via the `/agents` bind). + # Re-fire the daemon when the matrix token appears (hive-c0re + # provisions it after agent containers come up). Without this + # the daemon would exit 0 silently on first boot and the MCP + # would have no backend until next restart. See + # `docs/persistence.md` (same section as above). systemd.paths.hive-matrix-daemon = lib.mkIf config.hyperhive.matrix.enable { description = "trigger hive-matrix-daemon when matrix-token appears"; wantedBy = [ "multi-user.target" ];