vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

hive-c0re: scrub lifecycle + meta cookies (#715 batch 9)

Browse source
This commit is contained in:
damocles 2026-05-31 18:04:39 +02:00 • committed by mara
commit b0495c5167
2 changed files with 73 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

78
hive-c0re/src/lifecycle.rs
View file

@ -20,16 +20,13 @@ pub const MANAGER_NAME: &str = "hm1nd";
pub const CONTAINER_RUNTIME_MOUNT: &str = "/run/hive";
/// Where the per-agent Claude credentials dir mounts inside the
/// container. Pre-#658 this was a constant (`/root/.claude`, because
/// every agent ran as root). With the user-named-after-agent shape
/// the harness service runs as a non-root unix user whose home is
/// `/home/<agent>/`, so the mount path now varies per agent —
/// `container_claude_mount(name)` returns `/home/<name>/.claude`
/// for sub-agents and `/home/hm1nd/.claude` for the manager.
/// `claude` inside the container reads `$HOME/.claude` and the
/// service environment sets `HOME` to the same path, so the OAuth
/// session survives container restarts the same way the constant
/// did.
/// container. The harness service runs as a non-root unix user
/// whose home is `/home/<agent>/`, so the mount path varies per
/// agent — `container_claude_mount(name)` returns
/// `/home/<name>/.claude` for sub-agents and `/home/hm1nd/.claude`
/// for the manager. `claude` inside the container reads
/// `$HOME/.claude` and the service environment sets `HOME` to the
/// same path, so the OAuth session survives container restarts.
#[must_use]
pub fn container_claude_mount(name: &str) -> String {
format!("/home/{name}/.claude")
@ -55,13 +52,12 @@ const DEFAULT_MEMORY_MAX: &str = "2G";
const DEFAULT_CPU_QUOTA: &str = "50%";
/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) %
/// WEB_PORT_RANGE` for every agent including the manager (#753
/// dropped the pre-#753 "manager pinned at 8000" special case so
/// the port allocation rule reads the same for every name).
/// Collisions are possible (birthday paradox at ~30 agents); the
/// operator resolves them by renaming an agent (different hash →
/// different port). Stable across hosts, restarts, and dashboard
/// renders — no state-file dance.
/// WEB_PORT_RANGE` for every agent including the manager. The port
/// allocation rule reads the same for every name; collisions are
/// possible (birthday paradox at ~30 agents) and the operator
/// resolves them by renaming an agent (different hash → different
/// port). Stable across hosts, restarts, and dashboard renders —
/// no state-file dance.
#[must_use]
pub fn agent_web_port(name: &str) -> u16 {
let mut hash: u32 = 2_166_136_261;
@ -90,12 +86,12 @@ pub fn is_manager(name: &str) -> bool {
/// Read the agent user's `(uid, gid)` from the container's nixos-managed
/// `/etc/passwd`. Returns `None` when the container hasn't been built
/// yet, the passwd file is unparseable, or the agent user is missing
/// (e.g. legacy pre-#658 container that still runs as root).
/// (e.g. legacy container that still runs as root).
///
/// Used by `forge` + `matrix` after writing per-agent state files so
/// the bind-mounted host file ends up readable by the agent user
/// without waiting for the next container activation to run the chown
/// fixup (#673).
/// fixup.
///
/// Notes:
/// - Reads the *container-local* passwd at
@ -130,7 +126,7 @@ pub fn agent_uid_gid(agent_name: &str) -> Option<(u32, u32)> {
/// activation script in `harness-base.nix` is the steady-state safety
/// net. Used by per-agent state writers in `forge` + `matrix` so the
/// agent can read the file without waiting for the next container
/// rebuild (#673).
/// rebuild.
pub fn chown_to_agent(name: &str, path: &Path, subsystem: &str) {
let Some((uid, gid)) = agent_uid_gid(name) else {
return;
@ -406,7 +402,7 @@ pub async fn rebuild_no_meta(
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
if container_exists(name).await {
// Existing container: preserve the prior running state across
// rebuild (closes #371) and apply both the new system profile
// rebuild, and apply both the new system profile
// AND any `/etc/nixos-containers/<c>.conf` / drop-in changes
// in a single start rather than `update`'s reload-then-outer-
// restart double-bounce.
@ -428,16 +424,15 @@ pub async fn rebuild_no_meta(
// Pre-build the system toplevel **before** stopping the
// running container so the agent keeps serving its
// previous generation while the eval + fetch + build
// happens out-of-band (#706). `nixos-container update`
// then finds the toplevel cached and skips straight to
// the profile-swap + restart — downtime collapses to that
// happens out-of-band. `nixos-container update` then
// finds the toplevel cached and skips straight to the
// profile-swap + restart — downtime collapses to that
// window only. Build failures surface here, before we
// touch the container.
//
// When the container is already stopped there's no
// downtime to shave (mara on #721#9007) — let `update`
// do the build inline rather than evaluating the flake
// twice for nothing.
// downtime to shave — let `update` do the build inline
// rather than evaluating the flake twice for nothing.
prebuild_toplevel(name, &flake_ref).await?;
run(&["stop", &container]).await?;
}
@ -452,7 +447,7 @@ pub async fn rebuild_no_meta(
// `nixos-container create` builds + creates atomically — if
// the build fails, no container record is left around to
// clean up — so a pre-build adds nothing but a duplicate
// eval (mara on #721#9007).
// eval.
run(&["create", &container, "--flake", &flake_ref]).await?;
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
set_resource_limits(&container)?;
@ -480,8 +475,7 @@ pub async fn rebuild_no_meta(
/// back to `meta#<name>` (the shape `nixos-container update --flake
/// meta#<name>` uses) makes nix look for `packages.<system>.<name>`,
/// `legacyPackages.<system>.<name>`, or `<name>` at the flake root —
/// none of which exist in the rendered meta flake (closes #735, the
/// argus-prebuild regression after the first cold rebuild post-#721).
/// none of which exist in the rendered meta flake.
///
/// Returns the same error shape as the other nixos-container
/// helpers so callers can use `?` without translation.
@ -1065,9 +1059,7 @@ fn set_nspawn_flags(
// `claude` CLI (which reads `$HOME/.claude`) finds them. The
// harness service's environment sets `HOME` to the same path
// (`agent-base.nix` / `manager.nix`), so no `--setenv` plumbing
// is needed here — the bind alone is enough. Pre-#658 the mount
// was the constant `/root/.claude` because the service ran as
// root.
// is needed here — the bind alone is enough.
let claude_mount = container_claude_mount(agent_name);
let mut binds = format!(
@ -1079,8 +1071,7 @@ fn set_nspawn_flags(
// Per-agent state at `/agents/<container>/state`. Skipped for
// the manager — the `/agents` bind below already exposes its
// own state (along with every sub-agent's). Pre-#604 the manager
// had a bespoke `/state` legacy alias bind; that's gone.
// own state (along with every sub-agent's).
if container != MANAGER_NAME {
let _ = write!(
binds,
@ -1142,13 +1133,12 @@ fn set_nspawn_flags(
std::fs::create_dir_all(&config_dir).with_context(|| format!("create {config_dir}"))?;
let _ = write!(binds, " --bind-ro={config_dir}:/agents/{agent_name}/config");
// Per-agent socket subdir (#784 phase 2 step 2b). Bind-mounts
// `/run/hive-agent/<name>/` into the container at the same
// path so the harness's `HIVE_WEB_SOCKET` bind has a stable
// location both sides can see. Sub-agents only — the
// manager's UI is served at `/` via the c0re dashboard
// upstream, not via `/agent/<name>/`, so it never needs the
// per-agent socket dir.
// Per-agent socket subdir. Bind-mounts `/run/hive-agent/<name>/`
// into the container at the same path so the harness's
// `HIVE_WEB_SOCKET` bind has a stable location both sides can
// see. Sub-agents only — the manager's UI is served at `/`
// via the c0re dashboard upstream, not via `/agent/<name>/`,
// so it never needs the per-agent socket dir.
//
// Bind-mounting the SUBDIR (not the socket file) is mandatory:
// the harness's `bind_unix` helper unlinks any stale socket
@ -1161,8 +1151,8 @@ fn set_nspawn_flags(
//
// Per-agent dir (rather than a shared `/run/hive-agent/`
// mount) means the agent's container only sees its own
// subdir — never siblings' (mara on #800: "agents can only
// access their own sockets").
// subdir — never siblings'. See `docs/gateway.md::Per-agent
// unix-socket upstream`.
//
// mkdir source defensively: nspawn refuses to start when the
// bind source is missing, and on a fresh host `/run/hive-agent/`

81
hive-c0re/src/meta.rs
View file

@ -94,19 +94,19 @@ pub async fn sync_agents(
// Reconcile topology.json against the live agent set — adds
// entries for newly-spawned agents (default: manager as parent,
// manager itself as root) and drops removed agents. Operator
// overrides via the write API (#361 follow-up) are preserved
// because reconcile only fills in missing entries. Idempotent;
// when nothing changed the file isn't touched.
// overrides via the write API are preserved because reconcile
// only fills in missing entries. Idempotent; when nothing changed
// the file isn't touched.
let agent_names: Vec<String> = agents.iter().map(|a| a.name.clone()).collect();
let topology_changed = crate::topology::reconcile(&agent_names)
.with_context(|| format!("reconcile {}", crate::topology::topology_path().display()))?;
// Refresh /var/lib/hyperhive/agent-ports.json so the hive-gateway
// nginx sees the new agent set (#15 / #740). The file is the
// single source of truth for which agents the gateway proxies to,
// since the gateway container lives in system config and can't be
// rebuilt from meta-flake events. Atomic write (tmp + rename) so
// a partial write never trips the gateway's read.
// nginx sees the new agent set. The file is the single source of
// truth for which agents the gateway proxies to, since the
// gateway container lives in system config and can't be rebuilt
// from meta-flake events. Atomic write (tmp + rename) so a
// partial write never trips the gateway's read.
if let Err(e) = crate::agent_ports::write(&agent_names) {
// Best-effort: a failed write doesn't block the meta-flake
// regen + container ops that follow. The gateway falls back
@ -117,10 +117,11 @@ pub async fn sync_agents(
// Refresh /var/lib/hyperhive/agent-sockets.json — sibling to the
// ports map, drives the gateway's unix-socket upstreams once
// agents opt in to `HIVE_WEB_SOCKET` (PR #800 / #784 phase 1).
// Coexists with the TCP-port map during the transition: the
// gateway picks the socket upstream when one exists, falls back
// to the TCP port otherwise. Same best-effort + non-fatal shape.
// agents opt in to `HIVE_WEB_SOCKET`. Coexists with the TCP-port
// map during the transition: the gateway picks the socket
// upstream when one exists, falls back to the TCP port otherwise.
// Same best-effort + non-fatal shape. See
// `docs/gateway.md::Per-agent unix-socket upstream`.
if let Err(e) = crate::agent_sockets::write(&agent_names) {
tracing::warn!(error = ?e, "agent_sockets::write failed (non-fatal)");
}
@ -301,7 +302,7 @@ fn render_flake(
/// of these as a top-level input in its own `flake.nix` will get a
/// `follows = "<name>"` line emitted in meta — collapsing the
/// otherwise-separate-but-identical `nixpkgs_N` nodes into a single
/// meta-level reference (#355).
/// meta-level reference.
const CANONICAL_INPUTS: &[&str] = &["nixpkgs", "nixpkgs-unstable"];
/// Env vars hive-c0re forwards from its own systemd unit env into every
@ -391,10 +392,8 @@ where
// resolves transitively to hyperhive's pin. One channel decision
// in the whole tree, no second source to drift.
//
// Per mara via triage on #619: "all nixpkgs follow the one
// hyperhive was deployed with — if not, that's what we should
// fix." This is the fix (flip from the concrete-url shape
// #619 originally shipped with).
// All nixpkgs follow the one hyperhive was deployed with —
// anything else would drift.
//
// Operators who want to slide the whole swarm onto a different
// channel do it at the host level via
@ -402,9 +401,9 @@ where
// makes hyperhive's nixpkgs = the host's nixpkgs and cascades
// through to every agent.
//
// closes #317 invariant still satisfied — `nixpkgs` is still a
// single canonical name in the meta tree, it just resolves
// through hyperhive instead of being its own root input.
// `nixpkgs` is still a single canonical name in the meta tree,
// it just resolves through hyperhive instead of being its own
// root input.
let _ = writeln!(out, " hyperhive.url = \"{hyperhive_flake}\";");
out.push_str(" nixpkgs.follows = \"hyperhive/nixpkgs\";\n");
out.push_str(" nixpkgs-unstable.follows = \"hyperhive/nixpkgs-unstable\";\n");
@ -417,11 +416,11 @@ where
// For each canonical input the agent declares in its own
// `flake.nix` (detected by reading its applied `flake.lock`),
// emit `inputs.agent-<name>.inputs.<canon>.follows = "<canon>"`.
// Collapses three otherwise-separate-but-identical nixpkgs
// nodes (root + agent-bitburner's + agent-dmatrix's) into one
// (closes #355). Skipped silently for agents that don't
// declare the input — emitting follows on a non-existent
// input would error at `nix flake lock` time.
// Collapses otherwise-separate-but-identical nixpkgs nodes
// (root + every agent's own nixpkgs) into one. Skipped
// silently for agents that don't declare the input — emitting
// follows on a non-existent input would error at
// `nix flake lock` time.
for canon in lookup(&spec.name) {
let _ = writeln!(
out,
@ -452,13 +451,12 @@ where
modules = [
input.nixosModules.default
{
# Drop root (#658): the harness service inside the
# container runs as a non-root unix user named after
# the agent (`damocles`, `iris`, `hm1nd`, …). UID
# auto-assigned by NixOS per mara on #658; the per-
# agent override here is what makes `hyperhive.user.name`
# match the agent's identity instead of the harness-
# base default of `"agent"`.
# The harness service inside the container runs as a
# non-root unix user named after the agent (`damocles`,
# `iris`, `hm1nd`, …). UID auto-assigned by NixOS; the
# per-agent override here is what makes
# `hyperhive.user.name` match the agent's identity
# instead of the harness-base default of `"agent"`.
hyperhive.user.name = name;
programs.git.config.user = {
name = name;
@ -470,7 +468,7 @@ where
# shells); `systemd.globalEnvironment` is the analogue for
# systemd units so tea-login / forge-avatar-sync /
# matrix-avatar-sync etc. can read `$HYPERHIVE_STATE_DIR`
# without each service having to redeclare it (#604).
# without each service having to redeclare it.
environment.variables = {
HIVE_LABEL = name;
HYPERHIVE_STATE_DIR = "/agents/${name}/state";
@ -506,9 +504,9 @@ where
// → in-container accessors fall back to None / defaults gracefully.
//
// - HIVE_FORGE_URL: agents poll this for Forgejo notifications.
// - HYPERHIVE_HIVE_DOMAIN: machine-readable hive DNS (#589).
// - HYPERHIVE_HIVE_DOMAIN: machine-readable hive DNS.
// - HYPERHIVE_HIVE_NAME / HYPERHIVE_SWARM_NAME: human display
// names for hive + swarm (#701).
// names for hive + swarm.
for (var, val) in forwarded_env_vars() {
let escaped = val.replace('\\', "\\\\").replace('"', "\\\"");
let _ = writeln!(out, " {var} = \"{escaped}\";");
@ -526,9 +524,8 @@ where
);
// Pull the topology map once and look up each agent's parent. An
// empty / absent topology.json yields `parent = null` for everyone
// — equivalent to the pre-#361 status quo (every container at root).
// `meta::sync_agents` seeds the file on first run with manager as
// root + everyone else under manager.
// (every container at root). `meta::sync_agents` seeds the file
// on first run with manager as root + everyone else under manager.
let topology = crate::topology::read();
for spec in agents {
let parent_attr = topology
@ -648,8 +645,8 @@ mod tests {
);
// Meta's `nixpkgs` + `nixpkgs-unstable` are aliases for
// hyperhive's sub-inputs. Single channel-pin authority:
// hyperhive's own flake.nix. Per mara via triage on #619:
// "all nixpkgs follow the one hyperhive was deployed with."
// hyperhive's own flake.nix. All nixpkgs follow the one
// hyperhive was deployed with.
assert!(
out.contains("nixpkgs.follows = \"hyperhive/nixpkgs\""),
"missing nixpkgs follows alias:\n{out}"
@ -659,8 +656,8 @@ mod tests {
"missing nixpkgs-unstable follows alias:\n{out}"
);
// And conversely: no literal channel ref baked into meta. If
// this fails, someone reintroduced a hardcoded ref — see
// #619 follow-up rationale for why that drifts.
// this fails, someone reintroduced a hardcoded ref — would
// drift away from hyperhive's pin.
assert!(
!out.contains("nixpkgs.url ="),
"no literal `nixpkgs.url` should be emitted (hyperhive owns the pin):\n{out}"