diff --git a/Cargo.lock b/Cargo.lock index 446ea995..2e4e59c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,6 +1029,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -1381,6 +1387,7 @@ dependencies = [ "hive-sh4re", "libc", "listenfd", + "petgraph", "problem_details", "reqwest", "rusqlite", @@ -2554,6 +2561,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "phf" version = "0.11.3" diff --git a/Cargo.toml b/Cargo.toml index 128ba120..873c0434 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ reqwest = { version = "0.12", default-features = false, features = [ "json", "rustls-tls", ] } +petgraph = { version = "0.8", default-features = false, features = ["std"] } matrix-sdk = { version = "0.14", default-features = false, features = [ "rustls-tls", "sqlite", diff --git a/docs/approvals.md b/docs/approvals.md index b0274283..b730db34 100644 --- a/docs/approvals.md +++ b/docs/approvals.md @@ -336,50 +336,49 @@ approval id to retry. Because tags are first-class git objects, rejected and failed trees stay browsable forever — `git log --tags` in the applied repo is the audit trail. -### Dispatch via `rebuild_queue` +### Dispatch via the job queue Long-running approval work — `ApplyCommit`, `UpdateMetaInputs`, `Spawn` — no longer runs inline inside `actions::approve`. Instead -the approval handler enqueues a `QueueEntry` into the global -`rebuild_queue`: +the approval handler submits a DAG to the global job queue +(`docs/coordinator.md::Job queue`): -| `ApprovalKind` | `QueueKind` queued | `QueueSource` | +| `ApprovalKind` | DAG submitted | source | |---|---|---| -| `ApplyCommit` | `Rebuild` | `Approval` | -| `MergeConfigPr` | `Rebuild` | `Approval` | -| `UpdateMetaInputs` | `MetaUpdate` | `Approval` | -| `Spawn` | `Spawn` | `Approval` | +| `ApplyCommit` | `rebuild` (single opaque `ApprovalDeploy` node) | `approval` | +| `MergeConfigPr` | `rebuild` (single opaque `ApprovalDeploy` node) | `approval` | +| `UpdateMetaInputs` | `meta_update` (`MetaLock` + rebuild fan-out) | `approval` | +| `Spawn` | `spawn` (`Create → WriteDropin → Reconcile`) | `approval` | | `InitConfig` | — runs inline (sub-second git seed) | — | | `SchedulePrompt` | — runs inline (single sqlite insert) | — | -Each queue entry carries the originating `approval_id` so the -worker can re-fetch the approval row when it dispatches, run the -kind-specific pipeline (`run_approval_apply_commit` / -`run_approval_merge_config_pr` / `run_approval_update_meta_inputs` / -`run_approval_spawn`), and -fire the matching `HelperEvent::*` on completion via -`finish_approval`. +The DAG carries the originating `approval_id`. The `ApprovalDeploy` +node runs the kind-specific pipeline (`run_approval_apply_commit` / +`run_approval_merge_config_pr` — the two-phase meta deploy stays +inside `actions.rs`) and fires the matching `HelperEvent::*` via +`finish_approval` itself; `Spawn` and `UpdateMetaInputs` DAGs resolve +through `actions::resolve_approval_dag` when the DAG settles terminal +(a spawn additionally runs the post-spawn forge bookkeeping there). Two visible consequences: - **Operator dashboard**: after clicking APPR0VE the work-in-progress - shows up on the *rebuild queue* card (`POST /api/state.rebuild_queue` + shows up on the *rebuild queue* card (`/api/state.rebuild_queue` + live `rebuild_queue_changed` events), not on the approvals panel (which already moved the row to "approved"). A long meta-update - cascade renders as a parent entry with one child per per-agent - rebuild — see `docs/web-ui.md` for the layout. -- **Cancellation**: the dashboard's *× cancel* button on a `Queued` - entry calls `POST /api/rebuild-queue/{id}/cancel`, which flips the - entry to `Cancelled` before the worker dispatches it. Returns - `{"cancelled": true}` on success, `{"cancelled": false}` if the - entry already left `Queued` (running / done / failed) — terminal - states can't be retroactively rewritten. + cascade renders as a parent DAG with one child rebuild per affected + agent — see `docs/web-ui.md` for the layout. +- **Cancellation**: the dashboard's *× cancel* button on a still-queued + DAG calls `POST /api/rebuild-queue/{id}/cancel`, which flips it to + `Cancelled` before any node runs (and fails the approval row instead + of leaving it dangling). Returns `{"cancelled": true}` on success, + `{"cancelled": false}` once any node started — terminal states can't + be retroactively rewritten. -`QueueSource::Approval` carries the `approval_id` so a tail-end -build failure surfaces back as a failed approval row, not just a -silent queue entry. `QueueSource::Manual` (dashboard ↻ R3BU1LD) -and `QueueSource::AutoUpdate` (boot-time sweep) use the same -queue but skip the approval row plumbing. +The `approval` source + `approval_id` mean a tail-end build failure +surfaces back as a failed approval row, not just a silent queue +entry. `manual` (dashboard ↻ R3BU1LD) and `auto_update` (boot +reconcile) DAGs use the same queue but skip the approval plumbing. ### Forge mirror @@ -414,7 +413,7 @@ tool group can therefore edit, commit, and submit changes for any of its direct children directly inside its container at `/agents//config/`. Agents holding the `can_manage_top_level_agents` topology role -(defined as `ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS` in `hive-c0re/src/topology.rs`) +(defined as `ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS` in `hive-c0re/src/agent_config/topology.rs`) get additional host-side bind mounts via `set_nspawn_flags`: - `/var/lib/hyperhive/agents/` → `/agents/` (RW) — all top-level diff --git a/docs/coordinator.md b/docs/coordinator.md index 539aa159..4c83785a 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -6,102 +6,191 @@ and `docs/persistence.md`. --- -## Rebuild queue +## Job queue -Every long-running container/meta operation (rebuild, meta-update, first-spawn) -goes through the global rebuild queue (`hive-c0re/src/rebuild_queue.rs`). A single -background worker drains it in FIFO order so two `nixos-container update` runs on -the same agent never overlap, and a fresh agent rebuild never races a meta-update's -lock bump. +Every container/meta operation (rebuild, meta-update, first-spawn, power +changes) is submitted to the global job-DAG queue (`hive-c0re/src/job_queue/`) +as a **DAG of primitive nodes**. One scheduler task drives all DAGs; +concurrency comes from the resource classes below, not from multiple workers. +The old special cases — the graceful-stop watcher thread, the deferred-start +fast-lane follow-up, the meta-update cascade pre-enqueue — are all just DAG +*shapes* now. -### Why one queue +### Two levels: DAG and node -Before the rebuild queue landed, four independent call paths could fire -`auto_update::rebuild_agent` concurrently: +The **DAG** is the unit of dedup / cancel / approval-resolution and the +dashboard group; the **node** is the unit of scheduling / execution / +build-log / step label. Deps are intra-DAG edges only (`AfterOk` by default: +the dep must succeed, a failed/cancelled dep cancels the dependent — +cancel-downstream). Cross-DAG ordering comes from the per-agent lease + dedup, +never from edges between DAGs. Submit-time validation (petgraph `toposort`) +rejects cyclic specs outright, fixing the old queue's "circular dep silently +deadlocks" caveat. -- Dashboard manual rebuild button -- `update-all` / `meta-update` cascade -- Approval handler (apply-commit / spawn) -- Startup auto-update sweep +### Node inventory (primitives) -Nothing serialised them. `nix-daemon` serialises the actual store ops, but the rest -of `rebuild_agent` (token sync, kick, rescan, lock-bump emit) interleaved -unpredictably. The single-worker queue gives operators a visible, ordered runway and -lets the UI render "what's about to happen" instead of "something might be happening -somewhere." +Nix-heavy — hold one of the `buildSlots` permits for the node's duration: -### Queue kinds +| Node | Wraps | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `Prebuild` | meta `sync_agents` + optional per-agent relock + `lifecycle::prebuild_toplevel` — build the toplevel out-of-band while the container keeps serving | +| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped) + the post-rebuild bookkeeping tail (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan) | +| `Create` | first-spawn provisioning + `nixos-container create` (atomic build+create) | +| `MetaLock` | meta flake lock bump (`lock_update` / boot-sweep `lock_update_hyperhive`, commit fused — see below); fans out child `Rebuild` DAGs on completion | +| `ApprovalDeploy` | the opaque apply-commit / merge-config-PR pipeline (see _Approvals_ below) | -| Kind | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `Rebuild` | Single-agent rebuild. Covers manual, approval-driven, auto-update, and meta-update cascade variants — all funnel through the same path. The start-after-rebuild is **deferred to a fast-lane `Start` follow-up** (`parent_id` = this entry) so the build lane is freed as soon as the profile-swap finishes instead of waiting out the container boot — see _Deferred start_ under the rebuild path below. | -| `MetaUpdate` | `nix flake update` on the meta flake. The worker runs the lock bump itself, then enqueues a cascade of `Rebuild` entries with `parent_id` set to the meta-update's id. | -| `Spawn` | First-deploy of a new agent (approval-driven). Same serialisation as `Rebuild` from the operator's POV. | -| `Destroy` | For future use (`destroy --purge` does real I/O). Variant exists so the wire shape doesn't change later; not currently routed through the queue. | -| `Restart` | Stop + start a container without touching config (~5-10s). Routed through the queue so it serialises against in-flight rebuilds for the same agent — prevents a restart racing a rebuild mid-flight. Sources: dashboard ↺ button, the `restart` MCP tool. | -| `PermChange` | Write a tool-group or capability change to the shared JSON file (`tool-groups.json` / `capabilities.json`), then rebuild the agent so the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes effect. Serialising the file write through the queue prevents concurrent dashboard batch-apply actions from racing on the shared file. After a successful file write, emits `CapabilitiesChanged` or `ToolGroupsChanged` SSE snapshot so the P3RM1SS10NS tab updates live. | -| `GracefulStop` | Quiesce then stop a container (the `?graceful=true` path on `/api/kill/`). Signals the harness (its next `Recv` returns `GracefulStop` — the inbound fence — so it runs a stop-checkpoint turn that flushes durable `/state`, then takes the normal post-turn compaction path if it crossed the watermark, then exits) and **immediately releases the build lane**, spawning a detached watcher that holds the `Stopping` transient across the drain (bounded by a 3-min timeout → hard-stop fallback) and then enqueues a fast-lane `Stop` (`parent_id` = this entry) for the actual `nixos-container stop`. Net: a whole-hive graceful stop signals every agent up front, drains overlap, and only the container teardowns serialise (on the fast lane). Queued so the signal can't race an in-flight rebuild for the same agent. | +Cheap — no build slot: -**Intentionally not queued** (sub-second ops): the _hard_ `start`, `stop`, `kill` via the direct API paths. (A _graceful_ stop is the `GracefulStop` kind above — it takes a checkpoint turn, so it rides the queue.) The queue's fast lane does carry `Start` / `Stop` kinds, but only as **follow-ups** other entries enqueue for themselves — the graceful-stop teardown and the deferred start-after-rebuild — so the container op groups under its parent entry on the dashboard. +| Node | Behavior | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop | +| `StopForUpdate` | mechanical `nixos-container stop` for the profile swap; never touches `wanted`; noop if already stopped | +| `Signal` | set the graceful fence + kick, so the harness runs one stop-checkpoint turn | +| `Drain` | await the harness clearing the fence, bounded by the 3-min graceful-stop timeout; resolves ok either way | +| `WriteDropin` | `set_nspawn_flags` + `set_resource_limits` + daemon-reload | +| `WritePermFile` | commit `tool-groups.json` / `capabilities.json` (single git commit under `META_LOCK`) + emit the P3RM1SS10NS snapshots | -### Dedup +There is deliberately **no `GitCommit` node**: `meta.rs` fuses each mutation +with its commit under its internal `META_LOCK` mutex, so a standalone commit +node would open a dirty-working-tree window between nodes. -Enqueueing `(kind, agent)` that already has a `Queued` entry returns the existing -entry's id and appends the new request as an "also requested by …" line. Running -entries do not dedup — a re-queue during a run is legitimate (something changed -since the current run started). +Two further layers protect the meta repo across *windows* that span multiple +`META_LOCK` acquisitions — above all the approval deploy's prepare→finalize +span, which keeps a bumped `flake.lock` **staged uncommitted** for the whole +container build: -### Sources +- **The deploy-window gate** (`meta::exclusive()`): every executor that + mutates the meta repo (`Prebuild`'s sync+relock, `MetaLock`, + `WritePermFile`, `Create`'s agent registration, and `ApprovalDeploy` for + its whole span) holds this async mutex for its mutation span, so no commit + can land inside another node's staged window. `Prebuild` drops it before + the long toplevel build (store reads only), preserving `buildSlots > 1` + concurrency. +- **Path-limited commits**: the targeted meta committers (perm files, + topology, lock bumps, finalize) commit `-- ` with path-scoped + dirty checks, so even a non-queue caller (boot migration, destroy's + `sync_agents`) can never sweep someone else's staged content into its + commit. -| Source | Meaning | -| -------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| `Manual` | Operator clicked rebuild / update-all / meta-update on the dashboard, or any other direct human action (CLI, an agent MCP tool). | -| `AutoUpdate` | Legacy startup-sweep source (flat, no parent). Replaced by `StartupSweep` for new boots. | -| `StartupSweep` | Child of a `StartupSweep` parent entry; boot-time per-agent rebuild with the sweep as the visual group header. | -| `Approval` | Triggered by an operator-approved `ApprovalKind::{Spawn, ApplyCommit}`. | +### Every operation as a DAG -### Cascade parent tracking +```text +rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a) +graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a) +restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a) +start(a): [wanted=Up] Reconcile(a) (stale rev ⇒ upgraded to rebuild) +stop(a): [wanted=Offline] Reconcile(a) +spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a) +perm-change(a): WritePermFile(a) → «rebuild subgraph» +meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected agent» +boot: (if any rev marker stale) MetaLock(hyperhive) → «fan-out rebuild»; + plus Reconcile(a) for every drifted agent +``` -`MetaUpdate` and `StartupSweep` entries fan out `Rebuild` children, each carrying -`parent_id = `. The dashboard groups children under their parent -in the queue panel so the operator sees the whole cascade as a tree, -not a flat list. +Notable collapses: -### Step labels +- **`rebuild` is one uniform shape** — no `was_running` branch. + `StopForUpdate` noops when already down; the tail `Reconcile` auto-noops the + start when `wanted = Offline` (a rebuild of a deliberately-stopped agent + leaves it stopped). +- **The swap-failure recovery-start is structural**: `Reconcile` deps on + `Swap` with the one `AfterAny` edge in the system — it runs after `Swap` + terminal ok *or* fail, bringing a wanted-up agent back on its old config. +- **Deferred start is automatic**: `Reconcile` holds no build slot, so the + next DAG's `Prebuild` starts as soon as `Swap` frees the slot. +- **Graceful stop needs no watcher thread**: `Signal`/`Drain` are cheap, so a + whole-hive graceful stop fires every agent's signal immediately and all + drains overlap; each DAG's tail `Reconcile` does the actual stop. +- **The meta-update cascade fans out on completion**: `MetaLock`'s executor + computes the affected agent set after the bump lands and appends child + `rebuild` DAGs (`parent_id` set, `relock = false` so the children don't + revert the bump). A failed bump fans out nothing — no cancel-children dance. -Each queue entry has a mutable `step: Option` field that the worker updates -as it progresses through lifecycle phases (`"nix build"`, `"nixos-container stop"`, -`"nixos-container update"`, `"nixos-container start"`). The dashboard polls -`/api/state` and renders the current step beneath the running entry so the operator -can see which phase is taking time. +### Desired-state (spec vs status) -### Dependency tracking +Per-agent power *intent* — `wanted: Up | Offline` — is durable as the +`agent_power` table in the coordinator DB (`hive-c0re/src/stores/power.rs`). +`container_view` remains the observed *status*; `Reconcile` nodes converge the +two. Setting `wanted` is never a queued node: the submit layer +(`job_queue/submit.rs`) writes the row synchronously, then submits the DAG +whose `Reconcile` reads the fresh value — rapid toggles are last-writer-wins. +Power toggles never commit to the meta repo. Every operator power surface — +dashboard buttons, the MCP tools, and `hivectl stop/start/restart/kill` — +rides the queue through that submit layer, so intent, lease serialization, +and crash-watch suppression can't drift per surface; the only direct starts +left are the root-agent bootstrap and infra containers (no lease, no +harness). Cancelling a still-queued power DAG reverts `wanted` to the +observed state — a cancel means "don't do it", not "do it later". Agents +without a row are seeded from observed state on first touch (running ⇒ +`Up`); destroy removes the row. -Each entry carries a `depends_on: Vec` field. The worker skips entries whose -dependencies are not yet resolved — a dependency is resolved when its id is either -in the queue as a terminal entry (`Done` / `Failed` / `Cancelled`) or no longer in -the queue at all (evicted by `trim_history`, which only evicts terminals). +The admin-socket responses carry the submitted DAG ids; `hivectl` polls +`HostRequest::QueueDag` (~1s) and prints a progress line per DAG — roll-up +glyph, template, agent, node chain with the running node's step label — so +CLI verbs block until their jobs finish (`--no-wait` opts out; failures exit +non-zero). Fan-out children joining a polled parent show up in the same +loop. -Use cases: +### Scheduler semantics -- Chain a `Rebuild` after an explicit prerequisite step without coupling them through - the `parent_id` cascade mechanism. -- Sequence a `PermChange` + `Rebuild` pair where the rebuild must not start until the - perm-file write commits (already handled by the single-worker FIFO today, but - `depends_on` allows explicit cross-kind sequencing when parallel workers are added). +A node is **ready** when it's `Queued`, every dep is satisfied, and its +resources are free. Resources: -`depends_on` is part of the dedup key: two entries with the same `(kind, agent, -parent_id, inputs, approval_id)` but different dep sets are treated as distinct work. +1. **Build slots** — `services.hyperhive.c0re.buildSlots` permits (default 1), + held by nix-heavy nodes for the node's duration. +2. **Per-agent lifecycle lease** — DAG-scoped: acquired at the DAG's first + container-affecting node (`StopForUpdate`, `Swap`, `Signal`, `Drain`, + `Reconcile`, `WriteDropin`, `Create`, `ApprovalDeploy`), held until the DAG + is terminal, so two lifecycle DAGs for one agent never interleave their + container ops. **Lease-exempt**: `Prebuild`, `MetaLock`, `WritePermFile` — + they touch the store / meta, not the running container, which is exactly + why a stop can land while another DAG's prebuild is still building. -**Worker re-notification**: the worker drains `take_next()` in a tight loop after -each entry finishes. When a dep entry transitions to terminal, the loop re-evaluates -the queue immediately, so downstream entries are unblocked with no extra wakeup. No -additional `notify_one()` call is needed. +Among simultaneously-ready nodes competing for a resource, DAG-submit order +wins (FIFO) so bulk operations drain predictably. The scheduler also owns the +DAG-lifetime transient guard (dashboard pill + crash-watch suppression), +created on lease acquisition and dropped when the DAG settles terminal. -**Circular-dep caveat**: if A depends on B and B depends on A, neither entry ever -becomes runnable — the worker skips both indefinitely with no error. Callers must -ensure acyclic dep graphs. Cycle detection is deferred to a future iteration (when -parallel workers make a stuck queue more visible). +The queue is in-memory only and lost on hive-c0re restart — deliberate: +desired state is re-derived at boot from the DB + rev markers (see _Boot +reconcile_), so there is no durable-recovery machinery to go wrong. + +### Dedup, cancel, history + +Dedup at **DAG granularity**: a repeat submit against a DAG whose roll-up is +still `Queued` with the same `(template, agent, parent_id, approval_id)` — +plus `inputs` for meta-updates and the perm-type discriminant for perm +changes — returns the existing id and appends an "also requested by …" line. +`parent_id` in the key keeps a cascade child from collapsing into a +standalone or sweep rebuild. Running/terminal DAGs never dedup. + +Cancel only applies to still-fully-queued DAGs (an in-flight nix build isn't +interruptible); `cancel_children` cancels a parent's still-queued child DAGs. +Roll-up state: `Failed` if any node failed, else `Running` / `Queued` / +`Cancelled` / `Done`. The snapshot retains the 5 most recent terminal DAGs +per template. + +### Approvals + +`ApplyCommit` / `MergeConfigPr` approvals ride as single-node +`ApprovalDeploy` DAGs: the two-phase `prepare_deploy` / `finalize_deploy` / +`abort_deploy` meta orchestration stays inside `actions.rs` in v1 +(deliberately not modeled as scheduler nodes) and resolves the approval +itself. `Spawn` and `UpdateMetaInputs` approvals map onto the ordinary +`spawn` / `meta-update` shapes; the scheduler fires +`actions::resolve_approval_dag` exactly once when such a DAG settles +terminal (including cancelled-while-queued, which fails the approval instead +of dangling it). + +### Wire shape + +`RebuildQueueChanged { seq, queue: [DagView…] }` (event name kept). Each +`DagView` carries the old entry-level fields (`id`, `kind` = template string, +roll-up `state`, `agent`, `source`, `parent_id`, `reason`, timestamps, +`inputs`, `approval_id`) plus `nodes: [NodeView…]` — per-node `kind`, `deps`, +`state`, `step`, `build_log_id`, timestamps, `error`. Step labels and build +logs are **per-node**; the dashboard renders the node chain on each queue +card and keys the live-log panel off the running node. --- @@ -115,31 +204,31 @@ render. --- -## Auto-update sweep +## Boot reconcile -On startup, `auto_update.rs` rebuilds containers that actually need it. Two skip -rules keep boot-time work minimal: +On startup, `auto_update::run` classifies every agent by rev freshness (the +per-agent `.{name}.hyperhive-rev` marker under `/var/lib/hyperhive/applied/` +vs the current flake path) and persisted `wanted` intent, then: -1. **Stopped containers** are deferred: the startup sweep enqueues nothing for them. - When the operator later starts a stopped container (via the dashboard or the - `start` MCP tool), both `run_start` (queue path) and `handle_start` (socket path) - check the rev marker first — if it's stale, the start is silently upgraded to a - full rebuild+start so the container runs current nix derivations. +1. **Config path** — when *any* marker is stale, submit one `StartupSweep` + DAG: a `MetaLock` (hyperhive input bump, non-fatal) that fans out `Rebuild` + children for the stale agents whose `wanted = Up` (topology-sorted, parents + first). Stale but wanted-offline agents get no boot-time nix work — their + rebuild happens on their next start (the start submit path upgrades a + stale start to rebuild+start), which is also why the lock bump runs even + when every stale agent is offline: those later start-upgrades must build + against the bumped lock. Each child rebuild's tail `Reconcile` brings the + agent (back) up, covering both the running-stale and stopped-but-wanted-up + cases. -2. **Running containers with a matching rev marker** are skipped: if the per-agent - `.{name}.hyperhive-rev` file under `/var/lib/hyperhive/applied/` already holds - the current flake rev, no nix work is needed and the entry is omitted entirely. +2. **Power path** — every agent whose observed state drifted from `wanted` + gets a plain `Reconcile` DAG (`kind = reconcile`, source `auto_update`). -`auto_update::run` enqueues a single `StartupSweep` parent entry (`kind = -startup_sweep`, `agent = "hyperhive"`) followed by per-agent `Rebuild` children -for the agents that do need rebuilding (`source = startup_sweep`, `parent_id = -sweep_id`). The sweep description records the rebuild / deferred / skipped counts -so the operator can see at a glance how much work the boot triggered. The child -rebuilds drain sequentially through the queue; the dashboard renders them nested -under the parent. - -Before the sweep-grouping change, each boot enqueued flat `Rebuild` entries with -`source = AutoUpdate` and no parent — visible but ungrouped. +Booting with no config change performs **no meta commit** — only reconciles. +The sweep reason records the rebuild / deferred / up-to-date counts so the +operator sees at a glance how much work the boot triggered. Agents without an +`agent_power` row are seeded from observed state during classification (the +one-time migration; thereafter the DB is authoritative). ## Meta flake @@ -157,8 +246,13 @@ Key operations: `RequestApplyCommit` path so a failed `nixos-container update` leaves no orphan commit in meta. Prepare writes the new lock without committing; finalize commits with the deploy message; abort restores the lock. -- **`lock_update_hyperhive`** — one-shot for the auto-update path: bumps the - `hyperhive` input lock, commits, cascades agent rebuilds. +- **`lock_update_hyperhive`** — one-shot for the boot-reconcile path (the + sweep DAG's `MetaLock` node): bumps the `hyperhive` input lock and commits; + the scheduler fans out the agent rebuilds on completion. + +Every public `meta.rs` operation takes the module's internal `META_LOCK` +mutex, so concurrent job-queue nodes (and the approval deploy pipeline) never +race on the repo's `.git/index.lock`. --- @@ -179,34 +273,26 @@ new profile and skips the in-container `switch-to-configuration`. The subsequent `start` then applies both the new profile and any `EXTRA_NSPAWN_FLAGS` changes in one go, rather than the double-bounce a live `update` would trigger. -Sequence for a running container: +Sequence for a rebuild DAG (each step is its own queue node): -1. `prebuild_toplevel` — build the new `system.build.toplevel` **before** stopping. - The container keeps serving the previous generation while eval + fetch + build - happen out-of-band. `nixos-container update` then finds the result cached and - skips straight to the profile-swap. Build failures surface here, before the - running container is touched. -2. `nixos-container stop` — bring the container down. -3. `nixos-container update --flake meta#` — profile-swap (near-instant after - the prebuild). -4. `nixos-container start` — boot into the new generation; the in-container - activation script transitions old → new. +1. `Prebuild` — build the new `system.build.toplevel` **before** stopping. + The container keeps serving the previous generation while eval + fetch + + build happen out-of-band. `nixos-container update` then finds the result + cached and skips straight to the profile-swap. Build failures surface + here, before the running container is touched. (Runs even for a stopped + container — same total nix work, one uniform DAG shape.) +2. `StopForUpdate` — bring the container down (noop when already stopped). +3. `Swap` — `nixos-container update --flake meta#` profile-swap + (near-instant after the prebuild). +4. `Reconcile` — boot into the new generation when `wanted = Up`; the + in-container activation script transitions old → new. Holds no build + slot, so the next DAG's `Prebuild` overlaps the container boot — the old + "deferred start" split, now structural. -If the container is already stopped, step 1 is skipped (no downtime to shave — no -point evaluating the flake twice). - -**Deferred start (queue-dispatched rebuilds):** step 4 can take a while -(container boot), and holding the serialized build lane through it delays the -next queued rebuild's nix build for no reason. Queue-dispatched rebuilds -therefore pass `defer_start` — `rebuild_no_meta` skips the start and returns -`true`, and `rebuild_agent` enqueues a fast-lane `Start` entry instead -(`parent_id` = the rebuild entry, so the dashboard groups the follow-up under -it — the same split the graceful-stop path uses for its container stop). The -build-lane entry completes when the profile-swap finishes; a start failure -surfaces on the `Start` entry, which runs with the cold-start fallback. Direct -callers (admin-socket CLI, root-agent migration nudge, the apply-commit deploy -flow which verifies the agent comes back up before finalizing) keep the start -inline. +The approval apply-commit pipeline still drives `lifecycle::rebuild_no_meta` +(the fused stop/update/start path with an inline start) inside its +`ApprovalDeploy` node, because it verifies the agent comes back up before +finalizing the deploy tag. ### Cold-start fallback @@ -218,9 +304,8 @@ half-started at that point. Fallback: `stop` (graceful SIGTERM drain) → `kill` (SIGKILL any lingering processes) → `start` (clean cold-start, no generation transition, new activation runs cleanly). Both errors are preserved and surfaced if the cold-start also fails. The fallback -lives in `lifecycle::start_with_fallback`, shared by the inline start-after-rebuild -path and the queue's fast-lane `Start` handler (which the deferred -start-after-rebuild rides). +lives in `lifecycle::start_with_fallback`, shared by the apply-commit deploy's +inline start and every `Reconcile` node's start action. ### Spawn path (new container) @@ -241,9 +326,18 @@ or the flake root directly — none of which exist in the rendered meta flake. ## Host-level resource + performance options -Three `services.hyperhive.c0re.*` options tune container resource -limits and first-spawn latency. All three apply uniformly to every -agent container. +A handful of `services.hyperhive.c0re.*` options tune container resource +limits, build parallelism, and first-spawn latency. + +### Build slots + +`buildSlots` (default `1`) sets how many nix-heavy job-queue nodes +(prebuilds, profile swaps, first-spawn creates, meta lock bumps) run +concurrently. The default serializes all heavy nix work like the pre-DAG +rebuild queue did; raise it on hosts with the cores/RAM to build several +agent toplevels at once. Per-agent correctness is independent of the count — +each agent's container-affecting ops serialize on its lifecycle lease +regardless. ### Container resource limits diff --git a/docs/persistence.md b/docs/persistence.md index af900b95..c9aa1506 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -2,12 +2,12 @@ Where state lives, what survives what, and how it's bounded. -## Three sqlite databases +## Sqlite databases ### `/var/lib/hyperhive/db/broker.sqlite` (host) -Six tables, all in one file — four queues plus the schedule -header/targets split: +Seven tables, all in one file — four queues, the schedule +header/targets split, and the per-agent power-intent registry: - `messages` — every inter-agent / operator-bound message. `sender / recipient / body / sent_at / delivered_at / acked_at / @@ -44,6 +44,17 @@ header/targets split: last_fired_at_unix / last_result`. `ON DELETE CASCADE` from `scheduled_prompts(id)` — requires `PRAGMA foreign_keys = ON` per connection (set at open). +- `agent_power` — one tiny row per agent: `agent PK / wanted (up | + offline) / updated_at` — the durable power *intent* behind the job + queue's desired-state reconciliation + (`docs/coordinator.md::Job queue`; owner: `hive-c0re/src/stores/power.rs`). + Written synchronously by every operator/agent power action + (dashboard start/stop, `hivectl stop`, the MCP kill/start tools, + spawn approval); read by `Reconcile` nodes and the boot reconcile. + Intent survives hive-c0re restarts — in-flight queue work + deliberately does not. Agents without a row are seeded from + observed state on first touch (running ⇒ `up`); destroy deletes + the row. Retention: @@ -63,6 +74,8 @@ Retention: (`cancel_schedule` MCP / dashboard ✗) which tombstones via `cancelled_at_unix`, then `reap_cancelled` drops the row on the next worker pass. +- `agent_power` rows live until the agent is destroyed (one row per + agent — nothing to vacuum). ### `/harness/hyperhive-events.sqlite` (per agent) diff --git a/docs/tools/hivectl-cli.md b/docs/tools/hivectl-cli.md index ace96cb9..6c3fe955 100644 --- a/docs/tools/hivectl-cli.md +++ b/docs/tools/hivectl-cli.md @@ -271,8 +271,8 @@ Agent container management. Requires the hive-c0re daemon to be running (connect ###### **Subcommands:** * `list` — Show all managed agents with their status (running / needs-login / needs-update) and technical state (deployed sha, parent, pending reminders). The host roster overview; reuses the dashboard's per-agent aggregation. Requires the daemon running -* `restart` — Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config -* `restart-all` — Stop and restart ALL managed agent containers in sequence. Iterates the live container list and restarts each one. Any per-agent failure is reported at the end rather than stopping mid-run, so all containers get a restart attempt +* `restart` — Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config. Rides the job queue (serialized against in-flight rebuilds for the same agent); waits with live progress unless `--no-wait` +* `restart-all` — Restart ALL managed agent containers via one restart DAG each — unrelated agents overlap, each serializes on its own lease. Waits for the whole set with live progress unless `--no-wait` @@ -290,21 +290,29 @@ Show all managed agents with their status (running / needs-login / needs-update) ## `hivectl agents restart` -Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config +Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config. Rides the job queue (serialized against in-flight rebuilds for the same agent); waits with live progress unless `--no-wait` -**Usage:** `hivectl agents restart ` +**Usage:** `hivectl agents restart [OPTIONS] ` ###### **Arguments:** * `` — Agent name (e.g. `damocles`, `ruth`) +###### **Options:** + +* `--no-wait` — Return immediately after the restart DAG is queued + ## `hivectl agents restart-all` -Stop and restart ALL managed agent containers in sequence. Iterates the live container list and restarts each one. Any per-agent failure is reported at the end rather than stopping mid-run, so all containers get a restart attempt +Restart ALL managed agent containers via one restart DAG each — unrelated agents overlap, each serializes on its own lease. Waits for the whole set with live progress unless `--no-wait` -**Usage:** `hivectl agents restart-all` +**Usage:** `hivectl agents restart-all [OPTIONS]` + +###### **Options:** + +* `--no-wait` — Return immediately after the restart DAGs are queued @@ -407,7 +415,8 @@ Stop containers hive-wide in one operator action. Bare `hivectl stop` stops **ev * `--forge` — The forge container (`hive-forge`) * `--gateway` — The gateway container (`hive-gateway`) * `--matrix` — The matrix container (`hive-matrix`) -* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent is enqueued as a `GracefulStop` on the rebuild queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). Applies to agents only +* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent gets a graceful-stop DAG on the job queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). All drains overlap. Applies to agents only +* `--no-wait` — Return immediately after the stop DAGs are queued instead of waiting for them with live per-node progress @@ -425,6 +434,7 @@ Start containers hive-wide — the inverse of `hivectl stop`. Bare `hivectl star * `--forge` — The forge container (`hive-forge`) * `--gateway` — The gateway container (`hive-gateway`) * `--matrix` — The matrix container (`hive-matrix`) +* `--no-wait` — Return immediately after the start DAGs are queued instead of waiting for them with live per-node progress diff --git a/frontend/packages/dashboard/src/builds.js b/frontend/packages/dashboard/src/builds.js index 37f4a1be..bb881809 100644 --- a/frontend/packages/dashboard/src/builds.js +++ b/frontend/packages/dashboard/src/builds.js @@ -131,6 +131,7 @@ const QUEUE_KIND_GLYPH = { graceful_stop: '⏹', start: '▶', stop: '■', + reconcile: '⇄', }; const QUEUE_STATE_GLYPH = { queued: '⏸', @@ -139,11 +140,33 @@ const QUEUE_STATE_GLYPH = { failed: '✖', cancelled: '⊘', }; +// Short display labels for the per-DAG node kinds (the primitive ops). +const NODE_KIND_LABEL = { + prebuild: 'prebuild', + stop_for_update: 'stop', + swap: 'swap', + create: 'create', + meta_lock: 'meta lock', + reconcile: 'reconcile', + signal: 'signal', + drain: 'drain', + write_dropin: 'dropin', + write_perm_file: 'perm file', + approval_deploy: 'deploy', +}; + +// The currently-running node of a DAG (per-node `step` labels + build +// logs live on nodes now; the DAG's `state` is a roll-up). +function runningNode(entry) { + return (entry.nodes || []).find((n) => n.state === 'running') || null; +} +function firstFailedNode(entry) { + return (entry.nodes || []).find((n) => n.state === 'failed') || null; +} function rebuildQueueEntryFingerprint(entry, isChild) { return JSON.stringify({ state: entry.state, - step: entry.step, kind: entry.kind, agent: entry.agent, source: entry.source, @@ -151,8 +174,7 @@ function rebuildQueueEntryFingerprint(entry, isChild) { enqueued_at: entry.enqueued_at, finished_at: entry.finished_at, reason: entry.reason, - error: entry.error, - build_log_id: entry.build_log_id, + nodes: (entry.nodes || []).map((n) => [n.kind, n.state, n.step, n.build_log_id, n.error]), isChild, }); } @@ -257,22 +279,39 @@ function renderQueueEntry(entry, _byId, isChild) { const r = entry.reason.split('\n')[0]; li.append(' ', el('span', { class: 'rqe-reason', title: entry.reason }, '— ' + truncate(r, 60))); } - if (entry.step) { - li.append(el('div', { class: 'rqe-step' }, '↳ ' + entry.step)); + // Per-node chain: every DAG node in dependency order with its own + // state glyph, live step label, and build-log link. This is the + // node-aware render that makes queue jumps / partial progress + // visible (e.g. reconcile running while swap failed). + const nodes = entry.nodes || []; + if (nodes.length) { + const chain = el('div', { class: 'rqe-nodes' }); + nodes.forEach((n, i) => { + if (i > 0) chain.append(el('span', { class: 'rqe-node-arrow' }, ' → ')); + const chip = el('span', { + class: 'rqe-node rqe-node-' + n.state, + title: n.kind + ' · ' + n.state + (n.error ? ' — ' + n.error : ''), + }, + (QUEUE_STATE_GLYPH[n.state] || '?') + ' ' + (NODE_KIND_LABEL[n.kind] || n.kind)); + chain.append(chip); + if (n.build_log_id != null) { + chain.append(el('a', { + class: 'rqe-log-link rqe-node-log', + href: '/builds.html?id=' + n.build_log_id + '#buildlogs', + target: '_blank', + title: 'view build log #' + n.build_log_id, + }, '⎙')); + } + }); + li.append(chain); } - if (entry.build_log_id != null) { - li.append( - ' ', - el('a', { - class: 'rqe-log-link', - href: '/builds.html?id=' + entry.build_log_id + '#buildlogs', - target: '_blank', - title: 'view build log #' + entry.build_log_id, - }, 'logs →'), - ); + const running = runningNode(entry); + if (running && running.step) { + li.append(el('div', { class: 'rqe-step' }, '↳ ' + running.step)); } - if (entry.error) { - li.append(el('pre', { class: 'rqe-error', title: entry.error }, truncate(entry.error, 200))); + const failed = firstFailedNode(entry); + if (failed && failed.error) { + li.append(el('pre', { class: 'rqe-error', title: failed.error }, truncate(failed.error, 200))); } if (entry.state === 'queued') { const cancelForm = el('form', { @@ -296,11 +335,12 @@ function renderQueueEntry(entry, _byId, isChild) { } // ─── running-rebuild live log ───────────────────────────────────────────────── -// One persistent live-log panel for the currently-running rebuild (the queue -// runs one build at a time). Keyed to the running entry's build_log_id and -// kept in its own container (#rebuild-live-log) so the queue's row re-render -// — which rebuilds rows as the build step advances — never tears down the open -// SSE stream. +// One persistent live-log panel for the first currently-building node +// (build logs are per-node now; with buildSlots = 1 at most one nix +// build runs at a time). Keyed to that node's build_log_id and kept in +// its own container (#rebuild-live-log) so the queue's row re-render — +// which rebuilds rows as the build step advances — never tears down the +// open SSE stream. let liveLogEs = null; let liveLogId = null; let liveLogDone = false; @@ -310,23 +350,36 @@ function closeLiveLogStream() { if (liveLogEs) { liveLogEs.close(); liveLogEs = null; } } +// First (dag, node) pair with a running node that opened a build log. +function findLiveBuild(queue) { + for (const e of queue || []) { + if (e.state !== 'running') continue; + for (const n of e.nodes || []) { + if (n.state === 'running' && n.build_log_id != null) { + return { entry: e, node: n }; + } + } + } + return null; +} + function renderRebuildLiveLog(queue) { const root = $('rebuild-live-log'); if (!root) return; - const running = (queue || []).find( - (e) => e.state === 'running' && e.build_log_id != null); + const live = findLiveBuild(queue); - if (!running) { + if (!live) { closeLiveLogStream(); liveLogId = null; liveLogDone = false; if (!root.hidden) { root.hidden = true; root.replaceChildren(); } return; } - if (running.build_log_id === liveLogId && (liveLogEs || liveLogDone)) return; + const { entry: running, node: liveNode } = live; + if (liveNode.build_log_id === liveLogId && (liveLogEs || liveLogDone)) return; closeLiveLogStream(); - liveLogId = running.build_log_id; + liveLogId = liveNode.build_log_id; liveLogDone = false; root.hidden = false; root.replaceChildren(); @@ -351,17 +404,18 @@ function renderRebuildLiveLog(queue) { toggle, ' ', el('span', { class: 'rebuild-live-log-title' }, 'live build log — '), el('code', { class: 'rqe-agent' }, running.agent), - ' ', el('span', { class: 'rqe-kind' }, running.kind || 'rebuild'), + ' ', el('span', { class: 'rqe-kind' }, + (running.kind || 'rebuild') + ' · ' + (NODE_KIND_LABEL[liveNode.kind] || liveNode.kind)), ' ', badge, ' ', el('a', { class: 'rebuild-live-log-raw', - href: '/api/build-logs/id/' + running.build_log_id + '/raw', - download: 'build-log-' + running.build_log_id + '.txt', + href: '/api/build-logs/id/' + liveNode.build_log_id + '/raw', + download: 'build-log-' + liveNode.build_log_id + '.txt', }, '↓ raw'), ); root.append(header, pre); - liveLogEs = openBuildLogStream(running.build_log_id, pre, { + liveLogEs = openBuildLogStream(liveNode.build_log_id, pre, { onDone: (status) => { liveLogDone = true; badge.className = 'rebuild-live-log-badge ' + (status === 'ok' ? 'rll-ok' : 'rll-fail'); diff --git a/frontend/packages/dashboard/src/swarm.js b/frontend/packages/dashboard/src/swarm.js index ce2014d0..27c2a4a9 100644 --- a/frontend/packages/dashboard/src/swarm.js +++ b/frontend/packages/dashboard/src/swarm.js @@ -811,6 +811,7 @@ export function renderContainers(s) { : op.kind === 'start' ? 'starting' : op.kind === 'stop' ? 'stopping' : op.kind === 'graceful_stop' ? 'stopping' + : op.kind === 'reconcile' ? 'reconciling' : 'rebuilding') : (op.kind === 'meta_update' ? 'meta-update queued' : op.kind === 'destroy' ? 'destroy queued' @@ -818,6 +819,7 @@ export function renderContainers(s) { : op.kind === 'start' ? 'start queued' : op.kind === 'stop' ? 'stop queued' : op.kind === 'graceful_stop' ? 'stop queued' + : op.kind === 'reconcile' ? 'reconcile queued' : 'rebuild queued'))); const opRunning = transientKind != null || (op != null && op.state === 'running'); diff --git a/frontend/packages/dashboard/src/system-sections.css b/frontend/packages/dashboard/src/system-sections.css index 480eb0c9..019b9216 100644 --- a/frontend/packages/dashboard/src/system-sections.css +++ b/frontend/packages/dashboard/src/system-sections.css @@ -134,6 +134,33 @@ .rqe-source-approval { color: var(--green); border-color: var(--green); } .rqe-when { color: var(--muted); font-size: 0.85em; } .rqe-reason { color: var(--muted); font-size: 0.85em; flex: 1 1 auto; } +/* Per-node DAG chain: one chip per primitive node, dependency order. */ +.rqe-nodes { + flex-basis: 100%; + margin: 0.15em 0 0 1.8em; + font-size: 0.85em; + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 0.15em; +} +.rqe-node { + padding: 0.05em 0.45em; + border: 1px solid var(--border); + border-radius: 0.7em; + color: var(--muted); + white-space: nowrap; +} +.rqe-node-running { + color: var(--purple); + border-color: var(--purple); + animation: badge-pulse 1.6s ease-in-out infinite; +} +.rqe-node-done { color: var(--green); border-color: color-mix(in srgb, var(--green) 45%, transparent); } +.rqe-node-failed { color: var(--red); border-color: var(--red); } +.rqe-node-cancelled { opacity: 0.55; text-decoration: line-through; } +.rqe-node-arrow { color: var(--muted); } +.rqe-node-log { margin-left: 0.1em; text-decoration: none; } .rqe-step { flex-basis: 100%; margin: 0.1em 0 0 1.8em; diff --git a/hive-ag3nt/src/events.rs b/hive-ag3nt/src/events.rs index 2aa32d7f..f55494be 100644 --- a/hive-ag3nt/src/events.rs +++ b/hive-ag3nt/src/events.rs @@ -13,6 +13,7 @@ use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use hive_claude::TokenUsage; +use hive_sh4re::wire_time::now_unix; use rusqlite::{Connection, params}; use serde::{Deserialize, Serialize}; use tokio::sync::broadcast; @@ -213,14 +214,6 @@ pub fn write_forge_cursor( write_harness_json(&v); } -fn now_unix() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/hive-ag3nt/src/mcp/args.rs b/hive-ag3nt/src/mcp/args.rs new file mode 100644 index 00000000..007504cf --- /dev/null +++ b/hive-ag3nt/src/mcp/args.rs @@ -0,0 +1,359 @@ +//! Argument structs for the MCP tools: `serde::Deserialize` + +//! `schemars::JsonSchema` derives whose field doc-comments become the +//! parameter descriptions claude sees in each tool's input schema. + +use rmcp::schemars; + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct SendArgs { + /// Logical agent name to deliver the message to (e.g. `"manager"`, + /// `"alice"`, or the literal `"operator"` for the dashboard's T4LK box). + pub to: String, + /// Message body. Plain text; the broker doesn't parse it. + pub body: String, + /// Optional broker row-id of the message this is a reply to. Lets + /// the dashboard render conversation threads. Pass the `id` from the + /// `DeliveredMessage` you're responding to; omit for new threads. + /// Silently ignored if the id is unknown or out of retention. + #[serde(default)] + pub in_reply_to: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct RecvArgs { + /// How long to long-poll for the FIRST message before returning + /// the empty marker. Capped at 60s server-side. Default (None) + /// is 30s. Useful when an agent wants to park its turn waiting + /// for any new work — pick a longer wait to coalesce bursts. + #[serde(default)] + pub wait_seconds: Option, + /// Maximum number of messages to pop in this round-trip. Default + /// (None) is 1 (single-message behaviour — exactly what you want + /// when you're called to drive a turn off the first wake). Pass + /// a higher value (capped at 5 server-side) when you've been + /// told the inbox has more queued (the wake prompt mentions + /// pending count) and want to drain everything in one tool call. + /// Once the long-poll wakes up, the call drains up to `max` in + /// total before returning — no extra round-trip needed. + #[serde(default)] + pub max: Option, +} + +/// MCP tool args for `ack_until`. +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct AckUntilArgs { + /// Highest broker message id to mark handled: every inbox message + /// with `id <= up_to` (ids show as `[msg #]` in recv output) + /// is acked in one sweep. Pass the highest id you've actually + /// seen/triaged — anything above it stays queued for later turns. + pub up_to: i64, +} + +/// MCP tool args for `remind`. Exactly one of `delay_seconds` or +/// `at_unix_timestamp` must be set; both / neither is a tool-side error. +/// Hides the tagged `ReminderTiming` enum behind a flatter schema so the +/// model picks one field instead of building `{"timing_type": "in_seconds", +/// "seconds": 60}` shaped objects. +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct RemindArgs { + /// Body that lands in your inbox when the reminder fires (sender + /// will appear as `reminder`). Soft cap at 4 KiB inline — anything + /// larger gets auto-persisted to a file under + /// `/agents//state/reminders/auto-.md` and the inbox + /// message becomes a short pointer. Pass `file_path` if you want + /// to control the destination yourself. + pub message: String, + /// Fire `delay_seconds` from now (relative). Set this OR + /// `at_unix_timestamp`, not both. + #[serde(default)] + pub delay_seconds: Option, + /// Fire at this absolute unix timestamp (seconds since epoch). Set + /// this OR `delay_seconds`, not both. + #[serde(default)] + pub at_unix_timestamp: Option, + /// Optional path to a file the scheduler should reference instead of + /// inlining a long `message`. Use this for large payloads (research + /// notes, file lists, intermediate state). Path must be reachable from + /// the agent's container — typically under `/agents//state/`. + #[serde(default)] + pub file_path: Option, +} + +// ----------------------------------------------------------------------------- +// Privileged tool arg types (lifecycle, approvals, scheduling, diagnostics) +// ----------------------------------------------------------------------------- + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct RequestInitConfigArgs { + /// New sub-agent name (≤9 chars). Queues an `InitConfig` approval; on + /// approval hive-c0re seeds the proposed config repo at + /// `/agents//config/agent.nix` with the default template. After + /// the approval the manager edits + commits the config and calls + /// `request_apply_commit` to pin the customised sha for the container's + /// first build. + pub name: String, + /// Optional description shown on the dashboard approval card. + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct KillArgs { + /// Sub-agent name (without the `h-` container prefix). + pub name: String, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct SetStatusArgs { + /// Status text to display on the dashboard card. Pass an empty string to clear. + pub text: String, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct CreateRepoArgs { + /// Repo name — a single segment of letters, digits, `-`, `_`, `.` + /// (no leading `-`/`.`). The repo is created as `agents/`. + pub repo: String, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct GetAgentMetaArgs { + /// Logical name of the agent to query (e.g. `"iris"`, `"manager"`). + /// Omit to query your own identity + status — replaces the + /// previous `whoami` self-introspection tool. + #[serde(default)] + pub name: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct StartArgs { + /// Sub-agent name (without the `h-` container prefix). + pub name: String, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct RestartArgs { + /// Sub-agent name (without the `h-` container prefix). + pub name: String, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct UpdateArgs { + /// Sub-agent name (without the `h-` container prefix). + pub name: String, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct AskArgs { + /// The question to surface. + pub question: String, + /// Optional fixed-choice answers. The dashboard renders these as + /// chips alongside a free-text fallback ("Other…") so the operator + /// is never trapped by an incomplete list; peer-agent recipients + /// see the list in their inbox event and can return any string. + #[serde(default)] + pub options: Vec, + /// When true, options are rendered as checkboxes — the answerer + /// can pick any subset. The answer comes back as a single string + /// with selections joined by ", ". Ignored when `options` is empty. + #[serde(default)] + pub multi: bool, + /// Optional auto-cancel after `ttl_seconds` (capped server-side at + /// 6 hours). On expiry the question resolves with answer + /// `[expired]` and the asker receives the usual + /// `question_answered` system event (with `answerer: + /// "ttl-watchdog"`). `None` (default) = wait indefinitely. + #[serde(default)] + pub ttl_seconds: Option, + /// Recipient. Omit (or pass `"operator"`) to ask the human + /// operator via the dashboard. Pass another agent's logical name + /// to ask that peer — they receive a `question_asked` event in + /// their inbox and answer via `mcp__hyperhive__answer`. + #[serde(default)] + pub to: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct AnswerArgs { + /// Id of the question being answered — comes from the + /// `question_asked` event in your inbox. + pub id: i64, + /// Free-text answer body. Soft-capped at 4 KiB by the same + /// `MESSAGE_MAX_BYTES` limit as `send`; keep it short or write the + /// detail to a file and pass a path. + pub answer: String, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct CancelLooseEndArgs { + /// Which kind of thread to cancel — `"question"` for an open + /// `ask` that's still waiting on an answer, `"reminder"` for a + /// scheduled `remind` that hasn't fired yet. Use the `kind` + /// field straight off the `get_loose_ends` row. + pub kind: String, + /// Row id from the matching `get_loose_ends` entry (or the + /// `question_queued` reply when you submitted it). + pub id: i64, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct AgentGetLooseEndsArgs { + /// Whose loose ends to list. Omit (or `null`) for your own. You may + /// also pass a direct child agent's name without any extra capability. + /// Pass any other agent name to inspect their threads — requires the + /// `query_agent_state` capability; without it the request is rejected + /// with an error. The `"*"` hive-wide value is not available on the + /// agent socket. + #[serde(default)] + pub agent: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct RequestApplyCommitArgs { + /// Logical agent name whose config repo the commit lives in. + pub agent: String, + /// Commit sha (full or short, 7-40 hex chars) in that agent's + /// proposed config repo. Must be a sha — a branch or tag name + /// (e.g. `main`) is rejected; the approval pins the exact commit. + pub commit_ref: String, + /// Optional description shown on the dashboard approval card so the + /// operator knows what the change does without opening the diff. + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct UpdateMetaInputsArgs { + /// Flake input names to update (e.g. `["bitburner-agent", "nixpkgs"]`). + /// Pass an empty list to update ALL inputs. + #[serde(default)] + pub inputs: Vec, + /// Optional description shown on the dashboard approval card. + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct RequestSchedulePromptArgs { + /// Recipient agents — one schedule fires to many inboxes at the + /// scheduled time. `operator` is a legitimate target (mara: "we + /// want to get rid of the manager special case so yes manager + /// can be recipient" — the operator slot follows the same rule). + pub targets: Vec, + /// Message body delivered to each target's inbox at fire time. + /// Same size budget as `send` bodies. + pub body: String, + /// Absolute unix timestamp (seconds) for the FIRST fire. For + /// recurring schedules the worker re-arms in + /// `interval_seconds` steps from this point on. + pub first_fire_at_unix: i64, + /// `None` / absent = one-shot. `Some(n > 0)` = recurring every + /// `n` seconds. The worker clamps catch-up so a long downtime + /// fires ONCE on resume (skipped-cycle count surfaces in the + /// per-target `last_result`), not N delayed pulses in a row. + #[serde(default)] + pub interval_seconds: Option, + /// Optional description shown on the dashboard approval card + + /// preserved on the schedule row for later operator reference. + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct FireScheduleNowArgs { + /// Schedule id to fire out of band. Get this from a prior + /// `list_schedules` call or the approval-resolved event for + /// the originating `request_schedule_prompt`. + pub id: i64, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct CancelScheduleArgs { + /// Schedule id from a prior `list_schedules` call or the + /// approval-resolved event for a `request_schedule_prompt`. + pub id: i64, + /// Optional target list. `None` / empty = cancel the entire + /// schedule. `Some(["alice", "bob"])` = cancel just those + /// recipients (the schedule keeps firing for any remaining + /// active targets, and auto-cancels its parent row when every + /// target is gone). + #[serde(default)] + pub targets: Option>, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct EditScheduleArgs { + /// Schedule id from a prior `list_schedules` call or the + /// approval-resolved event for a `request_schedule_prompt`. + pub id: i64, + /// New body text. Omit to keep the existing one. + #[serde(default)] + pub body: Option, + /// New description. Omit to keep the existing one. (To CLEAR + /// the description, use the dashboard PATCH endpoint + /// directly — the agent surface intentionally keeps the args + /// flat / non-nullable to dodge the doubly-wrapped Option + /// schemars quirk; clearing fields is rare and operator-side.) + #[serde(default)] + pub description: Option, + /// Recurring interval in seconds. Omit to keep the existing + /// cadence; pass an explicit value to set a new one. Toggling + /// recurring↔one-shot (clearing the interval) is operator-only + /// for the same reason as `description` above. + #[serde(default)] + pub interval_seconds: Option, + /// New absolute unix timestamp for the next fire. Omit to + /// leave the schedule on its current cadence. + #[serde(default)] + pub next_fire_at_unix: Option, + /// Names of new targets to add. Replace-on-conflict: re-adding + /// a previously cancelled target resets its history (operator + /// intent on re-add = "this target is active again"). + #[serde(default)] + pub targets_add: Option>, + /// Names of targets to cancel. Tombstones preserve per-target + /// audit; when no active targets remain the schedule + /// auto-cancels. + #[serde(default)] + pub targets_remove: Option>, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct GetLogsArgs { + /// Logical agent name to fetch logs for (e.g. `gui`, `iris`). + /// hive-c0re maps it to the underlying machine name (`h-gui`) + /// itself — pass the plain agent name, not the `h-` form. + pub agent: String, + /// How many journal lines to return (default: 50, max: 500). + #[serde(default)] + pub lines: Option, +} + +/// Arguments for `get_host_journal` (capability-gated: `read_host_journal`). +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct GetHostJournalArgs { + /// Systemd unit to filter (e.g. `hive-c0re.service`). Omit for all units. + #[serde(default)] + pub unit: Option, + /// nspawn machine name verbatim (e.g. `h-iris`). Omit for host journal. + /// Agent containers use the `h-` prefix (e.g. `h-iris`); infrastructure + /// containers use their full name (e.g. `hive-ci`, `hive-forge`, + /// `hive-matrix`, `hive-gateway`). + #[serde(default)] + pub container: Option, + /// Number of lines to return (default 30, max 100). + #[serde(default)] + pub lines: Option, + /// Minimum syslog priority level. + #[serde(default)] + pub priority: Option, + /// Regex to match against log message fields (journalctl --grep). + #[serde(default)] + pub grep: Option, + /// Show entries on or newer than this timestamp (e.g. `-1h`). + #[serde(default)] + pub since: Option, + /// Show entries on or older than this timestamp. + #[serde(default)] + pub until: Option, +} diff --git a/hive-ag3nt/src/mcp.rs b/hive-ag3nt/src/mcp/mod.rs similarity index 59% rename from hive-ag3nt/src/mcp.rs rename to hive-ag3nt/src/mcp/mod.rs index 2f7fd95b..7bd00c49 100644 --- a/hive-ag3nt/src/mcp.rs +++ b/hive-ag3nt/src/mcp/mod.rs @@ -18,12 +18,31 @@ use std::path::PathBuf; use anyhow::Result; use rmcp::{ - ServerHandler, ServiceExt, handler::server::wrapper::Parameters, schemars, tool, tool_handler, + ServerHandler, ServiceExt, handler::server::wrapper::Parameters, tool, tool_handler, tool_router, transport::stdio, }; use crate::client; +mod args; +mod render; + +pub use args::{ + AckUntilArgs, AgentGetLooseEndsArgs, AnswerArgs, AskArgs, CancelLooseEndArgs, + CancelScheduleArgs, CreateRepoArgs, EditScheduleArgs, FireScheduleNowArgs, GetAgentMetaArgs, + GetHostJournalArgs, GetLogsArgs, KillArgs, RecvArgs, RemindArgs, RequestApplyCommitArgs, + RequestInitConfigArgs, RequestSchedulePromptArgs, RestartArgs, SendArgs, SetStatusArgs, + StartArgs, UpdateArgs, UpdateMetaInputsArgs, +}; +pub use render::{ + IDLE_WAIT_HINT, REDELIVERY_HINT, annotate_retries, format_ack, format_agent_meta, format_recv, +}; + +use render::{ + format_matrix_summary, loose_end_kind_label, matrix_unread_summary, parse_loose_end_kind, + render_loose_ends, reply_err, +}; + /// Write (or remove) the status file in the agent's own `state/` directory. /// Called by `AgentServer::set_status` for both agent and manager flavors /// before dispatching the wire `SetStatus` request (which only triggers a @@ -69,406 +88,6 @@ fn write_status_file(text: &str) -> Result<(), String> { result.map_err(|e| format!("set_status write failed: {e}")) } -/// Render the three identical failure arms every data-returning tool handler -/// repeats: a broker `Err` → `"{tool} failed: {m}"`, an unexpected `Ok` variant -/// → `"{tool} unexpected response: …"`, and a transport error → `"{tool} -/// transport error: …"`. Handlers match their own happy-path variant and route -/// everything else here via a catch-all arm (`other => reply_err(other, tool)`), -/// so the triplet lives in exactly one place. -fn reply_err(resp: Result, tool: &str) -> String { - match resp { - Ok(hive_sh4re::Response::Err { message }) => format!("{tool} failed: {message}"), - Ok(other) => format!("{tool} unexpected response: {other:?}"), - Err(e) => format!("{tool} transport error: {e:#}"), - } -} - -/// Format helper for "send-like" tools (anything that expects an `Ok`). -/// `tool` and `ok_msg` only appear in the result string; they don't change -/// behavior. -#[must_use] -pub fn format_ack( - resp: Result, - tool: &str, - ok_msg: String, -) -> String { - match resp { - Ok(hive_sh4re::Response::Ok) => ok_msg, - other => reply_err(other, tool), - } -} - -/// Format helper for `recv`: renders zero, one, or many popped -/// messages. Empty list collapses to "(empty)" so claude doesn't go -/// hunting for content; when `waited` is set (the call parked on a -/// long-poll that timed out) the empty result also carries -/// [`IDLE_WAIT_HINT`] nudging the model toward other work. A single -/// message renders as the historical `from: X\n\nbody` block (banner -/// first if `redelivered`). A multi-message batch renders with a -/// `popped N message(s):` header and `---` separators between bodies -/// so the model can tell where one ends and the next begins; -/// per-message redelivery banners included. -#[must_use] -pub fn format_recv(resp: Result, waited: bool) -> String { - match resp { - Ok(hive_sh4re::Response::Messages { messages }) => render_recv_messages(&messages, waited), - // A graceful stop is pending — the inbox is fenced. Render a single - // explicit directive (not an empty inbox, which claude's "park on recv" - // habit would long-poll again, stalling the stop-checkpoint turn until - // the drain wait times out into a hard stop) so every recv during the - // stop unmissably tells claude to flush + end. - Ok(hive_sh4re::Response::GracefulStop) => { - render_recv_messages(&[graceful_stop_message()], waited) - } - other => reply_err(other, "recv"), - } -} - -/// The synthetic single-message directive rendered for a fenced (graceful-stop) -/// inbox — see the `GracefulStop` arm of [`format_recv`]. -fn graceful_stop_message() -> hive_sh4re::DeliveredMessage { - hive_sh4re::DeliveredMessage { - from: "graceful-stop".into(), - body: "⛔ GRACEFUL STOP IN PROGRESS — the container shuts down as soon as this turn \ - ends. Flush anything worth keeping to your durable /state files, then END \ - YOUR TURN now. Do NOT call recv again: the inbox is fenced and recv will \ - only keep returning this same notice." - .into(), - id: 0, - redelivered: false, - in_reply_to: None, - } -} - -/// Render the popped-message payload of a successful `recv` (see `format_recv` -/// for the empty/single/batch shapes). -fn render_recv_messages(messages: &[hive_sh4re::DeliveredMessage], waited: bool) -> String { - use std::fmt::Write as _; - if messages.is_empty() { - return if waited { - format!("(empty){IDLE_WAIT_HINT}") - } else { - "(empty)".to_owned() - }; - } - if messages.len() == 1 { - let m = &messages[0]; - let banner = if m.redelivered { REDELIVERY_HINT } else { "" }; - return format!("{banner}{}from: {}\n\n{}", msg_id_tag(m.id), m.from, m.body); - } - let n = messages.len(); - let mut out = format!("popped {n} message(s):\n\n"); - for (i, m) in messages.iter().enumerate() { - if i > 0 { - out.push_str("\n---\n\n"); - } - let banner = if m.redelivered { REDELIVERY_HINT } else { "" }; - let _ = write!( - out, - "{banner}{}from: {}\n\n{}", - msg_id_tag(m.id), - m.from, - m.body - ); - } - out -} - -/// `[msg #] ` marker prefixed to each recv row so the agent knows -/// what to pass to `ack_until` when bulk-triaging a backlog. Transient -/// pings carry the sentinel id 0 (in-memory only, nothing in the -/// broker to ack) and render without the marker. -fn msg_id_tag(id: i64) -> String { - if id > 0 { - format!("[msg #{id}] ") - } else { - String::new() - } -} - -/// Header prepended to message bodies that were popped by a prior -/// harness session, never acked (turn crash / OOM / restart), and -/// resurfaced by `RequeueInflight` on this session's boot. Same -/// string surfaces in the wake prompt (see the bin loops) and the -/// in-turn `recv` tool result so claude sees the warning either way. -pub const REDELIVERY_HINT: &str = "[redelivered after harness restart — may already be handled]\n"; - -/// Appended to the `recv` empty result when the agent parked on a -/// long-poll (`wait_seconds > 0`) that timed out with nothing new. -/// Nudges the model to spend the idle time on other useful work -/// instead of immediately re-blocking on `recv`. -pub const IDLE_WAIT_HINT: &str = " — nothing arrived before the wait timed out. \ -If you have other useful work (assigned issues, in-flight PRs, a docs sweep, \ -notes to update), do that now rather than immediately parking on recv again."; - -/// Inner renderer for a `Vec` already extracted from the socket -/// reply. Called by the `get_loose_ends` handler, which injects the -/// `UnreadMatrix` entry before formatting. -fn render_loose_ends(loose_ends: &[hive_sh4re::LooseEnd]) -> String { - use std::fmt::Write as _; - if loose_ends.is_empty() { - return "(no loose ends)".to_owned(); - } - let mut out = format!("{} loose end(s):\n", loose_ends.len()); - for t in loose_ends { - match t { - hive_sh4re::LooseEnd::Approval { - id, - agent, - commit_ref, - description, - age_seconds, - } => { - let desc = description - .as_deref() - .map(|d| format!(" — {d}")) - .unwrap_or_default(); - let _ = writeln!( - out, - "- approval #{id} ({agent} @ {commit_ref}, {age_seconds}s old){desc}" - ); - } - hive_sh4re::LooseEnd::Question { - id, - asker, - target, - question, - age_seconds, - } => { - let to = target.as_deref().unwrap_or("operator"); - let _ = writeln!( - out, - "- question #{id} ({asker} → {to}, {age_seconds}s old): {question}" - ); - } - hive_sh4re::LooseEnd::Reminder { - id, - owner, - message, - due_at, - age_seconds, - } => { - let _ = writeln!( - out, - "- reminder #{id} ({owner}, scheduled {age_seconds}s ago, due_at={due_at}): {message}" - ); - } - hive_sh4re::LooseEnd::PendingMessages { count } => { - let _ = writeln!( - out, - "- {count} pending inbox message(s) — drain with recv (recv(max: {count}) to batch)" - ); - } - hive_sh4re::LooseEnd::UnreadMatrix { rooms, summary } => { - let _ = write!(out, "- unread matrix messages in {rooms} room(s)"); - if summary.is_empty() { - let _ = writeln!( - out, - " — use list_rooms + read_room to view, mark_read to clear" - ); - } else { - let _ = writeln!(out, ":"); - for line in summary.lines() { - let _ = writeln!(out, " {line}"); - } - let _ = writeln!( - out, - " use list_rooms + read_room to view, mark_read to clear" - ); - } - } - } - } - out -} - -/// Per-room unread entry returned by `matrix_unread_summary`. Mirrors -/// `hive-matrix-mcp`'s `RoomUnread` but defined locally to avoid a -/// cross-crate dep on the matrix-sdk crate tree. -#[derive(Debug, serde::Deserialize)] -struct MatrixRoomUnread { - label: String, - count: u32, - last_body: Option, - last_sender: Option, -} - -/// Query the local matrix daemon for per-room unread summaries. Returns -/// `None` if the daemon socket is absent or the query fails. Best-effort: -/// agents without matrix configured are not penalised. -async fn matrix_unread_summary() -> Option> { - use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; - use tokio::net::UnixStream; - let socket = std::env::var_os("HIVE_MATRIX_SOCKET").map_or_else( - || std::path::PathBuf::from("/run/hive-matrix/socket"), - std::path::PathBuf::from, - ); - if !socket.exists() { - return None; - } - let mut stream = UnixStream::connect(&socket).await.ok()?; - stream - .write_all(b"{\"method\":\"unread_summary\"}\n") - .await - .ok()?; - let mut lines = BufReader::new(stream).lines(); - let line = lines.next_line().await.ok()??; - let val: serde_json::Value = serde_json::from_str(&line).ok()?; - // Response: {"kind":"ok","payload":[{label, count, last_body?, last_sender?}]} - let arr = val.get("payload")?.as_array()?; - serde_json::from_value(serde_json::Value::Array(arr.clone())).ok() -} - -/// Format a `Vec` into a per-room summary string. -/// Single room / single message collapses to one line; multi-room -/// expands to a bulleted list. Returns an empty string for empty input. -fn format_matrix_summary(rooms: &[MatrixRoomUnread]) -> String { - use std::fmt::Write as _; - if rooms.is_empty() { - return String::new(); - } - let mut out = String::new(); - for r in rooms { - if r.count == 1 - && let (Some(body), Some(sender)) = (&r.last_body, &r.last_sender) - { - let _ = writeln!(out, "- {}: {sender}: {body}", r.label); - continue; - } - let _ = writeln!(out, "- {}: {} unread", r.label, r.count); - } - // Remove trailing newline. - if out.ends_with('\n') { - out.pop(); - } - out -} - -/// Parse the user-facing `kind` string for `cancel_loose_end` into the -/// wire enum. Accepts a small alias set so claude doesn't have to -/// remember the exact spelling (`"q"` / `"r"` shorthand falls out -/// for free). -fn parse_loose_end_kind(raw: &str) -> Result { - match raw.trim().to_ascii_lowercase().as_str() { - "question" | "q" => Ok(hive_sh4re::CancelLooseEndKind::Question), - "reminder" | "r" => Ok(hive_sh4re::CancelLooseEndKind::Reminder), - "approval" | "a" => Ok(hive_sh4re::CancelLooseEndKind::Approval), - other => Err(format!( - "cancel_loose_end: unknown kind '{other}' \ - (expected \"question\", \"reminder\", or \"approval\")" - )), - } -} - -/// Canonical user-facing label for a `CancelLooseEndKind` — used in -/// the success ack so the caller always sees `"question"` / -/// `"reminder"` instead of whatever alias they passed in (`"q"` / -/// `"r"`). -fn loose_end_kind_label(kind: hive_sh4re::CancelLooseEndKind) -> &'static str { - match kind { - hive_sh4re::CancelLooseEndKind::Question => "question", - hive_sh4re::CancelLooseEndKind::Reminder => "reminder", - hive_sh4re::CancelLooseEndKind::Approval => "approval", - } -} - -/// Format helper for `get_agent_meta`: renders an agent's identity + -/// current status as a short human-readable block. `name`, -/// `hyperhive_rev`, and `running` are always shown; `status` only -/// appears when one is set, otherwise the line reads `status: `. -/// When `running` is false the host has already cleared `status_text` -/// (it would be a stale snapshot from before the stop) so the status -/// line is implicitly `` in that case — but the explicit -/// `running: no` line tells the caller WHY. See -/// `docs/turn-loop/mcp.md::Core tools` (`get_agent_meta`). -#[must_use] -pub fn format_agent_meta(resp: Result) -> String { - match resp { - Ok(hive_sh4re::Response::AgentMeta { - name, - running, - hyperhive_rev, - status_text, - status_set_at, - hive_name, - swarm_name, - matrix_accounts, - }) => { - let rev = hyperhive_rev.as_deref().unwrap_or(""); - let run = if running { "yes" } else { "no" }; - let mut out = format!("name: {name}\nhyperhive_rev: {rev}\nrunning: {run}"); - // Surface hive + swarm display names only when set, so - // single-hive deployments don't see noisy `` lines. - if let Some(hn) = hive_name.as_deref() { - use std::fmt::Write as _; - let _ = write!(out, "\nhive_name: {hn}"); - } - if let Some(sn) = swarm_name.as_deref() { - use std::fmt::Write as _; - let _ = write!(out, "\nswarm_name: {sn}"); - } - match status_text { - None => out.push_str("\nstatus: "), - Some(s) => { - use std::fmt::Write as _; - let age = status_set_at.and_then(|ts| { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok()? - .as_secs(); - // `ts` is a unix epoch second the agent itself - // sourced from `SystemTime` — always positive - // in normal operation. Clamp the negative - // (clock-skew) edge to 0 before the unsigned - // cast so the cast loses no real precision. - let ts_secs = u64::try_from(ts).unwrap_or(0); - let secs = now.saturating_sub(ts_secs); - Some(format_age_secs(secs)) - }); - // `write!` into the buffer instead of `push_str(&format!(…))` — - // avoids the intermediate allocation clippy::format_push_string - // flags. The infallible `String` writer makes this safe to - // `let _ =`-ignore. - match age { - Some(a) => { - let _ = write!(out, "\nstatus: {s} (set {a} ago)"); - } - None => { - let _ = write!(out, "\nstatus: {s}"); - } - } - } - } - // Matrix identities the agent can act as (the `account` arg on - // the matrix tools). Listed only when matrix is provisioned, so - // non-matrix agents don't see an empty line. - if !matrix_accounts.is_empty() { - use std::fmt::Write as _; - out.push_str("\nmatrix_accounts:"); - for acct in &matrix_accounts { - let uid = acct.user_id.as_deref().unwrap_or("?"); - let _ = write!(out, "\n {} ({uid}) on {}", acct.name, acct.homeserver); - } - } - out - } - other => reply_err(other, "get_agent_meta"), - } -} - -/// Format a duration in seconds as a human-readable age string. -fn format_age_secs(secs: u64) -> String { - if secs < 60 { - format!("{secs}s") - } else if secs < 3600 { - format!("{}m", secs / 60) - } else if secs < 86400 { - format!("{}h", secs / 3600) - } else { - format!("{}d", secs / 86400) - } -} - /// Common envelope around every MCP tool handler: pre-log → run → /// post-log. Tool results stay clean — the inbox-status hint lives in /// the wake prompt + UI header, not appended here. @@ -482,100 +101,6 @@ where result } -/// Append a short note to a tool result when the underlying socket call -/// took retries to land. Lets claude distinguish "my request was wrong" -/// from "c0re flickered and the harness rode it out" — without the -/// hint, a tool result that took 30s to come back looks identical to a -/// content failure and the model would burn a turn retrying it. -#[must_use] -pub fn annotate_retries(mut s: String, retries: u32) -> String { - if retries > 0 { - use std::fmt::Write as _; - let suffix = if retries == 1 { "retry" } else { "retries" }; - let _ = write!( - s, - "\n\n(note: hive socket connect needed {retries} {suffix} — c0re likely \ - restarted. Your request did succeed on the final attempt; no action needed.)" - ); - } - s -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct SendArgs { - /// Logical agent name to deliver the message to (e.g. `"manager"`, - /// `"alice"`, or the literal `"operator"` for the dashboard's T4LK box). - pub to: String, - /// Message body. Plain text; the broker doesn't parse it. - pub body: String, - /// Optional broker row-id of the message this is a reply to. Lets - /// the dashboard render conversation threads. Pass the `id` from the - /// `DeliveredMessage` you're responding to; omit for new threads. - /// Silently ignored if the id is unknown or out of retention. - #[serde(default)] - pub in_reply_to: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct RecvArgs { - /// How long to long-poll for the FIRST message before returning - /// the empty marker. Capped at 60s server-side. Default (None) - /// is 30s. Useful when an agent wants to park its turn waiting - /// for any new work — pick a longer wait to coalesce bursts. - #[serde(default)] - pub wait_seconds: Option, - /// Maximum number of messages to pop in this round-trip. Default - /// (None) is 1 (single-message behaviour — exactly what you want - /// when you're called to drive a turn off the first wake). Pass - /// a higher value (capped at 5 server-side) when you've been - /// told the inbox has more queued (the wake prompt mentions - /// pending count) and want to drain everything in one tool call. - /// Once the long-poll wakes up, the call drains up to `max` in - /// total before returning — no extra round-trip needed. - #[serde(default)] - pub max: Option, -} - -/// MCP tool args for `ack_until`. -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct AckUntilArgs { - /// Highest broker message id to mark handled: every inbox message - /// with `id <= up_to` (ids show as `[msg #]` in recv output) - /// is acked in one sweep. Pass the highest id you've actually - /// seen/triaged — anything above it stays queued for later turns. - pub up_to: i64, -} - -/// MCP tool args for `remind`. Exactly one of `delay_seconds` or -/// `at_unix_timestamp` must be set; both / neither is a tool-side error. -/// Hides the tagged `ReminderTiming` enum behind a flatter schema so the -/// model picks one field instead of building `{"timing_type": "in_seconds", -/// "seconds": 60}` shaped objects. -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct RemindArgs { - /// Body that lands in your inbox when the reminder fires (sender - /// will appear as `reminder`). Soft cap at 4 KiB inline — anything - /// larger gets auto-persisted to a file under - /// `/agents//state/reminders/auto-.md` and the inbox - /// message becomes a short pointer. Pass `file_path` if you want - /// to control the destination yourself. - pub message: String, - /// Fire `delay_seconds` from now (relative). Set this OR - /// `at_unix_timestamp`, not both. - #[serde(default)] - pub delay_seconds: Option, - /// Fire at this absolute unix timestamp (seconds since epoch). Set - /// this OR `delay_seconds`, not both. - #[serde(default)] - pub at_unix_timestamp: Option, - /// Optional path to a file the scheduler should reference instead of - /// inlining a long `message`. Use this for large payloads (research - /// notes, file lists, intermediate state). Path must be reachable from - /// the agent's container — typically under `/agents//state/`. - #[serde(default)] - pub file_path: Option, -} - /// Unified MCP tool surface for both sub-agent and manager roles. /// /// `AgentRequest = ManagerRequest = Request` and `AgentResponse = @@ -1531,305 +1056,3 @@ pub async fn serve_http(socket: PathBuf, addr: std::net::SocketAddr) -> Result<( axum::serve(listener, app).await?; Ok(()) } - -// ----------------------------------------------------------------------------- -// Privileged tool arg types (lifecycle, approvals, scheduling, diagnostics) -// ----------------------------------------------------------------------------- - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct RequestInitConfigArgs { - /// New sub-agent name (≤9 chars). Queues an `InitConfig` approval; on - /// approval hive-c0re seeds the proposed config repo at - /// `/agents//config/agent.nix` with the default template. After - /// the approval the manager edits + commits the config and calls - /// `request_apply_commit` to pin the customised sha for the container's - /// first build. - pub name: String, - /// Optional description shown on the dashboard approval card. - #[serde(default)] - pub description: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct KillArgs { - /// Sub-agent name (without the `h-` container prefix). - pub name: String, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct SetStatusArgs { - /// Status text to display on the dashboard card. Pass an empty string to clear. - pub text: String, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct CreateRepoArgs { - /// Repo name — a single segment of letters, digits, `-`, `_`, `.` - /// (no leading `-`/`.`). The repo is created as `agents/`. - pub repo: String, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct GetAgentMetaArgs { - /// Logical name of the agent to query (e.g. `"iris"`, `"manager"`). - /// Omit to query your own identity + status — replaces the - /// previous `whoami` self-introspection tool. - #[serde(default)] - pub name: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct StartArgs { - /// Sub-agent name (without the `h-` container prefix). - pub name: String, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct RestartArgs { - /// Sub-agent name (without the `h-` container prefix). - pub name: String, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct UpdateArgs { - /// Sub-agent name (without the `h-` container prefix). - pub name: String, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct AskArgs { - /// The question to surface. - pub question: String, - /// Optional fixed-choice answers. The dashboard renders these as - /// chips alongside a free-text fallback ("Other…") so the operator - /// is never trapped by an incomplete list; peer-agent recipients - /// see the list in their inbox event and can return any string. - #[serde(default)] - pub options: Vec, - /// When true, options are rendered as checkboxes — the answerer - /// can pick any subset. The answer comes back as a single string - /// with selections joined by ", ". Ignored when `options` is empty. - #[serde(default)] - pub multi: bool, - /// Optional auto-cancel after `ttl_seconds` (capped server-side at - /// 6 hours). On expiry the question resolves with answer - /// `[expired]` and the asker receives the usual - /// `question_answered` system event (with `answerer: - /// "ttl-watchdog"`). `None` (default) = wait indefinitely. - #[serde(default)] - pub ttl_seconds: Option, - /// Recipient. Omit (or pass `"operator"`) to ask the human - /// operator via the dashboard. Pass another agent's logical name - /// to ask that peer — they receive a `question_asked` event in - /// their inbox and answer via `mcp__hyperhive__answer`. - #[serde(default)] - pub to: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct AnswerArgs { - /// Id of the question being answered — comes from the - /// `question_asked` event in your inbox. - pub id: i64, - /// Free-text answer body. Soft-capped at 4 KiB by the same - /// `MESSAGE_MAX_BYTES` limit as `send`; keep it short or write the - /// detail to a file and pass a path. - pub answer: String, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct CancelLooseEndArgs { - /// Which kind of thread to cancel — `"question"` for an open - /// `ask` that's still waiting on an answer, `"reminder"` for a - /// scheduled `remind` that hasn't fired yet. Use the `kind` - /// field straight off the `get_loose_ends` row. - pub kind: String, - /// Row id from the matching `get_loose_ends` entry (or the - /// `question_queued` reply when you submitted it). - pub id: i64, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct AgentGetLooseEndsArgs { - /// Whose loose ends to list. Omit (or `null`) for your own. You may - /// also pass a direct child agent's name without any extra capability. - /// Pass any other agent name to inspect their threads — requires the - /// `query_agent_state` capability; without it the request is rejected - /// with an error. The `"*"` hive-wide value is not available on the - /// agent socket. - #[serde(default)] - pub agent: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct RequestApplyCommitArgs { - /// Logical agent name whose config repo the commit lives in. - pub agent: String, - /// Commit sha (full or short, 7-40 hex chars) in that agent's - /// proposed config repo. Must be a sha — a branch or tag name - /// (e.g. `main`) is rejected; the approval pins the exact commit. - pub commit_ref: String, - /// Optional description shown on the dashboard approval card so the - /// operator knows what the change does without opening the diff. - #[serde(default)] - pub description: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct UpdateMetaInputsArgs { - /// Flake input names to update (e.g. `["bitburner-agent", "nixpkgs"]`). - /// Pass an empty list to update ALL inputs. - #[serde(default)] - pub inputs: Vec, - /// Optional description shown on the dashboard approval card. - #[serde(default)] - pub description: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct RequestSchedulePromptArgs { - /// Recipient agents — one schedule fires to many inboxes at the - /// scheduled time. `operator` is a legitimate target (mara: "we - /// want to get rid of the manager special case so yes manager - /// can be recipient" — the operator slot follows the same rule). - pub targets: Vec, - /// Message body delivered to each target's inbox at fire time. - /// Same size budget as `send` bodies. - pub body: String, - /// Absolute unix timestamp (seconds) for the FIRST fire. For - /// recurring schedules the worker re-arms in - /// `interval_seconds` steps from this point on. - pub first_fire_at_unix: i64, - /// `None` / absent = one-shot. `Some(n > 0)` = recurring every - /// `n` seconds. The worker clamps catch-up so a long downtime - /// fires ONCE on resume (skipped-cycle count surfaces in the - /// per-target `last_result`), not N delayed pulses in a row. - #[serde(default)] - pub interval_seconds: Option, - /// Optional description shown on the dashboard approval card + - /// preserved on the schedule row for later operator reference. - #[serde(default)] - pub description: Option, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct FireScheduleNowArgs { - /// Schedule id to fire out of band. Get this from a prior - /// `list_schedules` call or the approval-resolved event for - /// the originating `request_schedule_prompt`. - pub id: i64, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct CancelScheduleArgs { - /// Schedule id from a prior `list_schedules` call or the - /// approval-resolved event for a `request_schedule_prompt`. - pub id: i64, - /// Optional target list. `None` / empty = cancel the entire - /// schedule. `Some(["alice", "bob"])` = cancel just those - /// recipients (the schedule keeps firing for any remaining - /// active targets, and auto-cancels its parent row when every - /// target is gone). - #[serde(default)] - pub targets: Option>, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct EditScheduleArgs { - /// Schedule id from a prior `list_schedules` call or the - /// approval-resolved event for a `request_schedule_prompt`. - pub id: i64, - /// New body text. Omit to keep the existing one. - #[serde(default)] - pub body: Option, - /// New description. Omit to keep the existing one. (To CLEAR - /// the description, use the dashboard PATCH endpoint - /// directly — the agent surface intentionally keeps the args - /// flat / non-nullable to dodge the doubly-wrapped Option - /// schemars quirk; clearing fields is rare and operator-side.) - #[serde(default)] - pub description: Option, - /// Recurring interval in seconds. Omit to keep the existing - /// cadence; pass an explicit value to set a new one. Toggling - /// recurring↔one-shot (clearing the interval) is operator-only - /// for the same reason as `description` above. - #[serde(default)] - pub interval_seconds: Option, - /// New absolute unix timestamp for the next fire. Omit to - /// leave the schedule on its current cadence. - #[serde(default)] - pub next_fire_at_unix: Option, - /// Names of new targets to add. Replace-on-conflict: re-adding - /// a previously cancelled target resets its history (operator - /// intent on re-add = "this target is active again"). - #[serde(default)] - pub targets_add: Option>, - /// Names of targets to cancel. Tombstones preserve per-target - /// audit; when no active targets remain the schedule - /// auto-cancels. - #[serde(default)] - pub targets_remove: Option>, -} - -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct GetLogsArgs { - /// Logical agent name to fetch logs for (e.g. `gui`, `iris`). - /// hive-c0re maps it to the underlying machine name (`h-gui`) - /// itself — pass the plain agent name, not the `h-` form. - pub agent: String, - /// How many journal lines to return (default: 50, max: 500). - #[serde(default)] - pub lines: Option, -} - -/// Arguments for `get_host_journal` (capability-gated: `read_host_journal`). -#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] -pub struct GetHostJournalArgs { - /// Systemd unit to filter (e.g. `hive-c0re.service`). Omit for all units. - #[serde(default)] - pub unit: Option, - /// nspawn machine name verbatim (e.g. `h-iris`). Omit for host journal. - /// Agent containers use the `h-` prefix (e.g. `h-iris`); infrastructure - /// containers use their full name (e.g. `hive-ci`, `hive-forge`, - /// `hive-matrix`, `hive-gateway`). - #[serde(default)] - pub container: Option, - /// Number of lines to return (default 30, max 100). - #[serde(default)] - pub lines: Option, - /// Minimum syslog priority level. - #[serde(default)] - pub priority: Option, - /// Regex to match against log message fields (journalctl --grep). - #[serde(default)] - pub grep: Option, - /// Show entries on or newer than this timestamp (e.g. `-1h`). - #[serde(default)] - pub since: Option, - /// Show entries on or older than this timestamp. - #[serde(default)] - pub until: Option, -} -#[cfg(test)] -mod tests { - use super::{IDLE_WAIT_HINT, format_recv}; - - #[test] - fn empty_recv_after_wait_appends_idle_hint() { - let out = format_recv( - Ok(hive_sh4re::Response::Messages { messages: vec![] }), - true, - ); - assert!(out.starts_with("(empty)")); - assert!(out.contains(IDLE_WAIT_HINT)); - } - - #[test] - fn empty_recv_without_wait_has_no_hint() { - let out = format_recv( - Ok(hive_sh4re::Response::Messages { messages: vec![] }), - false, - ); - assert_eq!(out, "(empty)"); - } -} diff --git a/hive-ag3nt/src/mcp/render.rs b/hive-ag3nt/src/mcp/render.rs new file mode 100644 index 00000000..0fe22983 --- /dev/null +++ b/hive-ag3nt/src/mcp/render.rs @@ -0,0 +1,448 @@ +//! Formatting / render helpers for the MCP tool surface: ack / recv / +//! loose-end / agent-meta reply shaping plus the retry annotation. +//! Stateless string builders, with one exception — +//! [`matrix_unread_summary`] queries the local matrix daemon socket +//! (best-effort) so `get_loose_ends` can prepend an unread-rooms entry. + +/// Render the three identical failure arms every data-returning tool handler +/// repeats: a broker `Err` → `"{tool} failed: {m}"`, an unexpected `Ok` variant +/// → `"{tool} unexpected response: …"`, and a transport error → `"{tool} +/// transport error: …"`. Handlers match their own happy-path variant and route +/// everything else here via a catch-all arm (`other => reply_err(other, tool)`), +/// so the triplet lives in exactly one place. +pub(super) fn reply_err(resp: Result, tool: &str) -> String { + match resp { + Ok(hive_sh4re::Response::Err { message }) => format!("{tool} failed: {message}"), + Ok(other) => format!("{tool} unexpected response: {other:?}"), + Err(e) => format!("{tool} transport error: {e:#}"), + } +} + +/// Format helper for "send-like" tools (anything that expects an `Ok`). +/// `tool` and `ok_msg` only appear in the result string; they don't change +/// behavior. +#[must_use] +pub fn format_ack( + resp: Result, + tool: &str, + ok_msg: String, +) -> String { + match resp { + Ok(hive_sh4re::Response::Ok) => ok_msg, + other => reply_err(other, tool), + } +} + +/// Format helper for `recv`: renders zero, one, or many popped +/// messages. Empty list collapses to "(empty)" so claude doesn't go +/// hunting for content; when `waited` is set (the call parked on a +/// long-poll that timed out) the empty result also carries +/// [`IDLE_WAIT_HINT`] nudging the model toward other work. A single +/// message renders as the historical `from: X\n\nbody` block (banner +/// first if `redelivered`). A multi-message batch renders with a +/// `popped N message(s):` header and `---` separators between bodies +/// so the model can tell where one ends and the next begins; +/// per-message redelivery banners included. +#[must_use] +pub fn format_recv(resp: Result, waited: bool) -> String { + match resp { + Ok(hive_sh4re::Response::Messages { messages }) => render_recv_messages(&messages, waited), + // A graceful stop is pending — the inbox is fenced. Render a single + // explicit directive (not an empty inbox, which claude's "park on recv" + // habit would long-poll again, stalling the stop-checkpoint turn until + // the drain wait times out into a hard stop) so every recv during the + // stop unmissably tells claude to flush + end. + Ok(hive_sh4re::Response::GracefulStop) => { + render_recv_messages(&[graceful_stop_message()], waited) + } + other => reply_err(other, "recv"), + } +} + +/// The synthetic single-message directive rendered for a fenced (graceful-stop) +/// inbox — see the `GracefulStop` arm of [`format_recv`]. +fn graceful_stop_message() -> hive_sh4re::DeliveredMessage { + hive_sh4re::DeliveredMessage { + from: "graceful-stop".into(), + body: "⛔ GRACEFUL STOP IN PROGRESS — the container shuts down as soon as this turn \ + ends. Flush anything worth keeping to your durable /state files, then END \ + YOUR TURN now. Do NOT call recv again: the inbox is fenced and recv will \ + only keep returning this same notice." + .into(), + id: 0, + redelivered: false, + in_reply_to: None, + } +} + +/// Render the popped-message payload of a successful `recv` (see `format_recv` +/// for the empty/single/batch shapes). +fn render_recv_messages(messages: &[hive_sh4re::DeliveredMessage], waited: bool) -> String { + use std::fmt::Write as _; + if messages.is_empty() { + return if waited { + format!("(empty){IDLE_WAIT_HINT}") + } else { + "(empty)".to_owned() + }; + } + if messages.len() == 1 { + let m = &messages[0]; + let banner = if m.redelivered { REDELIVERY_HINT } else { "" }; + return format!("{banner}{}from: {}\n\n{}", msg_id_tag(m.id), m.from, m.body); + } + let n = messages.len(); + let mut out = format!("popped {n} message(s):\n\n"); + for (i, m) in messages.iter().enumerate() { + if i > 0 { + out.push_str("\n---\n\n"); + } + let banner = if m.redelivered { REDELIVERY_HINT } else { "" }; + let _ = write!( + out, + "{banner}{}from: {}\n\n{}", + msg_id_tag(m.id), + m.from, + m.body + ); + } + out +} + +/// `[msg #] ` marker prefixed to each recv row so the agent knows +/// what to pass to `ack_until` when bulk-triaging a backlog. Transient +/// pings carry the sentinel id 0 (in-memory only, nothing in the +/// broker to ack) and render without the marker. +fn msg_id_tag(id: i64) -> String { + if id > 0 { + format!("[msg #{id}] ") + } else { + String::new() + } +} + +/// Header prepended to message bodies that were popped by a prior +/// harness session, never acked (turn crash / OOM / restart), and +/// resurfaced by `RequeueInflight` on this session's boot. Same +/// string surfaces in the wake prompt (see the bin loops) and the +/// in-turn `recv` tool result so claude sees the warning either way. +pub const REDELIVERY_HINT: &str = "[redelivered after harness restart — may already be handled]\n"; + +/// Appended to the `recv` empty result when the agent parked on a +/// long-poll (`wait_seconds > 0`) that timed out with nothing new. +/// Nudges the model to spend the idle time on other useful work +/// instead of immediately re-blocking on `recv`. +pub const IDLE_WAIT_HINT: &str = " — nothing arrived before the wait timed out. \ +If you have other useful work (assigned issues, in-flight PRs, a docs sweep, \ +notes to update), do that now rather than immediately parking on recv again."; + +/// Inner renderer for a `Vec` already extracted from the socket +/// reply. Called by the `get_loose_ends` handler, which injects the +/// `UnreadMatrix` entry before formatting. +pub(super) fn render_loose_ends(loose_ends: &[hive_sh4re::LooseEnd]) -> String { + use std::fmt::Write as _; + if loose_ends.is_empty() { + return "(no loose ends)".to_owned(); + } + let mut out = format!("{} loose end(s):\n", loose_ends.len()); + for t in loose_ends { + match t { + hive_sh4re::LooseEnd::Approval { + id, + agent, + commit_ref, + description, + age_seconds, + } => { + let desc = description + .as_deref() + .map(|d| format!(" — {d}")) + .unwrap_or_default(); + let _ = writeln!( + out, + "- approval #{id} ({agent} @ {commit_ref}, {age_seconds}s old){desc}" + ); + } + hive_sh4re::LooseEnd::Question { + id, + asker, + target, + question, + age_seconds, + } => { + let to = target.as_deref().unwrap_or("operator"); + let _ = writeln!( + out, + "- question #{id} ({asker} → {to}, {age_seconds}s old): {question}" + ); + } + hive_sh4re::LooseEnd::Reminder { + id, + owner, + message, + due_at, + age_seconds, + } => { + let _ = writeln!( + out, + "- reminder #{id} ({owner}, scheduled {age_seconds}s ago, due_at={due_at}): {message}" + ); + } + hive_sh4re::LooseEnd::PendingMessages { count } => { + let _ = writeln!( + out, + "- {count} pending inbox message(s) — drain with recv (recv(max: {count}) to batch)" + ); + } + hive_sh4re::LooseEnd::UnreadMatrix { rooms, summary } => { + let _ = write!(out, "- unread matrix messages in {rooms} room(s)"); + if summary.is_empty() { + let _ = writeln!( + out, + " — use list_rooms + read_room to view, mark_read to clear" + ); + } else { + let _ = writeln!(out, ":"); + for line in summary.lines() { + let _ = writeln!(out, " {line}"); + } + let _ = writeln!( + out, + " use list_rooms + read_room to view, mark_read to clear" + ); + } + } + } + } + out +} + +/// Per-room unread entry returned by `matrix_unread_summary`. Mirrors +/// `hive-matrix-mcp`'s `RoomUnread` but defined locally to avoid a +/// cross-crate dep on the matrix-sdk crate tree. +#[derive(Debug, serde::Deserialize)] +pub(super) struct MatrixRoomUnread { + label: String, + count: u32, + last_body: Option, + last_sender: Option, +} + +/// Query the local matrix daemon for per-room unread summaries. Returns +/// `None` if the daemon socket is absent or the query fails. Best-effort: +/// agents without matrix configured are not penalised. +pub(super) async fn matrix_unread_summary() -> Option> { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + use tokio::net::UnixStream; + let socket = std::env::var_os("HIVE_MATRIX_SOCKET").map_or_else( + || std::path::PathBuf::from("/run/hive-matrix/socket"), + std::path::PathBuf::from, + ); + if !socket.exists() { + return None; + } + let mut stream = UnixStream::connect(&socket).await.ok()?; + stream + .write_all(b"{\"method\":\"unread_summary\"}\n") + .await + .ok()?; + let mut lines = BufReader::new(stream).lines(); + let line = lines.next_line().await.ok()??; + let val: serde_json::Value = serde_json::from_str(&line).ok()?; + // Response: {"kind":"ok","payload":[{label, count, last_body?, last_sender?}]} + let arr = val.get("payload")?.as_array()?; + serde_json::from_value(serde_json::Value::Array(arr.clone())).ok() +} + +/// Format a `Vec` into a per-room summary string. +/// Single room / single message collapses to one line; multi-room +/// expands to a bulleted list. Returns an empty string for empty input. +pub(super) fn format_matrix_summary(rooms: &[MatrixRoomUnread]) -> String { + use std::fmt::Write as _; + if rooms.is_empty() { + return String::new(); + } + let mut out = String::new(); + for r in rooms { + if r.count == 1 + && let (Some(body), Some(sender)) = (&r.last_body, &r.last_sender) + { + let _ = writeln!(out, "- {}: {sender}: {body}", r.label); + continue; + } + let _ = writeln!(out, "- {}: {} unread", r.label, r.count); + } + // Remove trailing newline. + if out.ends_with('\n') { + out.pop(); + } + out +} + +/// Parse the user-facing `kind` string for `cancel_loose_end` into the +/// wire enum. Accepts a small alias set so claude doesn't have to +/// remember the exact spelling (`"q"` / `"r"` shorthand falls out +/// for free). +pub(super) fn parse_loose_end_kind(raw: &str) -> Result { + match raw.trim().to_ascii_lowercase().as_str() { + "question" | "q" => Ok(hive_sh4re::CancelLooseEndKind::Question), + "reminder" | "r" => Ok(hive_sh4re::CancelLooseEndKind::Reminder), + "approval" | "a" => Ok(hive_sh4re::CancelLooseEndKind::Approval), + other => Err(format!( + "cancel_loose_end: unknown kind '{other}' \ + (expected \"question\", \"reminder\", or \"approval\")" + )), + } +} + +/// Canonical user-facing label for a `CancelLooseEndKind` — used in +/// the success ack so the caller always sees `"question"` / +/// `"reminder"` instead of whatever alias they passed in (`"q"` / +/// `"r"`). +pub(super) fn loose_end_kind_label(kind: hive_sh4re::CancelLooseEndKind) -> &'static str { + match kind { + hive_sh4re::CancelLooseEndKind::Question => "question", + hive_sh4re::CancelLooseEndKind::Reminder => "reminder", + hive_sh4re::CancelLooseEndKind::Approval => "approval", + } +} + +/// Format helper for `get_agent_meta`: renders an agent's identity + +/// current status as a short human-readable block. `name`, +/// `hyperhive_rev`, and `running` are always shown; `status` only +/// appears when one is set, otherwise the line reads `status: `. +/// When `running` is false the host has already cleared `status_text` +/// (it would be a stale snapshot from before the stop) so the status +/// line is implicitly `` in that case — but the explicit +/// `running: no` line tells the caller WHY. See +/// `docs/turn-loop/mcp.md::Core tools` (`get_agent_meta`). +#[must_use] +pub fn format_agent_meta(resp: Result) -> String { + match resp { + Ok(hive_sh4re::Response::AgentMeta { + name, + running, + hyperhive_rev, + status_text, + status_set_at, + hive_name, + swarm_name, + matrix_accounts, + }) => { + let rev = hyperhive_rev.as_deref().unwrap_or(""); + let run = if running { "yes" } else { "no" }; + let mut out = format!("name: {name}\nhyperhive_rev: {rev}\nrunning: {run}"); + // Surface hive + swarm display names only when set, so + // single-hive deployments don't see noisy `` lines. + if let Some(hn) = hive_name.as_deref() { + use std::fmt::Write as _; + let _ = write!(out, "\nhive_name: {hn}"); + } + if let Some(sn) = swarm_name.as_deref() { + use std::fmt::Write as _; + let _ = write!(out, "\nswarm_name: {sn}"); + } + match status_text { + None => out.push_str("\nstatus: "), + Some(s) => { + use std::fmt::Write as _; + let age = status_set_at.and_then(|ts| { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok()? + .as_secs(); + // `ts` is a unix epoch second the agent itself + // sourced from `SystemTime` — always positive + // in normal operation. Clamp the negative + // (clock-skew) edge to 0 before the unsigned + // cast so the cast loses no real precision. + let ts_secs = u64::try_from(ts).unwrap_or(0); + let secs = now.saturating_sub(ts_secs); + Some(format_age_secs(secs)) + }); + // `write!` into the buffer instead of `push_str(&format!(…))` — + // avoids the intermediate allocation clippy::format_push_string + // flags. The infallible `String` writer makes this safe to + // `let _ =`-ignore. + match age { + Some(a) => { + let _ = write!(out, "\nstatus: {s} (set {a} ago)"); + } + None => { + let _ = write!(out, "\nstatus: {s}"); + } + } + } + } + // Matrix identities the agent can act as (the `account` arg on + // the matrix tools). Listed only when matrix is provisioned, so + // non-matrix agents don't see an empty line. + if !matrix_accounts.is_empty() { + use std::fmt::Write as _; + out.push_str("\nmatrix_accounts:"); + for acct in &matrix_accounts { + let uid = acct.user_id.as_deref().unwrap_or("?"); + let _ = write!(out, "\n {} ({uid}) on {}", acct.name, acct.homeserver); + } + } + out + } + other => reply_err(other, "get_agent_meta"), + } +} + +/// Format a duration in seconds as a human-readable age string. +fn format_age_secs(secs: u64) -> String { + if secs < 60 { + format!("{secs}s") + } else if secs < 3600 { + format!("{}m", secs / 60) + } else if secs < 86400 { + format!("{}h", secs / 3600) + } else { + format!("{}d", secs / 86400) + } +} + +/// Append a short note to a tool result when the underlying socket call +/// took retries to land. Lets claude distinguish "my request was wrong" +/// from "c0re flickered and the harness rode it out" — without the +/// hint, a tool result that took 30s to come back looks identical to a +/// content failure and the model would burn a turn retrying it. +#[must_use] +pub fn annotate_retries(mut s: String, retries: u32) -> String { + if retries > 0 { + use std::fmt::Write as _; + let suffix = if retries == 1 { "retry" } else { "retries" }; + let _ = write!( + s, + "\n\n(note: hive socket connect needed {retries} {suffix} — c0re likely \ + restarted. Your request did succeed on the final attempt; no action needed.)" + ); + } + s +} + +#[cfg(test)] +mod tests { + use super::{IDLE_WAIT_HINT, format_recv}; + + #[test] + fn empty_recv_after_wait_appends_idle_hint() { + let out = format_recv( + Ok(hive_sh4re::Response::Messages { messages: vec![] }), + true, + ); + assert!(out.starts_with("(empty)")); + assert!(out.contains(IDLE_WAIT_HINT)); + } + + #[test] + fn empty_recv_without_wait_has_no_hint() { + let out = format_recv( + Ok(hive_sh4re::Response::Messages { messages: vec![] }), + false, + ); + assert_eq!(out, "(empty)"); + } +} diff --git a/hive-ag3nt/src/serve_common.rs b/hive-ag3nt/src/serve_common.rs index 4ea4b570..9e011798 100644 --- a/hive-ag3nt/src/serve_common.rs +++ b/hive-ag3nt/src/serve_common.rs @@ -7,6 +7,7 @@ use crate::events::Bus; use crate::mcp::REDELIVERY_HINT; use crate::turn::{TurnError, TurnOutcome}; use crate::turn_stats::TurnStatRow; +pub use hive_sh4re::wire_time::now_unix; /// Assemble the per-turn wake prompt string. The role/tools/etc. live in the /// system prompt; this is just the wake signal body. `id` is the broker row @@ -44,16 +45,6 @@ pub fn format_wake_prompt( format!("{banner}{tag}Incoming message from `{from}`:\n---\n{body}\n---{pending}") } -/// Current time as a Unix timestamp (seconds). Returns 0 on any error. -#[must_use] -pub fn now_unix() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - /// Field-named args for [`build_row`]. Mirrors the turn-stats row /// columns; `outcome` and `bus` borrow for the duration of the call. pub struct TurnRowArgs<'a> { diff --git a/hive-ag3nt/src/stats.rs b/hive-ag3nt/src/stats.rs index eb3f6152..b244e4ab 100644 --- a/hive-ag3nt/src/stats.rs +++ b/hive-ag3nt/src/stats.rs @@ -15,6 +15,7 @@ use rusqlite::{Connection, OpenFlags}; use serde::Serialize; use hive_sh4re::ReminderStats; +use hive_sh4re::wire_time::now_unix; /// Window param accepted by `/api/stats?window=`. Each maps to a /// total span + the bucket width used to roll up trend series. @@ -208,7 +209,7 @@ fn default_path() -> PathBuf { } fn empty_snapshot(window: Window) -> Snapshot { - let now = now_secs(); + let now = now_unix(); let from = now - window.span_secs(); let buckets = fill_buckets(from, now, window.bucket_secs(), &HashMap::new()); Snapshot { @@ -239,7 +240,7 @@ fn snapshot(path: &Path, window: Window) -> Result { // matches hive-c0re's host-side reader (`hive_stats::read_agent`). conn.busy_timeout(std::time::Duration::from_millis(500)) .with_context(|| format!("set busy_timeout on {}", path.display()))?; - let now = now_secs(); + let now = now_unix(); // Fixed windows look back a constant span; `all` starts at the earliest // recorded turn (`MIN(started_at)`, falling back to `now` on an empty // table) and sizes its buckets adaptively from that span. @@ -567,12 +568,6 @@ fn u64_from_i64(v: i64) -> u64 { u64::try_from(v).unwrap_or(0) } -fn now_secs() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX)) -} - #[cfg(test)] mod tests { use super::*; @@ -638,7 +633,7 @@ mod tests { fn snapshot_aggregates_rows() { let db = tmp_db(); let _ = std::fs::remove_file(&db); - let now = now_secs(); + let now = now_unix(); seed_db( &db, &[ @@ -727,7 +722,7 @@ mod tests { fn bash_breakdown_empty_without_table() { let db = tmp_db(); let _ = std::fs::remove_file(&db); - seed_db(&db, &[(now_secs() - 100, 1000, "opus", "recv", "ok", "{}")]); + seed_db(&db, &[(now_unix() - 100, 1000, "opus", "recv", "ok", "{}")]); let s = snapshot(&db, Window::Day).unwrap(); assert!(s.bash_breakdown.is_empty()); } @@ -739,7 +734,7 @@ mod tests { let db = tmp_db(); let _ = std::fs::remove_file(&db); seed_db(&db, &[]); - let now = now_secs(); + let now = now_unix(); let conn = Connection::open(&db).unwrap(); conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);") .unwrap(); diff --git a/hive-ag3nt/src/vacuum.rs b/hive-ag3nt/src/vacuum.rs index 8455a371..c33ec8ad 100644 --- a/hive-ag3nt/src/vacuum.rs +++ b/hive-ag3nt/src/vacuum.rs @@ -15,8 +15,9 @@ //! the honest fix is to clean them up where they live. use std::path::Path; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; +use hive_sh4re::wire_time::now_unix; use rusqlite::{Connection, Result, params}; /// How often the sweep runs. @@ -130,11 +131,3 @@ fn vacuum_events(path: &Path) -> Result { )?; Ok(u64::try_from(removed).unwrap_or(0)) } - -fn now_unix() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} diff --git a/hive-bash-mcp/src/runner.rs b/hive-bash-mcp/src/runner.rs index e2e9e25f..ac1915b1 100644 --- a/hive-bash-mcp/src/runner.rs +++ b/hive-bash-mcp/src/runner.rs @@ -90,13 +90,7 @@ fn signal_group(pgid: Option, sig: i32) { // Helpers // --------------------------------------------------------------------------- -fn now_unix() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - .cast_signed() -} +use hive_sh4re::wire_time::now_unix; /// Generate a task ID: ``. #[must_use] diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index 86502340..afa5ef54 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -18,6 +18,7 @@ clap-markdown = "0.1" hive-sh4re.workspace = true libc.workspace = true listenfd = "1" +petgraph.workspace = true rusqlite.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/hive-c0re/src/actions.rs b/hive-c0re/src/actions.rs index 5fc88933..27238df8 100644 --- a/hive-c0re/src/actions.rs +++ b/hive-c0re/src/actions.rs @@ -13,19 +13,20 @@ use crate::lifecycle; /// Approve a pending request. Marks the approval row durably, then /// either runs the work inline (`InitConfig`, sub-second git ops) or -/// enqueues it into `rebuild_queue` so the dashboard POST returns +/// submits it to the job queue so the dashboard POST returns /// immediately while the long-running pipeline runs off-thread /// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`). /// /// Dispatch: -/// - `ApplyCommit` → `QueueKind::Rebuild` (~30-90s wall time) -/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s) -/// - `Spawn` → `QueueKind::Spawn` (~30-90s) +/// - `ApplyCommit` / `MergeConfigPr` → a single-node `ApprovalDeploy` +/// DAG (the two-phase meta deploy stays opaque in v1; ~30-90s) +/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion) +/// - `Spawn` → a `Spawn` DAG (`Create → WriteDropin → Reconcile`) /// - `InitConfig` → inline (<1s; queue card would be noise) /// -/// The queue worker re-fetches the approval row on dispatch, runs -/// the kind-specific pipeline, and fires `ApprovalResolved` / -/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`. +/// `ApprovalDeploy` resolves the approval inside its pipeline; the +/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`] +/// when their DAG settles terminal. pub async fn approve(coord: Arc, id: i64) -> Result<()> { let approval = coord.approvals.mark_approved(id)?; tracing::info!( @@ -56,54 +57,41 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { } ApprovalKind::UpdateMetaInputs => { // Inputs JSON-encoded into commit_ref by the manager's - // submit path — surface them on the queue entry so the - // dashboard can show *which* inputs are about to bump. + // submit path — surface them on the DAG so the dashboard + // can show *which* inputs are about to bump. The cascade + // rebuilds fan out when the lock bump lands (so they build + // against the post-bump lock, and a failed bump fans out + // nothing). let inputs: Vec = serde_json::from_str(&approval.commit_ref).unwrap_or_default(); - let parent_id = coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::MetaUpdate, - agent: approval.agent.clone(), - source: crate::rebuild_queue::QueueSource::Approval, - reason: format!("approval #{id} meta input update"), - parent_id: None, - inputs: inputs.clone(), - approval_id: Some(id), - perm_payload: None, - depends_on: Vec::new(), - }); - // Pre-enqueue cascade rebuilds in topological order so - // agents depending on updated inputs are rebuilt after the - // lock bump, matching the dashboard post_meta_update path. - let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await; - let cascade_reason = format!("approval #{id} meta input cascade"); - for name in cascade_agents { - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name, - crate::rebuild_queue::QueueSource::MetaUpdate, - cascade_reason.clone(), - Some(parent_id), - ); + let submitted = coord + .job_queue + .submit(crate::job_queue::templates::meta_update( + inputs, + crate::job_queue::Source::Approval, + format!("approval #{id} meta input update"), + Some(id), + )); + if let Err(e) = submitted { + return Err(e.context("submit meta-update dag")); } coord.emit_rebuild_queue_snapshot(); Ok(()) } ApprovalKind::Spawn => { - coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::Spawn, - agent: approval.agent.clone(), - source: crate::rebuild_queue::QueueSource::Approval, - reason: format!("approval #{id} spawn"), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(id), - perm_payload: None, - depends_on: Vec::new(), - }); + // The spawn's tail `Reconcile` starts the container, so the + // new agent's power intent is `Up` from the outset. + if let Err(e) = coord.power.set(&approval.agent, crate::power::Wanted::Up) { + tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed"); + } + let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn( + &approval.agent, + id, + format!("approval #{id} spawn"), + )); + if let Err(e) = submitted { + return Err(e.context("submit spawn dag")); + } coord.emit_rebuild_queue_snapshot(); Ok(()) } @@ -137,29 +125,27 @@ pub async fn approve(coord: Arc, id: i64) -> Result<()> { } } -/// Enqueue a `Rebuild` queue entry tied to an approval id. Shared by the -/// `ApplyCommit` and `MergeConfigPr` dispatch arms — both end in a container -/// rebuild routed through the queue, differing only in the queue `reason`. -/// The queue worker branches on the approval's kind to pick the right handler. +/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id. +/// Shared by the `ApplyCommit` and `MergeConfigPr` dispatch arms — both +/// end in a container rebuild routed through the queue, differing only +/// in the `reason`. The node executor branches on the approval's kind +/// to pick the right handler. fn enqueue_approval_rebuild( coord: &Arc, agent: &str, approval_id: i64, reason: String, ) { - coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::Rebuild, - agent: agent.to_owned(), - source: crate::rebuild_queue::QueueSource::Approval, + if let Err(e) = coord + .job_queue + .submit(crate::job_queue::templates::approval_deploy( + agent, + approval_id, reason, - parent_id: None, - inputs: Vec::new(), - approval_id: Some(approval_id), - perm_payload: None, - depends_on: Vec::new(), - }); + )) + { + tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed"); + } coord.emit_rebuild_queue_snapshot(); } @@ -375,69 +361,60 @@ async fn run_approval_schedule_prompt( finish_approval(coord, &approval, result, None, false) } -/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue -/// entries. Inputs come from the approval row's `commit_ref` field -/// (JSON-encoded by the manager submit path), not the queue entry's -/// `inputs` — the queue copy is for dashboard display only. -pub async fn run_approval_update_meta_inputs( +/// Terminal hook for approval-carrying DAGs — the job queue's +/// scheduler calls this exactly once when such a DAG settles terminal. +/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is +/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves +/// *inside* its node, so its DAG is skipped — unless it was cancelled +/// while still queued, in which case the node never ran and the row +/// would otherwise dangle forever. +pub(crate) async fn resolve_approval_dag( coord: &Arc, - queue_entry_id: Option, - approval_id: i64, -) -> Result<()> { - let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?; - let inputs: Vec = serde_json::from_str(&approval.commit_ref).unwrap_or_default(); - coord.set_queue_step(queue_entry_id, "nix flake update"); - let result = crate::meta::lock_update(&inputs).await; - finish_approval(coord, &approval, result, None, false) -} - -/// Worker entry point for `ApprovalKind::Spawn` queue entries. -/// Differs from `run_approval_apply_commit` only in routing through -/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous -/// in the queue worker — the previous `tokio::spawn` wrapper is gone -/// (the queue worker itself is the async task). -pub async fn run_approval_spawn( - coord: &Arc, - queue_entry_id: Option, - approval_id: i64, -) -> Result<()> { - let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?; - let agent_dir = coord.ensure_runtime(&approval.agent)?; - let hive = coord.hive_env(); - let paths = Coordinator::agent_paths(&approval.agent, agent_dir); - // Transient guard keeps the per-container "Spawning" pill lit while - // the worker is doing the actual nixos-container create. Auto-clears - // on the function's scope exit (success or panic). - let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning); - coord.set_queue_step(queue_entry_id, "lifecycle::spawn"); - let result = lifecycle::spawn(&approval.agent, &hive, &paths).await; - if result.is_ok() { - coord.set_queue_step(queue_entry_id, "forge user"); - if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed"); + terminal: &crate::job_queue::TerminalDag, +) { + use crate::job_queue::{State, Template}; + let Some(approval_id) = terminal.approval_id else { + return; + }; + if terminal.template == Template::Rebuild && terminal.state != State::Cancelled { + return; // ApprovalDeploy resolved inside the node. + } + let approval = match coord.approvals.get(approval_id) { + Ok(Some(a)) => a, + Ok(None) => { + tracing::warn!(approval_id, "approval dag terminal: row no longer exists"); + return; } - coord.set_queue_step(queue_entry_id, "forge config repo"); - if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed"); + Err(e) => { + tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed"); + return; } - coord.set_queue_step(queue_entry_id, "forge push"); - if let Err(e) = crate::forge::push_config(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed"); - } - coord.set_queue_step(queue_entry_id, "forge meta access"); - if let Some(core_token) = crate::forge::core_token() - && let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await - { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed"); - } - if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await { - tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed"); + }; + let result: Result<()> = match terminal.state { + State::Done => Ok(()), + State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")), + _ => Err(anyhow::anyhow!( + "{}", + terminal + .error + .clone() + .unwrap_or_else(|| "job dag failed".to_owned()) + )), + }; + if approval.kind == ApprovalKind::Spawn { + // Post-spawn forge bookkeeping (user, config repo mirror, meta + // access) — warn-only, then the resolution events + a rescan so + // the dashboard reflects the post-spawn state either way. + if result.is_ok() { + forge_after_first_spawn(coord, &approval.agent).await; + } else { + coord.rescan_containers_and_emit().await; + crate::dashboard::emit_tombstones_snapshot(coord).await; } } - let final_result = finish_approval(coord, &approval, result, None, false); - coord.rescan_containers_and_emit().await; - crate::dashboard::emit_tombstones_snapshot(coord).await; - final_result + if let Err(e) = finish_approval(coord, &approval, result, None, false) { + tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure"); + } } /// Re-fetch an approval row from sqlite for a queue-worker dispatch. @@ -563,14 +540,7 @@ fn finish_approval( // snapshot refetch. `approved` rows that succeed get the // approval's logged resolved_at indirectly via `now_unix()`; // failures already wrote it via mark_failed above. - let approval_kind = match approval.kind { - ApprovalKind::Spawn => "spawn", - ApprovalKind::ApplyCommit => "apply_commit", - ApprovalKind::InitConfig => "init_config", - ApprovalKind::UpdateMetaInputs => "update_meta_inputs", - ApprovalKind::SchedulePrompt => "schedule_prompt", - ApprovalKind::MergeConfigPr => "merge_config_pr", - }; + let approval_kind = approval.kind.as_str(); let sha_short = approval .fetched_sha .as_deref() @@ -867,13 +837,7 @@ async fn deploy_applied_target( // part of this entry rather than a deferred fast-lane follow-up. false, &|step| coord.set_queue_step(queue_entry_id, step), - &|log_id| { - if let Some(qid) = queue_entry_id - && coord.rebuild_queue.set_build_log_id(qid, log_id) - { - coord.emit_rebuild_queue_snapshot(); - } - }, + &|log_id| coord.set_queue_build_log(queue_entry_id, log_id), ) .await; @@ -984,6 +948,11 @@ pub async fn destroy(coord: &Arc, name: &str, purge: bool) -> Resul "agent destroyed" }, ); + // Drop the durable power intent — a future agent of the same name + // seeds fresh from its observed state. + if let Err(e) = coord.power.remove(name) { + tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed"); + } drop(guard); coord.notify_manager(&HelperEvent::Destroyed { agent: name.to_owned(), @@ -1045,14 +1014,7 @@ pub async fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<() tracing::warn!(%id, agent = %a.agent, error = ?e, "forge: push_config after deny failed"); } } - let approval_kind = match a.kind { - ApprovalKind::Spawn => "spawn", - ApprovalKind::ApplyCommit => "apply_commit", - ApprovalKind::InitConfig => "init_config", - ApprovalKind::UpdateMetaInputs => "update_meta_inputs", - ApprovalKind::SchedulePrompt => "schedule_prompt", - ApprovalKind::MergeConfigPr => "merge_config_pr", - }; + let approval_kind = a.kind.as_str(); let sha_short = sha.as_deref().map(|s| s[..s.len().min(12)].to_owned()); let description = a.description.clone(); let agent_owned = a.agent.clone(); diff --git a/hive-c0re/src/capabilities.rs b/hive-c0re/src/agent_config/capabilities.rs similarity index 100% rename from hive-c0re/src/capabilities.rs rename to hive-c0re/src/agent_config/capabilities.rs diff --git a/hive-c0re/src/limits.rs b/hive-c0re/src/agent_config/limits.rs similarity index 100% rename from hive-c0re/src/limits.rs rename to hive-c0re/src/agent_config/limits.rs diff --git a/hive-c0re/src/agent_config/mod.rs b/hive-c0re/src/agent_config/mod.rs new file mode 100644 index 00000000..7ef28556 --- /dev/null +++ b/hive-c0re/src/agent_config/mod.rs @@ -0,0 +1,10 @@ +//! Per-agent configuration registries: tool groups, capabilities, +//! topology (all JSON files under `/var/lib/hyperhive/meta/`) and the +//! shared wire-protocol size limits. Each submodule is re-exported at +//! the crate root, so `crate::topology::…` etc. keep working +//! unchanged. + +pub mod capabilities; +pub mod limits; +pub mod tool_groups; +pub mod topology; diff --git a/hive-c0re/src/tool_groups.rs b/hive-c0re/src/agent_config/tool_groups.rs similarity index 100% rename from hive-c0re/src/tool_groups.rs rename to hive-c0re/src/agent_config/tool_groups.rs diff --git a/hive-c0re/src/topology.rs b/hive-c0re/src/agent_config/topology.rs similarity index 100% rename from hive-c0re/src/topology.rs rename to hive-c0re/src/agent_config/topology.rs diff --git a/hive-c0re/src/auto_update.rs b/hive-c0re/src/auto_update.rs deleted file mode 100644 index d17a1887..00000000 --- a/hive-c0re/src/auto_update.rs +++ /dev/null @@ -1,418 +0,0 @@ -//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that -//! actually need it. Two skip rules keep boot-time work minimal: -//! -//! 1. **Stopped containers** are deferred — they will be rebuilt the first -//! time the operator starts them (see `rebuild_queue::run_start` and -//! `socket_server::handle_start`). -//! 2. **Running containers whose rev marker matches** the current hyperhive -//! flake path are skipped — nothing changed, no nix work to do. -//! -//! See `docs/coordinator.md::Auto-update sweep`. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use anyhow::{Context, Result}; - -use crate::coordinator::Coordinator; -use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME}; - -/// Marker file recording the hyperhive rev a sub-agent's container was last -/// built against. Sibling of `applied//` (rather than inside it) to -/// keep it out of the applied repo's git history. Uses a leading dot so a -/// glob over `applied/*` doesn't include it. -pub fn rev_marker_path(name: &str) -> PathBuf { - PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev")) -} - -/// Resolve the current rev of `hyperhive_flake`. For a path on disk we -/// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/... -/// update yields a different string. For anything else we return None. -#[must_use] -pub fn current_flake_rev(hyperhive_flake: &str) -> Option { - let path = Path::new(hyperhive_flake); - if !path.exists() { - return None; - } - std::fs::canonicalize(path) - .ok() - .map(|p| p.display().to_string()) -} - -/// Returns true when the applied repo has commits that have not yet been -/// deployed (i.e. the applied HEAD differs from the sha currently locked in -/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip -/// conveys: "there is a config change ready to apply via rebuild." -/// -/// Async on purpose: this runs per agent inside `container_view::build_all`, -/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and -/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous -/// `git` fork here blocks a tokio worker for the whole exec — under -/// nix-build disk saturation that's long enough that concurrent sweeps -/// starved the runtime and stalled the per-agent sockets. -pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool { - let applied_head = tokio::process::Command::new("git") - .args([ - "-C", - &format!("/var/lib/hyperhive/applied/{name}"), - "rev-parse", - "HEAD", - ]) - .output() - .await - .ok() - .filter(|o| o.status.success()) - .and_then(|o| String::from_utf8(o.stdout).ok()) - .map(|s| s.trim().to_owned()); - - match (applied_head.as_deref(), deployed_sha) { - (Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head), - _ => false, - } -} - -/// Rebuild one sub-agent and refresh its marker. Used by both the startup -/// scanner and the dashboard's manual "update" button so the two paths -/// can't diverge. -/// -/// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from -/// the `rebuild_queue` worker (lets the function annotate its phase via -/// `coord.set_queue_step`) and `None` when called directly (e.g. the -/// root-agent migration nudge in `ensure_root_agent`). -/// -/// `relock` bumps the agent's meta input to `applied//main` before -/// the container rebuild. Pass `false` only for meta-update cascade -/// rebuilds, where re-locking would revert the bump the cascade just -/// committed (see `lifecycle::rebuild`). -/// -/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds: -/// instead of holding the serialized build lane through the container -/// boot, the start-after-rebuild is enqueued as a fast-lane `Start` -/// entry (grouped under this rebuild via `parent_id`, same split as the -/// graceful-stop follow-up). Pass `None` for direct callers to keep the -/// start inline. -/// -/// # Errors -/// -/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`. -pub async fn rebuild_agent( - coord: &Arc, - name: &str, - current_rev: &str, - queue_entry_id: Option, - relock: bool, - defer_start_source: Option, -) -> Result<()> { - tracing::info!(%name, rev = %current_rev, "rebuild agent"); - let agent_dir = coord - .ensure_runtime(name) - .with_context(|| format!("ensure_runtime {name}"))?; - let hive = coord.hive_env(); - let paths = Coordinator::agent_paths(name, agent_dir); - // Suppress crash_watch during the stop+start window inside - // lifecycle::rebuild. Dashboard rebuilds already do this via - // lifecycle_action; this catches the auto-update scan + any - // other direct caller. - let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding); - let result = lifecycle::rebuild( - name, - &hive, - &paths, - relock, - defer_start_source.is_some(), - &|step| coord.set_queue_step(queue_entry_id, step), - &|log_id| { - if let Some(qid) = queue_entry_id - && coord.rebuild_queue.set_build_log_id(qid, log_id) - { - coord.emit_rebuild_queue_snapshot(); - } - }, - ) - .await; - drop(guard); - match &result { - Ok(needs_start) => { - if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) { - tracing::warn!(%name, error = ?e, "write rev marker failed"); - } - // Deferred start: hand the container boot to the fast lane so - // this build-lane entry completes now and the next queued - // rebuild's nix build overlaps with the boot. `parent_id` - // groups the follow-up under this rebuild on the dashboard — - // same split the graceful-stop path uses for its container - // stop. A start failure surfaces on the Start entry (with - // the cold-start fallback) instead of failing the rebuild. - if *needs_start && let Some(source) = defer_start_source { - coord - .rebuild_queue - .enqueue_full(crate::rebuild_queue::FullEnqueue { - kind: crate::rebuild_queue::QueueKind::Start, - agent: name.to_owned(), - source, - reason: format!("start after rebuild of {name}"), - parent_id: queue_entry_id, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }); - coord.emit_rebuild_queue_snapshot(); - } - coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: name.to_owned(), - ok: true, - note: None, - sha: None, - tag: None, - }); - coord.set_queue_step(queue_entry_id, "forge sync"); - // Run the full forge sync on every successful rebuild so - // the rebuild path is equivalent to the hive-c0re startup - // sweep: token, config-repo mirror, meta read access, and - // meta remote are all kept in sync. Recovers missing tokens - // (e.g. first-spawn seeding failed transiently) without - // requiring a full hive-c0re restart. - crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; - // Mirror the matrix side of the startup sweep: if hive-matrix - // is present, ensure this agent has a registered user + token. - // Idempotent (skips if token file already exists). Keeps the - // rebuild path equivalent to the startup sweep for newly-spawned - // agents that missed ensure_all(). - crate::matrix::sync_agent_standalone(name).await; - // Wake the agent on its next turn so claude sees a - // "you were rebuilt — check /state/ for notes, --continue - // session intact" hint. Covers dashboard rebuild, admin - // CLI rebuild, auto-update startup scan, and the - // dashboard's meta-input update path — all of which - // route through rebuild_agent. - coord.kick_agent(name, "container rebuilt"); - // Container state (needs_update, deployed_sha) may have - // shifted — rescan so dashboards drop the "needs update" - // chip without waiting for the next /api/state poll. - coord.rescan_containers_and_emit().await; - // Lock bump → meta-inputs panel needs to re-render. - crate::dashboard::emit_meta_inputs_snapshot(coord); - } - Err(e) => { - coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: name.to_owned(), - ok: false, - note: Some(format!("{e:#}")), - sha: None, - tag: None, - }); - coord.rescan_containers_and_emit().await; - } - } - result.map(|_| ()) -} - -/// Whether this hive is "ruthless" — running with no root/manager agent at -/// all (no ruth). When true, hive-c0re skips the root-agent create/start -/// sweep entirely. Controlled by the host option -/// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS` -/// env var. Defaults to `false` when the var is unset (back-compat: the -/// root agent was always auto-managed before this opt-out existed); only -/// an explicit `true` / `1` / `yes` enables ruthless mode. -fn ruthless() -> bool { - match std::env::var("HYPERHIVE_RUTHLESS") { - Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"), - Err(_) => false, - } -} - -/// Auto-create the manager container on startup if it isn't already there. -/// hive-c0re manages the manager end-to-end: operators no longer declare -/// `containers.h-ruth` in their host NixOS config. Bypasses the approval -/// queue — the root/manager is auto-managed by default. Operators who -/// don't want a root agent at all set `services.hyperhive.ruthless = true`, -/// which short-circuits this whole function. Idempotent. -pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { - if ruthless() { - tracing::info!( - "ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start" - ); - return Ok(()); - } - let existing = lifecycle::list().await.unwrap_or_default(); - let current_rev = current_flake_rev(&coord.hyperhive_flake); - if existing - .iter() - .any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME)) - { - // Container exists already. If it predates the unified lifecycle - // (no applied flake on disk) we must rebuild — otherwise it's - // running whatever the host-declarative config was at create - // time, with a wrong systemd unit and port. - let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix"); - if !applied_flake.exists() - && let Some(rev) = current_rev.as_ref() - { - tracing::warn!( - "manager container exists but no applied flake — forcing rebuild to migrate" - ); - let coord_clone = coord.clone(); - if let Err(e) = - rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await - { - tracing::warn!(error = ?e, "manager migration rebuild failed"); - } - } else { - tracing::debug!("manager container already present"); - } - // hive-c0re auto-manages the root/manager by default, so a - // present-but-stopped root (e.g. a first-start failure on a fresh - // install) is brought back up here: the startup sweep's rebuild only - // restarts a container that was already running, so without this it - // stays down until a manual `nixos-container start`. The sub-agent - // `was_running` guard is intentionally left untouched. (Operators - // opt out of this whole auto-management with - // `services.hyperhive.ruthless = true`, gated at the top of - // this function.) - if !lifecycle::is_running(MANAGER_NAME).await { - tracing::info!("manager container present but not running — starting"); - if let Err(e) = lifecycle::start(MANAGER_NAME).await { - tracing::warn!(error = ?e, "manager start failed"); - } - } - return Ok(()); - } - tracing::info!("manager container missing — spawning"); - let runtime = coord.ensure_runtime(MANAGER_NAME)?; - let hive = coord.hive_env(); - let paths = Coordinator::agent_paths(MANAGER_NAME, runtime); - lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?; - if let Some(rev) = current_rev { - let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev); - } - Ok(()) -} - -/// Sort `names` in-place so parents precede their children in the topology. -/// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last, -/// alphabetically within their tier. Stable within each depth tier. -pub fn topology_sort( - names: &mut [String], - topo: &std::collections::BTreeMap>, -) { - use std::collections::{HashMap, VecDeque}; - // Build depth map using owned clones so the borrow on `names` is released - // before the sort_by mutable borrow. - let name_set: Vec = names.to_vec(); - let mut depth: HashMap = HashMap::new(); - let mut queue: VecDeque = VecDeque::new(); - // Seed roots: entries with no parent, or names not present in topo at all. - for name in &name_set { - if topo.get(name).is_none_or(Option::is_none) { - depth.insert(name.clone(), 0); - queue.push_back(name.clone()); - } - } - // BFS to assign depths to children. - while let Some(parent) = queue.pop_front() { - let d = depth[&parent] + 1; - for name in &name_set { - let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str()); - if is_child && !depth.contains_key(name) { - depth.insert(name.clone(), d); - queue.push_back(name.clone()); - } - } - } - names.sort_by(|a, b| { - let da = depth.get(a).copied().unwrap_or(usize::MAX); - let db = depth.get(b).copied().unwrap_or(usize::MAX); - da.cmp(&db).then(a.cmp(b)) - }); -} - -/// Rebuild containers that need it on startup. Skips: -/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start` -/// upgrades a plain start to rebuild+start when the rev marker is stale). -/// - **Running containers with a matching rev marker**: no nix work needed. -/// -/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild` -/// children linked via `parent_id`. Returns Ok even if some rebuilds failed. -pub async fn run(coord: Arc) -> Result<()> { - let containers = match lifecycle::list().await { - Ok(c) => c, - Err(e) => { - tracing::warn!(error = ?e, "auto-update: nixos-container list failed"); - return Ok(()); - } - }; - - let current_rev = current_flake_rev(&coord.hyperhive_flake); - - // Resolve container names to logical agent names, then sort by - // topology depth so parents are always rebuilt before their - // children. Root agents (depth 0) go first; agents absent from - // the topology file sort last (stable, alphabetical within tier). - let mut logical_names: Vec = containers - .iter() - .filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned)) - .collect(); - let topo = crate::topology::read(); - topology_sort(&mut logical_names, &topo); - - // Pre-classify: decide which agents need a rebuild now vs can be skipped. - let mut to_rebuild: Vec = Vec::new(); - let mut n_deferred = 0usize; - let mut n_skipped = 0usize; - for name in &logical_names { - // Idea 2: stopped containers are deferred — rebuild happens the first - // time the operator starts them. - if !lifecycle::is_running(name).await { - n_deferred += 1; - tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start"); - continue; - } - // Idea 1: running containers with a matching rev marker need no rebuild. - if let Some(ref rev) = current_rev { - let stored = std::fs::read_to_string(rev_marker_path(name)).ok(); - if stored.as_deref() == Some(rev.as_str()) { - n_skipped += 1; - tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild"); - continue; - } - } - to_rebuild.push(name.clone()); - } - - // Enqueue the parent sweep entry. The worker processes it trivially - // (no-op dispatch); its purpose is to give the dashboard a "why" header. - let sweep_id = coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::StartupSweep, - "hyperhive".to_owned(), - crate::rebuild_queue::QueueSource::AutoUpdate, - format!( - "startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)", - to_rebuild.len(), - n_deferred, - n_skipped, - ), - None, - ); - - tracing::info!( - total = containers.len(), - rebuilds = to_rebuild.len(), - deferred = n_deferred, - skipped = n_skipped, - sweep_id, - "auto-update: startup sweep" - ); - - for name in to_rebuild { - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name, - crate::rebuild_queue::QueueSource::StartupSweep, - "startup sweep".to_owned(), - Some(sweep_id), - ); - } - coord.emit_rebuild_queue_snapshot(); - Ok(()) -} diff --git a/hive-c0re/src/bin/hivectl.rs b/hive-c0re/src/bin/hivectl.rs index 3a5a743a..f2b20942 100644 --- a/hive-c0re/src/bin/hivectl.rs +++ b/hive-c0re/src/bin/hivectl.rs @@ -139,13 +139,17 @@ enum Cmd { #[command(flatten)] scope: ScopeArgs, /// Gracefully quiesce each agent before stopping, instead of a - /// hard stop. Each agent is enqueued as a `GracefulStop` on the - /// rebuild queue: the harness is signalled, runs one - /// stop-checkpoint turn to flush durable `/state`, drains, then - /// the container is stopped (bounded by a 3-min timeout that - /// falls back to a hard stop). Applies to agents only. + /// hard stop. Each agent gets a graceful-stop DAG on the job + /// queue: the harness is signalled, runs one stop-checkpoint + /// turn to flush durable `/state`, drains, then the container + /// is stopped (bounded by a 3-min timeout that falls back to a + /// hard stop). All drains overlap. Applies to agents only. #[arg(long)] graceful: bool, + /// Return immediately after the stop DAGs are queued instead of + /// waiting for them with live per-node progress. + #[arg(long)] + no_wait: bool, }, /// Start containers hive-wide — the inverse of `hivectl stop`. Bare /// `hivectl start` starts everything back up; the same scope flags as @@ -154,6 +158,10 @@ enum Cmd { Start { #[command(flatten)] scope: ScopeArgs, + /// Return immediately after the start DAGs are queued instead + /// of waiting for them with live per-node progress. + #[arg(long)] + no_wait: bool, }, /// Restart containers hive-wide — `stop` then `start` over the same /// scope. Bare `hivectl restart` restarts **everything** (all sub-agents @@ -531,15 +539,23 @@ enum AgentsCmd { /// Stop and start a single agent container without rebuilding config. /// Useful for "kick the container" when the process is stuck or the /// container needs a clean restart without changing the NixOS config. + /// Rides the job queue (serialized against in-flight rebuilds for + /// the same agent); waits with live progress unless `--no-wait`. Restart { /// Agent name (e.g. `damocles`, `ruth`). name: String, + /// Return immediately after the restart DAG is queued. + #[arg(long)] + no_wait: bool, + }, + /// Restart ALL managed agent containers via one restart DAG each — + /// unrelated agents overlap, each serializes on its own lease. + /// Waits for the whole set with live progress unless `--no-wait`. + RestartAll { + /// Return immediately after the restart DAGs are queued. + #[arg(long)] + no_wait: bool, }, - /// Stop and restart ALL managed agent containers in sequence. - /// Iterates the live container list and restarts each one. Any per-agent - /// failure is reported at the end rather than stopping mid-run, so all - /// containers get a restart attempt. - RestartAll, } #[derive(Subcommand)] @@ -602,8 +618,8 @@ async fn main() -> Result<()> { }, Cmd::Agents { cmd } => match cmd { AgentsCmd::List { json } => agents_list(&socket, json).await, - AgentsCmd::Restart { name } => agents_restart(&socket, &name).await, - AgentsCmd::RestartAll => agents_restart_all(&socket).await, + AgentsCmd::Restart { name, no_wait } => agents_restart(&socket, &name, no_wait).await, + AgentsCmd::RestartAll { no_wait } => agents_restart_all(&socket, no_wait).await, }, Cmd::Wg { cmd } => match cmd { WgCmd::Init { address } => wg_init(&socket, address.as_deref()).await, @@ -626,8 +642,12 @@ async fn main() -> Result<()> { peer_config(&domain, wg_address.as_deref(), wg_endpoint.as_deref()); Ok(()) } - Cmd::Stop { scope, graceful } => stop(&socket, scope.to_scope(), graceful).await, - Cmd::Start { scope } => start(&socket, scope.to_scope()).await, + Cmd::Stop { + scope, + graceful, + no_wait, + } => stop(&socket, scope.to_scope(), graceful, no_wait).await, + Cmd::Start { scope, no_wait } => start(&socket, scope.to_scope(), no_wait).await, Cmd::Restart { scope, graceful } => restart(&socket, scope.to_scope(), graceful).await, Cmd::Subvol { cmd } => match cmd { SubvolCmd::Upgrade { name, yes } => subvol_upgrade(&socket, &name, yes).await, @@ -1406,7 +1426,7 @@ fn gateway_list_users(file: &Path) -> Result<()> { // Agent management helpers (require daemon via host admin socket) // --------------------------------------------------------------------------- -async fn agents_restart(socket: &Path, name: &str) -> Result<()> { +async fn agents_restart(socket: &Path, name: &str, no_wait: bool) -> Result<()> { let resp = hive_c0re::client::request( socket, hive_sh4re::HostRequest::Restart { @@ -1416,8 +1436,8 @@ async fn agents_restart(socket: &Path, name: &str) -> Result<()> { .await .with_context(|| format!("connect to daemon socket {}", socket.display()))?; if resp.ok { - println!("restarted: {name}"); - Ok(()) + println!("restart queued: {name}"); + wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await } else { bail!( "restart {name}: {}", @@ -1426,6 +1446,107 @@ async fn agents_restart(socket: &Path, name: &str) -> Result<()> { } } +// --------------------------------------------------------------------------- +// Job-queue wait/progress loop — shared by every verb that submits DAGs +// --------------------------------------------------------------------------- + +/// Poll the submitted DAG ids (`HostRequest::QueueDag`, ~1s interval) +/// and print a progress line whenever a DAG's rendered state changes — +/// including fan-out children that appear under a polled parent. Exits +/// non-zero when any DAG (or child) ends `failed`; a `cancelled` DAG +/// terminates the wait but is an operator action, not an error. +async fn wait_for_dags(socket: &Path, ids: Vec, no_wait: bool) -> Result<()> { + if no_wait || ids.is_empty() { + return Ok(()); + } + let mut pending: std::collections::BTreeSet = ids.into_iter().collect(); + let mut last: std::collections::HashMap = std::collections::HashMap::new(); + let mut failed: Vec = Vec::new(); + while !pending.is_empty() { + for id in pending.clone() { + let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::QueueDag { id }) + .await + .with_context(|| format!("connect to daemon socket {}", socket.display()))?; + let dags = resp.dags.unwrap_or_default(); + if dags.is_empty() { + // Evicted from the queue's history tail — it finished a + // while ago; nothing left to report on. + println!("job #{id}: gone from queue history"); + pending.remove(&id); + continue; + } + let mut all_terminal = true; + for d in &dags { + let line = render_dag_line(d); + if last.get(&d.id) != Some(&line) { + println!("{line}"); + last.insert(d.id, line); + } + // Node-level terminality, not the roll-up: a DAG rolls + // up `failed` the moment one node fails while its + // after-any recovery node (rebuild's tail Reconcile) + // may still be running — keep watching so the operator + // sees whether the agent came back. + if !d.nodes.iter().all(|n| n.state.is_terminal()) { + all_terminal = false; + } else if d.state == hive_sh4re::jobs::State::Failed { + failed.push(format!("{} {}", d.kind.as_str(), d.agent)); + } + } + if all_terminal { + pending.remove(&id); + } + } + if !pending.is_empty() { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + if failed.is_empty() { + Ok(()) + } else { + failed.sort(); + failed.dedup(); + bail!("queued job(s) failed: {}", failed.join(", ")) + } +} + +fn state_glyph(state: hive_sh4re::jobs::State) -> &'static str { + match state { + hive_sh4re::jobs::State::Queued => "⏸", + hive_sh4re::jobs::State::Running => "▶", + hive_sh4re::jobs::State::Done => "✔", + hive_sh4re::jobs::State::Failed => "✖", + hive_sh4re::jobs::State::Cancelled => "⊘", + } +} + +/// One progress line for a DAG: roll-up glyph, template, agent, then +/// the node chain with the running node's live step label — the CLI +/// twin of the dashboard's queue card. +fn render_dag_line(d: &hive_sh4re::jobs::DagView) -> String { + use std::fmt::Write as _; + let mut out = format!( + "{} {} {:<12}", + state_glyph(d.state), + d.kind.as_str(), + d.agent + ); + for (i, n) in d.nodes.iter().enumerate() { + let sep = if i == 0 { " " } else { " → " }; + let _ = write!(out, "{sep}{} {}", state_glyph(n.state), n.kind); + if n.state == hive_sh4re::jobs::State::Running + && let Some(step) = &n.step + { + let _ = write!(out, " ({step})"); + } + } + if let Some(err) = d.nodes.iter().find_map(|n| n.error.as_deref()) { + let short: String = err.chars().take(120).collect(); + let _ = write!(out, " — {short}"); + } + out +} + /// `hivectl agents list` — fetch the per-agent status roster from the /// daemon (`HostRequest::AgentStatus`) and render it as a padded table, /// or the raw JSON rows with `--json`. Reuses the dashboard's @@ -1502,7 +1623,7 @@ async fn agents_list(socket: &Path, json: bool) -> Result<()> { Ok(()) } -async fn agents_restart_all(socket: &Path) -> Result<()> { +async fn agents_restart_all(socket: &Path, no_wait: bool) -> Result<()> { let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::RestartAll) .await .with_context(|| format!("connect to daemon socket {}", socket.display()))?; @@ -1511,7 +1632,7 @@ async fn agents_restart_all(socket: &Path) -> Result<()> { println!("restart-all: no managed containers found"); } else { for a in agents { - println!("restarted: {a}"); + println!("restart queued: {a}"); } } if !resp.ok { @@ -1520,22 +1641,34 @@ async fn agents_restart_all(socket: &Path) -> Result<()> { resp.error.as_deref().unwrap_or("unknown error") ); } - Ok(()) + wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await } -async fn stop(socket: &Path, scope: hive_sh4re::LifecycleScope, graceful: bool) -> Result<()> { +async fn stop( + socket: &Path, + scope: hive_sh4re::LifecycleScope, + graceful: bool, + no_wait: bool, +) -> Result<()> { let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::Stop { scope, graceful }) .await .with_context(|| format!("connect to daemon socket {}", socket.display()))?; - render_lifecycle(&resp, "stopped") + // Render first, but even when an infra failure makes it bail, + // watch the already-queued agent DAGs before surfacing the error — + // they run regardless. + let rendered = render_lifecycle(&resp, "stop queued"); + wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?; + rendered } -async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope) -> Result<()> { +async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope, no_wait: bool) -> Result<()> { let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::Start { scope }) .await .with_context(|| format!("connect to daemon socket {}", socket.display()))?; - render_lifecycle(&resp, "started") + let rendered = render_lifecycle(&resp, "start queued"); + wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?; + rendered } /// Restart = `stop` then `start` over the same scope, composed client-side @@ -1543,9 +1676,13 @@ async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope) -> Result<()> { /// `--graceful`; if it reports a failure (`stop` returns `Err`) the `?` /// short-circuits before the start phase, so a half-stopped hive isn't /// blindly started over — the operator sees the stop errors and can recover. +/// +/// No `--no-wait` here on purpose: the stop DAGs must complete before +/// the start submits, otherwise the start's `wanted = Up` write would +/// land before the queued stops execute and turn them into noops. async fn restart(socket: &Path, scope: hive_sh4re::LifecycleScope, graceful: bool) -> Result<()> { - stop(socket, scope.clone(), graceful).await?; - start(socket, scope).await + stop(socket, scope.clone(), graceful, false).await?; + start(socket, scope, false).await } /// A [`LifecycleScope`](hive_sh4re::LifecycleScope) targeting exactly one @@ -1594,6 +1731,12 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> { stop_resp.error.as_deref().unwrap_or("unknown error") ); } + // The stop is a queued DAG now — the migration below snapshots + + // swaps the state dir and MUST NOT run under a live bind mount, so + // wait for the stop to actually execute before touching anything. + wait_for_dags(socket, stop_resp.queued_dags.unwrap_or_default(), false) + .await + .with_context(|| format!("waiting for {name} to stop before the migration"))?; println!("migrating {name} state dir to a btrfs subvolume…"); let upgrade = hive_c0re::priv_client::upgrade_agent_subvolume(name).await; @@ -1633,6 +1776,14 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> { start_resp.error.as_deref().unwrap_or("unknown error") ); } + wait_for_dags(socket, start_resp.queued_dags.unwrap_or_default(), false) + .await + .with_context(|| { + format!( + "{name} migrated to a btrfs subvolume, but its restart job failed — run \ + `hivectl start --agent {name}` to retry" + ) + })?; println!("upgraded {name} to a btrfs subvolume and restarted it"); Ok(()) } @@ -1673,3 +1824,82 @@ fn validate_htpasswd_username(username: &str) -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use hive_sh4re::jobs::{DagView, NodeView, Source, State, Template}; + + use super::render_dag_line; + + fn node(id: u32, kind: &str, state: State, step: Option<&str>) -> NodeView { + NodeView { + id, + kind: kind.to_owned(), + deps: if id == 0 { vec![] } else { vec![id - 1] }, + state, + step: step.map(str::to_owned), + build_log_id: None, + started_at: None, + finished_at: None, + error: None, + } + } + + #[test] + fn render_dag_line_shows_chain_and_running_step() { + let dag = DagView { + id: 7, + agent: "alice".to_owned(), + kind: Template::Rebuild, + state: State::Running, + source: Source::Manual, + parent_id: None, + reason: "manual".to_owned(), + enqueued_at: 0, + started_at: Some(1), + finished_at: None, + inputs: vec![], + approval_id: None, + perm_payload: None, + nodes: vec![ + node(0, "prebuild", State::Done, None), + node(1, "stop_for_update", State::Done, None), + node(2, "swap", State::Running, Some("nixos-container update")), + node(3, "reconcile", State::Queued, None), + ], + }; + let line = render_dag_line(&dag); + assert!(line.starts_with("▶ rebuild alice"), "{line}"); + assert!( + line.contains( + "✔ prebuild → ✔ stop_for_update → ▶ swap (nixos-container update) → ⏸ reconcile" + ), + "{line}" + ); + } + + #[test] + fn render_dag_line_surfaces_first_node_error() { + let mut failed = node(0, "prebuild", State::Failed, None); + failed.error = Some("nix build exploded".to_owned()); + let dag = DagView { + id: 8, + agent: "bob".to_owned(), + kind: Template::Rebuild, + state: State::Failed, + source: Source::Manual, + parent_id: None, + reason: "manual".to_owned(), + enqueued_at: 0, + started_at: Some(1), + finished_at: Some(2), + inputs: vec![], + approval_id: None, + perm_payload: None, + nodes: vec![failed], + }; + let line = render_dag_line(&dag); + assert!(line.contains("✖ rebuild"), "{line}"); + assert!(line.contains("— nix build exploded"), "{line}"); + } +} diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index c19d1629..868e97af 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -173,12 +173,17 @@ pub struct Coordinator { /// tokio mutex so the rescan can `await` `lifecycle::list` / /// `is_running` without blocking other coordinator paths. last_containers: tokio::sync::Mutex>, - /// Global rebuild queue. Every long-running container/meta op - /// (rebuild, meta-update, first-spawn) goes through this queue so - /// hive-c0re runs at most one at a time and the dashboard can - /// render a single ordered view of pending + running work. See - /// `rebuild_queue.rs` for the dedup rules + history retention. - pub rebuild_queue: Arc, + /// Global job-DAG queue. Every container/meta op (rebuild, + /// meta-update, first-spawn, power changes) is submitted as a DAG + /// of primitive nodes; a single scheduler drives them with + /// build-slot + per-agent-lease gating so the dashboard renders one + /// ordered view of pending + running work. See `job_queue/` for + /// the dedup rules, resource classes, and history retention. + pub job_queue: Arc, + /// Durable per-agent power intent (`wanted: Up | Offline`) — the + /// spec half of desired-state reconciliation; the queue's + /// `Reconcile` nodes converge observed state to it. + pub power: Arc, /// Shutdown signal broadcast to all background tasks. Sending /// `true` asks every loop to exit after its current work item. /// Use `shutdown_rx()` to subscribe; `request_shutdown()` to fire. @@ -244,12 +249,27 @@ impl Default for HiveEnv { /// instead of every host-level setting as its own JSON-blob argument. /// `#[serde(default)]` lets any field be omitted and fall back to its /// canonical default. -#[derive(Clone, Debug, Default, serde::Deserialize)] +#[derive(Clone, Debug, serde::Deserialize)] #[serde(default)] pub struct ServeConfig { #[serde(flatten)] pub env: HiveEnv, pub model_prices: crate::hive_stats::PriceTable, + /// Number of concurrent nix-heavy job-queue nodes (prebuild / + /// profile-swap / create / meta lock). hive-c0re-local like + /// `model_prices` — never injected into containers. Set via + /// `services.hyperhive.c0re.buildSlots`. + pub build_slots: usize, +} + +impl Default for ServeConfig { + fn default() -> Self { + Self { + env: HiveEnv::default(), + model_prices: crate::hive_stats::PriceTable::default(), + build_slots: 1, + } + } } #[cfg(test)] @@ -433,6 +453,7 @@ impl Coordinator { db_path: &Path, env: HiveEnv, model_prices: crate::hive_stats::PriceTable, + build_slots: usize, ) -> Result { let HiveEnv { hyperhive_flake, @@ -469,6 +490,7 @@ impl Coordinator { let audit_log = Arc::new(crate::audit_log::AuditLog::open(build_logs_dir).context("open audit_log")?); crate::audit_log::install(audit_log.clone()); + let power = Arc::new(crate::power::PowerStore::open(db_path).context("open agent_power")?); let (dashboard_events, _) = broadcast::channel(DASHBOARD_CHANNEL); let (shutdown_tx, _) = watch::channel(false); Ok(Self { @@ -497,7 +519,8 @@ impl Coordinator { event_seq: AtomicU64::new(0), meta_updates_active: AtomicU64::new(0), last_containers: tokio::sync::Mutex::new(HashMap::new()), - rebuild_queue: Arc::new(crate::rebuild_queue::RebuildQueue::new()), + job_queue: Arc::new(crate::job_queue::JobQueue::new(build_slots)), + power, shutdown_tx, }) } @@ -543,7 +566,7 @@ impl Coordinator { /// wrappers below) and the worker so every state transition /// surfaces on the dashboard without extra plumbing. pub fn emit_rebuild_queue_snapshot(self: &Arc) { - let queue = self.rebuild_queue.snapshot(); + let queue = self.job_queue.snapshot(); self.emit_dashboard_event(DashboardEvent::RebuildQueueChanged { seq: self.next_seq(), queue, @@ -665,15 +688,27 @@ impl Coordinator { }); } - /// Update the `step` label on a running queue entry and (if it - /// actually changed) re-emit the queue snapshot so the dashboard - /// renders the new phase. Returns `true` when the label was new - /// and an emit fired, mostly for tracing/logging callers; safe to - /// ignore. No-op when `id` is `None` (e.g. callers that aren't - /// running from the queue worker) or when the row isn't `Running`. + /// Update the `step` label on the currently-running node of DAG + /// `id` and (if it actually changed) re-emit the queue snapshot so + /// the dashboard renders the new phase. DAG-id-only surface for + /// the opaque approval pipeline in `actions.rs`, whose callbacks + /// don't know node ids (its DAGs are single-node, so the lookup is + /// exact); queue executors use the precise per-node sink in + /// `job_queue::exec` instead. No-op when `id` is `None` (callers + /// not running from the queue) or when nothing is `Running`. pub fn set_queue_step(self: &Arc, id: Option, step: &str) { let Some(id) = id else { return }; - if self.rebuild_queue.set_step(id, step) { + if self.job_queue.set_step_running(id, step) { + self.emit_rebuild_queue_snapshot(); + } + } + + /// Link a `build_logs` row to the currently-running node of DAG + /// `id` and re-emit the snapshot. Same DAG-id-only compatibility + /// surface as [`Self::set_queue_step`]. + pub fn set_queue_build_log(self: &Arc, id: Option, log_id: i64) { + let Some(id) = id else { return }; + if self.job_queue.set_build_log_id_running(id, log_id) { self.emit_rebuild_queue_snapshot(); } } diff --git a/hive-c0re/src/dashboard.rs b/hive-c0re/src/dashboard.rs deleted file mode 100644 index 9ba989db..00000000 --- a/hive-c0re/src/dashboard.rs +++ /dev/null @@ -1,1746 +0,0 @@ -//! Hyperhive dashboard. Lists managed containers (with deep-links to each -//! container's web UI), pending approvals (with unified diff vs the applied -//! repo, plus approve/deny buttons), and the manager. - -use std::convert::Infallible; -use std::net::SocketAddr; -use std::path::Path; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use axum::extract::Form; -use axum::{ - Router, - extract::{Path as AxumPath, State}, - http::{HeaderMap, StatusCode}, - response::{ - IntoResponse, Response, - sse::{Event, KeepAlive, Sse}, - }, - routing::{get, post}, -}; -use hive_sh4re::Approval; -use serde::{Deserialize, Serialize}; -use tokio_stream::wrappers::BroadcastStream; -use tokio_stream::{Stream, StreamExt}; - -use crate::container_view::{ContainerView, claude_has_session}; -use crate::coordinator::Coordinator; -use crate::lifecycle; -use chrono::{DateTime, Utc}; - -mod approvals; -mod build_logs; -mod journal; -mod lifecycle_ops; -mod matrix_accounts; -pub(crate) mod permissions; -mod questions; -mod reminders; -mod schedules; -mod state_files; -mod topology; -mod webhook; - -// Pre-computed at approval-submit time by the manager-socket handler -// (`socket_server.rs`) and embedded in the `ApprovalAdded` event, so -// re-exported at the module root to preserve the `crate::dashboard::approval_diff` -// path across the submodule split. -pub(crate) use approvals::approval_diff; -// Run at broker-message ingest by the coordinator + the operator-msg path -// (`main.rs`); re-exported to preserve the `crate::dashboard::scan_validated_paths` -// path across the split. -pub use state_files::scan_validated_paths; - -#[derive(Clone)] -struct AppState { - coord: Arc, -} - -#[allow( - clippy::too_many_lines, - reason = "the body is dominated by the flat axum route table — one line \ - per endpoint mapping a URL to its (now per-concern submodule) \ - handler; splitting that exhaustive list across helpers would \ - obscure the route map for no readability gain" -)] -pub async fn serve(port: u16, coord: Arc) -> Result<()> { - // API-only: the gateway static-serves the dashboard dist and proxies - // non-static requests here (see hive-gateway.nix). Unmatched paths 404. - let app = Router::new() - .route("/api/state", get(api_state)) - .route("/api/journal/{name}", get(journal::get_journal)) - .route("/api/journal-host", get(journal::get_journal_host)) - .route("/api/approval-diff/{id}", get(approvals::get_approval_diff)) - .route("/api/state-file", get(state_files::get_state_file)) - .route( - "/api/matrix-accounts", - get(matrix_accounts::get_matrix_accounts), - ) - .route("/api/reminders", get(reminders::api_reminders)) - .route("/api/operator-inbox", get(api_operator_inbox)) - .route("/api/stats-hive", get(api_stats_hive)) - .route("/api/container-resources", get(api_container_resources)) - .route("/api/audit-log", get(api_audit_log)) - .route("/api/build-logs", get(build_logs::get_build_logs_all)) - .route( - "/api/build-logs/{agent}", - get(build_logs::get_build_logs_agent), - ) - .route( - "/api/build-logs/id/{id}", - get(build_logs::get_build_log_full), - ) - .route( - "/api/build-logs/id/{id}/stream", - get(build_logs::get_build_log_stream), - ) - .route( - "/api/build-logs/id/{id}/raw", - get(build_logs::get_build_log_raw), - ) - .route("/api/agent/{name}/mark-all-read", post(post_mark_all_read)) - .route("/api/topology/set-parent", post(topology::post_set_parent)) - .route( - "/api/topology/set-parent-bulk", - post(topology::post_set_parent_bulk), - ) - .route("/api/tool-groups", get(permissions::get_tool_groups)) - .route( - "/api/tool-groups/{agent}", - post(permissions::post_tool_groups), - ) - .route("/api/capabilities", get(permissions::get_capabilities)) - .route( - "/api/capabilities/{agent}", - post(permissions::post_capabilities), - ) - .route("/api/permissions", post(permissions::post_permissions)) - .route( - "/api/permissions/stale", - get(permissions::get_stale_permissions), - ) - .route( - "/api/permissions/{agent}", - axum::routing::delete(permissions::delete_agent_permissions), - ) - .route( - "/api/schedules", - get(schedules::api_schedules).post(schedules::post_schedule_new), - ) - .route( - "/api/schedules/{id}", - axum::routing::patch(schedules::patch_schedule), - ) - .route( - "/api/schedules/{id}/cancel", - post(schedules::post_schedule_cancel), - ) - .route( - "/api/schedules/{id}/pause", - post(schedules::post_schedule_pause), - ) - .route( - "/api/schedules/{id}/resume", - post(schedules::post_schedule_resume), - ) - .route( - "/api/schedules/{id}/fire-now", - post(schedules::post_schedule_fire_now), - ) - .route( - "/api/rebuild-queue/{id}/cancel", - post(schedules::post_rebuild_queue_cancel), - ) - .route("/webhook/knowledge", post(webhook::post_webhook_knowledge)) - // Backend routes — the frontend calls these `/api/` paths. The - // transitional bare top-level aliases were removed once the - // frontend migrated. `/webhook/knowledge` keeps its own prefix - // (forge-driven, not the SPA). - .route("/api/approve/{id}", post(approvals::post_approve)) - .route("/api/deny/{id}", post(approvals::post_deny)) - .route("/api/destroy/{name}", post(lifecycle_ops::post_destroy)) - .route("/api/kill/{name}", post(lifecycle_ops::post_kill)) - .route("/api/restart/{name}", post(lifecycle_ops::post_restart)) - .route("/api/start/{name}", post(lifecycle_ops::post_start)) - .route("/api/rebuild/{name}", post(lifecycle_ops::post_rebuild)) - .route("/api/update-all", post(lifecycle_ops::post_update_all)) - .route( - "/api/answer-question/{id}", - post(questions::post_answer_question), - ) - .route( - "/api/cancel-question/{id}", - post(questions::post_cancel_question), - ) - .route("/api/purge-tombstone/{name}", post(post_purge_tombstone)) - .route( - "/api/matrix-account-login", - post(matrix_accounts::post_matrix_account_login), - ) - .route( - "/api/cancel-reminder/{id}", - post(reminders::post_cancel_reminder), - ) - .route( - "/api/retry-reminder/{id}", - post(reminders::post_retry_reminder), - ) - .route("/api/request-spawn", post(post_request_spawn)) - .route("/api/op-send", post(post_op_send)) - .route("/api/meta-update", post(post_meta_update)) - .route("/api/dashboard/stream", get(dashboard_stream)) - .route("/api/dashboard/history", get(dashboard_history)) - // No static fallback — the gateway owns the dist; unmatched paths 404. - .with_state(AppState { coord }); - // Binds loopback-only; external access via gateway. - // Rationale: docs/gateway.md::Firewall posture. - let addr = SocketAddr::from(([127, 0, 0, 1], port)); - let listener = bind_with_retry(addr).await?; - tracing::info!(%addr, "dashboard listening"); - axum::serve(listener, app).await?; - Ok(()) -} - -// SPA shape + SSE channels: docs/web-ui/shape.md. - -/// `SO_REUSEADDR` bind with retry. Retry mechanics, attempt-cap -/// rationale, and log-level cadence: `docs/web-ui/shape.md::Listener bind`. -async fn bind_with_retry(addr: SocketAddr) -> Result { - let mut delay_ms = 250u64; - let mut attempts = 0u32; - loop { - match try_bind(addr) { - Ok(l) => { - if attempts > 0 { - tracing::info!( - %addr, attempts, - "dashboard: bind succeeded after retry" - ); - } - return Ok(l); - } - Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => { - let attempt = attempts + 1; - if attempt <= 12 { - tracing::warn!( - %addr, attempt, - "dashboard: AddrInUse, retrying in {delay_ms}ms" - ); - } else { - tracing::info!( - %addr, attempt, - "dashboard: AddrInUse still holding, retrying in {delay_ms}ms" - ); - } - tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; - attempts += 1; - delay_ms = (delay_ms * 2).min(2000); - } - Err(e) => { - return Err(e).with_context(|| format!("bind dashboard on {addr}")); - } - } - } -} - -fn try_bind(addr: SocketAddr) -> std::io::Result { - let sock = match addr { - SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4()?, - SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6()?, - }; - sock.set_reuseaddr(true)?; - sock.bind(addr)?; - sock.listen(1024) -} - -#[allow(clippy::struct_excessive_bools)] -#[derive(Serialize)] -struct StateSnapshot { - /// Broker seq at the moment this snapshot was assembled. Clients - /// dedupe their buffered SSE traffic against this value: any - /// `MessageEvent` with `seq <= snapshot.seq` is already reflected in - /// the snapshot (or pre-dates it); anything with `seq > snapshot.seq` - /// is post-snapshot and should be applied. Set to 0 in the - /// pre-emit case (no events ever fired) — clients treat that as - /// "apply everything you've buffered". - seq: u64, - hostname: String, - any_stale: bool, - containers: Vec, - transients: Vec, - approvals: Vec, - /// Last 30 resolved approvals (approved / denied / failed), newest- - /// first. Drives the "history" tab on the approvals section. - approval_history: Vec, - /// Pending operator-targeted questions (`target IS NULL`). Any - /// agent can `ask` the operator and `ask` returns immediately with - /// the id; on `/answer-question` we mark the row answered and - /// fire `HelperEvent::QuestionAnswered` back into the asker's - /// inbox. Peer-to-peer questions live in the same table but never - /// surface here (see `OperatorQuestions::pending`). - questions: Vec, - /// Last 20 answered questions, newest-first. - question_history: Vec, - /// State dirs (config history + claude creds + /state/ notes) that - /// survive after a destroy-without-purge. The operator can re-spawn - /// with the same name to resume, or PURG3 to wipe them. - tombstones: Vec, - /// Sub-agents whose FNV-1a hashed web UI port collides with at - /// least one other agent. Operator resolves by renaming. The - /// dashboard renders a banner at the top listing each cluster. - port_conflicts: Vec, - /// Inputs in `meta/flake.lock` the operator can selectively - /// `nix flake update`. Hyperhive first, then `agent-` rows. - meta_inputs: Vec, - /// True while a dashboard-triggered `meta-update` (flake lock bump + - /// agent rebuild ripple) is running in the background. Lets a - /// client that cold-loads mid-update render the META INPUTS panel's - /// disabled "updating…" state; live transitions arrive via the - /// `MetaUpdateRunning` event. - meta_update_running: bool, - /// Current state of the global rebuild queue — pending + running - /// long-lived ops (rebuild / meta-update / spawn) plus the most - /// recent few terminal entries the queue retains for history. - /// Live transitions arrive via the `RebuildQueueChanged` event. - /// See `rebuild_queue.rs`. - rebuild_queue: Vec, - /// Whether the hive-forge container is up. When true the dashboard - /// links each container's config + each approval's commit into the - /// forge's `agent-configs` repos. - forge_present: bool, - /// Whether the matrix GUI is reachable at `/matrix/`. Sourced from - /// `HIVE_MATRIX_GUI_ENABLED` env var (set by the c0re NixOS module - /// when `services.hyperhive.matrix.gui.enable` is on). The gateway - /// (hive-gateway.nix) does the actual `/matrix/` static serving; - /// this flag is just an availability signal for iris's dashboard - /// chrome so the `M4TR1X →` tab doesn't flash when the GUI is off. - matrix_gui_enabled: bool, - /// Whether `hive-gateway` is in front of this dashboard. Sourced - /// from the `HIVE_GATEWAY_ENABLED` env var, which the c0re NixOS - /// module now always sets (the gateway runs unconditionally - /// alongside hyperhive), so this is effectively always true: the - /// dashboard frontend builds same-origin `/agent//` links to - /// the per-agent web UI (the gateway routes them via the - /// runtime-generated `agents.conf` include file — see - /// `gateway_nginx.rs`). The `false` branch (direct - /// `http://:/` TCP links) is retained as a defensive - /// fallback for the env being unset. See `docs/gateway.md::Vhost map`. - gateway_enabled: bool, - /// Public URL of the forge vhost served by hive-gateway (e.g. - /// `"https://forge.pr1ma.darkest.space"`). Sourced from the - /// `HIVE_FORGE_PUBLIC_URL` env var, which the c0re NixOS module - /// sets when `forge.behindGateway = true`. `None` when absent — - /// the frontend falls back to `http://:3000`. - forge_public_url: Option, - /// Human name of this single-host hive instance (e.g. `"pr1ma"`). - /// Sourced from `HYPERHIVE_HIVE_NAME` env var, set by the c0re - /// NixOS module from `services.hyperhive.hiveName`. `None` when - /// the option is unset — chrome falls back to `hostname`. - hive_name: Option, - /// Human name of the wider swarm this hive belongs to (e.g. - /// `"constellat1on"`). Sourced from `HYPERHIVE_SWARM_NAME` env - /// var, set from `services.hyperhive.swarmName`. `None` when - /// unset — chrome omits the swarm segment of the breadcrumb. - swarm_name: Option, - /// Peer hives in the same swarm. Parsed from `HYPERHIVE_PEERS` - /// (JSON array of `{domain,cert_fingerprint}` objects, emitted by - /// the c0re NixOS module from `services.hyperhive.swarm.peers`). - /// Empty on single-hive deploys. Feeds the P33RS dashboard tab. - peer_hives: Vec, - /// Server-level warnings for the dashboard's top-of-page banner - /// (currently host disk-pressure; more producers can be added - /// backend-side). Empty when all clear. Built by - /// `host_stats::server_warnings`; the frontend renders this list - /// generically, so new warning kinds need no frontend change. - server_warnings: Vec, -} - -/// One peer hive for the P33RS dashboard tab. Derived from -/// `HYPERHIVE_PEERS` env; `url` is the peer's HTTPS dashboard root. -/// `cert_fingerprint` is `Some("sha256:")` when the peer uses a -/// self-signed cert and the operator pinned its fingerprint in -/// `services.hyperhive.swarm.peers`. -#[derive(Serialize)] -struct PeerHiveView { - name: String, - url: String, - cert_fingerprint: Option, -} - -/// `OpQuestion` + computed `question_refs` / `answer_refs`. Built -/// from the snapshot read; the live channel attaches the same -/// fields directly on `QuestionAdded` / `QuestionResolved`. -#[derive(Serialize)] -struct QuestionView { - #[serde(flatten)] - inner: crate::operator_questions::OpQuestion, - #[serde(skip_serializing_if = "Vec::is_empty")] - question_refs: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - answer_refs: Vec, -} - -impl QuestionView { - fn from_question(q: crate::operator_questions::OpQuestion) -> Self { - let question_refs = scan_validated_paths(&q.question); - let answer_refs = q - .answer - .as_deref() - .map(scan_validated_paths) - .unwrap_or_default(); - Self { - inner: q, - question_refs, - answer_refs, - } - } -} - -#[derive(Serialize)] -struct PortConflict { - port: u16, - /// All agent names sharing this port (sorted, ≥2 entries). - agents: Vec, -} - -#[derive(Serialize, Clone, Debug)] -pub struct TombstoneView { - pub name: String, - /// Bytes used by the state dir tree. Cheap-ish to compute; let the - /// operator know how much they're holding onto. - pub state_bytes: u64, - /// Mtime (unix seconds) of the state dir; rough "last seen". - pub last_seen: i64, - pub has_creds: bool, -} - -#[derive(Serialize)] -struct TransientView { - name: String, - kind: &'static str, - secs: u64, -} - -#[derive(Serialize)] -struct ApprovalHistoryView { - id: i64, - agent: String, - kind: &'static str, - /// First 12 chars of the canonical sha (preferred) or - /// manager-supplied ref. None for resolved spawn approvals. - sha_short: Option, - /// `approved` / `denied` / `failed`. - status: &'static str, - /// RFC 3339 UTC. Renders as a relative time on the dashboard. - resolved_at: DateTime, - /// Operator-supplied deny reason (for `denied`) or build error - /// (for `failed`). None on `approved`. - #[serde(skip_serializing_if = "Option::is_none")] - note: Option, -} - -#[derive(Serialize)] -struct ApprovalView { - id: i64, - agent: String, - kind: &'static str, - /// First 12 chars of the `commit_ref`, for `ApplyCommit` only. - sha_short: Option, - /// Raw unified diff text, for `ApplyCommit` only. The client splits - /// on `\n` and per-line classifies (`+` / `-` / `@@` / `--- ` / `+++ ` - /// → diff-add / diff-del / diff-hunk / diff-file). Shipping raw - /// instead of pre-rendered HTML saves bytes on the wire (no - /// per-line `` markup) and removes the only HTML-escape - /// surface from the snapshot. - diff: Option, - /// Manager-supplied description shown on the approval card. - #[serde(skip_serializing_if = "Option::is_none")] - description: Option, - /// Forge PR number, for `MergeConfigPr` only. Lets the frontend - /// build a "review PR on forge" link - /// (`{forgeBase}/agent-configs/{agent}/pulls/{pr_number}`) the same - /// way it builds the `apply_commit` "commit on forge" link from the - /// sha. `None` for every other kind. - #[serde(skip_serializing_if = "Option::is_none")] - pr_number: Option, - /// Raw `commit_ref` payload for `UpdateMetaInputs` (JSON-encoded - /// `Vec` of input names; `"[]"` = all inputs) and - /// `SchedulePrompt` (JSON-encoded `SchedulePromptPayload`). The - /// frontend parses this to render a human-readable card body. - /// `None` for every other kind. - #[serde(skip_serializing_if = "Option::is_none")] - commit_ref: Option, - /// RFC 3339 UTC time the approval was queued. Rendered as a - /// relative time on the card so the operator can spot a stale - /// request. - requested_at: DateTime, -} - -/// Replace silent `.unwrap_or_default()` on the data sources behind -/// `/api/state` so that whichever query degrades surfaces in journald -/// instead of leaving the operator staring at an empty list. The -/// dashboard still degrades to a sensible default value; the warn -/// is just the diagnostic breadcrumb the old code swallowed. -fn log_default(what: &str, result: std::result::Result) -> T -where - T: Default, - E: std::fmt::Debug, -{ - match result { - Ok(v) => v, - Err(e) => { - tracing::warn!(target: "api_state", source = %what, error = ?e, "snapshot source failed; using default"); - T::default() - } - } -} - -/// Window over which container crashes count toward the `agents_crashing` -/// banner warning. Wide enough that a crash-looping container (restarted -/// by `Restart=on-failure` every few seconds) keeps the warning lit -/// between flaps, short enough that a single recovered crash clears within -/// minutes. -const CRASH_WARNING_WINDOW: std::time::Duration = std::time::Duration::from_mins(10); - -async fn api_state(headers: HeaderMap, State(state): State) -> axum::Json { - let host = headers - .get("host") - .and_then(|h| h.to_str().ok()) - .unwrap_or("localhost"); - let hostname = host.split(':').next().unwrap_or(host).to_owned(); - - // Capture the unified dashboard-channel seq *before* any read so the - // dedupe contract is "events with seq > snapshot.seq are - // post-snapshot, never missed." An event landing during snapshot - // construction may be doubly applied (snapshot caught the write + - // client also applies the SSE frame) — that's a renderer's problem - // to make idempotent, not ours to avoid here. - let seq = state.coord.current_seq(); - - // Refresh the coordinator's cached container snapshot before - // reading. Cold-load clients then see whatever the latest rescan - // produced; live clients converge via the matching - // `ContainerStateChanged` / `ContainerRemoved` events the rescan - // emits. - // - // Bound the rescan: it shells out (`nixos-container list` etc.), so a - // saturated/wedged build backend — e.g. hive-c0re mid-startup-sweep - // hammering slow `nixos-container update` subprocesses — can stall it - // long enough that `/api/state` hangs for the whole request (the - // ~minute-long /state reported in the field). On timeout we skip the - // fresh rescan and serve the last cached snapshot instead; live - // clients still converge via the SSE events a later successful rescan - // emits, and the next /state call retries the refresh. Introspection - // stays responsive regardless of the build backend's health. - if tokio::time::timeout( - std::time::Duration::from_secs(3), - state.coord.rescan_containers_and_emit(), - ) - .await - .is_err() - { - tracing::warn!( - "api_state: container rescan exceeded 3s (build backend likely saturated); \ - serving last cached snapshot" - ); - } - let containers = state.coord.containers_snapshot().await; - let any_stale = containers.iter().any(|c| c.needs_update); - let transient_snapshot = state.coord.transient_snapshot(); - let pending_approvals = approvals::gc_orphans( - &state.coord, - log_default("approvals.pending", state.coord.approvals.pending()), - ); - let transients = build_transient_views(&containers, &transient_snapshot); - let approvals = build_approval_views(pending_approvals).await; - let approval_history = log_default( - "approvals.recent_resolved", - state.coord.approvals.recent_resolved(30), - ) - .into_iter() - .map(history_view) - .collect(); - let tombstones = build_tombstone_views(&state.coord, &containers, &transient_snapshot); - let port_conflicts = build_port_conflicts(&containers); - - // Both operator-targeted and peer threads surface on the dashboard - // (the client filters by target). Each row is wrapped in QuestionView - // so the snapshot carries the same file_refs the live event variants - // attach. - let questions: Vec = - log_default("questions.pending_all", state.coord.questions.pending_all()) - .into_iter() - .map(QuestionView::from_question) - .collect(); - let question_history: Vec = log_default( - "questions.recent_answered_all", - state.coord.questions.recent_answered_all(20), - ) - .into_iter() - .map(QuestionView::from_question) - .collect(); - - // Banner warnings: host probes (disk) + agent-state (pending logins, - // crashing agents). Built before the response struct because the - // agent-state producer borrows `containers`, which moves in below. - let server_warnings = { - let mut w = crate::host_stats::server_warnings(); - w.extend(crate::host_stats::agent_state_warnings( - &containers, - &state.coord.recent_crash_counts(CRASH_WARNING_WINDOW), - )); - w - }; - - axum::Json(StateSnapshot { - seq, - hostname, - any_stale, - containers, - transients, - approvals, - approval_history, - meta_inputs: read_meta_inputs(), - meta_update_running: state.coord.meta_update_in_progress(), - questions, - question_history, - tombstones, - port_conflicts, - rebuild_queue: state.coord.rebuild_queue.snapshot(), - forge_present: crate::forge::is_present().await, - matrix_gui_enabled: std::env::var_os("HIVE_MATRIX_GUI_ENABLED").is_some_and(|v| { - // Accept any truthy string ("1", "true", "yes") since the - // env var is set by NixOS module wiring with the literal - // "1"; defensive parse so manual overrides also work. - let s = v.to_string_lossy().to_ascii_lowercase(); - matches!(s.as_str(), "1" | "true" | "yes") - }), - gateway_enabled: std::env::var_os("HIVE_GATEWAY_ENABLED").is_some_and(|v| { - // Same truthy-string parse as `matrix_gui_enabled`; the - // env var is set by the c0re NixOS module to the literal - // "1" — the gateway always runs alongside hyperhive. - let s = v.to_string_lossy().to_ascii_lowercase(); - matches!(s.as_str(), "1" | "true" | "yes") - }), - forge_public_url: std::env::var("HIVE_FORGE_PUBLIC_URL") - .ok() - .filter(|s| !s.is_empty()), - hive_name: std::env::var("HYPERHIVE_HIVE_NAME") - .ok() - .filter(|s| !s.is_empty()), - swarm_name: std::env::var("HYPERHIVE_SWARM_NAME") - .ok() - .filter(|s| !s.is_empty()), - peer_hives: parse_peer_hives(), - server_warnings, - }) -} - -/// Parse `HYPERHIVE_PEERS` env var into dashboard-ready `PeerHiveView` -/// entries. The env var is a JSON array of `{domain, cert_fingerprint}` -/// objects emitted by the c0re NixOS module from -/// `services.hyperhive.swarm.peers`. Each entry becomes -/// `{ name: domain, url: "https://domain/" }` for the P33RS tab. -/// Returns empty vec when unset (single-hive deploy). -fn parse_peer_hives() -> Vec { - #[derive(serde::Deserialize)] - struct Raw { - domain: String, - cert_fingerprint: Option, - } - let Ok(json) = std::env::var("HYPERHIVE_PEERS") else { - return Vec::new(); - }; - let Ok(raw): Result, _> = serde_json::from_str(&json) else { - tracing::warn!("HYPERHIVE_PEERS is not valid JSON; ignoring"); - return Vec::new(); - }; - raw.into_iter() - .map(|r| { - let cert_fingerprint = r.cert_fingerprint.and_then(|fp| { - if validate_cert_fingerprint(&fp) { - Some(fp) - } else { - tracing::warn!( - domain = %r.domain, - fingerprint = %fp, - "HYPERHIVE_PEERS: invalid cert_fingerprint format \ - (expected `sha256:<64 hex chars>`); ignoring fingerprint" - ); - None - } - }); - PeerHiveView { - name: r.domain.clone(), - url: format!("https://{}/", r.domain), - cert_fingerprint, - } - }) - .collect() -} - -/// Validate a TLS certificate fingerprint string from `HYPERHIVE_PEERS`. -/// Accepts `sha256:<64 hex chars>` (upper or lower case). -fn validate_cert_fingerprint(fp: &str) -> bool { - let Some(hex) = fp.strip_prefix("sha256:") else { - return false; - }; - hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit()) -} - -/// Group live containers by their assigned web UI port; clusters with -/// more than one member are port-hash collisions the operator needs -/// to resolve by renaming. Manager (fixed at 8000) and sub-agents -/// (8100..8999) can't collide with each other — collisions are -/// strictly between sub-agents. -fn build_port_conflicts(containers: &[ContainerView]) -> Vec { - let mut by_port: std::collections::BTreeMap> = - std::collections::BTreeMap::new(); - for c in containers { - by_port.entry(c.port).or_default().push(c.name.clone()); - } - by_port - .into_iter() - .filter(|(_, agents)| agents.len() > 1) - .map(|(port, mut agents)| { - agents.sort(); - PortConflict { port, agents } - }) - .collect() -} - -#[derive(Serialize, Clone, Debug)] -pub struct MetaInputView { - /// Input key in meta's `flake.nix` — `hyperhive`, `agent-`, etc. - pub name: String, - /// Full locked sha. Not displayed verbatim; the dashboard - /// truncates to the first 12 chars for the chip. - pub rev: String, - /// Unix seconds — `locked.lastModified`. Drives the relative - /// "2h ago" timestamp on each input row. - pub last_modified: i64, - /// `original.url` if available, for the tooltip / row meta text. - #[serde(skip_serializing_if = "Option::is_none")] - pub url: Option, -} - -/// Walk `flake.lock`'s `nodes` graph from `root` and emit one -/// `MetaInputView` per fetched input, at **every** depth. That -/// surfaces the direct meta inputs (`hyperhive`, `agent-`), the -/// agent flakes' own inputs (`agent-dmatrix/mcp-matrix`, -/// `hyperhive/nixpkgs`), and any deeper transitive inputs — so the -/// operator can bump any of them individually. Names are -/// slash-separated paths from root, the syntax `nix flake update` -/// accepts for transitive inputs. -/// -/// Filtering: -/// - Inputs that resolve via a `follows` chain (lock value is an -/// array) are skipped — they alias another node, not their own -/// fetched derivation, so updating them does nothing. -/// - A node is emitted only when it carries a `locked.rev`. -/// - Each fetched node is walked exactly once (a `visited` set): -/// the lock graph shares nodes (many flakes reference one -/// nixpkgs), so without this a shared subtree re-walks per parent -/// and a cycle would recurse forever. The result is a spanning -/// tree — every input shown once, at its shallowest path. -fn read_meta_inputs() -> Vec { - let mut out = Vec::new(); - let Ok(raw) = std::fs::read_to_string("/var/lib/hyperhive/meta/flake.lock") else { - return out; - }; - let Ok(json) = serde_json::from_str::(&raw) else { - return out; - }; - let Some(nodes) = json.get("nodes").and_then(|v| v.as_object()) else { - return out; - }; - let Some(root_name) = json.get("root").and_then(|v| v.as_str()) else { - return out; - }; - let mut visited = std::collections::HashSet::new(); - visited.insert(root_name.to_owned()); - walk_meta_inputs(nodes, root_name, "", &mut visited, &mut out); - // hyperhive first, then alphabetical. String-sorting the - // slash-paths puts every node directly above its own children - // (`agent-foo`, `agent-foo/bar`, `agent-foo/bar/baz`), so the - // result is a pre-order traversal the tree renderer can consume. - out.sort_by(|a, b| match (a.name.as_str(), b.name.as_str()) { - ("hyperhive", _) => std::cmp::Ordering::Less, - (_, "hyperhive") => std::cmp::Ordering::Greater, - _ => a.name.cmp(&b.name), - }); - out -} - -fn walk_meta_inputs( - nodes: &serde_json::Map, - node_name: &str, - prefix: &str, - visited: &mut std::collections::HashSet, - out: &mut Vec, -) { - let Some(node) = nodes.get(node_name) else { - return; - }; - let Some(inputs_map) = node.get("inputs").and_then(|v| v.as_object()) else { - return; - }; - // Two passes: claim (and emit) every direct input of this node - // before descending into any of them. A shallow input that a - // deeper flake also references then keeps its shallow path - // rather than being captured first by the deep walk. - let mut to_recurse: Vec<(String, String)> = Vec::new(); - for (alias, target) in inputs_map { - // Inputs map value is either a string (node name) or an - // array (a `follows` chain). The latter just aliases another - // node — we can't `nix flake update` it directly, so skip. - let serde_json::Value::String(target_name) = target else { - continue; - }; - // Walk each fetched node once — guards shared subtrees and - // cycles, and keeps the panel free of duplicate rows. - if !visited.insert(target_name.clone()) { - continue; - } - let Some(target_node) = nodes.get(target_name) else { - continue; - }; - let path = if prefix.is_empty() { - alias.clone() - } else { - format!("{prefix}/{alias}") - }; - if let Some(rev) = target_node - .get("locked") - .and_then(|v| v.get("rev")) - .and_then(|v| v.as_str()) - { - let last_modified = target_node - .get("locked") - .and_then(|v| v.get("lastModified")) - .and_then(serde_json::Value::as_i64) - .unwrap_or(0); - let url = target_node - .get("original") - .and_then(|v| v.get("url")) - .and_then(|v| v.as_str()) - .map(str::to_owned); - out.push(MetaInputView { - name: path.clone(), - rev: rev.to_owned(), - last_modified, - url, - }); - } - to_recurse.push((target_name.clone(), path)); - } - // Recurse hyperhive's subtree before any agent's — without this, - // when meta's top-level `nixpkgs` is a `follows` alias the - // `String` check above skips it, and the alphabetical BTreeMap - // iteration descends into `agent-*` first. The agent walk then - // claims `nixpkgs` at `agent-X/nixpkgs` instead of - // `hyperhive/nixpkgs`, which is where the operator expects it. - // Sort by the same "hyperhive first, then alpha" - // priority `read_meta_inputs` uses for the final output. - to_recurse.sort_by(|(a, _), (b, _)| match (a.as_str(), b.as_str()) { - ("hyperhive", _) => std::cmp::Ordering::Less, - (_, "hyperhive") => std::cmp::Ordering::Greater, - _ => a.cmp(b), - }); - for (target_name, path) in to_recurse { - walk_meta_inputs(nodes, &target_name, &path, visited, out); - } -} - -/// Transient state for agents whose container does NOT yet exist -/// (`Spawning`). Lifecycle ops on existing containers surface as -/// `ContainerView.pending` inline; this list only catches pre-creation. -fn build_transient_views( - containers: &[ContainerView], - transient_snapshot: &std::collections::HashMap, -) -> Vec { - transient_snapshot - .iter() - .filter(|(name, _)| !containers.iter().any(|c| &c.name == *name)) - .map(|(name, st)| TransientView { - name: name.clone(), - kind: transient_label(st.kind), - secs: st.since.elapsed().as_secs(), - }) - .collect() -} - -/// Render each pending approval into its dashboard view (short sha + -/// unified diff for `ApplyCommit`, just the name for `Spawn`). -/// Project a resolved sqlite row into the lean shape the dashboard -/// history tab consumes — no `diff_html` (rendering 30 of them -/// per /api/state poll would mean 30 git diffs per refresh). -fn history_view(a: Approval) -> ApprovalHistoryView { - let displayed = a.fetched_sha.as_deref().unwrap_or(&a.commit_ref); - let sha_short = if displayed.is_empty() { - None - } else { - Some(displayed[..displayed.len().min(12)].to_owned()) - }; - let status = match a.status { - hive_sh4re::ApprovalStatus::Approved => "approved", - hive_sh4re::ApprovalStatus::Denied => "denied", - hive_sh4re::ApprovalStatus::Failed => "failed", - hive_sh4re::ApprovalStatus::Cancelled => "cancelled", - // Pending shouldn't appear in recent_resolved, but be defensive. - hive_sh4re::ApprovalStatus::Pending => "pending", - }; - let kind = match a.kind { - hive_sh4re::ApprovalKind::ApplyCommit => "apply_commit", - hive_sh4re::ApprovalKind::Spawn => "spawn", - hive_sh4re::ApprovalKind::InitConfig => "init_config", - hive_sh4re::ApprovalKind::UpdateMetaInputs => "update_meta_inputs", - hive_sh4re::ApprovalKind::SchedulePrompt => "schedule_prompt", - hive_sh4re::ApprovalKind::MergeConfigPr => "merge_config_pr", - }; - ApprovalHistoryView { - id: a.id, - agent: a.agent, - kind, - sha_short, - status, - resolved_at: a.resolved_at.unwrap_or_default(), - note: a.note, - } -} - -async fn build_approval_views(approvals: Vec) -> Vec { - let mut out = Vec::with_capacity(approvals.len()); - for a in approvals { - out.push(match a.kind { - hive_sh4re::ApprovalKind::ApplyCommit => { - // Prefer the canonical fetched sha from applied; - // commit_ref is only the manager's claim and may be - // amended out from under us. - let displayed = a.fetched_sha.as_deref().unwrap_or(&a.commit_ref); - let sha = displayed[..displayed.len().min(12)].to_owned(); - let diff = approval_diff(&a.agent, a.id).await; - ApprovalView { - id: a.id, - agent: a.agent.clone(), - kind: "apply_commit", - sha_short: Some(sha), - diff: Some(diff), - description: a.description, - pr_number: None, - commit_ref: None, - requested_at: a.requested_at, - } - } - hive_sh4re::ApprovalKind::Spawn => ApprovalView { - id: a.id, - agent: a.agent, - kind: "spawn", - sha_short: None, - diff: None, - description: a.description, - pr_number: None, - commit_ref: None, - requested_at: a.requested_at, - }, - hive_sh4re::ApprovalKind::InitConfig => ApprovalView { - id: a.id, - agent: a.agent, - kind: "init_config", - sha_short: None, - diff: None, - description: a.description, - pr_number: None, - commit_ref: None, - requested_at: a.requested_at, - }, - hive_sh4re::ApprovalKind::UpdateMetaInputs => ApprovalView { - id: a.id, - agent: a.agent, - kind: "update_meta_inputs", - sha_short: None, - diff: None, - description: a.description, - pr_number: None, - commit_ref: Some(a.commit_ref), - requested_at: a.requested_at, - }, - hive_sh4re::ApprovalKind::SchedulePrompt => ApprovalView { - id: a.id, - agent: a.agent, - kind: "schedule_prompt", - sha_short: None, - diff: None, - description: a.description, - pr_number: None, - commit_ref: Some(a.commit_ref), - requested_at: a.requested_at, - }, - hive_sh4re::ApprovalKind::MergeConfigPr => { - // commit_ref = PR number; fetched_sha = the reviewed PR - // head. Show the head sha; the forge PR diff surface is - // a later phase of the PR-based config flow — None for now. - let sha = a - .fetched_sha - .as_deref() - .map(|s| s[..s.len().min(12)].to_owned()); - // Surface the PR number so the frontend can link to the - // PR on the forge. commit_ref holds the number as text. - let pr_number = a.commit_ref.parse::().ok(); - ApprovalView { - id: a.id, - agent: a.agent, - kind: "merge_config_pr", - sha_short: sha, - diff: None, - description: a.description, - pr_number, - commit_ref: None, - requested_at: a.requested_at, - } - } - }); - } - out -} - -/// State-dir names that don't appear in the live container list. Each -/// one surfaces in the dashboard as a row with R3V1V3 + PURG3 actions. -fn build_tombstone_views( - coord: &Coordinator, - containers: &[ContainerView], - transient_snapshot: &std::collections::HashMap, -) -> Vec { - let _ = coord; // kept_state_names is a free fn but takes &self by future plan - let live: std::collections::HashSet<&str> = containers - .iter() - .map(|c| c.name.as_str()) - .chain(transient_snapshot.keys().map(String::as_str)) - .collect(); - Coordinator::kept_state_names() - .into_iter() - .filter(|name| !live.contains(name.as_str())) - .map(|name| { - let root = Coordinator::agent_state_root(&name); - let state_bytes = dir_size_bytes(&root); - let last_seen = std::fs::metadata(&root) - .and_then(|m| m.modified()) - .ok() - .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0); - let has_creds = claude_has_session(&Coordinator::agent_claude_dir(&name)); - TombstoneView { - name, - state_bytes, - last_seen, - has_creds, - } - }) - .collect() -} - -/// Sum the byte size of every regular file under `root`. Cheap to compute -/// for typical agent state (config repo + claude creds + notes file — -/// usually a few MB); fine to do inline on each /api/state. Returns 0 on -/// any error. -fn dir_size_bytes(root: &Path) -> u64 { - fn walk(p: &Path, acc: &mut u64) { - let Ok(rd) = std::fs::read_dir(p) else { return }; - for entry in rd.flatten() { - let Ok(ft) = entry.file_type() else { continue }; - if ft.is_dir() { - walk(&entry.path(), acc); - } else if ft.is_file() - && let Ok(meta) = entry.metadata() - { - *acc += meta.len(); - } - } - } - let mut total = 0u64; - walk(root, &mut total); - total -} - -async fn dashboard_history(State(state): State) -> Response { - // Backfill source for the dashboard terminal. Returns up to ~200 - // historical broker messages (no other event kinds are persisted) - // converted to `DashboardEvent::Sent` JSON so the client can replay - // through the same dispatch path as live frames. Wrapped in - // `{ seq, events }`: the seq is the dashboard channel's high-water - // mark at fetch time. Clients use it to dedupe their buffered live - // SSE traffic (drop anything with `seq <= history_seq`) so a frame - // that lands between SSE-subscribe and history-fetch isn't shown - // twice and isn't lost. Historical rows carry `seq = 0`; the - // boundary seq is what closes the dedupe window. - const HISTORY_LIMIT: u64 = 200; - let seq = state.coord.current_seq(); - match state.coord.broker.recent_all(HISTORY_LIMIT) { - Ok(mut messages) => { - messages.reverse(); - let events: Vec = messages - .into_iter() - .map(|m| match m { - crate::broker::MessageEvent::Sent { - id, - from, - to, - body, - at, - in_reply_to, - } => { - let file_refs = scan_validated_paths(&body); - crate::dashboard_events::DashboardEvent::Sent { - seq: 0, - id, - from, - to, - body, - at: hive_sh4re::wire_time::from_secs(at), - in_reply_to, - file_refs, - } - } - crate::broker::MessageEvent::Delivered { - id, - from, - to, - body, - at, - in_reply_to, - } => { - let file_refs = scan_validated_paths(&body); - crate::dashboard_events::DashboardEvent::Delivered { - seq: 0, - id, - from, - to, - body, - at: hive_sh4re::wire_time::from_secs(at), - in_reply_to, - file_refs, - } - } - }) - .collect(); - axum::Json(serde_json::json!({ "seq": seq, "events": events })).into_response() - } - Err(e) => error_response(&format!("dashboard/history failed: {e:#}")), - } -} - -/// `/dashboard/stream` query string. Today's only field is `kinds`: -/// a comma-separated allow-list of event-`kind` strings. -/// Empty / absent ⇒ no filter (current behaviour, all variants -/// forwarded). Set ⇒ only the named kinds reach the subscriber, -/// non-matches are skipped before the JSON serialise cost. -/// -/// Useful for narrow pages (e.g. `flow.js` only cares about `sent` -/// / `delivered` / `container_state_changed` / `container_removed`) -/// that want to drop the dispatch overhead on every unrelated mutation. -#[derive(Deserialize, Default)] -struct DashboardStreamQuery { - /// Comma-separated event kinds to forward. Each token is - /// trimmed; unknown kinds are silently ignored on lookup - /// (subscriber sees nothing instead of an error). - kinds: Option, -} - -async fn dashboard_stream( - State(state): State, - axum::extract::Query(q): axum::extract::Query, -) -> Sse>> { - let rx = state.coord.dashboard_subscribe(); - // Pre-parse the allow-list once at subscription time, so the - // per-event hot path is just a `HashSet::contains` on a - // `&'static str` — no string churn per frame. - let kind_filter: Option> = q.kinds.and_then(|raw| { - let set: std::collections::HashSet = raw - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(str::to_owned) - .collect(); - if set.is_empty() { None } else { Some(set) } - }); - let stream = BroadcastStream::new(rx).filter_map(move |res| { - // Drop lagged frames. Browsers reconnect; the seq dedupe on - // reconnect skips any frame already reflected in the snapshot. - let event = res.ok()?; - if let Some(filter) = kind_filter.as_ref() - && !filter.contains(event.kind_tag()) - { - return None; - } - let json = serde_json::to_string(&event).ok()?; - Some(Ok(Event::default().data(json))) - }); - Sse::new(stream).keep_alive(KeepAlive::default()) -} - -#[derive(Deserialize)] -struct RequestSpawnForm { - name: String, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn problem_details_carry_rfc9457_status_and_detail() { - // Contract the frontend depends on: the problem_details crate - // serialises the RFC 9457 members we rely on — `status` (numeric) - // and `detail` (the caller message; the FE reads `.detail`). - let pd = problem_details::ProblemDetails::from_status_code(StatusCode::BAD_REQUEST) - .with_detail("bad input"); - let v = serde_json::to_value(&pd).expect("problem details serialise"); - assert_eq!(v["status"], 400); - assert_eq!(v["detail"], "bad input"); - // The 500 wrapper path carries the internal-error status. - let five = - problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR) - .with_detail("boom"); - let fv = serde_json::to_value(&five).expect("problem details serialise"); - assert_eq!(fv["status"], 500); - } - - #[test] - fn walk_meta_inputs_keeps_nixpkgs_under_hyperhive_post_follows_refactor() { - // Reproduce the shape where meta has - // `nixpkgs.follows = "hyperhive/nixpkgs"` at the top level - // (rendered as an array — `["hyperhive" "nixpkgs"]` — which - // walk_meta_inputs skips because we can't `nix flake update` - // a follows alias). The remaining top-level inputs are - // `hyperhive` (string) and `agent-z` (string). Without the - // hyperhive-first recursion sort, the BTreeMap alphabetical - // order descends into `agent-z` first and claims - // `nixpkgs` at `agent-z/nixpkgs`. - let raw = r#"{ - "root": "root", - "version": 7, - "nodes": { - "root": { - "inputs": { - "hyperhive": "hyperhive", - "nixpkgs": ["hyperhive", "nixpkgs"], - "agent-z": "agent-z" - } - }, - "hyperhive": { - "inputs": { "nixpkgs": "nixpkgs" }, - "locked": {"rev": "hhrev", "lastModified": 1}, - "original": {"url": "git+file:///tmp/hyperhive"} - }, - "agent-z": { - "inputs": { "nixpkgs": "nixpkgs" }, - "locked": {"rev": "azrev", "lastModified": 2}, - "original": {"url": "git+file:///tmp/agent-z"} - }, - "nixpkgs": { - "locked": {"rev": "npkrev", "lastModified": 3}, - "original": {"url": "github:NixOS/nixpkgs/nixos-26.05"} - } - } - }"#; - let json: serde_json::Value = serde_json::from_str(raw).unwrap(); - let nodes = json.get("nodes").unwrap().as_object().unwrap(); - let root_name = json.get("root").unwrap().as_str().unwrap(); - let mut visited = std::collections::HashSet::new(); - visited.insert(root_name.to_owned()); - let mut out = Vec::new(); - walk_meta_inputs(nodes, root_name, "", &mut visited, &mut out); - - let nixpkgs = out - .iter() - .find(|v| v.rev == "npkrev") - .expect("nixpkgs node should be emitted exactly once"); - assert_eq!( - nixpkgs.name, "hyperhive/nixpkgs", - "nixpkgs should be claimed under hyperhive, not under agent-z. \ - got: {:?}", - nixpkgs.name - ); - // And the agent-z path should NOT also carry a nixpkgs entry — - // the spanning-tree visited set guarantees it's claimed once. - assert!( - !out.iter().any(|v| v.name == "agent-z/nixpkgs"), - "agent-z/nixpkgs should not be emitted (already claimed under hyperhive)" - ); - } - - #[test] - fn validate_agent_name_accepts_canonical_shapes() { - assert!(validate_agent_name("damocles").is_none()); - assert!(validate_agent_name("hm1nd").is_none()); - assert!(validate_agent_name("agent-with-dashes").is_none()); - assert!(validate_agent_name("snake_case").is_none()); - assert!(validate_agent_name("mixed_2-3").is_none()); - let max = "a".repeat(63); - assert!( - validate_agent_name(&max).is_none(), - "63-char name should pass" - ); - } - - // The two-axis guard (`guard_agent_name`) wires `validate_agent_name` - // + an async coordinator lookup. The lookup needs a populated - // `Coordinator`, which needs sqlite + tokio runtime; rather than - // build that scaffolding for an integration-flavoured test we cover - // the format axis here (the existence axis is enforced by the - // shared `containers_snapshot` API, tested in `coordinator.rs`'s - // own suite). 9 cases below cover the boundary-length case and - // other expected rejects to make the contract explicit. - #[test] - fn validate_agent_name_rejects_bad_input() { - assert!(validate_agent_name("").is_some()); - let too_long = "a".repeat(64); - assert!(validate_agent_name(&too_long).is_some()); - // Path-traversal attempts. - assert!(validate_agent_name("../etc/passwd").is_some()); - assert!(validate_agent_name("alice/bob").is_some()); - // Uppercase rejected — canonical lowercase convention. - assert!(validate_agent_name("Alice").is_some()); - // No spaces, dots, special chars. - assert!(validate_agent_name("alice bob").is_some()); - assert!(validate_agent_name("alice.bob").is_some()); - assert!(validate_agent_name("alice;DROP TABLE messages").is_some()); - // Non-ASCII (incl. unicode homoglyphs of ASCII dash). - assert!(validate_agent_name("damóclès").is_some()); - assert!(validate_agent_name("alice\u{2013}bob").is_some()); // en-dash - } -} - -/// Snapshot the current tombstone list and emit a -/// `TombstonesChanged` event. Call after any mutation that could -/// add or remove a tombstone (`actions::destroy`, -/// `post_purge_tombstone`, spawn finalisation). Cheap — the list -/// is tiny. -pub(crate) async fn emit_tombstones_snapshot(coord: &Arc) { - let containers = coord.containers_snapshot().await; - let transient_snapshot = coord.transient_snapshot(); - let tombstones = build_tombstone_views(coord, &containers, &transient_snapshot); - coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::TombstonesChanged { - seq: coord.next_seq(), - tombstones, - }); -} - -/// Snapshot meta/flake.lock's root inputs + emit -/// `MetaInputsChanged`. Call after any mutation that bumps a lock -/// (`run_meta_update`, `auto_update::rebuild_agent`). -pub(crate) fn emit_meta_inputs_snapshot(coord: &Coordinator) { - let inputs = read_meta_inputs(); - coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::MetaInputsChanged { - seq: coord.next_seq(), - inputs, - }); -} - -/// Unread operator-directed messages for the dashboard's Y3R C4LL inbox. -/// Returns messages addressed to `"operator"` that haven't been -/// acked yet (the operator clears them via the existing -/// `POST /api/agent/operator/mark-all-read`). Newest-first; path-shaped -/// tokens are validated so the client renders file links like the -/// terminal does. Shape: `{ "messages": [{ id, from, body, at, -/// in_reply_to, file_refs }] }`. -async fn api_operator_inbox(State(state): State) -> Response { - const INBOX_LIMIT: u64 = 100; - match state - .coord - .broker - .unread_for_recipient("operator", INBOX_LIMIT) - { - Ok(messages) => { - let items: Vec = messages - .into_iter() - .filter_map(|m| { - let crate::broker::MessageEvent::Sent { - id, - from, - body, - at, - in_reply_to, - .. - } = m - else { - return None; - }; - let file_refs = scan_validated_paths(&body); - Some(serde_json::json!({ - "id": id, - "from": from, - "body": body, - "at": hive_sh4re::wire_time::from_secs(at), - "in_reply_to": in_reply_to, - "file_refs": file_refs, - })) - }) - .collect(); - axum::Json(serde_json::json!({ "messages": items })).into_response() - } - Err(e) => error_response(&format!("operator-inbox failed: {e:#}")), - } -} - -#[derive(Deserialize)] -struct StatsHiveQuery { - window: Option, -} - -/// Hive-wide turn-stats rollup for the dashboard swarm-stats view. -/// Aggregates every agent's `hyperhive-turn-stats.sqlite` read-only -/// (skips missing/unreadable ones). Window defaults to `24h`. -async fn api_stats_hive( - State(state): State, - axum::extract::Query(q): axum::extract::Query, -) -> Response { - let window = crate::hive_stats::Window::parse(q.window.as_deref().unwrap_or("24h")); - axum::Json(crate::hive_stats::hive_snapshot( - window, - &state.coord.model_prices, - )) - .into_response() -} - -/// Live per-agent-container CPU + memory load from cgroup v2. Samples -/// CPU over a short interval (~200 ms), so this call briefly awaits. -async fn api_container_resources() -> Response { - axum::Json(crate::container_stats::gather().await).into_response() -} - -/// `GET /api/audit-log` — most-recent agent-initiated privileged-action -/// audit entries, newest first (server-clamped to 500). Backs the -/// operator dashboard's audit view. Returns -/// `{ "entries": [AuditEntry…], "total": N }` so the UI can show -/// "latest 500 of N" rather than silently capping. `ts_unix` is in -/// **seconds**. -async fn api_audit_log(State(state): State) -> Response { - const LIMIT: usize = 500; - let entries = match state.coord.audit_log.list_recent(LIMIT) { - Ok(rows) => rows, - Err(e) => return error_response(&format!("audit-log: {e:#}")), - }; - let total = match state.coord.audit_log.count_total() { - Ok(n) => n, - Err(e) => return error_response(&format!("audit-log count: {e:#}")), - }; - axum::Json(serde_json::json!({ "entries": entries, "total": total })).into_response() -} - -/// Validate that a path-param agent name conforms to the hyperhive -/// naming whitelist: 1-63 chars of `[a-z0-9_-]`. Rejects empty, -/// uppercase, slashes, dots, and any non-ASCII (incl. unicode -/// homoglyphs of dash/underscore). Returns `None` on accept, `Some(reason)` -/// on reject — caller wraps the reason in a 400 response. Conservative -/// whitelist matching `nixos-container` basename rules and the existing -/// agent-name convention across the codebase. -pub(crate) fn validate_agent_name(name: &str) -> Option<&'static str> { - if name.is_empty() { - return Some("agent name must not be empty"); - } - if name.len() > 63 { - return Some("agent name must be 63 characters or fewer"); - } - if !name - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_') - { - return Some("agent name must contain only [a-z0-9_-]"); - } - None -} - -/// Two-axis path-param guard for write routes. Combines: -/// -/// 1. **format validation** (`validate_agent_name`) — rejects path -/// traversal / unicode homoglyphs / empty + too-long names with -/// HTTP 400. -/// 2. **existence check** — looks up `name` in the coordinator's -/// container snapshot; unknown name → HTTP 404 with a clear -/// "no such agent" message. catches the operator-typo case where -/// a destructive POST would otherwise hit silently (mark-all-read -/// returning 0) or hit downstream lifecycle code that fails with -/// a confusing nspawn error. -/// -/// Returns `None` when both checks pass (caller proceeds), `Some(Response)` -/// when the request should be rejected. Use at the top of every write -/// handler taking a name path-param. Read-only GET handlers and -/// handlers that legitimately operate on tombstoned agents (e.g. -/// `mark-all-read` on broker rows for a destroyed agent) call -/// `validate_agent_name` directly and skip the existence check. -async fn guard_agent_name(state: &AppState, name: &str) -> Option { - if let Some(reason) = validate_agent_name(name) { - return Some( - (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(), - ); - } - let snapshot = state.coord.containers_snapshot().await; - if !snapshot.iter().any(|c| c.name == name) { - return Some((StatusCode::NOT_FOUND, format!("no such agent: {name}")).into_response()); - } - None -} - -/// Operator-driven "clear this agent's inbox" — backs the side-panel -/// "mark all read" button. Marks every message addressed to the -/// agent as acked (backfilling `delivered_at` for any still-pending -/// rows so vacuum can collect them). Returns `{ "marked": N }` so the -/// frontend can show "cleared N messages" feedback without an extra -/// fetch. -async fn post_mark_all_read( - State(state): State, - AxumPath(name): AxumPath, -) -> Response { - if let Some(reason) = validate_agent_name(&name) { - return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(); - } - match state.coord.broker.mark_all_read(&name) { - Ok(n) => { - tracing::info!(%name, marked = n, "operator marked all messages read"); - axum::Json(serde_json::json!({ "marked": n })).into_response() - } - Err(e) => error_response(&format!("mark-all-read {name} failed: {e:#}")), - } -} - -async fn post_purge_tombstone( - State(state): State, - AxumPath(name): AxumPath, -) -> Response { - // Format guard FIRST so a name like `..` can't traverse into the - // parent of `/var/lib/hyperhive/agents/{name}` and have - // `remove_dir_all` wipe `/var/lib/hyperhive/` itself. Existing - // manager + live-container checks below don't catch `..` — only - // the whitelist does. Existence check via - // `containers_snapshot()` is deliberately NOT used here: - // tombstoned agents are gone from the snapshot by design; that's - // the whole point of this endpoint. - if let Some(reason) = validate_agent_name(&name) { - return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(); - } - // Sanity: refuse to purge if a live container still exists with this - // name. The dashboard already filters tombstones to non-live names, - // but the operator could send a stale POST. - let live = lifecycle::list().await.unwrap_or_default(); - if live - .iter() - .any(|c| c == &format!("{}{name}", lifecycle::AGENT_PREFIX) || c == &name) - { - return error_response(&format!( - "refusing to purge {name}: container still exists — use DESTR0Y first" - )); - } - let mut errors = Vec::new(); - for dir in [ - Coordinator::agent_state_root(&name), - Coordinator::agent_applied_dir(&name), - ] { - if dir.exists() - && let Err(e) = std::fs::remove_dir_all(&dir) - { - errors.push(format!("{}: {e}", dir.display())); - } - } - let _ = state - .coord - .approvals - .fail_pending_for_agent(&name, "agent state purged"); - if errors.is_empty() { - tracing::info!(%name, "tombstone purged"); - // Fire the post-purge tombstones snapshot so dashboards - // drop the row live; matching form carries - // `data-no-refresh`. - emit_tombstones_snapshot(&state.coord).await; - (StatusCode::OK, "ok").into_response() - } else { - error_response(&format!("purge {name} partial: {}", errors.join(", "))) - } -} - -/// Operator-side compose form on the dashboard terminal. Drops a -/// message into the broker as `{from: "operator", to, body}`. Same -/// shape that per-agent web UIs use via `OperatorMsg`, but here the -/// operator picks the recipient explicitly with `@name`. No -/// validation that `to` resolves to a known agent — broker accepts -/// arbitrary recipients (and the agent's inbox grows whether or not -/// they exist, which is fine for spawn-then-greet flows). -#[derive(Deserialize)] -struct OpSendForm { - to: String, - body: String, -} - -/// Form for `POST /meta-update`. Inputs ride in as a comma-separated -/// list under the `inputs` field — the JS submitter joins the -/// checked boxes since axum's `Form` extractor doesn't natively -/// decode repeated keys without a helper. -#[derive(Deserialize)] -struct MetaUpdateForm { - inputs: String, -} - -/// Bulk-update selected meta flake inputs, then rebuild the affected -/// agents in the background. Idempotent w.r.t. selection — choosing -/// an input that's already at the latest sha is a no-op (no commit, -/// no rebuild ripple). Returns immediately after queueing the work; -/// dashboard polls for progress via container `pending` spinners + -/// the meta-inputs row sha update. -async fn post_meta_update( - State(state): State, - Form(form): Form, -) -> Response { - let inputs: Vec = form - .inputs - .split(',') - .map(|s| s.trim().to_owned()) - .filter(|s| !s.is_empty()) - .collect(); - if inputs.is_empty() { - return error_response("meta-update: no inputs selected"); - } - let inputs_label = inputs.join(", "); - let parent_id = state.coord.rebuild_queue.enqueue_with_inputs( - crate::rebuild_queue::QueueKind::MetaUpdate, - "hyperhive".to_owned(), - crate::rebuild_queue::QueueSource::Manual, - format!("meta-update via dashboard ({inputs_label})"), - None, - inputs.clone(), - ); - // Pre-enqueue cascade rebuilds NOW so they're visible in the queue - // alongside the parent. The worker's MetaUpdate arm - // no longer enqueues children — it just runs the lock bump and - // (on failure) cancels these pre-queued children. - let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await; - let cascade_reason = format!("meta-update cascade ({inputs_label})"); - for name in cascade_agents { - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name, - crate::rebuild_queue::QueueSource::MetaUpdate, - cascade_reason.clone(), - Some(parent_id), - ); - } - state.coord.emit_rebuild_queue_snapshot(); - (StatusCode::OK, "ok").into_response() -} - -async fn post_op_send(State(state): State, Form(form): Form) -> Response { - let to = form.to.trim().to_owned(); - let body = form.body.trim().to_owned(); - if to.is_empty() { - return error_response("op-send: `to` required"); - } - if body.is_empty() { - return error_response("op-send: `body` required"); - } - if to == "*" { - let errors = state - .coord - .broadcast_send(hive_sh4re::OPERATOR_RECIPIENT, &body); - if !errors.is_empty() { - return error_response(&format!( - "op-send broadcast partial fail: {}", - errors.join("; ") - )); - } - } else if let Err(e) = state.coord.broker.send(&hive_sh4re::Message { - from: hive_sh4re::OPERATOR_RECIPIENT.to_owned(), - to: to.clone(), - body, - in_reply_to: None, - }) { - return error_response(&format!("op-send to {to} failed: {e:#}")); - } - // 200 instead of 303 → the client doesn't refetch /api/state. The - // broker `send` already emitted a `MessageEvent` which the - // dashboard channel forwarder mirrors as `DashboardEvent::Sent`, - // and the page's terminal + inbox derive from that stream — so the - // operator's send shows up the same way an agent's send does, with - // no full-state refresh in between. - (axum::http::StatusCode::OK, "ok").into_response() -} - -async fn post_request_spawn( - State(state): State, - Form(form): Form, -) -> Response { - let name = form.name.trim().to_owned(); - if name.is_empty() { - return error_response("spawn: `name` required"); - } - match state.coord.approvals.submit_kind( - &name, - hive_sh4re::ApprovalKind::Spawn, - "", - None, - "operator", - ) { - Ok(id) => { - tracing::info!(%id, %name, "operator: spawn approval queued via dashboard"); - // Phase 5b: notify the dashboard event channel so live - // subscribers can append the row without a snapshot - // refetch. Spawn approvals carry no diff/sha. - state - .coord - .emit_approval_added(crate::coordinator::ApprovalAdded { - id, - agent: &name, - approval_kind: "spawn", - sha_short: None, - diff: None, - description: None, - pr_number: None, - }); - (StatusCode::OK, "ok").into_response() - } - Err(e) => error_response(&format!("request-spawn {name} failed: {e:#}")), - } -} - -fn transient_label(k: crate::coordinator::TransientKind) -> &'static str { - use crate::coordinator::TransientKind::{ - Destroying, Rebuilding, Restarting, Spawning, Starting, Stopping, - }; - match k { - Spawning => "spawning", - Starting => "starting", - Stopping => "stopping", - Restarting => "restarting", - Rebuilding => "rebuilding", - Destroying => "destroying", - } -} - -/// Convert either a logical name or a container name back to the logical -/// name. Sub-agents are `h-foo` → `foo`; manager stays `root`. -fn strip_container_prefix(name: &str) -> String { - name.strip_prefix(lifecycle::AGENT_PREFIX) - .unwrap_or(name) - .to_owned() -} - -/// The common internal-error case as a `ProblemDetails`: a 500 RFC 9457 -/// (`application/problem+json`) value via the `problem_details` crate. -/// `from_status_code` sets `status` + `title` (the canonical reason phrase) -/// and leaves `type` as the default `about:blank`; `with_detail` carries the -/// caller message; the crate's axum `IntoResponse` emits the -/// `application/problem+json` body the frontend parses (it reads `detail`). -/// Handlers that surface client failures return `Result<_, ProblemDetails>` -/// and hand this (or an inline `from_status_code(4xx)`) straight to `Err` — -/// no manual `.into_response()`. -fn error_problem(message: &str) -> problem_details::ProblemDetails { - problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR) - .with_detail(message) -} - -/// `Response` wrapper around [`error_problem`] for the many handlers typed -/// `-> Response` whose only failure mode is a 500 — they funnel errors -/// through here rather than threading a `Result` return type. -fn error_response(message: &str) -> Response { - error_problem(message).into_response() -} diff --git a/hive-c0re/src/dashboard/lifecycle_ops.rs b/hive-c0re/src/dashboard/lifecycle_ops.rs index b8b8b3a4..c139feaa 100644 --- a/hive-c0re/src/dashboard/lifecycle_ops.rs +++ b/hive-c0re/src/dashboard/lifecycle_ops.rs @@ -1,10 +1,12 @@ //! Container lifecycle endpoints for the dashboard. //! //! Rebuild / restart / start / stop (hard + graceful) / update-all all -//! enqueue onto the rebuild queue, so each shows a visible queued→running -//! transient on the dashboard — a direct sub-second start/stop only flashed -//! the badge; destroy delegates to `actions::destroy` (optionally -//! purging). +//! submit DAGs to the job queue (`job_queue::submit`), so each shows a +//! visible queued→running transient on the dashboard — a direct +//! sub-second start/stop only flashed the badge. Start/stop also +//! persist the agent's `wanted` power intent before submitting; the +//! DAG's `Reconcile` converges to it. Destroy delegates to +//! `actions::destroy` (optionally purging). use axum::{ extract::{Form, Path as AxumPath, Query, State}, @@ -23,6 +25,7 @@ pub(super) struct KillParams { } use super::{AppState, error_response, guard_agent_name, strip_container_prefix}; +use crate::job_queue::{Source, submit}; use crate::{actions, lifecycle}; pub(super) async fn post_rebuild( @@ -33,14 +36,12 @@ pub(super) async fn post_rebuild( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::rebuild( + &state.coord, + &logical, + Source::Manual, "manual via dashboard ↻ R3BU1LD button".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -54,19 +55,17 @@ pub(super) async fn post_kill( return reject; } if params.graceful { - // Graceful stop: enqueue the quiesce orchestration (signal the harness - // → one stop-checkpoint turn → drain → container stop, with a timeout - // fallback to a hard stop). Serialised through the rebuild queue so it - // can't race an in-flight rebuild for the same agent, and its per-step - // progress surfaces on the queue snapshot + build log. - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::GracefulStop, - logical, - crate::rebuild_queue::QueueSource::Manual, + // Graceful stop: submit the quiesce DAG (signal the harness → + // one stop-checkpoint turn → drain → container stop, with a + // timeout fallback to a hard stop). The agent's lifecycle + // lease keeps it from racing an in-flight rebuild for the same + // agent, and per-node progress surfaces on the queue snapshot. + submit::graceful_stop( + &state.coord, + &logical, + Source::Manual, "manual via dashboard graceful stop".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); return (StatusCode::OK, "ok").into_response(); } // Manager is stoppable from the dashboard like any other @@ -79,14 +78,12 @@ pub(super) async fn post_kill( // `socket_server.rs::ManagerRequest::Kill` stays in place: a // manager calling Kill on its own container is self-suicide // mid-call, not a legitimate operator action. - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Stop, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::stop( + &state.coord, + &logical, + Source::Manual, "manual via dashboard stop".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -98,14 +95,12 @@ pub(super) async fn post_restart( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Restart, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::restart( + &state.coord, + &logical, + Source::Manual, "manual via dashboard ↺ R3START button".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -117,14 +112,12 @@ pub(super) async fn post_start( if let Some(reject) = guard_agent_name(&state, &logical).await { return reject; } - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Start, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::start( + &state.coord, + &logical, + Source::Manual, "manual via dashboard start".to_owned(), - None, ); - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } @@ -137,15 +130,13 @@ pub(super) async fn post_update_all(State(state): State) -> Response { else { continue; }; - state.coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - logical, - crate::rebuild_queue::QueueSource::Manual, + submit::rebuild( + &state.coord, + &logical, + Source::Manual, "manual via dashboard 🌀 UPDATE ALL".to_owned(), - None, ); } - state.coord.emit_rebuild_queue_snapshot(); (StatusCode::OK, "ok").into_response() } diff --git a/hive-c0re/src/dashboard/meta_inputs.rs b/hive-c0re/src/dashboard/meta_inputs.rs new file mode 100644 index 00000000..b02fc82a --- /dev/null +++ b/hive-c0re/src/dashboard/meta_inputs.rs @@ -0,0 +1,279 @@ +//! META INPUTS panel backend: walks `meta/flake.lock` into +//! `MetaInputView` rows for the snapshot, emits the `MetaInputsChanged` +//! event after lock bumps, and handles `POST /api/meta-update` (bulk +//! flake-input update + rebuild ripple via the job queue). + +use axum::{ + extract::{Form, State}, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde::{Deserialize, Serialize}; + +use crate::coordinator::Coordinator; + +use super::{AppState, error_response}; + +#[derive(Serialize, Clone, Debug)] +pub struct MetaInputView { + /// Input key in meta's `flake.nix` — `hyperhive`, `agent-`, etc. + pub name: String, + /// Full locked sha. Not displayed verbatim; the dashboard + /// truncates to the first 12 chars for the chip. + pub rev: String, + /// Unix seconds — `locked.lastModified`. Drives the relative + /// "2h ago" timestamp on each input row. + pub last_modified: i64, + /// `original.url` if available, for the tooltip / row meta text. + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, +} + +/// Walk `flake.lock`'s `nodes` graph from `root` and emit one +/// `MetaInputView` per fetched input, at **every** depth. That +/// surfaces the direct meta inputs (`hyperhive`, `agent-`), the +/// agent flakes' own inputs (`agent-dmatrix/mcp-matrix`, +/// `hyperhive/nixpkgs`), and any deeper transitive inputs — so the +/// operator can bump any of them individually. Names are +/// slash-separated paths from root, the syntax `nix flake update` +/// accepts for transitive inputs. +/// +/// Filtering: +/// - Inputs that resolve via a `follows` chain (lock value is an +/// array) are skipped — they alias another node, not their own +/// fetched derivation, so updating them does nothing. +/// - A node is emitted only when it carries a `locked.rev`. +/// - Each fetched node is walked exactly once (a `visited` set): +/// the lock graph shares nodes (many flakes reference one +/// nixpkgs), so without this a shared subtree re-walks per parent +/// and a cycle would recurse forever. The result is a spanning +/// tree — every input shown once, at its shallowest path. +pub(super) fn read_meta_inputs() -> Vec { + let mut out = Vec::new(); + let Ok(raw) = std::fs::read_to_string("/var/lib/hyperhive/meta/flake.lock") else { + return out; + }; + let Ok(json) = serde_json::from_str::(&raw) else { + return out; + }; + let Some(nodes) = json.get("nodes").and_then(|v| v.as_object()) else { + return out; + }; + let Some(root_name) = json.get("root").and_then(|v| v.as_str()) else { + return out; + }; + let mut visited = std::collections::HashSet::new(); + visited.insert(root_name.to_owned()); + walk_meta_inputs(nodes, root_name, "", &mut visited, &mut out); + // hyperhive first, then alphabetical. String-sorting the + // slash-paths puts every node directly above its own children + // (`agent-foo`, `agent-foo/bar`, `agent-foo/bar/baz`), so the + // result is a pre-order traversal the tree renderer can consume. + out.sort_by(|a, b| match (a.name.as_str(), b.name.as_str()) { + ("hyperhive", _) => std::cmp::Ordering::Less, + (_, "hyperhive") => std::cmp::Ordering::Greater, + _ => a.name.cmp(&b.name), + }); + out +} + +fn walk_meta_inputs( + nodes: &serde_json::Map, + node_name: &str, + prefix: &str, + visited: &mut std::collections::HashSet, + out: &mut Vec, +) { + let Some(node) = nodes.get(node_name) else { + return; + }; + let Some(inputs_map) = node.get("inputs").and_then(|v| v.as_object()) else { + return; + }; + // Two passes: claim (and emit) every direct input of this node + // before descending into any of them. A shallow input that a + // deeper flake also references then keeps its shallow path + // rather than being captured first by the deep walk. + let mut to_recurse: Vec<(String, String)> = Vec::new(); + for (alias, target) in inputs_map { + // Inputs map value is either a string (node name) or an + // array (a `follows` chain). The latter just aliases another + // node — we can't `nix flake update` it directly, so skip. + let serde_json::Value::String(target_name) = target else { + continue; + }; + // Walk each fetched node once — guards shared subtrees and + // cycles, and keeps the panel free of duplicate rows. + if !visited.insert(target_name.clone()) { + continue; + } + let Some(target_node) = nodes.get(target_name) else { + continue; + }; + let path = if prefix.is_empty() { + alias.clone() + } else { + format!("{prefix}/{alias}") + }; + if let Some(rev) = target_node + .get("locked") + .and_then(|v| v.get("rev")) + .and_then(|v| v.as_str()) + { + let last_modified = target_node + .get("locked") + .and_then(|v| v.get("lastModified")) + .and_then(serde_json::Value::as_i64) + .unwrap_or(0); + let url = target_node + .get("original") + .and_then(|v| v.get("url")) + .and_then(|v| v.as_str()) + .map(str::to_owned); + out.push(MetaInputView { + name: path.clone(), + rev: rev.to_owned(), + last_modified, + url, + }); + } + to_recurse.push((target_name.clone(), path)); + } + // Recurse hyperhive's subtree before any agent's — without this, + // when meta's top-level `nixpkgs` is a `follows` alias the + // `String` check above skips it, and the alphabetical BTreeMap + // iteration descends into `agent-*` first. The agent walk then + // claims `nixpkgs` at `agent-X/nixpkgs` instead of + // `hyperhive/nixpkgs`, which is where the operator expects it. + // Sort by the same "hyperhive first, then alpha" + // priority `read_meta_inputs` uses for the final output. + to_recurse.sort_by(|(a, _), (b, _)| match (a.as_str(), b.as_str()) { + ("hyperhive", _) => std::cmp::Ordering::Less, + (_, "hyperhive") => std::cmp::Ordering::Greater, + _ => a.cmp(b), + }); + for (target_name, path) in to_recurse { + walk_meta_inputs(nodes, &target_name, &path, visited, out); + } +} + +/// Snapshot meta/flake.lock's root inputs + emit +/// `MetaInputsChanged`. Call after any mutation that bumps a lock +/// (`run_meta_update`, `auto_update::rebuild_agent`). +pub(crate) fn emit_meta_inputs_snapshot(coord: &Coordinator) { + let inputs = read_meta_inputs(); + coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::MetaInputsChanged { + seq: coord.next_seq(), + inputs, + }); +} + +/// Form for `POST /meta-update`. Inputs ride in as a comma-separated +/// list under the `inputs` field — the JS submitter joins the +/// checked boxes since axum's `Form` extractor doesn't natively +/// decode repeated keys without a helper. +#[derive(Deserialize)] +pub(super) struct MetaUpdateForm { + inputs: String, +} + +/// Bulk-update selected meta flake inputs, then rebuild the affected +/// agents in the background. Idempotent w.r.t. selection — choosing +/// an input that's already at the latest sha is a no-op (no commit, +/// no rebuild ripple). Returns immediately after queueing the work; +/// dashboard polls for progress via container `pending` spinners + +/// the meta-inputs row sha update. +pub(super) async fn post_meta_update( + State(state): State, + Form(form): Form, +) -> Response { + let inputs: Vec = form + .inputs + .split(',') + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) + .collect(); + if inputs.is_empty() { + return error_response("meta-update: no inputs selected"); + } + let inputs_label = inputs.join(", "); + // Cascade rebuild children fan out from the MetaLock node when the + // lock bump lands — appended by the scheduler so they build against + // the post-bump lock, and a failed bump simply fans out nothing. + crate::job_queue::submit::meta_update( + &state.coord, + inputs, + crate::job_queue::Source::Manual, + format!("meta-update via dashboard ({inputs_label})"), + ); + (StatusCode::OK, "ok").into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn walk_meta_inputs_keeps_nixpkgs_under_hyperhive_post_follows_refactor() { + // Reproduce the shape where meta has + // `nixpkgs.follows = "hyperhive/nixpkgs"` at the top level + // (rendered as an array — `["hyperhive" "nixpkgs"]` — which + // walk_meta_inputs skips because we can't `nix flake update` + // a follows alias). The remaining top-level inputs are + // `hyperhive` (string) and `agent-z` (string). Without the + // hyperhive-first recursion sort, the BTreeMap alphabetical + // order descends into `agent-z` first and claims + // `nixpkgs` at `agent-z/nixpkgs`. + let raw = r#"{ + "root": "root", + "version": 7, + "nodes": { + "root": { + "inputs": { + "hyperhive": "hyperhive", + "nixpkgs": ["hyperhive", "nixpkgs"], + "agent-z": "agent-z" + } + }, + "hyperhive": { + "inputs": { "nixpkgs": "nixpkgs" }, + "locked": {"rev": "hhrev", "lastModified": 1}, + "original": {"url": "git+file:///tmp/hyperhive"} + }, + "agent-z": { + "inputs": { "nixpkgs": "nixpkgs" }, + "locked": {"rev": "azrev", "lastModified": 2}, + "original": {"url": "git+file:///tmp/agent-z"} + }, + "nixpkgs": { + "locked": {"rev": "npkrev", "lastModified": 3}, + "original": {"url": "github:NixOS/nixpkgs/nixos-26.05"} + } + } + }"#; + let json: serde_json::Value = serde_json::from_str(raw).unwrap(); + let nodes = json.get("nodes").unwrap().as_object().unwrap(); + let root_name = json.get("root").unwrap().as_str().unwrap(); + let mut visited = std::collections::HashSet::new(); + visited.insert(root_name.to_owned()); + let mut out = Vec::new(); + walk_meta_inputs(nodes, root_name, "", &mut visited, &mut out); + + let nixpkgs = out + .iter() + .find(|v| v.rev == "npkrev") + .expect("nixpkgs node should be emitted exactly once"); + assert_eq!( + nixpkgs.name, "hyperhive/nixpkgs", + "nixpkgs should be claimed under hyperhive, not under agent-z. \ + got: {:?}", + nixpkgs.name + ); + // And the agent-z path should NOT also carry a nixpkgs entry — + // the spanning-tree visited set guarantees it's claimed once. + assert!( + !out.iter().any(|v| v.name == "agent-z/nixpkgs"), + "agent-z/nixpkgs should not be emitted (already claimed under hyperhive)" + ); + } +} diff --git a/hive-c0re/src/dashboard/misc_api.rs b/hive-c0re/src/dashboard/misc_api.rs new file mode 100644 index 00000000..d4c8d175 --- /dev/null +++ b/hive-c0re/src/dashboard/misc_api.rs @@ -0,0 +1,220 @@ +//! Remaining single-endpoint dashboard handlers: the operator inbox +//! (`Y3R C4LL`) + mark-all-read, operator compose (`op-send`), +//! spawn-request, hive-wide turn stats, container resources, and the +//! audit log. + +use axum::{ + extract::{Form, Path as AxumPath, State}, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde::Deserialize; + +use super::{AppState, error_response, scan_validated_paths, validate_agent_name}; + +/// Unread operator-directed messages for the dashboard's Y3R C4LL inbox. +/// Returns messages addressed to `"operator"` that haven't been +/// acked yet (the operator clears them via the existing +/// `POST /api/agent/operator/mark-all-read`). Newest-first; path-shaped +/// tokens are validated so the client renders file links like the +/// terminal does. Shape: `{ "messages": [{ id, from, body, at, +/// in_reply_to, file_refs }] }`. +pub(super) async fn api_operator_inbox(State(state): State) -> Response { + const INBOX_LIMIT: u64 = 100; + match state + .coord + .broker + .unread_for_recipient("operator", INBOX_LIMIT) + { + Ok(messages) => { + let items: Vec = messages + .into_iter() + .filter_map(|m| { + let crate::broker::MessageEvent::Sent { + id, + from, + body, + at, + in_reply_to, + .. + } = m + else { + return None; + }; + let file_refs = scan_validated_paths(&body); + Some(serde_json::json!({ + "id": id, + "from": from, + "body": body, + "at": hive_sh4re::wire_time::from_secs(at), + "in_reply_to": in_reply_to, + "file_refs": file_refs, + })) + }) + .collect(); + axum::Json(serde_json::json!({ "messages": items })).into_response() + } + Err(e) => error_response(&format!("operator-inbox failed: {e:#}")), + } +} + +#[derive(Deserialize)] +pub(super) struct StatsHiveQuery { + window: Option, +} + +/// Hive-wide turn-stats rollup for the dashboard swarm-stats view. +/// Aggregates every agent's `hyperhive-turn-stats.sqlite` read-only +/// (skips missing/unreadable ones). Window defaults to `24h`. +pub(super) async fn api_stats_hive( + State(state): State, + axum::extract::Query(q): axum::extract::Query, +) -> Response { + let window = crate::hive_stats::Window::parse(q.window.as_deref().unwrap_or("24h")); + axum::Json(crate::hive_stats::hive_snapshot( + window, + &state.coord.model_prices, + )) + .into_response() +} + +/// Live per-agent-container CPU + memory load from cgroup v2. Samples +/// CPU over a short interval (~200 ms), so this call briefly awaits. +pub(super) async fn api_container_resources() -> Response { + axum::Json(crate::container_stats::gather().await).into_response() +} + +/// `GET /api/audit-log` — most-recent agent-initiated privileged-action +/// audit entries, newest first (server-clamped to 500). Backs the +/// operator dashboard's audit view. Returns +/// `{ "entries": [AuditEntry…], "total": N }` so the UI can show +/// "latest 500 of N" rather than silently capping. `ts_unix` is in +/// **seconds**. +pub(super) async fn api_audit_log(State(state): State) -> Response { + const LIMIT: usize = 500; + let entries = match state.coord.audit_log.list_recent(LIMIT) { + Ok(rows) => rows, + Err(e) => return error_response(&format!("audit-log: {e:#}")), + }; + let total = match state.coord.audit_log.count_total() { + Ok(n) => n, + Err(e) => return error_response(&format!("audit-log count: {e:#}")), + }; + axum::Json(serde_json::json!({ "entries": entries, "total": total })).into_response() +} + +/// Operator-driven "clear this agent's inbox" — backs the side-panel +/// "mark all read" button. Marks every message addressed to the +/// agent as acked (backfilling `delivered_at` for any still-pending +/// rows so vacuum can collect them). Returns `{ "marked": N }` so the +/// frontend can show "cleared N messages" feedback without an extra +/// fetch. +pub(super) async fn post_mark_all_read( + State(state): State, + AxumPath(name): AxumPath, +) -> Response { + if let Some(reason) = validate_agent_name(&name) { + return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(); + } + match state.coord.broker.mark_all_read(&name) { + Ok(n) => { + tracing::info!(%name, marked = n, "operator marked all messages read"); + axum::Json(serde_json::json!({ "marked": n })).into_response() + } + Err(e) => error_response(&format!("mark-all-read {name} failed: {e:#}")), + } +} + +/// Operator-side compose form on the dashboard terminal. Drops a +/// message into the broker as `{from: "operator", to, body}`. Same +/// shape that per-agent web UIs use via `OperatorMsg`, but here the +/// operator picks the recipient explicitly with `@name`. No +/// validation that `to` resolves to a known agent — broker accepts +/// arbitrary recipients (and the agent's inbox grows whether or not +/// they exist, which is fine for spawn-then-greet flows). +#[derive(Deserialize)] +pub(super) struct OpSendForm { + to: String, + body: String, +} + +pub(super) async fn post_op_send( + State(state): State, + Form(form): Form, +) -> Response { + let to = form.to.trim().to_owned(); + let body = form.body.trim().to_owned(); + if to.is_empty() { + return error_response("op-send: `to` required"); + } + if body.is_empty() { + return error_response("op-send: `body` required"); + } + if to == "*" { + let errors = state + .coord + .broadcast_send(hive_sh4re::OPERATOR_RECIPIENT, &body); + if !errors.is_empty() { + return error_response(&format!( + "op-send broadcast partial fail: {}", + errors.join("; ") + )); + } + } else if let Err(e) = state.coord.broker.send(&hive_sh4re::Message { + from: hive_sh4re::OPERATOR_RECIPIENT.to_owned(), + to: to.clone(), + body, + in_reply_to: None, + }) { + return error_response(&format!("op-send to {to} failed: {e:#}")); + } + // 200 instead of 303 → the client doesn't refetch /api/state. The + // broker `send` already emitted a `MessageEvent` which the + // dashboard channel forwarder mirrors as `DashboardEvent::Sent`, + // and the page's terminal + inbox derive from that stream — so the + // operator's send shows up the same way an agent's send does, with + // no full-state refresh in between. + (axum::http::StatusCode::OK, "ok").into_response() +} + +#[derive(Deserialize)] +pub(super) struct RequestSpawnForm { + name: String, +} + +pub(super) async fn post_request_spawn( + State(state): State, + Form(form): Form, +) -> Response { + let name = form.name.trim().to_owned(); + if name.is_empty() { + return error_response("spawn: `name` required"); + } + match state.coord.approvals.submit_kind( + &name, + hive_sh4re::ApprovalKind::Spawn, + "", + None, + "operator", + ) { + Ok(id) => { + tracing::info!(%id, %name, "operator: spawn approval queued via dashboard"); + // Phase 5b: notify the dashboard event channel so live + // subscribers can append the row without a snapshot + // refetch. Spawn approvals carry no diff/sha. + state + .coord + .emit_approval_added(crate::coordinator::ApprovalAdded { + id, + agent: &name, + approval_kind: "spawn", + sha_short: None, + diff: None, + description: None, + pr_number: None, + }); + (StatusCode::OK, "ok").into_response() + } + Err(e) => error_response(&format!("request-spawn {name} failed: {e:#}")), + } +} diff --git a/hive-c0re/src/dashboard/mod.rs b/hive-c0re/src/dashboard/mod.rs new file mode 100644 index 00000000..18abd4e5 --- /dev/null +++ b/hive-c0re/src/dashboard/mod.rs @@ -0,0 +1,418 @@ +//! Hyperhive dashboard. Lists managed containers (with deep-links to each +//! container's web UI), pending approvals (with unified diff vs the applied +//! repo, plus approve/deny buttons), and the manager. + +use std::net::SocketAddr; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use axum::{ + Router, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post}, +}; + +use crate::coordinator::Coordinator; +use crate::lifecycle; + +mod approvals; +mod build_logs; +mod journal; +mod lifecycle_ops; +mod matrix_accounts; +mod meta_inputs; +mod misc_api; +pub(crate) mod permissions; +mod questions; +mod reminders; +mod schedules; +mod state_files; +mod state_snapshot; +mod tombstones; +mod topology; +mod webhook; + +// Pre-computed at approval-submit time by the manager-socket handler +// (`socket_server.rs`) and embedded in the `ApprovalAdded` event, so +// re-exported at the module root to preserve the `crate::dashboard::approval_diff` +// path across the submodule split. +pub(crate) use approvals::approval_diff; +// Run after lock bumps by the job queue (`job_queue/exec.rs`); the view +// type feeds `DashboardEvent::MetaInputsChanged` (`dashboard_events.rs`). +// Re-exported to preserve the `crate::dashboard::*` paths across the split. +pub use meta_inputs::MetaInputView; +pub(crate) use meta_inputs::emit_meta_inputs_snapshot; +// Run at broker-message ingest by the coordinator + the operator-msg path +// (`main.rs`); re-exported to preserve the `crate::dashboard::scan_validated_paths` +// path across the split. +pub use state_files::scan_validated_paths; +// Called after destroy/purge/spawn finalisation (`actions.rs`); the view +// type feeds `DashboardEvent::TombstonesChanged` (`dashboard_events.rs`). +// Re-exported to preserve the `crate::dashboard::*` paths across the split. +pub use tombstones::TombstoneView; +pub(crate) use tombstones::emit_tombstones_snapshot; + +#[derive(Clone)] +struct AppState { + coord: Arc, +} + +#[allow( + clippy::too_many_lines, + reason = "the body is dominated by the flat axum route table — one line \ + per endpoint mapping a URL to its (now per-concern submodule) \ + handler; splitting that exhaustive list across helpers would \ + obscure the route map for no readability gain" +)] +pub async fn serve(port: u16, coord: Arc) -> Result<()> { + // API-only: the gateway static-serves the dashboard dist and proxies + // non-static requests here (see hive-gateway.nix). Unmatched paths 404. + let app = Router::new() + .route("/api/state", get(state_snapshot::api_state)) + .route("/api/journal/{name}", get(journal::get_journal)) + .route("/api/journal-host", get(journal::get_journal_host)) + .route("/api/approval-diff/{id}", get(approvals::get_approval_diff)) + .route("/api/state-file", get(state_files::get_state_file)) + .route( + "/api/matrix-accounts", + get(matrix_accounts::get_matrix_accounts), + ) + .route("/api/reminders", get(reminders::api_reminders)) + .route("/api/operator-inbox", get(misc_api::api_operator_inbox)) + .route("/api/stats-hive", get(misc_api::api_stats_hive)) + .route( + "/api/container-resources", + get(misc_api::api_container_resources), + ) + .route("/api/audit-log", get(misc_api::api_audit_log)) + .route("/api/build-logs", get(build_logs::get_build_logs_all)) + .route( + "/api/build-logs/{agent}", + get(build_logs::get_build_logs_agent), + ) + .route( + "/api/build-logs/id/{id}", + get(build_logs::get_build_log_full), + ) + .route( + "/api/build-logs/id/{id}/stream", + get(build_logs::get_build_log_stream), + ) + .route( + "/api/build-logs/id/{id}/raw", + get(build_logs::get_build_log_raw), + ) + .route( + "/api/agent/{name}/mark-all-read", + post(misc_api::post_mark_all_read), + ) + .route("/api/topology/set-parent", post(topology::post_set_parent)) + .route( + "/api/topology/set-parent-bulk", + post(topology::post_set_parent_bulk), + ) + .route("/api/tool-groups", get(permissions::get_tool_groups)) + .route( + "/api/tool-groups/{agent}", + post(permissions::post_tool_groups), + ) + .route("/api/capabilities", get(permissions::get_capabilities)) + .route( + "/api/capabilities/{agent}", + post(permissions::post_capabilities), + ) + .route("/api/permissions", post(permissions::post_permissions)) + .route( + "/api/permissions/stale", + get(permissions::get_stale_permissions), + ) + .route( + "/api/permissions/{agent}", + axum::routing::delete(permissions::delete_agent_permissions), + ) + .route( + "/api/schedules", + get(schedules::api_schedules).post(schedules::post_schedule_new), + ) + .route( + "/api/schedules/{id}", + axum::routing::patch(schedules::patch_schedule), + ) + .route( + "/api/schedules/{id}/cancel", + post(schedules::post_schedule_cancel), + ) + .route( + "/api/schedules/{id}/pause", + post(schedules::post_schedule_pause), + ) + .route( + "/api/schedules/{id}/resume", + post(schedules::post_schedule_resume), + ) + .route( + "/api/schedules/{id}/fire-now", + post(schedules::post_schedule_fire_now), + ) + .route( + "/api/rebuild-queue/{id}/cancel", + post(schedules::post_rebuild_queue_cancel), + ) + .route("/webhook/knowledge", post(webhook::post_webhook_knowledge)) + // Backend routes — the frontend calls these `/api/` paths. The + // transitional bare top-level aliases were removed once the + // frontend migrated. `/webhook/knowledge` keeps its own prefix + // (forge-driven, not the SPA). + .route("/api/approve/{id}", post(approvals::post_approve)) + .route("/api/deny/{id}", post(approvals::post_deny)) + .route("/api/destroy/{name}", post(lifecycle_ops::post_destroy)) + .route("/api/kill/{name}", post(lifecycle_ops::post_kill)) + .route("/api/restart/{name}", post(lifecycle_ops::post_restart)) + .route("/api/start/{name}", post(lifecycle_ops::post_start)) + .route("/api/rebuild/{name}", post(lifecycle_ops::post_rebuild)) + .route("/api/update-all", post(lifecycle_ops::post_update_all)) + .route( + "/api/answer-question/{id}", + post(questions::post_answer_question), + ) + .route( + "/api/cancel-question/{id}", + post(questions::post_cancel_question), + ) + .route( + "/api/purge-tombstone/{name}", + post(tombstones::post_purge_tombstone), + ) + .route( + "/api/matrix-account-login", + post(matrix_accounts::post_matrix_account_login), + ) + .route( + "/api/cancel-reminder/{id}", + post(reminders::post_cancel_reminder), + ) + .route( + "/api/retry-reminder/{id}", + post(reminders::post_retry_reminder), + ) + .route("/api/request-spawn", post(misc_api::post_request_spawn)) + .route("/api/op-send", post(misc_api::post_op_send)) + .route("/api/meta-update", post(meta_inputs::post_meta_update)) + .route( + "/api/dashboard/stream", + get(state_snapshot::dashboard_stream), + ) + .route( + "/api/dashboard/history", + get(state_snapshot::dashboard_history), + ) + // No static fallback — the gateway owns the dist; unmatched paths 404. + .with_state(AppState { coord }); + // Binds loopback-only; external access via gateway. + // Rationale: docs/gateway.md::Firewall posture. + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + let listener = bind_with_retry(addr).await?; + tracing::info!(%addr, "dashboard listening"); + axum::serve(listener, app).await?; + Ok(()) +} + +// SPA shape + SSE channels: docs/web-ui/shape.md. + +/// `SO_REUSEADDR` bind with retry. Retry mechanics, attempt-cap +/// rationale, and log-level cadence: `docs/web-ui/shape.md::Listener bind`. +async fn bind_with_retry(addr: SocketAddr) -> Result { + let mut delay_ms = 250u64; + let mut attempts = 0u32; + loop { + match try_bind(addr) { + Ok(l) => { + if attempts > 0 { + tracing::info!( + %addr, attempts, + "dashboard: bind succeeded after retry" + ); + } + return Ok(l); + } + Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => { + let attempt = attempts + 1; + if attempt <= 12 { + tracing::warn!( + %addr, attempt, + "dashboard: AddrInUse, retrying in {delay_ms}ms" + ); + } else { + tracing::info!( + %addr, attempt, + "dashboard: AddrInUse still holding, retrying in {delay_ms}ms" + ); + } + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + attempts += 1; + delay_ms = (delay_ms * 2).min(2000); + } + Err(e) => { + return Err(e).with_context(|| format!("bind dashboard on {addr}")); + } + } + } +} + +fn try_bind(addr: SocketAddr) -> std::io::Result { + let sock = match addr { + SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4()?, + SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6()?, + }; + sock.set_reuseaddr(true)?; + sock.bind(addr)?; + sock.listen(1024) +} + +/// Validate that a path-param agent name conforms to the hyperhive +/// naming whitelist: 1-63 chars of `[a-z0-9_-]`. Rejects empty, +/// uppercase, slashes, dots, and any non-ASCII (incl. unicode +/// homoglyphs of dash/underscore). Returns `None` on accept, `Some(reason)` +/// on reject — caller wraps the reason in a 400 response. Conservative +/// whitelist matching `nixos-container` basename rules and the existing +/// agent-name convention across the codebase. +pub(crate) fn validate_agent_name(name: &str) -> Option<&'static str> { + if name.is_empty() { + return Some("agent name must not be empty"); + } + if name.len() > 63 { + return Some("agent name must be 63 characters or fewer"); + } + if !name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_') + { + return Some("agent name must contain only [a-z0-9_-]"); + } + None +} + +/// Two-axis path-param guard for write routes. Combines: +/// +/// 1. **format validation** (`validate_agent_name`) — rejects path +/// traversal / unicode homoglyphs / empty + too-long names with +/// HTTP 400. +/// 2. **existence check** — looks up `name` in the coordinator's +/// container snapshot; unknown name → HTTP 404 with a clear +/// "no such agent" message. catches the operator-typo case where +/// a destructive POST would otherwise hit silently (mark-all-read +/// returning 0) or hit downstream lifecycle code that fails with +/// a confusing nspawn error. +/// +/// Returns `None` when both checks pass (caller proceeds), `Some(Response)` +/// when the request should be rejected. Use at the top of every write +/// handler taking a name path-param. Read-only GET handlers and +/// handlers that legitimately operate on tombstoned agents (e.g. +/// `mark-all-read` on broker rows for a destroyed agent) call +/// `validate_agent_name` directly and skip the existence check. +async fn guard_agent_name(state: &AppState, name: &str) -> Option { + if let Some(reason) = validate_agent_name(name) { + return Some( + (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(), + ); + } + let snapshot = state.coord.containers_snapshot().await; + if !snapshot.iter().any(|c| c.name == name) { + return Some((StatusCode::NOT_FOUND, format!("no such agent: {name}")).into_response()); + } + None +} + +/// Convert either a logical name or a container name back to the logical +/// name. Sub-agents are `h-foo` → `foo`; manager stays `root`. +fn strip_container_prefix(name: &str) -> String { + name.strip_prefix(lifecycle::AGENT_PREFIX) + .unwrap_or(name) + .to_owned() +} + +/// The common internal-error case as a `ProblemDetails`: a 500 RFC 9457 +/// (`application/problem+json`) value via the `problem_details` crate. +/// `from_status_code` sets `status` + `title` (the canonical reason phrase) +/// and leaves `type` as the default `about:blank`; `with_detail` carries the +/// caller message; the crate's axum `IntoResponse` emits the +/// `application/problem+json` body the frontend parses (it reads `detail`). +/// Handlers that surface client failures return `Result<_, ProblemDetails>` +/// and hand this (or an inline `from_status_code(4xx)`) straight to `Err` — +/// no manual `.into_response()`. +fn error_problem(message: &str) -> problem_details::ProblemDetails { + problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR) + .with_detail(message) +} + +/// `Response` wrapper around [`error_problem`] for the many handlers typed +/// `-> Response` whose only failure mode is a 500 — they funnel errors +/// through here rather than threading a `Result` return type. +fn error_response(message: &str) -> Response { + error_problem(message).into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn problem_details_carry_rfc9457_status_and_detail() { + // Contract the frontend depends on: the problem_details crate + // serialises the RFC 9457 members we rely on — `status` (numeric) + // and `detail` (the caller message; the FE reads `.detail`). + let pd = problem_details::ProblemDetails::from_status_code(StatusCode::BAD_REQUEST) + .with_detail("bad input"); + let v = serde_json::to_value(&pd).expect("problem details serialise"); + assert_eq!(v["status"], 400); + assert_eq!(v["detail"], "bad input"); + // The 500 wrapper path carries the internal-error status. + let five = + problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR) + .with_detail("boom"); + let fv = serde_json::to_value(&five).expect("problem details serialise"); + assert_eq!(fv["status"], 500); + } + + #[test] + fn validate_agent_name_accepts_canonical_shapes() { + assert!(validate_agent_name("damocles").is_none()); + assert!(validate_agent_name("hm1nd").is_none()); + assert!(validate_agent_name("agent-with-dashes").is_none()); + assert!(validate_agent_name("snake_case").is_none()); + assert!(validate_agent_name("mixed_2-3").is_none()); + let max = "a".repeat(63); + assert!( + validate_agent_name(&max).is_none(), + "63-char name should pass" + ); + } + + // The two-axis guard (`guard_agent_name`) wires `validate_agent_name` + // + an async coordinator lookup. The lookup needs a populated + // `Coordinator`, which needs sqlite + tokio runtime; rather than + // build that scaffolding for an integration-flavoured test we cover + // the format axis here (the existence axis is enforced by the + // shared `containers_snapshot` API, tested in `coordinator.rs`'s + // own suite). 9 cases below cover the boundary-length case and + // other expected rejects to make the contract explicit. + #[test] + fn validate_agent_name_rejects_bad_input() { + assert!(validate_agent_name("").is_some()); + let too_long = "a".repeat(64); + assert!(validate_agent_name(&too_long).is_some()); + // Path-traversal attempts. + assert!(validate_agent_name("../etc/passwd").is_some()); + assert!(validate_agent_name("alice/bob").is_some()); + // Uppercase rejected — canonical lowercase convention. + assert!(validate_agent_name("Alice").is_some()); + // No spaces, dots, special chars. + assert!(validate_agent_name("alice bob").is_some()); + assert!(validate_agent_name("alice.bob").is_some()); + assert!(validate_agent_name("alice;DROP TABLE messages").is_some()); + // Non-ASCII (incl. unicode homoglyphs of ASCII dash). + assert!(validate_agent_name("damóclès").is_some()); + assert!(validate_agent_name("alice\u{2013}bob").is_some()); // en-dash + } +} diff --git a/hive-c0re/src/dashboard/permissions.rs b/hive-c0re/src/dashboard/permissions.rs index 1c41637c..1f133354 100644 --- a/hive-c0re/src/dashboard/permissions.rs +++ b/hive-c0re/src/dashboard/permissions.rs @@ -128,18 +128,19 @@ pub(super) async fn post_tool_groups( return Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST) .with_detail(format!("invalid tool-groups for {logical}: {e}"))); } - // Enqueue a PermChange so the JSON file write is serialised through - // the FIFO worker. Prevents concurrent batch-apply actions for - // different agents from racing on the shared tool-groups.json. - state.coord.rebuild_queue.enqueue_with_perm( - logical.clone(), - crate::rebuild_queue::QueueSource::Manual, + // Submit a PermChange DAG: the JSON file write commits under + // META_LOCK inside the WritePermFile node, so concurrent + // batch-apply actions for different agents never race on the + // shared tool-groups.json. + crate::job_queue::submit::perm_change( + &state.coord, + &logical, + crate::job_queue::Source::Manual, "tool-group change via permissions UI".to_owned(), - crate::rebuild_queue::PermPayload::ToolGroups { + crate::job_queue::PermPayload::ToolGroups { groups: body.groups.clone(), }, ); - state.coord.emit_rebuild_queue_snapshot(); tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard"); Ok((StatusCode::OK, "ok").into_response()) } @@ -214,18 +215,19 @@ pub(super) async fn post_capabilities( .with_detail(format!("unknown capability: {cap}"))); } } - // Enqueue a PermChange so the JSON file write is serialised through - // the FIFO worker. Prevents concurrent batch-apply actions for - // different agents from racing on the shared capabilities.json. - state.coord.rebuild_queue.enqueue_with_perm( - logical.clone(), - crate::rebuild_queue::QueueSource::Manual, + // Submit a PermChange DAG: the JSON file write commits under + // META_LOCK inside the WritePermFile node, so concurrent + // batch-apply actions for different agents never race on the + // shared capabilities.json. + crate::job_queue::submit::perm_change( + &state.coord, + &logical, + crate::job_queue::Source::Manual, "capability change via dashboard".to_owned(), - crate::rebuild_queue::PermPayload::Capabilities { + crate::job_queue::PermPayload::Capabilities { caps: body.caps.clone(), }, ); - state.coord.emit_rebuild_queue_snapshot(); tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard"); Ok((StatusCode::OK, "ok").into_response()) } @@ -298,17 +300,17 @@ pub(super) async fn post_permissions( )); } } - // Phase 2 — enqueue one combined PermChange per affected agent. + // Phase 2 — submit one combined PermChange DAG per affected agent. for (logical, groups, caps) in staged { - state.coord.rebuild_queue.enqueue_with_perm( - logical.clone(), - crate::rebuild_queue::QueueSource::Manual, + crate::job_queue::submit::perm_change( + &state.coord, + &logical, + crate::job_queue::Source::Manual, "batch permission change via permissions UI".to_owned(), - crate::rebuild_queue::PermPayload::Combined { groups, caps }, + crate::job_queue::PermPayload::Combined { groups, caps }, ); tracing::info!(agent = %logical, "operator: batch perm change via dashboard"); } - state.coord.emit_rebuild_queue_snapshot(); Ok((StatusCode::OK, "ok").into_response()) } diff --git a/hive-c0re/src/dashboard/schedules.rs b/hive-c0re/src/dashboard/schedules.rs index 967146f2..8ad220d9 100644 --- a/hive-c0re/src/dashboard/schedules.rs +++ b/hive-c0re/src/dashboard/schedules.rs @@ -115,20 +115,19 @@ pub(super) async fn post_schedule_fire_now( } } -/// `POST /api/rebuild-queue/{id}/cancel` — drop a `Queued` entry -/// from the rebuild queue. Refuses `Running` / terminal -/// entries: an in-flight rebuild owns the agent's nix store + -/// nixos-container update lock and can't be safely interrupted -/// from the queue side. Always returns 200; the body is -/// `{"cancelled": true}` on a successful flip from Queued → -/// Cancelled, `{"cancelled": false}` when the row was Running / -/// terminal / gone. On success a fresh `RebuildQueueChanged` -/// snapshot fires so the row's state flip surfaces live. +/// `POST /api/rebuild-queue/{id}/cancel` — drop a still-fully-queued +/// DAG from the job queue. Refuses `Running` / terminal DAGs: an +/// in-flight node owns the agent's nix store + nixos-container update +/// lock and can't be safely interrupted from the queue side. Always +/// returns 200; the body is `{"cancelled": true}` on a successful +/// flip to Cancelled, `{"cancelled": false}` when the DAG was +/// Running / terminal / gone. On success a fresh `RebuildQueueChanged` +/// snapshot fires so the state flip surfaces live. pub(super) async fn post_rebuild_queue_cancel( State(state): State, AxumPath(id): AxumPath, ) -> Response { - let cancelled = state.coord.rebuild_queue.cancel(id); + let cancelled = state.coord.job_queue.cancel(id); if cancelled { state.coord.emit_rebuild_queue_snapshot(); axum::Json(serde_json::json!({"cancelled": true})).into_response() diff --git a/hive-c0re/src/dashboard/state_snapshot.rs b/hive-c0re/src/dashboard/state_snapshot.rs new file mode 100644 index 00000000..e095b242 --- /dev/null +++ b/hive-c0re/src/dashboard/state_snapshot.rs @@ -0,0 +1,750 @@ +//! `/api/state` cold-load snapshot plus the dashboard's live read side: +//! the `StateSnapshot` shape and its view builders, the +//! `/api/dashboard/stream` SSE channel, and the `/api/dashboard/history` +//! backfill. SPA shape + SSE channels: docs/web-ui/shape.md. + +use std::convert::Infallible; + +use axum::{ + extract::State, + http::HeaderMap, + response::{ + IntoResponse, Response, + sse::{Event, KeepAlive, Sse}, + }, +}; +use chrono::{DateTime, Utc}; +use hive_sh4re::Approval; +use serde::{Deserialize, Serialize}; +use tokio_stream::wrappers::BroadcastStream; +use tokio_stream::{Stream, StreamExt}; + +use crate::container_view::ContainerView; + +use super::meta_inputs::{MetaInputView, read_meta_inputs}; +use super::tombstones::{TombstoneView, build_tombstone_views}; +use super::{AppState, approval_diff, approvals, error_response, scan_validated_paths}; + +#[allow(clippy::struct_excessive_bools)] +#[derive(Serialize)] +pub(super) struct StateSnapshot { + /// Broker seq at the moment this snapshot was assembled. Clients + /// dedupe their buffered SSE traffic against this value: any + /// `MessageEvent` with `seq <= snapshot.seq` is already reflected in + /// the snapshot (or pre-dates it); anything with `seq > snapshot.seq` + /// is post-snapshot and should be applied. Set to 0 in the + /// pre-emit case (no events ever fired) — clients treat that as + /// "apply everything you've buffered". + seq: u64, + hostname: String, + any_stale: bool, + containers: Vec, + transients: Vec, + approvals: Vec, + /// Last 30 resolved approvals (approved / denied / failed), newest- + /// first. Drives the "history" tab on the approvals section. + approval_history: Vec, + /// Pending operator-targeted questions (`target IS NULL`). Any + /// agent can `ask` the operator and `ask` returns immediately with + /// the id; on `/answer-question` we mark the row answered and + /// fire `HelperEvent::QuestionAnswered` back into the asker's + /// inbox. Peer-to-peer questions live in the same table but never + /// surface here (see `OperatorQuestions::pending`). + questions: Vec, + /// Last 20 answered questions, newest-first. + question_history: Vec, + /// State dirs (config history + claude creds + /state/ notes) that + /// survive after a destroy-without-purge. The operator can re-spawn + /// with the same name to resume, or PURG3 to wipe them. + tombstones: Vec, + /// Sub-agents whose FNV-1a hashed web UI port collides with at + /// least one other agent. Operator resolves by renaming. The + /// dashboard renders a banner at the top listing each cluster. + port_conflicts: Vec, + /// Inputs in `meta/flake.lock` the operator can selectively + /// `nix flake update`. Hyperhive first, then `agent-` rows. + meta_inputs: Vec, + /// True while a dashboard-triggered `meta-update` (flake lock bump + + /// agent rebuild ripple) is running in the background. Lets a + /// client that cold-loads mid-update render the META INPUTS panel's + /// disabled "updating…" state; live transitions arrive via the + /// `MetaUpdateRunning` event. + meta_update_running: bool, + /// Current state of the global job queue — pending + running DAGs + /// (rebuild / meta-update / spawn / power ops) with their per-node + /// breakdowns, plus the most recent few terminal DAGs the queue + /// retains for history. Live transitions arrive via the + /// `RebuildQueueChanged` event. See `job_queue/`. Field name kept + /// from the old flat queue for wire compatibility. + rebuild_queue: Vec, + /// Whether the hive-forge container is up. When true the dashboard + /// links each container's config + each approval's commit into the + /// forge's `agent-configs` repos. + forge_present: bool, + /// Whether the matrix GUI is reachable at `/matrix/`. Sourced from + /// `HIVE_MATRIX_GUI_ENABLED` env var (set by the c0re NixOS module + /// when `services.hyperhive.matrix.gui.enable` is on). The gateway + /// (hive-gateway.nix) does the actual `/matrix/` static serving; + /// this flag is just an availability signal for iris's dashboard + /// chrome so the `M4TR1X →` tab doesn't flash when the GUI is off. + matrix_gui_enabled: bool, + /// Whether `hive-gateway` is in front of this dashboard. Sourced + /// from the `HIVE_GATEWAY_ENABLED` env var, which the c0re NixOS + /// module now always sets (the gateway runs unconditionally + /// alongside hyperhive), so this is effectively always true: the + /// dashboard frontend builds same-origin `/agent//` links to + /// the per-agent web UI (the gateway routes them via the + /// runtime-generated `agents.conf` include file — see + /// `gateway_nginx.rs`). The `false` branch (direct + /// `http://:/` TCP links) is retained as a defensive + /// fallback for the env being unset. See `docs/gateway.md::Vhost map`. + gateway_enabled: bool, + /// Public URL of the forge vhost served by hive-gateway (e.g. + /// `"https://forge.pr1ma.darkest.space"`). Sourced from the + /// `HIVE_FORGE_PUBLIC_URL` env var, which the c0re NixOS module + /// sets when `forge.behindGateway = true`. `None` when absent — + /// the frontend falls back to `http://:3000`. + forge_public_url: Option, + /// Human name of this single-host hive instance (e.g. `"pr1ma"`). + /// Sourced from `HYPERHIVE_HIVE_NAME` env var, set by the c0re + /// NixOS module from `services.hyperhive.hiveName`. `None` when + /// the option is unset — chrome falls back to `hostname`. + hive_name: Option, + /// Human name of the wider swarm this hive belongs to (e.g. + /// `"constellat1on"`). Sourced from `HYPERHIVE_SWARM_NAME` env + /// var, set from `services.hyperhive.swarmName`. `None` when + /// unset — chrome omits the swarm segment of the breadcrumb. + swarm_name: Option, + /// Peer hives in the same swarm. Parsed from `HYPERHIVE_PEERS` + /// (JSON array of `{domain,cert_fingerprint}` objects, emitted by + /// the c0re NixOS module from `services.hyperhive.swarm.peers`). + /// Empty on single-hive deploys. Feeds the P33RS dashboard tab. + peer_hives: Vec, + /// Server-level warnings for the dashboard's top-of-page banner + /// (currently host disk-pressure; more producers can be added + /// backend-side). Empty when all clear. Built by + /// `host_stats::server_warnings`; the frontend renders this list + /// generically, so new warning kinds need no frontend change. + server_warnings: Vec, +} + +/// One peer hive for the P33RS dashboard tab. Derived from +/// `HYPERHIVE_PEERS` env; `url` is the peer's HTTPS dashboard root. +/// `cert_fingerprint` is `Some("sha256:")` when the peer uses a +/// self-signed cert and the operator pinned its fingerprint in +/// `services.hyperhive.swarm.peers`. +#[derive(Serialize)] +struct PeerHiveView { + name: String, + url: String, + cert_fingerprint: Option, +} + +/// `OpQuestion` + computed `question_refs` / `answer_refs`. Built +/// from the snapshot read; the live channel attaches the same +/// fields directly on `QuestionAdded` / `QuestionResolved`. +#[derive(Serialize)] +struct QuestionView { + #[serde(flatten)] + inner: crate::operator_questions::OpQuestion, + #[serde(skip_serializing_if = "Vec::is_empty")] + question_refs: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + answer_refs: Vec, +} + +impl QuestionView { + fn from_question(q: crate::operator_questions::OpQuestion) -> Self { + let question_refs = scan_validated_paths(&q.question); + let answer_refs = q + .answer + .as_deref() + .map(scan_validated_paths) + .unwrap_or_default(); + Self { + inner: q, + question_refs, + answer_refs, + } + } +} + +#[derive(Serialize)] +struct PortConflict { + port: u16, + /// All agent names sharing this port (sorted, ≥2 entries). + agents: Vec, +} + +#[derive(Serialize)] +struct TransientView { + name: String, + kind: &'static str, + secs: u64, +} + +#[derive(Serialize)] +struct ApprovalHistoryView { + id: i64, + agent: String, + kind: &'static str, + /// First 12 chars of the canonical sha (preferred) or + /// manager-supplied ref. None for resolved spawn approvals. + sha_short: Option, + /// `approved` / `denied` / `failed`. + status: &'static str, + /// RFC 3339 UTC. Renders as a relative time on the dashboard. + resolved_at: DateTime, + /// Operator-supplied deny reason (for `denied`) or build error + /// (for `failed`). None on `approved`. + #[serde(skip_serializing_if = "Option::is_none")] + note: Option, +} + +#[derive(Serialize)] +struct ApprovalView { + id: i64, + agent: String, + kind: &'static str, + /// First 12 chars of the `commit_ref`, for `ApplyCommit` only. + sha_short: Option, + /// Raw unified diff text, for `ApplyCommit` only. The client splits + /// on `\n` and per-line classifies (`+` / `-` / `@@` / `--- ` / `+++ ` + /// → diff-add / diff-del / diff-hunk / diff-file). Shipping raw + /// instead of pre-rendered HTML saves bytes on the wire (no + /// per-line `` markup) and removes the only HTML-escape + /// surface from the snapshot. + diff: Option, + /// Manager-supplied description shown on the approval card. + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + /// Forge PR number, for `MergeConfigPr` only. Lets the frontend + /// build a "review PR on forge" link + /// (`{forgeBase}/agent-configs/{agent}/pulls/{pr_number}`) the same + /// way it builds the `apply_commit` "commit on forge" link from the + /// sha. `None` for every other kind. + #[serde(skip_serializing_if = "Option::is_none")] + pr_number: Option, + /// Raw `commit_ref` payload for `UpdateMetaInputs` (JSON-encoded + /// `Vec` of input names; `"[]"` = all inputs) and + /// `SchedulePrompt` (JSON-encoded `SchedulePromptPayload`). The + /// frontend parses this to render a human-readable card body. + /// `None` for every other kind. + #[serde(skip_serializing_if = "Option::is_none")] + commit_ref: Option, + /// RFC 3339 UTC time the approval was queued. Rendered as a + /// relative time on the card so the operator can spot a stale + /// request. + requested_at: DateTime, +} + +/// Replace silent `.unwrap_or_default()` on the data sources behind +/// `/api/state` so that whichever query degrades surfaces in journald +/// instead of leaving the operator staring at an empty list. The +/// dashboard still degrades to a sensible default value; the warn +/// is just the diagnostic breadcrumb the old code swallowed. +fn log_default(what: &str, result: std::result::Result) -> T +where + T: Default, + E: std::fmt::Debug, +{ + match result { + Ok(v) => v, + Err(e) => { + tracing::warn!(target: "api_state", source = %what, error = ?e, "snapshot source failed; using default"); + T::default() + } + } +} + +/// Window over which container crashes count toward the `agents_crashing` +/// banner warning. Wide enough that a crash-looping container (restarted +/// by `Restart=on-failure` every few seconds) keeps the warning lit +/// between flaps, short enough that a single recovered crash clears within +/// minutes. +const CRASH_WARNING_WINDOW: std::time::Duration = std::time::Duration::from_mins(10); + +pub(super) async fn api_state( + headers: HeaderMap, + State(state): State, +) -> axum::Json { + let host = headers + .get("host") + .and_then(|h| h.to_str().ok()) + .unwrap_or("localhost"); + let hostname = host.split(':').next().unwrap_or(host).to_owned(); + + // Capture the unified dashboard-channel seq *before* any read so the + // dedupe contract is "events with seq > snapshot.seq are + // post-snapshot, never missed." An event landing during snapshot + // construction may be doubly applied (snapshot caught the write + + // client also applies the SSE frame) — that's a renderer's problem + // to make idempotent, not ours to avoid here. + let seq = state.coord.current_seq(); + + // Refresh the coordinator's cached container snapshot before + // reading. Cold-load clients then see whatever the latest rescan + // produced; live clients converge via the matching + // `ContainerStateChanged` / `ContainerRemoved` events the rescan + // emits. + // + // Bound the rescan: it shells out (`nixos-container list` etc.), so a + // saturated/wedged build backend — e.g. hive-c0re mid-startup-sweep + // hammering slow `nixos-container update` subprocesses — can stall it + // long enough that `/api/state` hangs for the whole request (the + // ~minute-long /state reported in the field). On timeout we skip the + // fresh rescan and serve the last cached snapshot instead; live + // clients still converge via the SSE events a later successful rescan + // emits, and the next /state call retries the refresh. Introspection + // stays responsive regardless of the build backend's health. + if tokio::time::timeout( + std::time::Duration::from_secs(3), + state.coord.rescan_containers_and_emit(), + ) + .await + .is_err() + { + tracing::warn!( + "api_state: container rescan exceeded 3s (build backend likely saturated); \ + serving last cached snapshot" + ); + } + let containers = state.coord.containers_snapshot().await; + let any_stale = containers.iter().any(|c| c.needs_update); + let transient_snapshot = state.coord.transient_snapshot(); + let pending_approvals = approvals::gc_orphans( + &state.coord, + log_default("approvals.pending", state.coord.approvals.pending()), + ); + let transients = build_transient_views(&containers, &transient_snapshot); + let approvals = build_approval_views(pending_approvals).await; + let approval_history = log_default( + "approvals.recent_resolved", + state.coord.approvals.recent_resolved(30), + ) + .into_iter() + .map(history_view) + .collect(); + let tombstones = build_tombstone_views(&state.coord, &containers, &transient_snapshot); + let port_conflicts = build_port_conflicts(&containers); + + // Both operator-targeted and peer threads surface on the dashboard + // (the client filters by target). Each row is wrapped in QuestionView + // so the snapshot carries the same file_refs the live event variants + // attach. + let questions: Vec = + log_default("questions.pending_all", state.coord.questions.pending_all()) + .into_iter() + .map(QuestionView::from_question) + .collect(); + let question_history: Vec = log_default( + "questions.recent_answered_all", + state.coord.questions.recent_answered_all(20), + ) + .into_iter() + .map(QuestionView::from_question) + .collect(); + + // Banner warnings: host probes (disk) + agent-state (pending logins, + // crashing agents). Built before the response struct because the + // agent-state producer borrows `containers`, which moves in below. + let server_warnings = { + let mut w = crate::host_stats::server_warnings(); + w.extend(crate::host_stats::agent_state_warnings( + &containers, + &state.coord.recent_crash_counts(CRASH_WARNING_WINDOW), + )); + w + }; + + axum::Json(StateSnapshot { + seq, + hostname, + any_stale, + containers, + transients, + approvals, + approval_history, + meta_inputs: read_meta_inputs(), + meta_update_running: state.coord.meta_update_in_progress(), + questions, + question_history, + tombstones, + port_conflicts, + rebuild_queue: state.coord.job_queue.snapshot(), + forge_present: crate::forge::is_present().await, + matrix_gui_enabled: std::env::var_os("HIVE_MATRIX_GUI_ENABLED").is_some_and(|v| { + // Accept any truthy string ("1", "true", "yes") since the + // env var is set by NixOS module wiring with the literal + // "1"; defensive parse so manual overrides also work. + let s = v.to_string_lossy().to_ascii_lowercase(); + matches!(s.as_str(), "1" | "true" | "yes") + }), + gateway_enabled: std::env::var_os("HIVE_GATEWAY_ENABLED").is_some_and(|v| { + // Same truthy-string parse as `matrix_gui_enabled`; the + // env var is set by the c0re NixOS module to the literal + // "1" — the gateway always runs alongside hyperhive. + let s = v.to_string_lossy().to_ascii_lowercase(); + matches!(s.as_str(), "1" | "true" | "yes") + }), + forge_public_url: std::env::var("HIVE_FORGE_PUBLIC_URL") + .ok() + .filter(|s| !s.is_empty()), + hive_name: std::env::var("HYPERHIVE_HIVE_NAME") + .ok() + .filter(|s| !s.is_empty()), + swarm_name: std::env::var("HYPERHIVE_SWARM_NAME") + .ok() + .filter(|s| !s.is_empty()), + peer_hives: parse_peer_hives(), + server_warnings, + }) +} + +/// Parse `HYPERHIVE_PEERS` env var into dashboard-ready `PeerHiveView` +/// entries. The env var is a JSON array of `{domain, cert_fingerprint}` +/// objects emitted by the c0re NixOS module from +/// `services.hyperhive.swarm.peers`. Each entry becomes +/// `{ name: domain, url: "https://domain/" }` for the P33RS tab. +/// Returns empty vec when unset (single-hive deploy). +fn parse_peer_hives() -> Vec { + #[derive(serde::Deserialize)] + struct Raw { + domain: String, + cert_fingerprint: Option, + } + let Ok(json) = std::env::var("HYPERHIVE_PEERS") else { + return Vec::new(); + }; + let Ok(raw): Result, _> = serde_json::from_str(&json) else { + tracing::warn!("HYPERHIVE_PEERS is not valid JSON; ignoring"); + return Vec::new(); + }; + raw.into_iter() + .map(|r| { + let cert_fingerprint = r.cert_fingerprint.and_then(|fp| { + if validate_cert_fingerprint(&fp) { + Some(fp) + } else { + tracing::warn!( + domain = %r.domain, + fingerprint = %fp, + "HYPERHIVE_PEERS: invalid cert_fingerprint format \ + (expected `sha256:<64 hex chars>`); ignoring fingerprint" + ); + None + } + }); + PeerHiveView { + name: r.domain.clone(), + url: format!("https://{}/", r.domain), + cert_fingerprint, + } + }) + .collect() +} + +/// Validate a TLS certificate fingerprint string from `HYPERHIVE_PEERS`. +/// Accepts `sha256:<64 hex chars>` (upper or lower case). +fn validate_cert_fingerprint(fp: &str) -> bool { + let Some(hex) = fp.strip_prefix("sha256:") else { + return false; + }; + hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// Group live containers by their assigned web UI port; clusters with +/// more than one member are port-hash collisions the operator needs +/// to resolve by renaming. Manager (fixed at 8000) and sub-agents +/// (8100..8999) can't collide with each other — collisions are +/// strictly between sub-agents. +fn build_port_conflicts(containers: &[ContainerView]) -> Vec { + let mut by_port: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for c in containers { + by_port.entry(c.port).or_default().push(c.name.clone()); + } + by_port + .into_iter() + .filter(|(_, agents)| agents.len() > 1) + .map(|(port, mut agents)| { + agents.sort(); + PortConflict { port, agents } + }) + .collect() +} + +/// Transient state for agents whose container does NOT yet exist +/// (`Spawning`). Lifecycle ops on existing containers surface as +/// `ContainerView.pending` inline; this list only catches pre-creation. +fn build_transient_views( + containers: &[ContainerView], + transient_snapshot: &std::collections::HashMap, +) -> Vec { + transient_snapshot + .iter() + .filter(|(name, _)| !containers.iter().any(|c| &c.name == *name)) + .map(|(name, st)| TransientView { + name: name.clone(), + kind: transient_label(st.kind), + secs: st.since.elapsed().as_secs(), + }) + .collect() +} + +fn transient_label(k: crate::coordinator::TransientKind) -> &'static str { + use crate::coordinator::TransientKind::{ + Destroying, Rebuilding, Restarting, Spawning, Starting, Stopping, + }; + match k { + Spawning => "spawning", + Starting => "starting", + Stopping => "stopping", + Restarting => "restarting", + Rebuilding => "rebuilding", + Destroying => "destroying", + } +} + +/// Render each pending approval into its dashboard view (short sha + +/// unified diff for `ApplyCommit`, just the name for `Spawn`). +/// Project a resolved sqlite row into the lean shape the dashboard +/// history tab consumes — no `diff_html` (rendering 30 of them +/// per /api/state poll would mean 30 git diffs per refresh). +fn history_view(a: Approval) -> ApprovalHistoryView { + let displayed = a.fetched_sha.as_deref().unwrap_or(&a.commit_ref); + let sha_short = if displayed.is_empty() { + None + } else { + Some(displayed[..displayed.len().min(12)].to_owned()) + }; + let status = match a.status { + hive_sh4re::ApprovalStatus::Approved => "approved", + hive_sh4re::ApprovalStatus::Denied => "denied", + hive_sh4re::ApprovalStatus::Failed => "failed", + hive_sh4re::ApprovalStatus::Cancelled => "cancelled", + // Pending shouldn't appear in recent_resolved, but be defensive. + hive_sh4re::ApprovalStatus::Pending => "pending", + }; + let kind = a.kind.as_str(); + ApprovalHistoryView { + id: a.id, + agent: a.agent, + kind, + sha_short, + status, + resolved_at: a.resolved_at.unwrap_or_default(), + note: a.note, + } +} + +async fn build_approval_views(approvals: Vec) -> Vec { + let mut out = Vec::with_capacity(approvals.len()); + for a in approvals { + out.push(match a.kind { + hive_sh4re::ApprovalKind::ApplyCommit => { + // Prefer the canonical fetched sha from applied; + // commit_ref is only the manager's claim and may be + // amended out from under us. + let displayed = a.fetched_sha.as_deref().unwrap_or(&a.commit_ref); + let sha = displayed[..displayed.len().min(12)].to_owned(); + let diff = approval_diff(&a.agent, a.id).await; + ApprovalView { + id: a.id, + agent: a.agent.clone(), + kind: "apply_commit", + sha_short: Some(sha), + diff: Some(diff), + description: a.description, + pr_number: None, + commit_ref: None, + requested_at: a.requested_at, + } + } + hive_sh4re::ApprovalKind::Spawn => ApprovalView { + id: a.id, + agent: a.agent, + kind: "spawn", + sha_short: None, + diff: None, + description: a.description, + pr_number: None, + commit_ref: None, + requested_at: a.requested_at, + }, + hive_sh4re::ApprovalKind::InitConfig => ApprovalView { + id: a.id, + agent: a.agent, + kind: "init_config", + sha_short: None, + diff: None, + description: a.description, + pr_number: None, + commit_ref: None, + requested_at: a.requested_at, + }, + hive_sh4re::ApprovalKind::UpdateMetaInputs => ApprovalView { + id: a.id, + agent: a.agent, + kind: "update_meta_inputs", + sha_short: None, + diff: None, + description: a.description, + pr_number: None, + commit_ref: Some(a.commit_ref), + requested_at: a.requested_at, + }, + hive_sh4re::ApprovalKind::SchedulePrompt => ApprovalView { + id: a.id, + agent: a.agent, + kind: "schedule_prompt", + sha_short: None, + diff: None, + description: a.description, + pr_number: None, + commit_ref: Some(a.commit_ref), + requested_at: a.requested_at, + }, + hive_sh4re::ApprovalKind::MergeConfigPr => { + // commit_ref = PR number; fetched_sha = the reviewed PR + // head. Show the head sha; the forge PR diff surface is + // a later phase of the PR-based config flow — None for now. + let sha = a + .fetched_sha + .as_deref() + .map(|s| s[..s.len().min(12)].to_owned()); + // Surface the PR number so the frontend can link to the + // PR on the forge. commit_ref holds the number as text. + let pr_number = a.commit_ref.parse::().ok(); + ApprovalView { + id: a.id, + agent: a.agent, + kind: "merge_config_pr", + sha_short: sha, + diff: None, + description: a.description, + pr_number, + commit_ref: None, + requested_at: a.requested_at, + } + } + }); + } + out +} + +pub(super) async fn dashboard_history(State(state): State) -> Response { + // Backfill source for the dashboard terminal. Returns up to ~200 + // historical broker messages (no other event kinds are persisted) + // converted to `DashboardEvent::Sent` JSON so the client can replay + // through the same dispatch path as live frames. Wrapped in + // `{ seq, events }`: the seq is the dashboard channel's high-water + // mark at fetch time. Clients use it to dedupe their buffered live + // SSE traffic (drop anything with `seq <= history_seq`) so a frame + // that lands between SSE-subscribe and history-fetch isn't shown + // twice and isn't lost. Historical rows carry `seq = 0`; the + // boundary seq is what closes the dedupe window. + const HISTORY_LIMIT: u64 = 200; + let seq = state.coord.current_seq(); + match state.coord.broker.recent_all(HISTORY_LIMIT) { + Ok(mut messages) => { + messages.reverse(); + let events: Vec = messages + .into_iter() + .map(|m| match m { + crate::broker::MessageEvent::Sent { + id, + from, + to, + body, + at, + in_reply_to, + } => { + let file_refs = scan_validated_paths(&body); + crate::dashboard_events::DashboardEvent::Sent { + seq: 0, + id, + from, + to, + body, + at: hive_sh4re::wire_time::from_secs(at), + in_reply_to, + file_refs, + } + } + crate::broker::MessageEvent::Delivered { + id, + from, + to, + body, + at, + in_reply_to, + } => { + let file_refs = scan_validated_paths(&body); + crate::dashboard_events::DashboardEvent::Delivered { + seq: 0, + id, + from, + to, + body, + at: hive_sh4re::wire_time::from_secs(at), + in_reply_to, + file_refs, + } + } + }) + .collect(); + axum::Json(serde_json::json!({ "seq": seq, "events": events })).into_response() + } + Err(e) => error_response(&format!("dashboard/history failed: {e:#}")), + } +} + +/// `/dashboard/stream` query string. Today's only field is `kinds`: +/// a comma-separated allow-list of event-`kind` strings. +/// Empty / absent ⇒ no filter (current behaviour, all variants +/// forwarded). Set ⇒ only the named kinds reach the subscriber, +/// non-matches are skipped before the JSON serialise cost. +/// +/// Useful for narrow pages (e.g. `flow.js` only cares about `sent` +/// / `delivered` / `container_state_changed` / `container_removed`) +/// that want to drop the dispatch overhead on every unrelated mutation. +#[derive(Deserialize, Default)] +pub(super) struct DashboardStreamQuery { + /// Comma-separated event kinds to forward. Each token is + /// trimmed; unknown kinds are silently ignored on lookup + /// (subscriber sees nothing instead of an error). + kinds: Option, +} + +pub(super) async fn dashboard_stream( + State(state): State, + axum::extract::Query(q): axum::extract::Query, +) -> Sse>> { + let rx = state.coord.dashboard_subscribe(); + // Pre-parse the allow-list once at subscription time, so the + // per-event hot path is just a `HashSet::contains` on a + // `&'static str` — no string churn per frame. + let kind_filter: Option> = q.kinds.and_then(|raw| { + let set: std::collections::HashSet = raw + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .collect(); + if set.is_empty() { None } else { Some(set) } + }); + let stream = BroadcastStream::new(rx).filter_map(move |res| { + // Drop lagged frames. Browsers reconnect; the seq dedupe on + // reconnect skips any frame already reflected in the snapshot. + let event = res.ok()?; + if let Some(filter) = kind_filter.as_ref() + && !filter.contains(event.kind_tag()) + { + return None; + } + let json = serde_json::to_string(&event).ok()?; + Some(Ok(Event::default().data(json))) + }); + Sse::new(stream).keep_alive(KeepAlive::default()) +} diff --git a/hive-c0re/src/dashboard/tombstones.rs b/hive-c0re/src/dashboard/tombstones.rs new file mode 100644 index 00000000..8cd6c408 --- /dev/null +++ b/hive-c0re/src/dashboard/tombstones.rs @@ -0,0 +1,159 @@ +//! Tombstone rows for the dashboard: state dirs surviving a +//! destroy-without-purge. Builds `TombstoneView`s for the snapshot, +//! emits the `TombstonesChanged` event after mutations, and handles +//! `POST /api/purge-tombstone/{name}`. + +use std::path::Path; +use std::sync::Arc; + +use axum::{ + extract::{Path as AxumPath, State}, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde::Serialize; + +use crate::container_view::{ContainerView, claude_has_session}; +use crate::coordinator::Coordinator; +use crate::lifecycle; + +use super::{AppState, error_response, validate_agent_name}; + +#[derive(Serialize, Clone, Debug)] +pub struct TombstoneView { + pub name: String, + /// Bytes used by the state dir tree. Cheap-ish to compute; let the + /// operator know how much they're holding onto. + pub state_bytes: u64, + /// Mtime (unix seconds) of the state dir; rough "last seen". + pub last_seen: i64, + pub has_creds: bool, +} + +/// State-dir names that don't appear in the live container list. Each +/// one surfaces in the dashboard as a row with R3V1V3 + PURG3 actions. +pub(super) fn build_tombstone_views( + coord: &Coordinator, + containers: &[ContainerView], + transient_snapshot: &std::collections::HashMap, +) -> Vec { + let _ = coord; // kept_state_names is a free fn but takes &self by future plan + let live: std::collections::HashSet<&str> = containers + .iter() + .map(|c| c.name.as_str()) + .chain(transient_snapshot.keys().map(String::as_str)) + .collect(); + Coordinator::kept_state_names() + .into_iter() + .filter(|name| !live.contains(name.as_str())) + .map(|name| { + let root = Coordinator::agent_state_root(&name); + let state_bytes = dir_size_bytes(&root); + let last_seen = std::fs::metadata(&root) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .and_then(|d| i64::try_from(d.as_secs()).ok()) + .unwrap_or(0); + let has_creds = claude_has_session(&Coordinator::agent_claude_dir(&name)); + TombstoneView { + name, + state_bytes, + last_seen, + has_creds, + } + }) + .collect() +} + +/// Sum the byte size of every regular file under `root`. Cheap to compute +/// for typical agent state (config repo + claude creds + notes file — +/// usually a few MB); fine to do inline on each /api/state. Returns 0 on +/// any error. +fn dir_size_bytes(root: &Path) -> u64 { + fn walk(p: &Path, acc: &mut u64) { + let Ok(rd) = std::fs::read_dir(p) else { return }; + for entry in rd.flatten() { + let Ok(ft) = entry.file_type() else { continue }; + if ft.is_dir() { + walk(&entry.path(), acc); + } else if ft.is_file() + && let Ok(meta) = entry.metadata() + { + *acc += meta.len(); + } + } + } + let mut total = 0u64; + walk(root, &mut total); + total +} + +/// Snapshot the current tombstone list and emit a +/// `TombstonesChanged` event. Call after any mutation that could +/// add or remove a tombstone (`actions::destroy`, +/// `post_purge_tombstone`, spawn finalisation). Cheap — the list +/// is tiny. +pub(crate) async fn emit_tombstones_snapshot(coord: &Arc) { + let containers = coord.containers_snapshot().await; + let transient_snapshot = coord.transient_snapshot(); + let tombstones = build_tombstone_views(coord, &containers, &transient_snapshot); + coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::TombstonesChanged { + seq: coord.next_seq(), + tombstones, + }); +} + +pub(super) async fn post_purge_tombstone( + State(state): State, + AxumPath(name): AxumPath, +) -> Response { + // Format guard FIRST so a name like `..` can't traverse into the + // parent of `/var/lib/hyperhive/agents/{name}` and have + // `remove_dir_all` wipe `/var/lib/hyperhive/` itself. Existing + // manager + live-container checks below don't catch `..` — only + // the whitelist does. Existence check via + // `containers_snapshot()` is deliberately NOT used here: + // tombstoned agents are gone from the snapshot by design; that's + // the whole point of this endpoint. + if let Some(reason) = validate_agent_name(&name) { + return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(); + } + // Sanity: refuse to purge if a live container still exists with this + // name. The dashboard already filters tombstones to non-live names, + // but the operator could send a stale POST. + let live = lifecycle::list().await.unwrap_or_default(); + if live + .iter() + .any(|c| c == &format!("{}{name}", lifecycle::AGENT_PREFIX) || c == &name) + { + return error_response(&format!( + "refusing to purge {name}: container still exists — use DESTR0Y first" + )); + } + let mut errors = Vec::new(); + for dir in [ + Coordinator::agent_state_root(&name), + Coordinator::agent_applied_dir(&name), + ] { + if dir.exists() + && let Err(e) = std::fs::remove_dir_all(&dir) + { + errors.push(format!("{}: {e}", dir.display())); + } + } + let _ = state + .coord + .approvals + .fail_pending_for_agent(&name, "agent state purged"); + if errors.is_empty() { + tracing::info!(%name, "tombstone purged"); + // Fire the post-purge tombstones snapshot so dashboards + // drop the row live; matching form carries + // `data-no-refresh`. + emit_tombstones_snapshot(&state.coord).await; + (StatusCode::OK, "ok").into_response() + } else { + error_response(&format!("purge {name} partial: {}", errors.join(", "))) + } +} diff --git a/hive-c0re/src/dashboard_events.rs b/hive-c0re/src/dashboard_events.rs index 21f5d820..04590dca 100644 --- a/hive-c0re/src/dashboard_events.rs +++ b/hive-c0re/src/dashboard_events.rs @@ -8,7 +8,7 @@ use serde::Serialize; use crate::container_view::ContainerView; use crate::dashboard::{MetaInputView, TombstoneView}; -use crate::rebuild_queue::QueueEntry; +use crate::job_queue::DagView; use chrono::{DateTime, Utc}; #[derive(Debug, Clone, Serialize)] @@ -210,7 +210,7 @@ pub enum DashboardEvent { /// the add/remove races a per-row event would have, and the /// dashboard's grouping (`parent_id`) is most naturally re-derived /// from the full list. - RebuildQueueChanged { seq: u64, queue: Vec }, + RebuildQueueChanged { seq: u64, queue: Vec }, /// Full snapshot of all scheduled prompts. Emitted after every /// operator mutation (new / edit / cancel / fire-now) and after the /// worker fires or rearms a row. Same snapshot-shape rationale as diff --git a/hive-c0re/src/forge.rs b/hive-c0re/src/forge.rs deleted file mode 100644 index fe44824b..00000000 --- a/hive-c0re/src/forge.rs +++ /dev/null @@ -1,1671 +0,0 @@ -//! Optional Forgejo wiring — per-agent user + token provisioning, -//! config-repo mirroring, meta read-access grants. Also seeds -//! `internal/docs` — a private repo every agent gets read-only -//! collaborator access to for operator-curated shared content. -//! No-op when `hive-forge` isn't running. Full design: `docs/forge.md`. - -use std::path::Path; - -use anyhow::{Context, Result}; -use base64::Engine; -use reqwest::StatusCode; -use tokio::process::Command; - -use crate::coordinator::Coordinator; - -const FORGE_CONTAINER: &str = "hive-forge"; -pub(crate) const FORGE_HTTP: &str = "http://localhost:3000"; -const TOKEN_NAME_PREFIX: &str = "hyperhive"; -/// Where the host-side `core` admin token lives. Used by hive-c0re -/// itself to push the meta repo + drive admin API calls (org -/// creation, future webhook setup, etc.). Root-only. -const CORE_TOKEN_PATH: &str = "/var/lib/hyperhive/forge-core-token"; -// Forge provisioning markers (`forge/core-avatar-set`, -// `forge/agent-configs-avatar-set`, `forge/email-aligned-`) live -// in `crate::paths` — one-shot guards: the upload/align runs once, the -// marker is written, subsequent startups skip. Delete one to force its -// step to re-run. -// Avatar PNGs are loaded at runtime from -// `$HIVE_ASSETS_DIR/branding/{hyperhive,agent-configs}.png` via the -// helpers in `hive_sh4re::assets`. The `agent-configs.png` is -// rendered from its SVG during the `hyperhive-assets` derivation's -// build. -/// Forgejo org grouping every agent's config repo. Core is a site admin -/// and reads + writes every repo here. As of the agent-config-PR flow each -/// agent is a **write collaborator on its own** `agent-configs/` repo — -/// the editable PR surface it pushes config-change branches to — but `main` is -/// branch-protected core-only, so only hive-c0re's verify-and-ff-push merge -/// handler lands on it (operator approval required; the agent can't push -/// `main` or self-merge). The repos remain private, so an agent still can't -/// reach *another* agent's config. `main` is fast-forward-only — hive-c0re -/// never force-pushes; the `push_config` mirror runs best-effort until the -/// PR-merge flow retires it. -const CONFIG_ORG: &str = "agent-configs"; -/// Forgejo org hosting the operator-curated shared docs/skills repo -/// that every agent gets read-only access to. Agents use it as a -/// common reference without the operator having to bake content into -/// the system prompt or rely on `/shared`. Only the manager + operator -/// (i.e. `core` user) can push. -const SHARED_ORG: &str = "internal"; -/// The shared docs repo inside `SHARED_ORG`. Cloneable by every agent -/// at `{FORGE_HTTP}/internal/docs.git`. -const SHARED_DOCS_REPO: &str = "docs"; -/// The hive-wide knowledge repo inside `SHARED_ORG`. Public — agents -/// can fork it and open PRs without explicit collaborator grants. -/// Bind-mounted read-only into every container at `/knowledge`. -/// See `hive-c0re/src/knowledge.rs`. -const KNOWLEDGE_REPO: &str = crate::knowledge::REPO; -/// Forgejo org that owns agent-created repos. Agents can't create -/// repos with their own token (`max_repo_creation = 0`); instead hive-c0re -/// creates them here and adds the requesting agent as a **write** member -/// (not owner/admin). Because the org — not the agent — owns the repo, -/// perms stay c0re-managed and branch protection (referencing -/// [`OPERATORS_TEAM`]) can block the author from merging their own PR. This -/// is the "agents namespace" repos land in by default. -const AGENTS_ORG: &str = "agents"; -/// Operator merge-gate team inside [`AGENTS_ORG`]. Provisioned **empty** by -/// hive-c0re (so perms can be set before anyone joins); the operator adds -/// herself via the forge UI / hivectl. Branch protection on agents-org repos -/// references this team by name for the merge/approval whitelist, so the -/// rule never hardcodes a specific reviewer agent (which may not exist). -const OPERATORS_TEAM: &str = "operators"; -/// Hive-managed Forgejo namespaces that agent-initiated repo creation must -/// never target. `internal` is operator-curated shared content; -/// `agent-configs` + `core` are hive-c0re-internal mirror/meta namespaces. -/// (`hyperhive` is NOT managed — it's just a repo that happens to be built -/// by this hive.) hive-c0re's create path forces [`AGENTS_ORG`], so this is -/// a defensive guard against any future caller passing an explicit owner. -const HIVE_MANAGED_NAMESPACES: &[&str] = &[SHARED_ORG, CONFIG_ORG, "core"]; -/// Forgejo orgs hive-c0re ensures on startup. The meta repo lives at -/// `core/meta` (the `core` user's own namespace — no org needed). -const SEEDED_ORGS: &[&str] = &[CONFIG_ORG, SHARED_ORG, AGENTS_ORG]; -/// Per-agent token scopes (broad-but-not-admin). See -/// `docs/forge.md::Token scopes` for the per-scope rationale. -const TOKEN_SCOPES: &str = "read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc"; - -/// Bootstrap `core` token scopes — adds `read:admin,write:admin` on -/// top of `TOKEN_SCOPES` so the host daemon can drive -/// `/api/v1/admin/*`. Site-admin membership alone isn't enough: the -/// token's own scope gate runs before the user-permission check. -/// See `docs/forge.md::Token scopes`. -const CORE_TOKEN_SCOPES: &str = "read:admin,write:admin,read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc"; - -/// Probe whether `hive-forge` exists as a nixos-container. Cheap — -/// `nixos-container list` is just a directory scan in /etc. Routed -/// through hive-priv: `nixos-container` needs root, and hive-c0re runs -/// unprivileged (privsep). -pub async fn is_present() -> bool { - let Ok(stdout) = crate::priv_client::list_containers().await else { - return false; - }; - stdout.lines().any(|l| l.trim() == FORGE_CONTAINER) -} - -/// Run `forgejo admin ` inside the hive-forge container as the -/// forgejo user (the only uid with write access to the state dir). -/// Returns stdout on success; bails with stderr context on failure. -async fn forge_admin(args: &[&str]) -> Result { - // Route through hive-priv (root helper) because `nixos-container run` - // uses nsenter to enter the container's namespaces, which requires root. - // hive-c0re runs as the unprivileged `hive-core` user and cannot call - // nsenter directly — doing so produces: - // nsenter: stat of /proc//ns/user failed: Permission denied - let (stdout, _stderr) = crate::priv_client::run_forge_admin(args) - .await - .with_context(|| format!("forgejo admin {} (via hive-priv)", args.join(" ")))?; - Ok(stdout) -} - -/// Pull the access token out of forgejo's success message. Format -/// has shifted across versions (table form vs. "Access token was -/// successfully created: "), so just hunt the output for the -/// first long hex-looking word. -fn extract_token(output: &str) -> Option { - output - .split(|c: char| c.is_whitespace() || c == ',' || c == ':') - .find(|w| w.len() >= 32 && w.chars().all(|c| c.is_ascii_hexdigit())) - .map(str::to_owned) -} - -/// Canonical email address for a hive agent's Forgejo account. -/// Must match the `user.email` set by `meta::render_flake` so commits -/// by the agent link back to their Forgejo profile page. -fn agent_email(name: &str) -> String { - format!("{name}@hyperhive.local") -} - -/// Thin Forgejo REST helper. Sends `method` to `url` with a JSON body -/// and `Authorization: token `, returns the HTTP status code. -/// All Forgejo API calls that don't shell out to `forgejo admin` go -/// through here — one place for auth header, content-type, error -/// propagation, and the shared reqwest Client. -/// Returns the response status **and body**. The body lets callers log -/// *why* Forgejo rejected a request (e.g. the validation message on a -/// 422); status-only callers just bind `(status, _)`. Body read is -/// best-effort — a read error yields an empty string rather than -/// failing the whole call. -async fn forge_http( - method: reqwest::Method, - url: &str, - token: &str, - body: &str, -) -> Result<(StatusCode, String)> { - let client = reqwest::Client::new(); - let resp = client - .request(method, url) - .header("Authorization", format!("token {token}")) - .header("Content-Type", "application/json") - .body(body.to_owned()) - .send() - .await - .with_context(|| format!("forge HTTP request to {url}"))?; - let status = resp.status(); - let text = resp.text().await.unwrap_or_default(); - Ok((status, text)) -} - -/// Ensure a forgejo user named `name` exists. Idempotent: forgejo -/// returns a "user already exists" error which we treat as success. -/// `admin` adds `--admin` (site admin) — used for the bootstrap -/// `core` user that drives the API. `password` picks the initial -/// account password: `None` uses `--random-password` (the existing -/// agent provisioning shape — the password is never read, agents auth -/// by token); `Some(pw)` uses `--password ` so the operator path -/// in `hivectl` can set a real password for matrix-style web-UI login. -async fn ensure_user_exists(name: &str, admin: bool, password: Option<&str>) -> Result<()> { - let email = agent_email(name); - let mut args = vec!["user", "create", "--username", name, "--email", &email]; - match password { - Some(pw) => args.extend(["--password", pw, "--must-change-password=false"]), - None => args.extend(["--random-password", "--must-change-password=false"]), - } - if admin { - args.push("--admin"); - } - let result = forge_admin(&args).await; - match result { - Ok(_) => { - tracing::info!(%name, "forge: created user"); - Ok(()) - } - Err(e) => { - // Forgejo's "already exists" error wording varies; just - // try the next step and let token issuance surface a - // real failure if the user truly isn't there. - let msg = format!("{e:#}"); - if msg.contains("already exists") || msg.contains("user already") { - tracing::debug!(%name, "forge: user already exists"); - Ok(()) - } else { - tracing::warn!(%name, error = %msg, "forge: user create unclear; trying token anyway"); - Ok(()) - } - } - } -} - -/// Set the forgejo password for an existing user. Used by the operator -/// path in `hivectl forge create-user --password` so re-running on an -/// already-created account still updates the password (covers the -/// "I forgot the password I set last week" case + the "argus retried -/// the verb to verify the fix" case — `forgejo admin user create` -/// silently skips a password change once the account exists). Idempotent -/// from the operator's point of view: same password input → same final -/// account state. -async fn change_user_password(name: &str, password: &str) -> Result<()> { - let args = [ - "user", - "change-password", - "--username", - name, - "--password", - password, - ]; - forge_admin(&args) - .await - .with_context(|| format!("forgejo admin user change-password {name}"))?; - tracing::info!(%name, "forge: changed user password"); - Ok(()) -} - -/// Idempotently align the Forgejo account email to `agent_email(name)`. -/// Existing agents were created with `{name}@hive.local`; this corrects -/// that so git commits (which use `{name}@hyperhive`) link to profiles. -/// Best-effort: failures are warned, not propagated. -/// -/// Marker-guarded: writes `EMAIL_ALIGNED_MARKER_PREFIX{name}` on first -/// success and skips the PATCH on all subsequent calls. This prevents -/// Forgejo's admin-user-edit endpoint from resetting `use_custom_avatar` -/// on every `sync_agent` tick. Delete the marker to force re-alignment. -/// -/// Uses the admin REST API (`PATCH /api/v1/admin/users/{name}`) rather -/// than `forgejo admin user edit` because the CLI dropped the `edit` -/// subcommand somewhere between forgejo 8 and current. Body includes -/// `login_name` (required by Forgejo's `EditUserOption` validator) and -/// `source_id = 0` (local auth, the default for users hive-c0re creates). -async fn ensure_user_email(name: &str) { - let marker = crate::paths::forge_email_aligned_marker(name); - if marker.exists() { - return; - } - let Some(token) = core_token() else { - tracing::debug!(%name, "forge: skipping ensure_user_email — no core token yet"); - return; - }; - let email = agent_email(name); - // `login_name` is required by Forgejo's EditUserOption validator. - // Omitting it caused Forgejo to reset use_custom_avatar on each call. - let body = format!(r#"{{"email":"{email}","login_name":"{name}","source_id":0}}"#); - let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}"); - match forge_http(reqwest::Method::PATCH, &url, &token, &body).await { - Ok((status, _)) if status.is_success() => { - if let Some(parent) = marker.parent() { - std::fs::create_dir_all(parent).ok(); - } - std::fs::write(&marker, "").ok(); - tracing::info!(%name, %email, "forge: user email aligned"); - } - Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => { - // Core token missing admin scope — see - // `docs/forge.md::Token scopes` migration note. - tracing::warn!( - %name, %email, %status, - "forge: PATCH user email forbidden — core token likely missing admin scope. \ - Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes." - ); - } - Ok((status, _)) => { - tracing::warn!(%name, %email, %status, "forge: PATCH user email returned non-success"); - } - Err(e) => tracing::warn!(%name, error = %e, "forge: PATCH user email transport error"), - } -} - -/// Disable direct repo creation for agent `name` by setting -/// `max_repo_creation = 0` on its Forgejo account. Agents must -/// create repos *through hive-c0re* (which owns the perms), never with -/// their own token — a write-scoped token can otherwise create + own -/// repos and self-merge, bypassing the operator-only-merge policy. -/// -/// `max_repo_creation = 0` means `CanCreateRepo()` is false for any -/// count (Forgejo: `MaxRepoCreation >= 0 && NumRepos >= MaxRepoCreation`), -/// so creation is refused while push / PR / clone stay intact. **Existing -/// repos are untouched** — this only blocks *new* direct creation. -/// -/// Marker-guarded like [`ensure_user_email`]: the PATCH runs once per -/// agent (delete the marker to re-apply). Body carries `login_name` + -/// `source_id` for the same reason `ensure_user_email` does — omitting -/// `login_name` makes Forgejo's `EditUserOption` validator reset -/// `use_custom_avatar`. Best-effort: failures warn, don't propagate. -async fn ensure_repo_creation_disabled(name: &str) { - let marker = crate::paths::forge_repo_creation_disabled_marker(name); - if marker.exists() { - return; - } - let Some(token) = core_token() else { - tracing::debug!(%name, "forge: skipping ensure_repo_creation_disabled — no core token yet"); - return; - }; - let body = format!(r#"{{"login_name":"{name}","source_id":0,"max_repo_creation":0}}"#); - let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}"); - match forge_http(reqwest::Method::PATCH, &url, &token, &body).await { - Ok((status, _)) if status.is_success() => { - if let Some(parent) = marker.parent() { - std::fs::create_dir_all(parent).ok(); - } - std::fs::write(&marker, "").ok(); - tracing::info!(%name, "forge: disabled direct repo creation (max_repo_creation=0)"); - } - Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => { - tracing::warn!( - %name, %status, - "forge: PATCH max_repo_creation forbidden — core token likely missing admin scope. \ - Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes." - ); - } - Ok((status, _)) => { - tracing::warn!(%name, %status, "forge: PATCH max_repo_creation returned non-success"); - } - Err(e) => { - tracing::warn!(%name, error = %e, "forge: PATCH max_repo_creation transport error"); - } - } -} - -/// Mint a fresh access token for `name`. Token name is suffixed with -/// a monotonic clock so re-issuing doesn't collide with an existing -/// token of the same name in the DB. `scopes` is the scope string -/// passed to `forgejo admin user generate-access-token --scopes`; -/// use `TOKEN_SCOPES` for agents, `CORE_TOKEN_SCOPES` for the -/// bootstrap `core` user. -async fn mint_token(name: &str, scopes: &str) -> Result { - let token_name = format!( - "{TOKEN_NAME_PREFIX}-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()) - ); - let stdout = forge_admin(&[ - "user", - "generate-access-token", - "--username", - name, - "--token-name", - &token_name, - "--scopes", - scopes, - ]) - .await?; - let token = extract_token(&stdout) - .with_context(|| format!("parse token from forgejo output: {stdout:?}"))?; - tracing::debug!(%name, %token_name, "forge: minted access token"); - Ok(token) -} - -/// Mint a fresh Forgejo access token for an agent and write it to the -/// agent's state dir via hive-priv. hive-c0re runs unprivileged and -/// cannot write to agent-owned (0755) state directories directly. -async fn mint_and_persist_agent_token(name: &str) -> Result<()> { - let token = mint_token(name, TOKEN_SCOPES).await?; - crate::priv_client::write_agent_forge_token(name, &token) - .await - .with_context(|| format!("write forge-token for {name} via hive-priv")) -} - -/// Mint a fresh Forgejo access token for the `core` admin user and -/// write it directly to `path`. Unlike agent tokens this path is owned -/// by hive-c0re itself (under `/var/lib/hyperhive/`), so a direct -/// write is both correct and necessary (no priv round-trip). -async fn mint_and_persist_core_token(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - let token = mint_token("core", CORE_TOKEN_SCOPES).await?; - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).ok(); - } - std::fs::write(path, format!("{token}\n")) - .with_context(|| format!("write core token to {}", path.display()))?; - let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)); - tracing::info!(path = %path.display(), "forge: persisted core access token"); - Ok(()) -} - -/// Ensure `name` has a forgejo user + token file. Always re-mints the -/// token so the on-disk file always reflects the current `TOKEN_SCOPES`. -/// Safe to call on every spawn and on every hive-c0re startup. -pub async fn ensure_user_for(name: &str) -> Result<()> { - if !is_present().await { - return Ok(()); - } - ensure_user_exists(name, false, None).await?; - ensure_user_email(name).await; - mint_and_persist_agent_token(name).await -} - -/// Provision a forgejo user for `name` and return the freshly-minted -/// token. Unlike [`ensure_user_for`], the token is **not** persisted to -/// disk — the caller is responsible for storing it. Used by `hivectl -/// forge create-user` for human (non-agent) accounts so we don't create -/// stray `/var/lib/hyperhive/agents//` directories for users that -/// aren't agents. -/// -/// `password` picks the account password. `None` keeps the existing -/// random-throwaway shape (caller doesn't need web UI access — token -/// alone is enough). `Some(pw)` sets `pw` as the password, including -/// running `forgejo admin user change-password` if the account already -/// exists, so the operator can log into the forge web UI afterwards. -/// Idempotent: re-running with the same `Some(pw)` lands on the same -/// final state. -pub async fn provision_user_token(name: &str, password: Option<&str>) -> Result { - if !is_present().await { - anyhow::bail!( - "hive-forge container not running — wait for hive-c0re to start it before provisioning forge users" - ); - } - ensure_user_exists(name, false, password).await?; - if let Some(pw) = password { - // `user create` silently no-ops on an existing account, so - // we run change-password unconditionally when the caller - // asked for a specific password — keeps the verb idempotent - // for "set or reset" use. - change_user_password(name, pw).await?; - } - ensure_user_email(name).await; - mint_token(name, TOKEN_SCOPES).await -} - -/// Set `core`'s Forgejo avatar to the hyperhive logo once, then -/// remember it so subsequent startups don't re-upload. Best-effort -/// — any non-2xx is logged at the caller; the project runs fine -/// with the default hash identicon. -async fn ensure_core_avatar(token: &str) -> Result<()> { - let marker = crate::paths::forge_core_avatar_marker(); - if marker.exists() { - return Ok(()); - } - let png_path = hive_sh4re::assets::core_avatar_png(); - let png_bytes = tokio::fs::read(&png_path) - .await - .with_context(|| format!("read core avatar PNG from {}", png_path.display()))?; - let body = format!( - r#"{{"image":"{}"}}"#, - base64::engine::general_purpose::STANDARD.encode(&png_bytes), - ); - let url = format!("{FORGE_HTTP}/api/v1/admin/users/core/avatar"); - let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; - if !status.is_success() { - anyhow::bail!("set core avatar: HTTP {status}"); - } - if let Some(parent) = marker.parent() { - std::fs::create_dir_all(parent).ok(); - } - std::fs::write(marker, "").ok(); - tracing::info!("forge: set core user avatar to hyperhive logo"); - Ok(()) -} - -/// Set the `agent-configs` org's Forgejo avatar to the -/// configs-stack glyph once. Sibling to `ensure_core_avatar`: -/// one-shot, marker-guarded, best-effort. Forgejo's per-org avatar -/// endpoint is `POST /api/v1/orgs/{org}/avatar` with a base64-PNG -/// JSON body — same shape as the admin user endpoint above. -async fn ensure_config_org_avatar(token: &str) -> Result<()> { - let marker = crate::paths::forge_config_org_avatar_marker(); - if marker.exists() { - return Ok(()); - } - let png_path = hive_sh4re::assets::config_org_avatar_png(); - let png_bytes = tokio::fs::read(&png_path) - .await - .with_context(|| format!("read {CONFIG_ORG} avatar PNG from {}", png_path.display()))?; - let body = format!( - r#"{{"image":"{}"}}"#, - base64::engine::general_purpose::STANDARD.encode(&png_bytes), - ); - let url = format!("{FORGE_HTTP}/api/v1/orgs/{CONFIG_ORG}/avatar"); - let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; - if !status.is_success() { - anyhow::bail!("set {CONFIG_ORG} avatar: HTTP {status}"); - } - if let Some(parent) = marker.parent() { - std::fs::create_dir_all(parent).ok(); - } - std::fs::write(marker, "").ok(); - tracing::info!( - org = CONFIG_ORG, - "forge: set org avatar to configs-stack logo" - ); - Ok(()) -} - -/// Outcome of probing whether the persisted core token still works -/// against the *current* forge. Existence on disk is not validity: a -/// token minted before a forge rebuild / re-provision is unknown to the -/// new forge's DB and 401s on every call — which silently breaks the -/// hive-ci runner-registration prefetch (it reads this same token to -/// fetch a runner registration token). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CoreTokenCheck { - /// Token authenticated successfully — keep using it. - Valid, - /// Forge explicitly rejected the token (401/403) — re-mint. - Invalid, - /// Couldn't determine (forge unreachable / 5xx). Don't re-mint on a - /// transient: keep the existing token and let a later ensure pass - /// re-check once the forge is responsive. Re-minting here would both - /// fail (mint needs the forge too) and churn tokens needlessly. - Indeterminate, -} - -/// Map the HTTP status of the token-probe call to a [`CoreTokenCheck`]. -/// Pure so the decision logic is unit-testable without a live forge. -fn classify_core_token_status(status: StatusCode) -> CoreTokenCheck { - if status.is_success() { - CoreTokenCheck::Valid - } else if status == StatusCode::UNAUTHORIZED || status == StatusCode::FORBIDDEN { - CoreTokenCheck::Invalid - } else { - CoreTokenCheck::Indeterminate - } -} - -/// Probe whether `token` is still accepted by the current forge with a -/// cheap authenticated `GET /api/v1/user` (covered by the core token's -/// `read:user` scope). See [`CoreTokenCheck`] for how the outcome is -/// interpreted. -async fn check_core_token(token: &str) -> CoreTokenCheck { - let url = format!("{FORGE_HTTP}/api/v1/user"); - match forge_http(reqwest::Method::GET, &url, token, "").await { - Ok((status, _)) => classify_core_token_status(status), - Err(e) => { - tracing::debug!( - error = %e, - "forge: core-token probe could not reach forge; treating as indeterminate" - ); - CoreTokenCheck::Indeterminate - } - } -} - -/// Ensure the bootstrap `core` admin user + a token at -/// `CORE_TOKEN_PATH`. The token is what hive-c0re uses for forgejo -/// API calls (org creation, meta-repo push, and the hive-ci -/// runner-registration prefetch). Returns the token. -/// -/// Idempotent, but validity-aware: when a token file is already present -/// it is **probed against the current forge** before being trusted. A -/// token persisted before a forge rebuild / re-provision is stale (the -/// new forge DB doesn't know it) and would 401 every caller — so on a -/// definitive rejection the token is re-minted. A merely-unreachable -/// forge leaves the existing token in place (a later ensure pass -/// re-checks) rather than churning tokens on a transient. -async fn ensure_core_user_and_token() -> Result { - let path = std::path::Path::new(CORE_TOKEN_PATH); - if let Ok(existing) = std::fs::read_to_string(path) { - let trimmed = existing.trim().to_owned(); - if !trimmed.is_empty() { - match check_core_token(&trimmed).await { - CoreTokenCheck::Valid | CoreTokenCheck::Indeterminate => return Ok(trimmed), - CoreTokenCheck::Invalid => { - tracing::warn!( - path = %path.display(), - "forge: persisted core token rejected by forge (stale after rebuild?); \ - re-minting" - ); - } - } - } - } - ensure_user_exists("core", true, None).await?; - mint_and_persist_core_token(path).await?; - let raw = std::fs::read_to_string(path) - .with_context(|| format!("read {CORE_TOKEN_PATH} after mint"))?; - Ok(raw.trim().to_owned()) -} - -/// JSON body for a private, empty repo defaulting to `main`. -fn repo_body(name: &str) -> String { - format!(r#"{{"name":"{name}","auto_init":false,"private":true,"default_branch":"main"}}"#) -} - -/// JSON body for a public, empty repo defaulting to `main`. -fn repo_body_public(name: &str) -> String { - format!(r#"{{"name":"{name}","auto_init":false,"private":false,"default_branch":"main"}}"#) -} - -/// Set an existing repo to public visibility. No-op if the repo is -/// already public. Used for `internal/knowledge` which may have been -/// created as private on an older deployment. -async fn set_repo_public(owner: &str, repo: &str, token: &str) -> Result<()> { - let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}"); - let (status, _) = - forge_http(reqwest::Method::PATCH, &url, token, r#"{"private":false}"#).await?; - match status.as_u16() { - 200 => { - tracing::debug!(%owner, %repo, "forge: repo set to public"); - Ok(()) - } - other => anyhow::bail!("PATCH {owner}/{repo} (set public) returned HTTP {other}"), - } -} - -/// Create `name` inside org `org` as a public repo. Idempotent. -async fn ensure_org_repo_public(org: &str, name: &str, token: &str) -> Result<()> { - create_repo( - &format!("{FORGE_HTTP}/api/v1/orgs/{org}/repos"), - &repo_body_public(name), - token, - &format!("{org}/{name}"), - ) - .await -} - -/// POST a repo-creation request to `url` and fold "already exists" -/// (HTTP 409 / 422) into success. `label` is `/` — purely -/// for log + error context. -async fn create_repo(url: &str, body: &str, token: &str, label: &str) -> Result<()> { - let (status, _) = forge_http(reqwest::Method::POST, url, token, body).await?; - match status.as_u16() { - 201 => { - tracing::info!(%label, "forge: created repo"); - Ok(()) - } - 409 | 422 => { - tracing::debug!(%label, "forge: repo already exists"); - Ok(()) - } - other => anyhow::bail!("POST {url} ({label}) returned HTTP {other}"), - } -} - -/// Create a repo in the token-owner's own namespace. `token` belongs -/// to the user we want the repo owned by (we use `core`'s token for -/// `core/meta`). Idempotent. -pub async fn ensure_repo(name: &str, token: &str) -> Result<()> { - create_repo( - &format!("{FORGE_HTTP}/api/v1/user/repos"), - &repo_body(name), - token, - &format!("core/{name}"), - ) - .await -} - -/// Create `name` inside org `org` (used for `agent-configs/`). -/// Idempotent. -async fn ensure_org_repo(org: &str, name: &str, token: &str) -> Result<()> { - create_repo( - &format!("{FORGE_HTTP}/api/v1/orgs/{org}/repos"), - &repo_body(name), - token, - &format!("{org}/{name}"), - ) - .await -} - -/// Read the persisted core token, or None when the forge isn't -/// seeded yet. Cheap — just a file read. -pub fn core_token() -> Option { - std::fs::read_to_string(CORE_TOKEN_PATH) - .ok() - .map(|s| s.trim().to_owned()) - .filter(|s| !s.is_empty()) -} - -/// Push `dir` (the meta repo) to `core/meta` on the local forge. -/// Best-effort: returns Err which callers log + ignore. No-op when -/// the core token isn't present yet (forge container not provisioned). -pub async fn push_meta(dir: &Path) -> Result<()> { - let Some(token) = core_token() else { - return Ok(()); - }; - // Token-in-URL push. Forgejo accepts `oauth2:` or just - // any-username:; using `core` matches the owner so the - // remote name is self-describing. - let url = format!("http://core:{token}@localhost:3000/core/meta.git"); - let out = Command::new("git") - .current_dir(dir) - .args(["push", "--force", &url, "HEAD:main"]) - .output() - .await - .context("invoke git push core/meta")?; - if !out.status.success() { - anyhow::bail!( - "git push core/meta failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!("forge: pushed meta to core/meta"); - Ok(()) -} - -/// Ensure the `agent-configs/` repo exists so the first -/// `push_config` doesn't 404, and wire it as the agent-editable PR surface: -/// the agent is a **write** collaborator (can push feature branches + -/// open config PRs) and `main` is branch-protected core-only (only hive-c0re's -/// merge handler lands on it; operator approval required). No-op when the forge -/// isn't running or the core token isn't minted yet. Safe to call on every -/// spawn and on every startup (all steps idempotent). -pub async fn ensure_config_repo(name: &str) -> Result<()> { - if !is_present().await { - return Ok(()); - } - let Some(token) = core_token() else { - return Ok(()); - }; - ensure_org_repo(CONFIG_ORG, name, &token).await?; - // Agent = write collaborator: it can push config-PR branches + open PRs, - // but the branch protection below keeps it off `main` directly. - add_collaborator(CONFIG_ORG, name, name, "write", &token).await?; - // Protect `main` core-only, fast-forward-only (no auto force-push). - apply_config_repo_branch_protection(name, &token).await -} - -/// Ensure the `internal/docs` repo exists. Called once at startup -/// after `ensure_org(SHARED_ORG)`. Idempotent — `ensure_org_repo` -/// treats 409 as success. -pub async fn ensure_shared_docs_repo(core_token: &str) -> Result<()> { - ensure_org_repo(SHARED_ORG, SHARED_DOCS_REPO, core_token).await -} - -/// Grant agent `name` read-only collaborator access to `internal/docs`. -/// Idempotent: HTTP 204 (already a collaborator) is treated as success. -/// Mirrors `meta_read_access` so agents can clone the shared docs repo -/// without authentication hassle. -pub async fn shared_docs_access(name: &str, core_token: &str) -> Result<()> { - let url = - format!("{FORGE_HTTP}/api/v1/repos/{SHARED_ORG}/{SHARED_DOCS_REPO}/collaborators/{name}"); - let body = r#"{"permission":"read"}"#; - let out = Command::new("curl") - .args([ - "-sS", - "-o", - "/dev/null", - "-w", - "%{http_code}", - "-X", - "PUT", - "-H", - "Content-Type: application/json", - "-H", - &format!("Authorization: token {core_token}"), - "-d", - body, - &url, - ]) - .output() - .await - .context("invoke curl PUT internal/docs/collaborators")?; - let code = String::from_utf8_lossy(&out.stdout).trim().to_owned(); - match code.as_str() { - "204" => { - tracing::info!(%name, "forge: granted shared-docs read access"); - Ok(()) - } - other => anyhow::bail!( - "PUT {SHARED_ORG}/{SHARED_DOCS_REPO}/collaborators/{name} returned HTTP {other}" - ), - } -} - -/// Ensure the `internal/knowledge` repo exists and is public. -/// Called once at startup after `ensure_org(SHARED_ORG)`. Idempotent. -/// -/// The repo is created as public so any agent with a forge account can -/// fork it and open PRs to contribute. Existing deployments that ended -/// up with a private repo are patched to public on the next hive-c0re -/// startup via `set_repo_public`. -pub async fn ensure_knowledge_repo(core_token: &str) -> Result<()> { - ensure_org_repo_public(SHARED_ORG, KNOWLEDGE_REPO, core_token).await?; - // Ensure public even if the repo already existed as private (older deployment). - set_repo_public(SHARED_ORG, KNOWLEDGE_REPO, core_token).await -} - -/// Grant agent `name` read-only collaborator access to `core/meta` on -/// the forge so the agent can clone/fetch the meta flake. Idempotent: -/// HTTP 204 (already a collaborator) is treated as success. -pub async fn meta_read_access(name: &str, core_token: &str) -> Result<()> { - let url = format!("{FORGE_HTTP}/api/v1/repos/core/meta/collaborators/{name}"); - let body = r#"{"permission":"read"}"#; - let out = Command::new("curl") - .args([ - "-sS", - "-o", - "/dev/null", - "-w", - "%{http_code}", - "-X", - "PUT", - "-H", - "Content-Type: application/json", - "-H", - &format!("Authorization: token {core_token}"), - "-d", - body, - &url, - ]) - .output() - .await - .context("invoke curl PUT core/meta/collaborators")?; - let code = String::from_utf8_lossy(&out.stdout).trim().to_owned(); - match code.as_str() { - "204" => { - tracing::info!(%name, "forge: granted meta read access"); - Ok(()) - } - other => anyhow::bail!("PUT core/meta/collaborators/{name} returned HTTP {other}"), - } -} - -/// Add `http://localhost:3000/core/meta.git` as the `meta` remote in -/// the agent's proposed config repo so the agent (and the manager) can -/// fetch the meta flake from the forge. Idempotent: no-op when the -/// remote already points at the right URL, or when the proposed repo -/// does not exist yet. No-op when the forge is not running. -pub async fn ensure_meta_remote(name: &str) -> Result<()> { - if !is_present().await { - return Ok(()); - } - let proposed_dir = Coordinator::agent_proposed_dir(name); - if !proposed_dir.join(".git").exists() { - return Ok(()); - } - let want = format!("{FORGE_HTTP}/core/meta.git"); - let existing = crate::lifecycle::git_command() - .current_dir(&proposed_dir) - .args(["remote", "get-url", "meta"]) - .output() - .await - .context("git remote get-url meta")?; - if existing.status.success() { - let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned(); - if current == want { - return Ok(()); - } - crate::lifecycle::git(&proposed_dir, &["remote", "set-url", "meta", &want]).await - } else { - crate::lifecycle::git(&proposed_dir, &["remote", "add", "meta", &want]).await - } -} - -/// Mirror agent `name`'s applied config repo — `main` plus every tag -/// (`proposal` / `approved` / `building` / `deployed` / `failed` / -/// `denied`) — to `agent-configs/` on the local forge. -/// Best-effort: returns Err which callers log + ignore. No-op when the -/// forge isn't seeded or the applied repo doesn't exist yet. -/// -/// Call this after every hive-c0re mutation of an applied repo's refs -/// so the forge copy always reflects what core actually did. -/// -/// Never force-pushes. The status tags are id-suffixed -/// (`proposal/`, `deployed/`, …) and therefore add-only, and -/// `main` is published history — after a failed deploy rolls the LOCAL -/// applied `main` back to last-good, the forge `main` may legitimately -/// be ahead (e.g. an operator-merged config PR whose rebuild failed). -/// Rewinding it would erase that merged commit from the forge, which -/// is exactly the incident this guards against: the local repo tracks -/// "what last built", the forge tracks "what was approved", and the -/// `failed/` tag records the divergence. A non-fast-forward -/// rejection of `main` is therefore expected + logged at info; the -/// tags in the same push still land (git pushes refspecs -/// independently). Any other failure is a real error. -/// -/// The tokenised URL is passed straight to `git push` and deliberately -/// never stored as a named remote: the applied repo is bind-mounted -/// READ-ONLY into the manager container (`/applied`), so a token in -/// `.git/config` would leak core's admin credential to an agent. -pub async fn push_config(name: &str) -> Result<()> { - let Some(token) = core_token() else { - return Ok(()); - }; - let dir = Coordinator::agent_applied_dir(name); - if !dir.join(".git").exists() { - return Ok(()); - } - let url = format!("http://core:{token}@localhost:3000/{CONFIG_ORG}/{name}.git"); - let out = crate::lifecycle::git_command() - .current_dir(&dir) - .args([ - "push", - &url, - "refs/heads/main:refs/heads/main", - "refs/tags/*:refs/tags/*", - ]) - .output() - .await - .context("invoke git push agent-configs")?; - if !out.status.success() { - let stderr = String::from_utf8_lossy(&out.stderr); - if stderr.contains("non-fast-forward") { - tracing::info!( - %name, - "forge: mirror push of main rejected (non-fast-forward) — forge main is \ - ahead of local applied main (rolled-back deploy); leaving forge history intact" - ); - return Ok(()); - } - anyhow::bail!( - "git push {CONFIG_ORG}/{name} failed ({}): {}", - out.status, - stderr.trim() - ); - } - tracing::info!(%name, "forge: mirrored applied config to agent-configs"); - Ok(()) -} - -/// POST `/api/v1/orgs` to create an org named `name`. Idempotent: -/// HTTP 422 ("user already exists") is treated as success. -async fn ensure_org(name: &str, admin_token: &str) -> Result<()> { - let body = format!(r#"{{"username":"{name}"}}"#); - let url = format!("{FORGE_HTTP}/api/v1/orgs"); - let (status, _) = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?; - match status.as_u16() { - 201 => { - tracing::info!(%name, "forge: created org"); - Ok(()) - } - 422 | 409 => { - tracing::debug!(%name, "forge: org already exists"); - Ok(()) - } - other => anyhow::bail!("POST /api/v1/orgs name={name} returned HTTP {other}"), - } -} - -/// One operator-declared pull-mirror, forwarded from the nix -/// `services.hyperhive.forge.mirrors` option as JSON in -/// `HYPERHIVE_FORGE_MIRRORS`. -#[derive(serde::Deserialize)] -struct Mirror { - /// Upstream clone URL to mirror from (e.g. `https://github.com/actions/checkout`). - upstream: String, - /// Local `/` the mirror is created at. - dest: String, -} - -/// Ensure each `HYPERHIVE_FORGE_MIRRORS` entry exists as a real Forgejo -/// pull-mirror. The env carries the JSON-encoded nix `forge.mirrors` list -/// (plus the CI-auto `actions/checkout` entry). Absent/empty env = no-op. -/// Per-mirror failures warn and continue — never abort the startup sweep. -async fn ensure_mirrors(admin_token: &str) { - let raw = match std::env::var("HYPERHIVE_FORGE_MIRRORS") { - Ok(s) if !s.trim().is_empty() => s, - _ => return, - }; - let mirrors: Vec = match serde_json::from_str(&raw) { - Ok(m) => m, - Err(e) => { - tracing::warn!(error = ?e, "forge: HYPERHIVE_FORGE_MIRRORS is not valid JSON; skipping mirror seed"); - return; - } - }; - for m in mirrors { - let Some((owner, repo)) = m.dest.split_once('/') else { - tracing::warn!(dest = %m.dest, "forge: mirror dest is not /; skipping"); - continue; - }; - // Create the dest org first (idempotent); the mirror can't land - // without its owner existing. - if let Err(e) = ensure_org(owner, admin_token).await { - tracing::warn!(%owner, error = ?e, "forge: ensure_org for mirror failed"); - continue; - } - if let Err(e) = ensure_mirror_repo(&m.upstream, owner, repo, admin_token).await { - tracing::warn!(dest = %m.dest, error = ?e, "forge: ensure_mirror_repo failed"); - } - } -} - -/// Periodic sync interval for pull-mirrors. Forgejo syncs mirrors -/// on-access by default, which re-introduces external DNS latency on -/// every `git clone` (the hive-ci runner shares the host netns and is -/// therefore affected by host resolver blips). A fixed periodic interval -/// isolates CI from transient DNS failures — a stale mirror is -/// acceptable; a broken clone because of a momentary DNS blip is not. -const MIRROR_INTERVAL: &str = "8h0m0s"; - -/// Create `owner/repo` as a pull-mirror of `upstream` via the migrate API. -/// Idempotent: if the repo already exists this function patches its -/// `mirror_interval` to ensure it matches (covers mirrors that were -/// created before the interval was introduced). A 409 on the migrate -/// POST (a race between the GET check and the POST) is also success. -async fn ensure_mirror_repo( - upstream: &str, - owner: &str, - repo: &str, - admin_token: &str, -) -> Result<()> { - let repo_url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}"); - let (status, _) = forge_http(reqwest::Method::GET, &repo_url, admin_token, "").await?; - if status.is_success() { - // Mirror already present. Patch interval so mirrors seeded before - // this field was introduced (or with a different value) converge. - let patch_body = serde_json::json!({ "mirror_interval": MIRROR_INTERVAL }).to_string(); - let (patch_status, patch_text) = - forge_http(reqwest::Method::PATCH, &repo_url, admin_token, &patch_body).await?; - if patch_status.is_success() { - tracing::debug!(%owner, %repo, interval = MIRROR_INTERVAL, "forge: pull-mirror interval updated"); - } else { - tracing::warn!( - %owner, %repo, status = %patch_status, body = %patch_text, - "forge: failed to set mirror_interval on existing pull-mirror" - ); - } - return Ok(()); - } - // serde_json::json! → the upstream URL is escaped safely (no string - // interpolation into the JSON body). - let body = serde_json::json!({ - "clone_addr": upstream, - "repo_owner": owner, - "repo_name": repo, - "mirror": true, - // Periodic refresh instead of on-access sync — keeps CI isolated - // from external DNS failures at clone time. - "interval": MIRROR_INTERVAL, - "service": "git", - "private": false, - }) - .to_string(); - let url = format!("{FORGE_HTTP}/api/v1/repos/migrate"); - let (status, text) = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?; - match status.as_u16() { - 201 => { - tracing::info!(%owner, %repo, %upstream, interval = MIRROR_INTERVAL, "forge: created pull-mirror"); - Ok(()) - } - // 409 = a race created it between our GET check and here (the GET - // is the real idempotency guard). NOT 422: for the migrate endpoint - // 422 is a validation error (bad clone_addr / service), so it must - // surface via the bail arm, not be swallowed as "already exists". - 409 => { - tracing::debug!(%owner, %repo, "forge: pull-mirror already exists (race)"); - Ok(()) - } - other => { - anyhow::bail!("POST /api/v1/repos/migrate {owner}/{repo} returned HTTP {other}: {text}") - } - } -} - -/// Whether `ns` is a hive-managed Forgejo namespace that agent-initiated -/// repo creation must never target — `internal` (operator-curated -/// shared content) + `agent-configs` / `core` (hive-c0re-internal). The -/// create path forces [`AGENTS_ORG`], so this guards a future surface that -/// might accept an explicit owner. -#[must_use] -pub fn is_hive_managed_namespace(ns: &str) -> bool { - HIVE_MANAGED_NAMESPACES.contains(&ns) -} - -/// Provision the [`OPERATORS_TEAM`] inside `org` as an **empty** team. -/// Branch protection on that org's repos references it as the -/// merge/approval whitelist; the operator adds herself as a member via the -/// forge UI / hivectl. `includes_all_repositories` so the gate applies to -/// every repo in the org; `write` is enough to approve + merge. hive-c0re -/// never manages membership. Idempotent (422/409 = already exists). -/// -/// Must run for BOTH [`AGENTS_ORG`] and [`CONFIG_ORG`]: Gitea teams are -/// org-scoped, so a config-repo branch-protection rule referencing -/// `operators` needs the team to exist in `agent-configs` too. Missing it -/// there 422'd every `apply_config_repo_branch_protection`, leaving config -/// repos unprotected — operator-merged config PRs then bypassed the deploy -/// pipeline and silently didn't apply. -async fn ensure_operators_team(org: &str, token: &str) -> Result<()> { - let url = format!("{FORGE_HTTP}/api/v1/orgs/{org}/teams"); - let body = format!( - r#"{{"name":"{OPERATORS_TEAM}","description":"hyperhive operators — merge gate for agent repos","permission":"write","includes_all_repositories":true,"can_create_org_repo":false}}"# - ); - let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; - match status.as_u16() { - 201 => { - tracing::info!(%org, "forge: created {OPERATORS_TEAM} team"); - Ok(()) - } - 409 | 422 => { - tracing::debug!(%org, "forge: {OPERATORS_TEAM} team already exists"); - Ok(()) - } - other => { - anyhow::bail!("POST /orgs/{org}/teams ({OPERATORS_TEAM}) returned HTTP {other}") - } - } -} - -/// Add `user` as a collaborator on `owner/repo` at `permission` -/// (`read` / `write` / `admin`). Idempotent: 201 (added) and 204 (already a -/// collaborator / permission updated) both count as success. -async fn add_collaborator( - owner: &str, - repo: &str, - user: &str, - permission: &str, - token: &str, -) -> Result<()> { - let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}/collaborators/{user}"); - let body = format!(r#"{{"permission":"{permission}"}}"#); - let (status, _) = forge_http(reqwest::Method::PUT, &url, token, &body).await?; - match status.as_u16() { - 201 | 204 => { - tracing::debug!(%owner, %repo, %user, %permission, "forge: collaborator set"); - Ok(()) - } - other => { - anyhow::bail!("PUT {owner}/{repo}/collaborators/{user} returned HTTP {other}") - } - } -} - -/// Apply the operator merge-gate branch protection to `repo`'s default -/// branch: only [`OPERATORS_TEAM`] members can merge, and an -/// approving review from that team is required — so the author (a write-level -/// agent, not in the team) cannot merge its own PR. Idempotent: an existing -/// rule for the branch (200/409/422) is treated as success. -async fn apply_operator_branch_protection(repo: &str, token: &str) -> Result<()> { - let url = format!("{FORGE_HTTP}/api/v1/repos/{AGENTS_ORG}/{repo}/branch_protections"); - let body = format!( - r#"{{"branch_name":"main","enable_merge_whitelist":true,"merge_whitelist_teams":["{OPERATORS_TEAM}"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true}}"# - ); - let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; - match status.as_u16() { - 201 => { - tracing::info!(%repo, "forge: applied operator branch protection"); - Ok(()) - } - 200 | 409 | 422 => { - tracing::debug!(%repo, "forge: branch protection already present"); - Ok(()) - } - other => anyhow::bail!("POST {AGENTS_ORG}/{repo}/branch_protections returned HTTP {other}"), - } -} - -/// Apply branch protection to an `agent-configs/` repo's `main` so it -/// can serve as the agent-editable, PR-merge config surface: -/// - **push + merge whitelists are `core`-only** — the agent (a write -/// collaborator) can push feature branches and open config PRs, but only -/// hive-c0re lands on `main`, via its verify-and-ff-push merge handler -/// (`run_merge_config_pr`). The agent can never push `main` directly. -/// - **operator-team approval is required** to merge, and the author (not in -/// the team) cannot self-approve. -/// - **`enable_force_push` is `false`** — `main` only ever advances by -/// fast-forward. The merge handler's `ff_push_to_main` is already a -/// non-force push, so it lands fine. The legacy `push_config` mirror DOES -/// force-push (it re-points status tags and rewinds `main` on a failed-build -/// rollback), so the protection now rejects those non-ff updates — that -/// mirror runs best-effort until the agent-opened PR-merge flow retires it. -/// (Auto force-push is intentionally not allowed: per operator directive a -/// silent force-push is a bug, not a feature.) -/// -/// Idempotent: an existing rule for the branch (200/409/422) is success. -async fn apply_config_repo_branch_protection(repo: &str, token: &str) -> Result<()> { - let url = format!("{FORGE_HTTP}/api/v1/repos/{CONFIG_ORG}/{repo}/branch_protections"); - let body = format!( - r#"{{"branch_name":"main","enable_push_whitelist":true,"push_whitelist_usernames":["core"],"enable_merge_whitelist":true,"merge_whitelist_usernames":["core"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true,"allow_manual_merge":true,"enable_force_push":false}}"# - ); - let (status, resp_body) = forge_http(reqwest::Method::POST, &url, token, &body).await?; - if status.as_u16() == 201 { - tracing::info!(%repo, "forge: applied config-repo branch protection"); - return Ok(()); - } - // Non-201 is ambiguous: it can mean "rule already exists" (idempotent - // success) OR a silent rejection — e.g. a 422 where Forgejo refused - // the request and created NO rule. The old code treated 200/409/422 - // all as success, so a rejected POST left the repo unprotected with - // no error (the reported case: a new agent's config repo had no - // `main` rule and nothing was logged). Don't trust the status code: - // verify the `main` rule actually exists, and on failure surface the - // POST's response body so the real reason is in the journal. - let main_url = format!("{url}/main"); - let (check, _) = forge_http(reqwest::Method::GET, &main_url, token, "").await?; - if check.as_u16() == 200 { - tracing::debug!(%repo, %status, "forge: config-repo branch protection already present"); - Ok(()) - } else { - anyhow::bail!( - "branch protection for {CONFIG_ORG}/{repo} not applied: POST -> HTTP {status} \ - (body: {body}); GET main -> HTTP {check}, no `main` rule present", - body = resp_body.trim(), - ) - } -} - -/// Create a repo for `agent` in the c0re-owned [`AGENTS_ORG`] and wire the -/// perms: the org owns it (perms stay c0re-managed), the agent is added -/// as a **write** collaborator (not owner — can push + open PRs but can't -/// bypass branch protection), and the default branch gets the operator -/// merge gate. This is the sanctioned create path now that agents can't -/// create repos directly (`max_repo_creation = 0`). Idempotent. -pub async fn create_agent_repo(agent: &str, repo: &str, core_token: &str) -> Result { - ensure_org_repo(AGENTS_ORG, repo, core_token).await?; - add_collaborator(AGENTS_ORG, repo, agent, "write", core_token).await?; - apply_operator_branch_protection(repo, core_token).await?; - tracing::info!(%agent, %repo, "forge: created agent repo in {AGENTS_ORG} with operator merge gate"); - Ok(format!("{AGENTS_ORG}/{repo}")) -} - -/// Per-agent forge sync: ensure the agent has a forgejo user + token, -/// a mirrored config repo, read access to `core/meta`, and the `meta` -/// remote in its proposed repo. All operations are idempotent; failures -/// are logged as warnings but don't abort the caller. -/// -/// `core_token` is `core_token()` — passed in so callers that already -/// fetched it don't re-read the file. Pass `None` to skip the -/// `meta_read_access` step (safe: the access grant is best-effort). -/// -/// Called by both `ensure_all()` (startup sweep) and `rebuild_agent` -/// (per-rebuild) so the two paths stay equivalent. -pub async fn sync_agent(name: &str, core_token: Option<&str>) { - if let Err(e) = ensure_user_for(name).await { - tracing::warn!(%name, error = ?e, "forge: ensure_user failed"); - } - // Align email to match the git user.email set by meta::render_flake - // so commits link to the agent's Forgejo profile. Best-effort; - // also patches up agents created before this fix (old @hive.local). - ensure_user_email(name).await; - // Block direct agent-initiated repo creation: agents create - // repos through hive-c0re, never with their own token. Idempotent + - // marker-guarded; also covers agents provisioned before this landed. - ensure_repo_creation_disabled(name).await; - // Mirror the agent's applied config repo into agent-configs. - // ensure_config_repo is idempotent; push_config catches any - // drift since the last run — e.g. the startup migration just - // relocated `deployed/0`, or a deploy landed while the forge - // was down. - if let Err(e) = ensure_config_repo(name).await { - tracing::warn!(%name, error = ?e, "forge: ensure_config_repo failed"); - } - if let Err(e) = push_config(name).await { - tracing::warn!(%name, error = ?e, "forge: push_config failed"); - } - // Grant read-only access to core/meta and wire the `meta` remote - // into the proposed repo so agents can fetch their deployment context. - if let Some(token) = core_token - && let Err(e) = meta_read_access(name, token).await - { - tracing::warn!(%name, error = ?e, "forge: ensure_meta_read_access failed"); - } - if let Err(e) = ensure_meta_remote(name).await { - tracing::warn!(%name, error = ?e, "forge: ensure_meta_remote failed"); - } - // Grant read-only access to internal/docs so the agent can clone - // the operator-curated shared skills/runbook repo. Best-effort. - if let Some(token) = core_token - && let Err(e) = shared_docs_access(name, token).await - { - tracing::warn!(%name, error = ?e, "forge: shared_docs_access failed"); - } - // internal/knowledge is public — no per-agent collaborator grant needed. -} - -/// Sweep every existing container (manager + sub-agents) and ensure -/// each has a forgejo user + token, plus an `agent-configs/` -/// repo mirroring its applied config. Also seeds the `core` admin -/// user (hive-c0re's own identity for pushing the meta repo + driving -/// the API), the `agent-configs` org, and the `core/meta` repo. -/// Called once at hive-c0re startup. Per-step failures are logged -/// but don't abort the sweep. -pub async fn ensure_all() { - if !is_present().await { - tracing::debug!("forge: hive-forge container absent, skipping user sweep"); - return; - } - let core_token = match ensure_core_user_and_token().await { - Ok(t) => Some(t), - Err(e) => { - tracing::warn!(error = ?e, "forge: ensure_core_user_and_token failed"); - None - } - }; - if let Some(token) = core_token.as_deref() { - for org in SEEDED_ORGS { - if let Err(e) = ensure_org(org, token).await { - tracing::warn!(%org, error = ?e, "forge: ensure_org failed"); - } - } - // Seed the operator-declared pull-mirrors (nix `forge.mirrors` + - // the CI-auto `actions/checkout`, forwarded via the - // `HYPERHIVE_FORGE_MIRRORS` env). Each ensures its own dest org, so - // this is independent of the SEEDED_ORGS loop above. - ensure_mirrors(token).await; - // Provision the operator merge-gate team (empty) inside BOTH the - // agents org and the agent-configs org so branch protection in each - // can reference it before anyone joins. Gitea teams are org-scoped — - // missing the agent-configs copy 422'd every config-repo protection - // apply, leaving those repos unprotected and letting operator-merged - // config PRs bypass the deploy pipeline. The operator adds herself as - // a member out-of-band. - for org in [AGENTS_ORG, CONFIG_ORG] { - if let Err(e) = ensure_operators_team(org, token).await { - tracing::warn!(%org, error = ?e, "forge: ensure_operators_team failed"); - } - } - // Meta repo lives at core/meta — pushed from git_commit in - // meta.rs on every deploy/lock-update. Make sure it exists - // before the first push hits a 404. - if let Err(e) = ensure_repo("meta", token).await { - tracing::warn!(error = ?e, "forge: ensure_repo core/meta failed"); - } - // Seed the shared docs repo. internal is already in - // SEEDED_ORGS above so the org exists; ensure the repo itself. - if let Err(e) = ensure_shared_docs_repo(token).await { - tracing::warn!(error = ?e, "forge: ensure_shared_docs_repo failed"); - } - // Seed the hive-wide knowledge repo. - if let Err(e) = ensure_knowledge_repo(token).await { - tracing::warn!(error = ?e, "forge: ensure_knowledge_repo failed"); - } - // Clone knowledge repo locally so it can be bind-mounted into agents. - if let Err(e) = crate::knowledge::ensure_local_clone(token).await { - tracing::warn!(error = ?e, "knowledge: ensure_local_clone failed"); - } - if let Err(e) = ensure_core_avatar(token).await { - tracing::warn!(error = ?e, "forge: ensure_core_avatar failed"); - } - if let Err(e) = ensure_config_org_avatar(token).await { - tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed"); - } - } - let Ok(containers) = crate::lifecycle::list().await else { - tracing::warn!("forge: nixos-container list failed; skipping user sweep"); - return; - }; - for c in containers { - let Some(name) = c.strip_prefix(crate::lifecycle::AGENT_PREFIX) else { - continue; - }; - sync_agent(name, core_token.as_deref()).await; - } -} - -// --------------------------------------------------------------------------- -// PR-based config-flow merge primitives (part of the -// dashboard-approve-driven config-change flow). -// -// The dashboard-approve-driven flow has hive-c0re verify an operator-approved -// config PR, then land it: ff-push the verified sha to the protected default -// branch (= the merge) and mark the PR merged manually. These three fns are -// the forge-side mechanics the c0re approve-handler (`run_merge_config_pr`) -// orchestrates; the orchestration fetches the verified sha into the agent's -// applied repo before calling `ff_push_to_main`. The core token is sourced -// internally (`core_token`), never passed in. `repo` is the agent's editable -// forge config repo in `owner/name` form (e.g. `agent-configs/`). -// --------------------------------------------------------------------------- - -/// Typed failure for the merge primitives so the c0re approve-handler can -/// `match` recoverable drift (refresh the request sha + re-verify) against a -/// hard failure (fail the approval). The two drift variants carry the observed -/// sha so the handler can re-pin to it; `Other` is everything else (transport, -/// API, unexpected) and is not auto-retried. -#[derive(Debug)] -pub enum ForgeMergeError { - /// The PR head moved off `expected` (now at `actual`), seen at - /// mark-merged time. The handler's pre-merge `pr_head_sha` re-read is the - /// primary drift gate; this is the belt-and-suspenders race catch. - HeadDrift { expected: String, actual: String }, - /// `main` (`actual_head`) is not a descendant of `expected_ancestor`, so - /// pushing the verified sha would not be a fast-forward — `main` raced. - NotFastForward { - expected_ancestor: String, - actual_head: String, - }, - /// Transport / API / unexpected failure — hard-fail, no auto-retry. - Other(anyhow::Error), -} - -impl std::fmt::Display for ForgeMergeError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::HeadDrift { expected, actual } => { - write!(f, "PR head drifted: expected {expected}, found {actual}") - } - Self::NotFastForward { - expected_ancestor, - actual_head, - } => write!( - f, - "not a fast-forward: main {actual_head} is not a descendant of {expected_ancestor}" - ), - Self::Other(e) => write!(f, "{e}"), - } - } -} - -impl std::error::Error for ForgeMergeError {} - -impl From for ForgeMergeError { - fn from(e: anyhow::Error) -> Self { - Self::Other(e) - } -} - -/// Agent name from an `owner/name` forge repo string (the trailing segment). -fn repo_agent_name(repo: &str) -> &str { - repo.rsplit('/').next().unwrap_or(repo) -} - -/// Token-in-URL clone/push URL for a forge repo (`owner/name`). Mirrors -/// `push_config`'s pattern; the token is passed straight to git and never -/// stored as a named remote. -fn tokenised_repo_url(repo: &str, token: &str) -> String { - format!("http://core:{token}@localhost:3000/{repo}.git") -} - -/// Resolve a PR's head sha via `git ls-remote refs/pull//head` -/// (Forgejo exposes PR heads there). Pure read, no mutation — the handler's -/// primary drift gate (compare against the approved sha), and `mark_pr_merged` -/// uses it to detect head-drift on failure. -/// -/// # Errors -/// `Other` on transport failure or an empty/missing ref. -pub async fn pr_head_sha(repo: &str, pr: u64) -> Result { - let token = core_token() - .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; - let url = tokenised_repo_url(repo, &token); - let refspec = format!("refs/pull/{pr}/head"); - let out = crate::lifecycle::git_command() - .args(["ls-remote", &url, &refspec]) - .output() - .await - .context("git ls-remote PR head")?; - if !out.status.success() { - return Err(ForgeMergeError::Other(anyhow::anyhow!( - "git ls-remote {repo} {refspec} failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ))); - } - let stdout = String::from_utf8_lossy(&out.stdout); - let sha = stdout - .split_whitespace() - .next() - .filter(|s| !s.is_empty()) - .ok_or_else(|| { - ForgeMergeError::Other(anyhow::anyhow!( - "no head ref for PR #{pr} in {repo} (ls-remote empty)" - )) - })?; - Ok(sha.to_string()) -} - -/// Full `owner/name` path of an agent's config repo on the forge — the -/// `agent-configs` org mirror that the PR-merge flow reads + fast-forwards. -pub fn config_repo(agent: &str) -> String { - format!("{CONFIG_ORG}/{agent}") -} - -/// Fetch PR #`pr`'s head into the agent's applied repo via -/// `refs/pull//head` (which Forgejo always serves — a bare-sha fetch can -/// be refused by uploadpack policy). This makes the reviewed head an object -/// in the applied repo so the ancestor check in [`ff_push_to_main`], the -/// `git_update_ref(main, …)` in the deploy tail, and the eval-verify all -/// resolve it locally before the irreversible push. -/// -/// # Errors -/// `Other` on transport failure or a non-zero git exit. -pub async fn fetch_pr_head_into_applied(repo: &str, pr: u64) -> Result<(), ForgeMergeError> { - let token = core_token() - .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; - let url = tokenised_repo_url(repo, &token); - let applied = Coordinator::agent_applied_dir(repo_agent_name(repo)); - let refspec = format!("refs/pull/{pr}/head"); - let out = crate::lifecycle::git_command() - .current_dir(&applied) - .args(["fetch", "--no-tags", &url, &refspec]) - .output() - .await - .context("git fetch PR head into applied")?; - if !out.status.success() { - return Err(ForgeMergeError::Other(anyhow::anyhow!( - "git fetch {repo} {refspec} into applied failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ))); - } - Ok(()) -} - -/// Fast-forward the forge repo's `main` to `sha` — THE merge in the PR flow. -/// Reads `main`'s current sha (`git ls-remote … refs/heads/main`), verifies it -/// is a strict ancestor of `sha` (`git merge-base --is-ancestor`, run in the -/// agent's applied repo where the orchestration has already fetched `sha`), -/// then does a **non-force** `git push :refs/heads/main`. The ancestor -/// pre-check and the non-force push are two independent guards: either catches -/// a raced `main` (→ `NotFastForward`) rather than clobbering reviewed history. -/// -/// # Errors -/// `NotFastForward` if `main` raced ahead of `sha`; `Other` on transport/other. -pub async fn ff_push_to_main(repo: &str, sha: &str) -> Result<(), ForgeMergeError> { - let token = core_token() - .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; - let url = tokenised_repo_url(repo, &token); - let applied = Coordinator::agent_applied_dir(repo_agent_name(repo)); - - // Current `main` on the forge repo. - let ls = crate::lifecycle::git_command() - .args(["ls-remote", &url, "refs/heads/main"]) - .output() - .await - .context("git ls-remote main")?; - if !ls.status.success() { - return Err(ForgeMergeError::Other(anyhow::anyhow!( - "git ls-remote {repo} refs/heads/main failed ({}): {}", - ls.status, - String::from_utf8_lossy(&ls.stderr).trim() - ))); - } - let ls_out = String::from_utf8_lossy(&ls.stdout); - let main_sha = ls_out.split_whitespace().next().unwrap_or("").to_string(); - - // Strict-ancestor check: `main` must be an ancestor of `sha` for a true - // fast-forward. Skip when `main` is unborn (empty) — the push creates it. - if !main_sha.is_empty() { - let anc = crate::lifecycle::git_command() - .current_dir(&applied) - .args(["merge-base", "--is-ancestor", &main_sha, sha]) - .output() - .await - .context("git merge-base --is-ancestor")?; - match anc.status.code() { - Some(0) => {} // ancestor → fast-forward safe - Some(1) => { - return Err(ForgeMergeError::NotFastForward { - expected_ancestor: main_sha, - actual_head: sha.to_string(), - }); - } - _ => { - return Err(ForgeMergeError::Other(anyhow::anyhow!( - "git merge-base --is-ancestor errored ({}): {}", - anc.status, - String::from_utf8_lossy(&anc.stderr).trim() - ))); - } - } - } - - // Non-force push `sha` → `main`. Without `--force`, git rejects a - // non-fast-forward (a race between the check above and now), surfaced as - // `NotFastForward` rather than clobbering the remote. - let push = crate::lifecycle::git_command() - .current_dir(&applied) - .args(["push", &url, &format!("{sha}:refs/heads/main")]) - .output() - .await - .context("git push sha:main")?; - if !push.status.success() { - let stderr = String::from_utf8_lossy(&push.stderr); - if stderr.contains("non-fast-forward") || stderr.contains("fetch first") { - return Err(ForgeMergeError::NotFastForward { - expected_ancestor: main_sha, - actual_head: sha.to_string(), - }); - } - return Err(ForgeMergeError::Other(anyhow::anyhow!( - "git push {repo} {sha}:main failed ({}): {}", - push.status, - stderr.trim() - ))); - } - Ok(()) -} - -/// Mark PR `pr` as **manually merged** at `sha` (Forgejo -/// `POST …/pulls/{pr}/merge` with `Do=manually-merged`, `MergeCommitID=sha`). -/// `ff_push_to_main` must have already set `main` to `sha` (Forgejo requires -/// the branch already be at the merge commit). On a non-2xx, re-reads the PR -/// head to distinguish drift (`HeadDrift`) from a generic failure (`Other`) — -/// best-effort, since the handler's pre-merge head re-read is the real gate. -/// -/// # Errors -/// `HeadDrift` if the PR head no longer matches `sha`; `Other` otherwise. -pub async fn mark_pr_merged(repo: &str, pr: u64, sha: &str) -> Result<(), ForgeMergeError> { - let token = core_token() - .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; - let url = format!("{FORGE_HTTP}/api/v1/repos/{repo}/pulls/{pr}/merge"); - let body = format!(r#"{{"Do":"manually-merged","MergeCommitID":"{sha}"}}"#); - let (status, _) = forge_http(reqwest::Method::POST, &url, &token, &body) - .await - .context("POST pulls//merge (manually-merged)")?; - if status.is_success() { - return Ok(()); - } - // Best-effort drift detection: if the live head no longer matches `sha`, - // that's a head-drift race; otherwise surface as a hard failure. - match pr_head_sha(repo, pr).await { - Ok(actual) if actual != sha => Err(ForgeMergeError::HeadDrift { - expected: sha.to_string(), - actual, - }), - _ => Err(ForgeMergeError::Other(anyhow::anyhow!( - "mark PR #{pr} in {repo} manually-merged at {sha} failed: HTTP {status}" - ))), - } -} - -#[cfg(test)] -mod tests { - use super::{CoreTokenCheck, classify_core_token_status, repo_agent_name, tokenised_repo_url}; - use reqwest::StatusCode; - - #[test] - fn success_statuses_are_valid() { - assert_eq!( - classify_core_token_status(StatusCode::OK), - CoreTokenCheck::Valid - ); - assert_eq!( - classify_core_token_status(StatusCode::NO_CONTENT), - CoreTokenCheck::Valid - ); - } - - #[test] - fn auth_rejection_statuses_are_invalid() { - // The whole point: a stale token (forge rebuilt out from under it) - // 401s, and 401/403 are the only outcomes that trigger a re-mint. - assert_eq!( - classify_core_token_status(StatusCode::UNAUTHORIZED), - CoreTokenCheck::Invalid - ); - assert_eq!( - classify_core_token_status(StatusCode::FORBIDDEN), - CoreTokenCheck::Invalid - ); - } - - #[test] - fn transient_and_unexpected_statuses_are_indeterminate() { - // Never re-mint on a transient — minting needs the forge too, and - // churning tokens on a blip is worse than keeping the existing one. - for s in [ - StatusCode::INTERNAL_SERVER_ERROR, - StatusCode::BAD_GATEWAY, - StatusCode::SERVICE_UNAVAILABLE, - StatusCode::GATEWAY_TIMEOUT, - StatusCode::NOT_FOUND, - ] { - assert_eq!( - classify_core_token_status(s), - CoreTokenCheck::Indeterminate, - "status {s} should be indeterminate" - ); - } - } - - #[test] - fn repo_agent_name_takes_trailing_segment() { - assert_eq!(repo_agent_name("agent-configs/atlas"), "atlas"); - assert_eq!(repo_agent_name("atlas"), "atlas"); - assert_eq!(repo_agent_name("a/b/c"), "c"); - } - - #[test] - fn tokenised_repo_url_shape() { - assert_eq!( - tokenised_repo_url("agent-configs/iris", "tok"), - "http://core:tok@localhost:3000/agent-configs/iris.git" - ); - } -} diff --git a/hive-c0re/src/forge/mod.rs b/hive-c0re/src/forge/mod.rs new file mode 100644 index 00000000..f5d70daa --- /dev/null +++ b/hive-c0re/src/forge/mod.rs @@ -0,0 +1,280 @@ +//! Optional Forgejo wiring — per-agent user + token provisioning, +//! config-repo mirroring, meta read-access grants. Also seeds +//! `internal/docs` — a private repo every agent gets read-only +//! collaborator access to for operator-curated shared content. +//! No-op when `hive-forge` isn't running. Full design: `docs/forge.md`. + +mod pr_merge; +mod repos; +mod users; + +pub use pr_merge::{ + ForgeMergeError, config_repo, fetch_pr_head_into_applied, ff_push_to_main, mark_pr_merged, + pr_head_sha, +}; +pub use repos::{ + create_agent_repo, ensure_config_repo, ensure_knowledge_repo, ensure_meta_remote, ensure_repo, + ensure_shared_docs_repo, meta_read_access, push_config, push_meta, shared_docs_access, +}; +pub use users::{core_token, ensure_user_for, provision_user_token}; + +use anyhow::{Context, Result}; +use reqwest::StatusCode; + +use repos::{ensure_mirrors, ensure_operators_team, ensure_org}; +use users::{ + ensure_config_org_avatar, ensure_core_avatar, ensure_core_user_and_token, + ensure_repo_creation_disabled, ensure_user_email, +}; + +const FORGE_CONTAINER: &str = "hive-forge"; +pub(crate) const FORGE_HTTP: &str = "http://localhost:3000"; +/// Forgejo org grouping every agent's config repo. Core is a site admin +/// and reads + writes every repo here. As of the agent-config-PR flow each +/// agent is a **write collaborator on its own** `agent-configs/` repo — +/// the editable PR surface it pushes config-change branches to — but `main` is +/// branch-protected core-only, so only hive-c0re's verify-and-ff-push merge +/// handler lands on it (operator approval required; the agent can't push +/// `main` or self-merge). The repos remain private, so an agent still can't +/// reach *another* agent's config. `main` is fast-forward-only — hive-c0re +/// never force-pushes; the `push_config` mirror runs best-effort until the +/// PR-merge flow retires it. +const CONFIG_ORG: &str = "agent-configs"; +/// Forgejo org hosting the operator-curated shared docs/skills repo +/// that every agent gets read-only access to. Agents use it as a +/// common reference without the operator having to bake content into +/// the system prompt or rely on `/shared`. Only the manager + operator +/// (i.e. `core` user) can push. +const SHARED_ORG: &str = "internal"; +/// The shared docs repo inside `SHARED_ORG`. Cloneable by every agent +/// at `{FORGE_HTTP}/internal/docs.git`. +const SHARED_DOCS_REPO: &str = "docs"; +/// The hive-wide knowledge repo inside `SHARED_ORG`. Public — agents +/// can fork it and open PRs without explicit collaborator grants. +/// Bind-mounted read-only into every container at `/knowledge`. +/// See `hive-c0re/src/knowledge.rs`. +const KNOWLEDGE_REPO: &str = crate::knowledge::REPO; +/// Forgejo org that owns agent-created repos. Agents can't create +/// repos with their own token (`max_repo_creation = 0`); instead hive-c0re +/// creates them here and adds the requesting agent as a **write** member +/// (not owner/admin). Because the org — not the agent — owns the repo, +/// perms stay c0re-managed and branch protection (referencing +/// [`OPERATORS_TEAM`]) can block the author from merging their own PR. This +/// is the "agents namespace" repos land in by default. +const AGENTS_ORG: &str = "agents"; +/// Operator merge-gate team inside [`AGENTS_ORG`]. Provisioned **empty** by +/// hive-c0re (so perms can be set before anyone joins); the operator adds +/// herself via the forge UI / hivectl. Branch protection on agents-org repos +/// references this team by name for the merge/approval whitelist, so the +/// rule never hardcodes a specific reviewer agent (which may not exist). +const OPERATORS_TEAM: &str = "operators"; +/// Hive-managed Forgejo namespaces that agent-initiated repo creation must +/// never target. `internal` is operator-curated shared content; +/// `agent-configs` + `core` are hive-c0re-internal mirror/meta namespaces. +/// (`hyperhive` is NOT managed — it's just a repo that happens to be built +/// by this hive.) hive-c0re's create path forces [`AGENTS_ORG`], so this is +/// a defensive guard against any future caller passing an explicit owner. +const HIVE_MANAGED_NAMESPACES: &[&str] = &[SHARED_ORG, CONFIG_ORG, "core"]; +/// Forgejo orgs hive-c0re ensures on startup. The meta repo lives at +/// `core/meta` (the `core` user's own namespace — no org needed). +const SEEDED_ORGS: &[&str] = &[CONFIG_ORG, SHARED_ORG, AGENTS_ORG]; + +/// Probe whether `hive-forge` exists as a nixos-container. Cheap — +/// `nixos-container list` is just a directory scan in /etc. Routed +/// through hive-priv: `nixos-container` needs root, and hive-c0re runs +/// unprivileged (privsep). +pub async fn is_present() -> bool { + let Ok(stdout) = crate::priv_client::list_containers().await else { + return false; + }; + stdout.lines().any(|l| l.trim() == FORGE_CONTAINER) +} + +/// Run `forgejo admin ` inside the hive-forge container as the +/// forgejo user (the only uid with write access to the state dir). +/// Returns stdout on success; bails with stderr context on failure. +async fn forge_admin(args: &[&str]) -> Result { + // Route through hive-priv (root helper) because `nixos-container run` + // uses nsenter to enter the container's namespaces, which requires root. + // hive-c0re runs as the unprivileged `hive-core` user and cannot call + // nsenter directly — doing so produces: + // nsenter: stat of /proc//ns/user failed: Permission denied + let (stdout, _stderr) = crate::priv_client::run_forge_admin(args) + .await + .with_context(|| format!("forgejo admin {} (via hive-priv)", args.join(" ")))?; + Ok(stdout) +} + +/// Thin Forgejo REST helper. Sends `method` to `url` with a JSON body +/// and `Authorization: token `, returns the HTTP status code. +/// All Forgejo API calls that don't shell out to `forgejo admin` go +/// through here — one place for auth header, content-type, error +/// propagation, and the shared reqwest Client. +/// Returns the response status **and body**. The body lets callers log +/// *why* Forgejo rejected a request (e.g. the validation message on a +/// 422); status-only callers just bind `(status, _)`. Body read is +/// best-effort — a read error yields an empty string rather than +/// failing the whole call. +async fn forge_http( + method: reqwest::Method, + url: &str, + token: &str, + body: &str, +) -> Result<(StatusCode, String)> { + let client = reqwest::Client::new(); + let resp = client + .request(method, url) + .header("Authorization", format!("token {token}")) + .header("Content-Type", "application/json") + .body(body.to_owned()) + .send() + .await + .with_context(|| format!("forge HTTP request to {url}"))?; + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + Ok((status, text)) +} + +/// Whether `ns` is a hive-managed Forgejo namespace that agent-initiated +/// repo creation must never target — `internal` (operator-curated +/// shared content) + `agent-configs` / `core` (hive-c0re-internal). The +/// create path forces [`AGENTS_ORG`], so this guards a future surface that +/// might accept an explicit owner. +#[must_use] +pub fn is_hive_managed_namespace(ns: &str) -> bool { + HIVE_MANAGED_NAMESPACES.contains(&ns) +} + +/// Per-agent forge sync: ensure the agent has a forgejo user + token, +/// a mirrored config repo, read access to `core/meta`, and the `meta` +/// remote in its proposed repo. All operations are idempotent; failures +/// are logged as warnings but don't abort the caller. +/// +/// `core_token` is `core_token()` — passed in so callers that already +/// fetched it don't re-read the file. Pass `None` to skip the +/// `meta_read_access` step (safe: the access grant is best-effort). +/// +/// Called by both `ensure_all()` (startup sweep) and `rebuild_agent` +/// (per-rebuild) so the two paths stay equivalent. +pub async fn sync_agent(name: &str, core_token: Option<&str>) { + if let Err(e) = ensure_user_for(name).await { + tracing::warn!(%name, error = ?e, "forge: ensure_user failed"); + } + // Align email to match the git user.email set by meta::render_flake + // so commits link to the agent's Forgejo profile. Best-effort; + // also patches up agents created before this fix (old @hive.local). + ensure_user_email(name).await; + // Block direct agent-initiated repo creation: agents create + // repos through hive-c0re, never with their own token. Idempotent + + // marker-guarded; also covers agents provisioned before this landed. + ensure_repo_creation_disabled(name).await; + // Mirror the agent's applied config repo into agent-configs. + // ensure_config_repo is idempotent; push_config catches any + // drift since the last run — e.g. the startup migration just + // relocated `deployed/0`, or a deploy landed while the forge + // was down. + if let Err(e) = ensure_config_repo(name).await { + tracing::warn!(%name, error = ?e, "forge: ensure_config_repo failed"); + } + if let Err(e) = push_config(name).await { + tracing::warn!(%name, error = ?e, "forge: push_config failed"); + } + // Grant read-only access to core/meta and wire the `meta` remote + // into the proposed repo so agents can fetch their deployment context. + if let Some(token) = core_token + && let Err(e) = meta_read_access(name, token).await + { + tracing::warn!(%name, error = ?e, "forge: ensure_meta_read_access failed"); + } + if let Err(e) = ensure_meta_remote(name).await { + tracing::warn!(%name, error = ?e, "forge: ensure_meta_remote failed"); + } + // Grant read-only access to internal/docs so the agent can clone + // the operator-curated shared skills/runbook repo. Best-effort. + if let Some(token) = core_token + && let Err(e) = shared_docs_access(name, token).await + { + tracing::warn!(%name, error = ?e, "forge: shared_docs_access failed"); + } + // internal/knowledge is public — no per-agent collaborator grant needed. +} + +/// Sweep every existing container (manager + sub-agents) and ensure +/// each has a forgejo user + token, plus an `agent-configs/` +/// repo mirroring its applied config. Also seeds the `core` admin +/// user (hive-c0re's own identity for pushing the meta repo + driving +/// the API), the `agent-configs` org, and the `core/meta` repo. +/// Called once at hive-c0re startup. Per-step failures are logged +/// but don't abort the sweep. +pub async fn ensure_all() { + if !is_present().await { + tracing::debug!("forge: hive-forge container absent, skipping user sweep"); + return; + } + let core_token = match ensure_core_user_and_token().await { + Ok(t) => Some(t), + Err(e) => { + tracing::warn!(error = ?e, "forge: ensure_core_user_and_token failed"); + None + } + }; + if let Some(token) = core_token.as_deref() { + for org in SEEDED_ORGS { + if let Err(e) = ensure_org(org, token).await { + tracing::warn!(%org, error = ?e, "forge: ensure_org failed"); + } + } + // Seed the operator-declared pull-mirrors (nix `forge.mirrors` + + // the CI-auto `actions/checkout`, forwarded via the + // `HYPERHIVE_FORGE_MIRRORS` env). Each ensures its own dest org, so + // this is independent of the SEEDED_ORGS loop above. + ensure_mirrors(token).await; + // Provision the operator merge-gate team (empty) inside BOTH the + // agents org and the agent-configs org so branch protection in each + // can reference it before anyone joins. Gitea teams are org-scoped — + // missing the agent-configs copy 422'd every config-repo protection + // apply, leaving those repos unprotected and letting operator-merged + // config PRs bypass the deploy pipeline. The operator adds herself as + // a member out-of-band. + for org in [AGENTS_ORG, CONFIG_ORG] { + if let Err(e) = ensure_operators_team(org, token).await { + tracing::warn!(%org, error = ?e, "forge: ensure_operators_team failed"); + } + } + // Meta repo lives at core/meta — pushed from git_commit in + // meta.rs on every deploy/lock-update. Make sure it exists + // before the first push hits a 404. + if let Err(e) = ensure_repo("meta", token).await { + tracing::warn!(error = ?e, "forge: ensure_repo core/meta failed"); + } + // Seed the shared docs repo. internal is already in + // SEEDED_ORGS above so the org exists; ensure the repo itself. + if let Err(e) = ensure_shared_docs_repo(token).await { + tracing::warn!(error = ?e, "forge: ensure_shared_docs_repo failed"); + } + // Seed the hive-wide knowledge repo. + if let Err(e) = ensure_knowledge_repo(token).await { + tracing::warn!(error = ?e, "forge: ensure_knowledge_repo failed"); + } + // Clone knowledge repo locally so it can be bind-mounted into agents. + if let Err(e) = crate::knowledge::ensure_local_clone(token).await { + tracing::warn!(error = ?e, "knowledge: ensure_local_clone failed"); + } + if let Err(e) = ensure_core_avatar(token).await { + tracing::warn!(error = ?e, "forge: ensure_core_avatar failed"); + } + if let Err(e) = ensure_config_org_avatar(token).await { + tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed"); + } + } + let Ok(containers) = crate::lifecycle::list().await else { + tracing::warn!("forge: nixos-container list failed; skipping user sweep"); + return; + }; + for c in containers { + let Some(name) = c.strip_prefix(crate::lifecycle::AGENT_PREFIX) else { + continue; + }; + sync_agent(name, core_token.as_deref()).await; + } +} diff --git a/hive-c0re/src/forge/pr_merge.rs b/hive-c0re/src/forge/pr_merge.rs new file mode 100644 index 00000000..02133085 --- /dev/null +++ b/hive-c0re/src/forge/pr_merge.rs @@ -0,0 +1,295 @@ +//! PR-based config-flow merge primitives — the forge-side mechanics +//! hive-c0re's approve-handler (`run_merge_config_pr`) orchestrates to +//! land an operator-approved config PR. Part of the operator trust +//! boundary; moved verbatim from the `forge` module root. + +use anyhow::Context; + +use crate::coordinator::Coordinator; + +use super::{CONFIG_ORG, FORGE_HTTP, core_token, forge_http}; + +// --------------------------------------------------------------------------- +// PR-based config-flow merge primitives (part of the +// dashboard-approve-driven config-change flow). +// +// The dashboard-approve-driven flow has hive-c0re verify an operator-approved +// config PR, then land it: ff-push the verified sha to the protected default +// branch (= the merge) and mark the PR merged manually. These three fns are +// the forge-side mechanics the c0re approve-handler (`run_merge_config_pr`) +// orchestrates; the orchestration fetches the verified sha into the agent's +// applied repo before calling `ff_push_to_main`. The core token is sourced +// internally (`core_token`), never passed in. `repo` is the agent's editable +// forge config repo in `owner/name` form (e.g. `agent-configs/`). +// --------------------------------------------------------------------------- + +/// Typed failure for the merge primitives so the c0re approve-handler can +/// `match` recoverable drift (refresh the request sha + re-verify) against a +/// hard failure (fail the approval). The two drift variants carry the observed +/// sha so the handler can re-pin to it; `Other` is everything else (transport, +/// API, unexpected) and is not auto-retried. +#[derive(Debug)] +pub enum ForgeMergeError { + /// The PR head moved off `expected` (now at `actual`), seen at + /// mark-merged time. The handler's pre-merge `pr_head_sha` re-read is the + /// primary drift gate; this is the belt-and-suspenders race catch. + HeadDrift { expected: String, actual: String }, + /// `main` (`actual_head`) is not a descendant of `expected_ancestor`, so + /// pushing the verified sha would not be a fast-forward — `main` raced. + NotFastForward { + expected_ancestor: String, + actual_head: String, + }, + /// Transport / API / unexpected failure — hard-fail, no auto-retry. + Other(anyhow::Error), +} + +impl std::fmt::Display for ForgeMergeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::HeadDrift { expected, actual } => { + write!(f, "PR head drifted: expected {expected}, found {actual}") + } + Self::NotFastForward { + expected_ancestor, + actual_head, + } => write!( + f, + "not a fast-forward: main {actual_head} is not a descendant of {expected_ancestor}" + ), + Self::Other(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for ForgeMergeError {} + +impl From for ForgeMergeError { + fn from(e: anyhow::Error) -> Self { + Self::Other(e) + } +} + +/// Agent name from an `owner/name` forge repo string (the trailing segment). +fn repo_agent_name(repo: &str) -> &str { + repo.rsplit('/').next().unwrap_or(repo) +} + +/// Token-in-URL clone/push URL for a forge repo (`owner/name`). Mirrors +/// `push_config`'s pattern; the token is passed straight to git and never +/// stored as a named remote. +fn tokenised_repo_url(repo: &str, token: &str) -> String { + format!("http://core:{token}@localhost:3000/{repo}.git") +} + +/// Resolve a PR's head sha via `git ls-remote refs/pull//head` +/// (Forgejo exposes PR heads there). Pure read, no mutation — the handler's +/// primary drift gate (compare against the approved sha), and `mark_pr_merged` +/// uses it to detect head-drift on failure. +/// +/// # Errors +/// `Other` on transport failure or an empty/missing ref. +pub async fn pr_head_sha(repo: &str, pr: u64) -> Result { + let token = core_token() + .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; + let url = tokenised_repo_url(repo, &token); + let refspec = format!("refs/pull/{pr}/head"); + let out = crate::lifecycle::git_command() + .args(["ls-remote", &url, &refspec]) + .output() + .await + .context("git ls-remote PR head")?; + if !out.status.success() { + return Err(ForgeMergeError::Other(anyhow::anyhow!( + "git ls-remote {repo} {refspec} failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ))); + } + let stdout = String::from_utf8_lossy(&out.stdout); + let sha = stdout + .split_whitespace() + .next() + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + ForgeMergeError::Other(anyhow::anyhow!( + "no head ref for PR #{pr} in {repo} (ls-remote empty)" + )) + })?; + Ok(sha.to_string()) +} + +/// Full `owner/name` path of an agent's config repo on the forge — the +/// `agent-configs` org mirror that the PR-merge flow reads + fast-forwards. +pub fn config_repo(agent: &str) -> String { + format!("{CONFIG_ORG}/{agent}") +} + +/// Fetch PR #`pr`'s head into the agent's applied repo via +/// `refs/pull//head` (which Forgejo always serves — a bare-sha fetch can +/// be refused by uploadpack policy). This makes the reviewed head an object +/// in the applied repo so the ancestor check in [`ff_push_to_main`], the +/// `git_update_ref(main, …)` in the deploy tail, and the eval-verify all +/// resolve it locally before the irreversible push. +/// +/// # Errors +/// `Other` on transport failure or a non-zero git exit. +pub async fn fetch_pr_head_into_applied(repo: &str, pr: u64) -> Result<(), ForgeMergeError> { + let token = core_token() + .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; + let url = tokenised_repo_url(repo, &token); + let applied = Coordinator::agent_applied_dir(repo_agent_name(repo)); + let refspec = format!("refs/pull/{pr}/head"); + let out = crate::lifecycle::git_command() + .current_dir(&applied) + .args(["fetch", "--no-tags", &url, &refspec]) + .output() + .await + .context("git fetch PR head into applied")?; + if !out.status.success() { + return Err(ForgeMergeError::Other(anyhow::anyhow!( + "git fetch {repo} {refspec} into applied failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ))); + } + Ok(()) +} + +/// Fast-forward the forge repo's `main` to `sha` — THE merge in the PR flow. +/// Reads `main`'s current sha (`git ls-remote … refs/heads/main`), verifies it +/// is a strict ancestor of `sha` (`git merge-base --is-ancestor`, run in the +/// agent's applied repo where the orchestration has already fetched `sha`), +/// then does a **non-force** `git push :refs/heads/main`. The ancestor +/// pre-check and the non-force push are two independent guards: either catches +/// a raced `main` (→ `NotFastForward`) rather than clobbering reviewed history. +/// +/// # Errors +/// `NotFastForward` if `main` raced ahead of `sha`; `Other` on transport/other. +pub async fn ff_push_to_main(repo: &str, sha: &str) -> Result<(), ForgeMergeError> { + let token = core_token() + .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; + let url = tokenised_repo_url(repo, &token); + let applied = Coordinator::agent_applied_dir(repo_agent_name(repo)); + + // Current `main` on the forge repo. + let ls = crate::lifecycle::git_command() + .args(["ls-remote", &url, "refs/heads/main"]) + .output() + .await + .context("git ls-remote main")?; + if !ls.status.success() { + return Err(ForgeMergeError::Other(anyhow::anyhow!( + "git ls-remote {repo} refs/heads/main failed ({}): {}", + ls.status, + String::from_utf8_lossy(&ls.stderr).trim() + ))); + } + let ls_out = String::from_utf8_lossy(&ls.stdout); + let main_sha = ls_out.split_whitespace().next().unwrap_or("").to_string(); + + // Strict-ancestor check: `main` must be an ancestor of `sha` for a true + // fast-forward. Skip when `main` is unborn (empty) — the push creates it. + if !main_sha.is_empty() { + let anc = crate::lifecycle::git_command() + .current_dir(&applied) + .args(["merge-base", "--is-ancestor", &main_sha, sha]) + .output() + .await + .context("git merge-base --is-ancestor")?; + match anc.status.code() { + Some(0) => {} // ancestor → fast-forward safe + Some(1) => { + return Err(ForgeMergeError::NotFastForward { + expected_ancestor: main_sha, + actual_head: sha.to_string(), + }); + } + _ => { + return Err(ForgeMergeError::Other(anyhow::anyhow!( + "git merge-base --is-ancestor errored ({}): {}", + anc.status, + String::from_utf8_lossy(&anc.stderr).trim() + ))); + } + } + } + + // Non-force push `sha` → `main`. Without `--force`, git rejects a + // non-fast-forward (a race between the check above and now), surfaced as + // `NotFastForward` rather than clobbering the remote. + let push = crate::lifecycle::git_command() + .current_dir(&applied) + .args(["push", &url, &format!("{sha}:refs/heads/main")]) + .output() + .await + .context("git push sha:main")?; + if !push.status.success() { + let stderr = String::from_utf8_lossy(&push.stderr); + if stderr.contains("non-fast-forward") || stderr.contains("fetch first") { + return Err(ForgeMergeError::NotFastForward { + expected_ancestor: main_sha, + actual_head: sha.to_string(), + }); + } + return Err(ForgeMergeError::Other(anyhow::anyhow!( + "git push {repo} {sha}:main failed ({}): {}", + push.status, + stderr.trim() + ))); + } + Ok(()) +} + +/// Mark PR `pr` as **manually merged** at `sha` (Forgejo +/// `POST …/pulls/{pr}/merge` with `Do=manually-merged`, `MergeCommitID=sha`). +/// `ff_push_to_main` must have already set `main` to `sha` (Forgejo requires +/// the branch already be at the merge commit). On a non-2xx, re-reads the PR +/// head to distinguish drift (`HeadDrift`) from a generic failure (`Other`) — +/// best-effort, since the handler's pre-merge head re-read is the real gate. +/// +/// # Errors +/// `HeadDrift` if the PR head no longer matches `sha`; `Other` otherwise. +pub async fn mark_pr_merged(repo: &str, pr: u64, sha: &str) -> Result<(), ForgeMergeError> { + let token = core_token() + .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; + let url = format!("{FORGE_HTTP}/api/v1/repos/{repo}/pulls/{pr}/merge"); + let body = format!(r#"{{"Do":"manually-merged","MergeCommitID":"{sha}"}}"#); + let (status, _) = forge_http(reqwest::Method::POST, &url, &token, &body) + .await + .context("POST pulls//merge (manually-merged)")?; + if status.is_success() { + return Ok(()); + } + // Best-effort drift detection: if the live head no longer matches `sha`, + // that's a head-drift race; otherwise surface as a hard failure. + match pr_head_sha(repo, pr).await { + Ok(actual) if actual != sha => Err(ForgeMergeError::HeadDrift { + expected: sha.to_string(), + actual, + }), + _ => Err(ForgeMergeError::Other(anyhow::anyhow!( + "mark PR #{pr} in {repo} manually-merged at {sha} failed: HTTP {status}" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::{repo_agent_name, tokenised_repo_url}; + + #[test] + fn repo_agent_name_takes_trailing_segment() { + assert_eq!(repo_agent_name("agent-configs/atlas"), "atlas"); + assert_eq!(repo_agent_name("atlas"), "atlas"); + assert_eq!(repo_agent_name("a/b/c"), "c"); + } + + #[test] + fn tokenised_repo_url_shape() { + assert_eq!( + tokenised_repo_url("agent-configs/iris", "tok"), + "http://core:tok@localhost:3000/agent-configs/iris.git" + ); + } +} diff --git a/hive-c0re/src/forge/repos.rs b/hive-c0re/src/forge/repos.rs new file mode 100644 index 00000000..d5fce8f5 --- /dev/null +++ b/hive-c0re/src/forge/repos.rs @@ -0,0 +1,621 @@ +//! Repo + org plumbing on the local Forgejo: org / repo creation, +//! the meta + shared-docs + knowledge repos, per-agent config-repo +//! mirroring (`push_config` / `push_meta`), collaborator grants, +//! pull-mirrors, and branch-protection rules. Shared HTTP helpers + +//! org-name constants live in the module root (`super`). + +use std::path::Path; + +use anyhow::{Context, Result}; +use tokio::process::Command; + +use crate::coordinator::Coordinator; + +use super::{ + AGENTS_ORG, CONFIG_ORG, FORGE_HTTP, KNOWLEDGE_REPO, OPERATORS_TEAM, SHARED_DOCS_REPO, + SHARED_ORG, core_token, forge_http, is_present, +}; + +/// JSON body for a private, empty repo defaulting to `main`. +fn repo_body(name: &str) -> String { + format!(r#"{{"name":"{name}","auto_init":false,"private":true,"default_branch":"main"}}"#) +} + +/// JSON body for a public, empty repo defaulting to `main`. +fn repo_body_public(name: &str) -> String { + format!(r#"{{"name":"{name}","auto_init":false,"private":false,"default_branch":"main"}}"#) +} + +/// Set an existing repo to public visibility. No-op if the repo is +/// already public. Used for `internal/knowledge` which may have been +/// created as private on an older deployment. +async fn set_repo_public(owner: &str, repo: &str, token: &str) -> Result<()> { + let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}"); + let (status, _) = + forge_http(reqwest::Method::PATCH, &url, token, r#"{"private":false}"#).await?; + match status.as_u16() { + 200 => { + tracing::debug!(%owner, %repo, "forge: repo set to public"); + Ok(()) + } + other => anyhow::bail!("PATCH {owner}/{repo} (set public) returned HTTP {other}"), + } +} + +/// Create `name` inside org `org` as a public repo. Idempotent. +async fn ensure_org_repo_public(org: &str, name: &str, token: &str) -> Result<()> { + create_repo( + &format!("{FORGE_HTTP}/api/v1/orgs/{org}/repos"), + &repo_body_public(name), + token, + &format!("{org}/{name}"), + ) + .await +} + +/// POST a repo-creation request to `url` and fold "already exists" +/// (HTTP 409 / 422) into success. `label` is `/` — purely +/// for log + error context. +async fn create_repo(url: &str, body: &str, token: &str, label: &str) -> Result<()> { + let (status, _) = forge_http(reqwest::Method::POST, url, token, body).await?; + match status.as_u16() { + 201 => { + tracing::info!(%label, "forge: created repo"); + Ok(()) + } + 409 | 422 => { + tracing::debug!(%label, "forge: repo already exists"); + Ok(()) + } + other => anyhow::bail!("POST {url} ({label}) returned HTTP {other}"), + } +} + +/// Create a repo in the token-owner's own namespace. `token` belongs +/// to the user we want the repo owned by (we use `core`'s token for +/// `core/meta`). Idempotent. +pub async fn ensure_repo(name: &str, token: &str) -> Result<()> { + create_repo( + &format!("{FORGE_HTTP}/api/v1/user/repos"), + &repo_body(name), + token, + &format!("core/{name}"), + ) + .await +} + +/// Create `name` inside org `org` (used for `agent-configs/`). +/// Idempotent. +async fn ensure_org_repo(org: &str, name: &str, token: &str) -> Result<()> { + create_repo( + &format!("{FORGE_HTTP}/api/v1/orgs/{org}/repos"), + &repo_body(name), + token, + &format!("{org}/{name}"), + ) + .await +} + +/// Push `dir` (the meta repo) to `core/meta` on the local forge. +/// Best-effort: returns Err which callers log + ignore. No-op when +/// the core token isn't present yet (forge container not provisioned). +pub async fn push_meta(dir: &Path) -> Result<()> { + let Some(token) = core_token() else { + return Ok(()); + }; + // Token-in-URL push. Forgejo accepts `oauth2:` or just + // any-username:; using `core` matches the owner so the + // remote name is self-describing. + let url = format!("http://core:{token}@localhost:3000/core/meta.git"); + let out = Command::new("git") + .current_dir(dir) + .args(["push", "--force", &url, "HEAD:main"]) + .output() + .await + .context("invoke git push core/meta")?; + if !out.status.success() { + anyhow::bail!( + "git push core/meta failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!("forge: pushed meta to core/meta"); + Ok(()) +} + +/// Ensure the `agent-configs/` repo exists so the first +/// `push_config` doesn't 404, and wire it as the agent-editable PR surface: +/// the agent is a **write** collaborator (can push feature branches + +/// open config PRs) and `main` is branch-protected core-only (only hive-c0re's +/// merge handler lands on it; operator approval required). No-op when the forge +/// isn't running or the core token isn't minted yet. Safe to call on every +/// spawn and on every startup (all steps idempotent). +pub async fn ensure_config_repo(name: &str) -> Result<()> { + if !is_present().await { + return Ok(()); + } + let Some(token) = core_token() else { + return Ok(()); + }; + ensure_org_repo(CONFIG_ORG, name, &token).await?; + // Agent = write collaborator: it can push config-PR branches + open PRs, + // but the branch protection below keeps it off `main` directly. + add_collaborator(CONFIG_ORG, name, name, "write", &token).await?; + // Protect `main` core-only, fast-forward-only (no auto force-push). + apply_config_repo_branch_protection(name, &token).await +} + +/// Ensure the `internal/docs` repo exists. Called once at startup +/// after `ensure_org(SHARED_ORG)`. Idempotent — `ensure_org_repo` +/// treats 409 as success. +pub async fn ensure_shared_docs_repo(core_token: &str) -> Result<()> { + ensure_org_repo(SHARED_ORG, SHARED_DOCS_REPO, core_token).await +} + +/// Grant agent `name` read-only collaborator access to `internal/docs`. +/// Idempotent: HTTP 204 (already a collaborator) is treated as success. +/// Mirrors `meta_read_access` so agents can clone the shared docs repo +/// without authentication hassle. +pub async fn shared_docs_access(name: &str, core_token: &str) -> Result<()> { + let url = + format!("{FORGE_HTTP}/api/v1/repos/{SHARED_ORG}/{SHARED_DOCS_REPO}/collaborators/{name}"); + let body = r#"{"permission":"read"}"#; + let out = Command::new("curl") + .args([ + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "-X", + "PUT", + "-H", + "Content-Type: application/json", + "-H", + &format!("Authorization: token {core_token}"), + "-d", + body, + &url, + ]) + .output() + .await + .context("invoke curl PUT internal/docs/collaborators")?; + let code = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + match code.as_str() { + "204" => { + tracing::info!(%name, "forge: granted shared-docs read access"); + Ok(()) + } + other => anyhow::bail!( + "PUT {SHARED_ORG}/{SHARED_DOCS_REPO}/collaborators/{name} returned HTTP {other}" + ), + } +} + +/// Ensure the `internal/knowledge` repo exists and is public. +/// Called once at startup after `ensure_org(SHARED_ORG)`. Idempotent. +/// +/// The repo is created as public so any agent with a forge account can +/// fork it and open PRs to contribute. Existing deployments that ended +/// up with a private repo are patched to public on the next hive-c0re +/// startup via `set_repo_public`. +pub async fn ensure_knowledge_repo(core_token: &str) -> Result<()> { + ensure_org_repo_public(SHARED_ORG, KNOWLEDGE_REPO, core_token).await?; + // Ensure public even if the repo already existed as private (older deployment). + set_repo_public(SHARED_ORG, KNOWLEDGE_REPO, core_token).await +} + +/// Grant agent `name` read-only collaborator access to `core/meta` on +/// the forge so the agent can clone/fetch the meta flake. Idempotent: +/// HTTP 204 (already a collaborator) is treated as success. +pub async fn meta_read_access(name: &str, core_token: &str) -> Result<()> { + let url = format!("{FORGE_HTTP}/api/v1/repos/core/meta/collaborators/{name}"); + let body = r#"{"permission":"read"}"#; + let out = Command::new("curl") + .args([ + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "-X", + "PUT", + "-H", + "Content-Type: application/json", + "-H", + &format!("Authorization: token {core_token}"), + "-d", + body, + &url, + ]) + .output() + .await + .context("invoke curl PUT core/meta/collaborators")?; + let code = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + match code.as_str() { + "204" => { + tracing::info!(%name, "forge: granted meta read access"); + Ok(()) + } + other => anyhow::bail!("PUT core/meta/collaborators/{name} returned HTTP {other}"), + } +} + +/// Add `http://localhost:3000/core/meta.git` as the `meta` remote in +/// the agent's proposed config repo so the agent (and the manager) can +/// fetch the meta flake from the forge. Idempotent: no-op when the +/// remote already points at the right URL, or when the proposed repo +/// does not exist yet. No-op when the forge is not running. +pub async fn ensure_meta_remote(name: &str) -> Result<()> { + if !is_present().await { + return Ok(()); + } + let proposed_dir = Coordinator::agent_proposed_dir(name); + if !proposed_dir.join(".git").exists() { + return Ok(()); + } + let want = format!("{FORGE_HTTP}/core/meta.git"); + let existing = crate::lifecycle::git_command() + .current_dir(&proposed_dir) + .args(["remote", "get-url", "meta"]) + .output() + .await + .context("git remote get-url meta")?; + if existing.status.success() { + let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned(); + if current == want { + return Ok(()); + } + crate::lifecycle::git(&proposed_dir, &["remote", "set-url", "meta", &want]).await + } else { + crate::lifecycle::git(&proposed_dir, &["remote", "add", "meta", &want]).await + } +} + +/// Mirror agent `name`'s applied config repo — `main` plus every tag +/// (`proposal` / `approved` / `building` / `deployed` / `failed` / +/// `denied`) — to `agent-configs/` on the local forge. +/// Best-effort: returns Err which callers log + ignore. No-op when the +/// forge isn't seeded or the applied repo doesn't exist yet. +/// +/// Call this after every hive-c0re mutation of an applied repo's refs +/// so the forge copy always reflects what core actually did. +/// +/// Never force-pushes. The status tags are id-suffixed +/// (`proposal/`, `deployed/`, …) and therefore add-only, and +/// `main` is published history — after a failed deploy rolls the LOCAL +/// applied `main` back to last-good, the forge `main` may legitimately +/// be ahead (e.g. an operator-merged config PR whose rebuild failed). +/// Rewinding it would erase that merged commit from the forge, which +/// is exactly the incident this guards against: the local repo tracks +/// "what last built", the forge tracks "what was approved", and the +/// `failed/` tag records the divergence. A non-fast-forward +/// rejection of `main` is therefore expected + logged at info; the +/// tags in the same push still land (git pushes refspecs +/// independently). Any other failure is a real error. +/// +/// The tokenised URL is passed straight to `git push` and deliberately +/// never stored as a named remote: the applied repo is bind-mounted +/// READ-ONLY into the manager container (`/applied`), so a token in +/// `.git/config` would leak core's admin credential to an agent. +pub async fn push_config(name: &str) -> Result<()> { + let Some(token) = core_token() else { + return Ok(()); + }; + let dir = Coordinator::agent_applied_dir(name); + if !dir.join(".git").exists() { + return Ok(()); + } + let url = format!("http://core:{token}@localhost:3000/{CONFIG_ORG}/{name}.git"); + let out = crate::lifecycle::git_command() + .current_dir(&dir) + .args([ + "push", + &url, + "refs/heads/main:refs/heads/main", + "refs/tags/*:refs/tags/*", + ]) + .output() + .await + .context("invoke git push agent-configs")?; + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + if stderr.contains("non-fast-forward") { + tracing::info!( + %name, + "forge: mirror push of main rejected (non-fast-forward) — forge main is \ + ahead of local applied main (rolled-back deploy); leaving forge history intact" + ); + return Ok(()); + } + anyhow::bail!( + "git push {CONFIG_ORG}/{name} failed ({}): {}", + out.status, + stderr.trim() + ); + } + tracing::info!(%name, "forge: mirrored applied config to agent-configs"); + Ok(()) +} + +/// POST `/api/v1/orgs` to create an org named `name`. Idempotent: +/// HTTP 422 ("user already exists") is treated as success. +pub(super) async fn ensure_org(name: &str, admin_token: &str) -> Result<()> { + let body = format!(r#"{{"username":"{name}"}}"#); + let url = format!("{FORGE_HTTP}/api/v1/orgs"); + let (status, _) = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?; + match status.as_u16() { + 201 => { + tracing::info!(%name, "forge: created org"); + Ok(()) + } + 422 | 409 => { + tracing::debug!(%name, "forge: org already exists"); + Ok(()) + } + other => anyhow::bail!("POST /api/v1/orgs name={name} returned HTTP {other}"), + } +} + +/// One operator-declared pull-mirror, forwarded from the nix +/// `services.hyperhive.forge.mirrors` option as JSON in +/// `HYPERHIVE_FORGE_MIRRORS`. +#[derive(serde::Deserialize)] +struct Mirror { + /// Upstream clone URL to mirror from (e.g. `https://github.com/actions/checkout`). + upstream: String, + /// Local `/` the mirror is created at. + dest: String, +} + +/// Ensure each `HYPERHIVE_FORGE_MIRRORS` entry exists as a real Forgejo +/// pull-mirror. The env carries the JSON-encoded nix `forge.mirrors` list +/// (plus the CI-auto `actions/checkout` entry). Absent/empty env = no-op. +/// Per-mirror failures warn and continue — never abort the startup sweep. +pub(super) async fn ensure_mirrors(admin_token: &str) { + let raw = match std::env::var("HYPERHIVE_FORGE_MIRRORS") { + Ok(s) if !s.trim().is_empty() => s, + _ => return, + }; + let mirrors: Vec = match serde_json::from_str(&raw) { + Ok(m) => m, + Err(e) => { + tracing::warn!(error = ?e, "forge: HYPERHIVE_FORGE_MIRRORS is not valid JSON; skipping mirror seed"); + return; + } + }; + for m in mirrors { + let Some((owner, repo)) = m.dest.split_once('/') else { + tracing::warn!(dest = %m.dest, "forge: mirror dest is not /; skipping"); + continue; + }; + // Create the dest org first (idempotent); the mirror can't land + // without its owner existing. + if let Err(e) = ensure_org(owner, admin_token).await { + tracing::warn!(%owner, error = ?e, "forge: ensure_org for mirror failed"); + continue; + } + if let Err(e) = ensure_mirror_repo(&m.upstream, owner, repo, admin_token).await { + tracing::warn!(dest = %m.dest, error = ?e, "forge: ensure_mirror_repo failed"); + } + } +} + +/// Periodic sync interval for pull-mirrors. Forgejo syncs mirrors +/// on-access by default, which re-introduces external DNS latency on +/// every `git clone` (the hive-ci runner shares the host netns and is +/// therefore affected by host resolver blips). A fixed periodic interval +/// isolates CI from transient DNS failures — a stale mirror is +/// acceptable; a broken clone because of a momentary DNS blip is not. +const MIRROR_INTERVAL: &str = "8h0m0s"; + +/// Create `owner/repo` as a pull-mirror of `upstream` via the migrate API. +/// Idempotent: if the repo already exists this function patches its +/// `mirror_interval` to ensure it matches (covers mirrors that were +/// created before the interval was introduced). A 409 on the migrate +/// POST (a race between the GET check and the POST) is also success. +async fn ensure_mirror_repo( + upstream: &str, + owner: &str, + repo: &str, + admin_token: &str, +) -> Result<()> { + let repo_url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}"); + let (status, _) = forge_http(reqwest::Method::GET, &repo_url, admin_token, "").await?; + if status.is_success() { + // Mirror already present. Patch interval so mirrors seeded before + // this field was introduced (or with a different value) converge. + let patch_body = serde_json::json!({ "mirror_interval": MIRROR_INTERVAL }).to_string(); + let (patch_status, patch_text) = + forge_http(reqwest::Method::PATCH, &repo_url, admin_token, &patch_body).await?; + if patch_status.is_success() { + tracing::debug!(%owner, %repo, interval = MIRROR_INTERVAL, "forge: pull-mirror interval updated"); + } else { + tracing::warn!( + %owner, %repo, status = %patch_status, body = %patch_text, + "forge: failed to set mirror_interval on existing pull-mirror" + ); + } + return Ok(()); + } + // serde_json::json! → the upstream URL is escaped safely (no string + // interpolation into the JSON body). + let body = serde_json::json!({ + "clone_addr": upstream, + "repo_owner": owner, + "repo_name": repo, + "mirror": true, + // Periodic refresh instead of on-access sync — keeps CI isolated + // from external DNS failures at clone time. + "interval": MIRROR_INTERVAL, + "service": "git", + "private": false, + }) + .to_string(); + let url = format!("{FORGE_HTTP}/api/v1/repos/migrate"); + let (status, text) = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?; + match status.as_u16() { + 201 => { + tracing::info!(%owner, %repo, %upstream, interval = MIRROR_INTERVAL, "forge: created pull-mirror"); + Ok(()) + } + // 409 = a race created it between our GET check and here (the GET + // is the real idempotency guard). NOT 422: for the migrate endpoint + // 422 is a validation error (bad clone_addr / service), so it must + // surface via the bail arm, not be swallowed as "already exists". + 409 => { + tracing::debug!(%owner, %repo, "forge: pull-mirror already exists (race)"); + Ok(()) + } + other => { + anyhow::bail!("POST /api/v1/repos/migrate {owner}/{repo} returned HTTP {other}: {text}") + } + } +} + +/// Provision the [`OPERATORS_TEAM`] inside `org` as an **empty** team. +/// Branch protection on that org's repos references it as the +/// merge/approval whitelist; the operator adds herself as a member via the +/// forge UI / hivectl. `includes_all_repositories` so the gate applies to +/// every repo in the org; `write` is enough to approve + merge. hive-c0re +/// never manages membership. Idempotent (422/409 = already exists). +/// +/// Must run for BOTH [`AGENTS_ORG`] and [`CONFIG_ORG`]: Gitea teams are +/// org-scoped, so a config-repo branch-protection rule referencing +/// `operators` needs the team to exist in `agent-configs` too. Missing it +/// there 422'd every `apply_config_repo_branch_protection`, leaving config +/// repos unprotected — operator-merged config PRs then bypassed the deploy +/// pipeline and silently didn't apply. +pub(super) async fn ensure_operators_team(org: &str, token: &str) -> Result<()> { + let url = format!("{FORGE_HTTP}/api/v1/orgs/{org}/teams"); + let body = format!( + r#"{{"name":"{OPERATORS_TEAM}","description":"hyperhive operators — merge gate for agent repos","permission":"write","includes_all_repositories":true,"can_create_org_repo":false}}"# + ); + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; + match status.as_u16() { + 201 => { + tracing::info!(%org, "forge: created {OPERATORS_TEAM} team"); + Ok(()) + } + 409 | 422 => { + tracing::debug!(%org, "forge: {OPERATORS_TEAM} team already exists"); + Ok(()) + } + other => { + anyhow::bail!("POST /orgs/{org}/teams ({OPERATORS_TEAM}) returned HTTP {other}") + } + } +} + +/// Add `user` as a collaborator on `owner/repo` at `permission` +/// (`read` / `write` / `admin`). Idempotent: 201 (added) and 204 (already a +/// collaborator / permission updated) both count as success. +async fn add_collaborator( + owner: &str, + repo: &str, + user: &str, + permission: &str, + token: &str, +) -> Result<()> { + let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}/collaborators/{user}"); + let body = format!(r#"{{"permission":"{permission}"}}"#); + let (status, _) = forge_http(reqwest::Method::PUT, &url, token, &body).await?; + match status.as_u16() { + 201 | 204 => { + tracing::debug!(%owner, %repo, %user, %permission, "forge: collaborator set"); + Ok(()) + } + other => { + anyhow::bail!("PUT {owner}/{repo}/collaborators/{user} returned HTTP {other}") + } + } +} + +/// Apply the operator merge-gate branch protection to `repo`'s default +/// branch: only [`OPERATORS_TEAM`] members can merge, and an +/// approving review from that team is required — so the author (a write-level +/// agent, not in the team) cannot merge its own PR. Idempotent: an existing +/// rule for the branch (200/409/422) is treated as success. +async fn apply_operator_branch_protection(repo: &str, token: &str) -> Result<()> { + let url = format!("{FORGE_HTTP}/api/v1/repos/{AGENTS_ORG}/{repo}/branch_protections"); + let body = format!( + r#"{{"branch_name":"main","enable_merge_whitelist":true,"merge_whitelist_teams":["{OPERATORS_TEAM}"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true}}"# + ); + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; + match status.as_u16() { + 201 => { + tracing::info!(%repo, "forge: applied operator branch protection"); + Ok(()) + } + 200 | 409 | 422 => { + tracing::debug!(%repo, "forge: branch protection already present"); + Ok(()) + } + other => anyhow::bail!("POST {AGENTS_ORG}/{repo}/branch_protections returned HTTP {other}"), + } +} + +/// Apply branch protection to an `agent-configs/` repo's `main` so it +/// can serve as the agent-editable, PR-merge config surface: +/// - **push + merge whitelists are `core`-only** — the agent (a write +/// collaborator) can push feature branches and open config PRs, but only +/// hive-c0re lands on `main`, via its verify-and-ff-push merge handler +/// (`run_merge_config_pr`). The agent can never push `main` directly. +/// - **operator-team approval is required** to merge, and the author (not in +/// the team) cannot self-approve. +/// - **`enable_force_push` is `false`** — `main` only ever advances by +/// fast-forward. The merge handler's `ff_push_to_main` is already a +/// non-force push, so it lands fine. The legacy `push_config` mirror DOES +/// force-push (it re-points status tags and rewinds `main` on a failed-build +/// rollback), so the protection now rejects those non-ff updates — that +/// mirror runs best-effort until the agent-opened PR-merge flow retires it. +/// (Auto force-push is intentionally not allowed: per operator directive a +/// silent force-push is a bug, not a feature.) +/// +/// Idempotent: an existing rule for the branch (200/409/422) is success. +async fn apply_config_repo_branch_protection(repo: &str, token: &str) -> Result<()> { + let url = format!("{FORGE_HTTP}/api/v1/repos/{CONFIG_ORG}/{repo}/branch_protections"); + let body = format!( + r#"{{"branch_name":"main","enable_push_whitelist":true,"push_whitelist_usernames":["core"],"enable_merge_whitelist":true,"merge_whitelist_usernames":["core"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true,"allow_manual_merge":true,"enable_force_push":false}}"# + ); + let (status, resp_body) = forge_http(reqwest::Method::POST, &url, token, &body).await?; + if status.as_u16() == 201 { + tracing::info!(%repo, "forge: applied config-repo branch protection"); + return Ok(()); + } + // Non-201 is ambiguous: it can mean "rule already exists" (idempotent + // success) OR a silent rejection — e.g. a 422 where Forgejo refused + // the request and created NO rule. The old code treated 200/409/422 + // all as success, so a rejected POST left the repo unprotected with + // no error (the reported case: a new agent's config repo had no + // `main` rule and nothing was logged). Don't trust the status code: + // verify the `main` rule actually exists, and on failure surface the + // POST's response body so the real reason is in the journal. + let main_url = format!("{url}/main"); + let (check, _) = forge_http(reqwest::Method::GET, &main_url, token, "").await?; + if check.as_u16() == 200 { + tracing::debug!(%repo, %status, "forge: config-repo branch protection already present"); + Ok(()) + } else { + anyhow::bail!( + "branch protection for {CONFIG_ORG}/{repo} not applied: POST -> HTTP {status} \ + (body: {body}); GET main -> HTTP {check}, no `main` rule present", + body = resp_body.trim(), + ) + } +} + +/// Create a repo for `agent` in the c0re-owned [`AGENTS_ORG`] and wire the +/// perms: the org owns it (perms stay c0re-managed), the agent is added +/// as a **write** collaborator (not owner — can push + open PRs but can't +/// bypass branch protection), and the default branch gets the operator +/// merge gate. This is the sanctioned create path now that agents can't +/// create repos directly (`max_repo_creation = 0`). Idempotent. +pub async fn create_agent_repo(agent: &str, repo: &str, core_token: &str) -> Result { + ensure_org_repo(AGENTS_ORG, repo, core_token).await?; + add_collaborator(AGENTS_ORG, repo, agent, "write", core_token).await?; + apply_operator_branch_protection(repo, core_token).await?; + tracing::info!(%agent, %repo, "forge: created agent repo in {AGENTS_ORG} with operator merge gate"); + Ok(format!("{AGENTS_ORG}/{repo}")) +} diff --git a/hive-c0re/src/forge/users.rs b/hive-c0re/src/forge/users.rs new file mode 100644 index 00000000..38bc382b --- /dev/null +++ b/hive-c0re/src/forge/users.rs @@ -0,0 +1,534 @@ +//! Per-agent Forgejo user + access-token provisioning, account +//! policy (email alignment, repo-creation lockdown), avatar uploads, +//! and the bootstrap `core` admin user + token lifecycle. Shared +//! HTTP / `forgejo admin` helpers live in the module root (`super`). + +use std::path::Path; + +use anyhow::{Context, Result}; +use base64::Engine; +use reqwest::StatusCode; + +use super::{CONFIG_ORG, FORGE_HTTP, forge_admin, forge_http, is_present}; + +const TOKEN_NAME_PREFIX: &str = "hyperhive"; +/// Where the host-side `core` admin token lives. Used by hive-c0re +/// itself to push the meta repo + drive admin API calls (org +/// creation, future webhook setup, etc.). Root-only. +const CORE_TOKEN_PATH: &str = "/var/lib/hyperhive/forge-core-token"; +// Forge provisioning markers (`forge/core-avatar-set`, +// `forge/agent-configs-avatar-set`, `forge/email-aligned-`) live +// in `crate::paths` — one-shot guards: the upload/align runs once, the +// marker is written, subsequent startups skip. Delete one to force its +// step to re-run. +// Avatar PNGs are loaded at runtime from +// `$HIVE_ASSETS_DIR/branding/{hyperhive,agent-configs}.png` via the +// helpers in `hive_sh4re::assets`. The `agent-configs.png` is +// rendered from its SVG during the `hyperhive-assets` derivation's +// build. +/// Per-agent token scopes (broad-but-not-admin). See +/// `docs/forge.md::Token scopes` for the per-scope rationale. +const TOKEN_SCOPES: &str = "read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc"; + +/// Bootstrap `core` token scopes — adds `read:admin,write:admin` on +/// top of `TOKEN_SCOPES` so the host daemon can drive +/// `/api/v1/admin/*`. Site-admin membership alone isn't enough: the +/// token's own scope gate runs before the user-permission check. +/// See `docs/forge.md::Token scopes`. +const CORE_TOKEN_SCOPES: &str = "read:admin,write:admin,read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc"; + +/// Pull the access token out of forgejo's success message. Format +/// has shifted across versions (table form vs. "Access token was +/// successfully created: "), so just hunt the output for the +/// first long hex-looking word. +fn extract_token(output: &str) -> Option { + output + .split(|c: char| c.is_whitespace() || c == ',' || c == ':') + .find(|w| w.len() >= 32 && w.chars().all(|c| c.is_ascii_hexdigit())) + .map(str::to_owned) +} + +/// Canonical email address for a hive agent's Forgejo account. +/// Must match the `user.email` set by `meta::render_flake` so commits +/// by the agent link back to their Forgejo profile page. +fn agent_email(name: &str) -> String { + format!("{name}@hyperhive.local") +} + +/// Ensure a forgejo user named `name` exists. Idempotent: forgejo +/// returns a "user already exists" error which we treat as success. +/// `admin` adds `--admin` (site admin) — used for the bootstrap +/// `core` user that drives the API. `password` picks the initial +/// account password: `None` uses `--random-password` (the existing +/// agent provisioning shape — the password is never read, agents auth +/// by token); `Some(pw)` uses `--password ` so the operator path +/// in `hivectl` can set a real password for matrix-style web-UI login. +async fn ensure_user_exists(name: &str, admin: bool, password: Option<&str>) -> Result<()> { + let email = agent_email(name); + let mut args = vec!["user", "create", "--username", name, "--email", &email]; + match password { + Some(pw) => args.extend(["--password", pw, "--must-change-password=false"]), + None => args.extend(["--random-password", "--must-change-password=false"]), + } + if admin { + args.push("--admin"); + } + let result = forge_admin(&args).await; + match result { + Ok(_) => { + tracing::info!(%name, "forge: created user"); + Ok(()) + } + Err(e) => { + // Forgejo's "already exists" error wording varies; just + // try the next step and let token issuance surface a + // real failure if the user truly isn't there. + let msg = format!("{e:#}"); + if msg.contains("already exists") || msg.contains("user already") { + tracing::debug!(%name, "forge: user already exists"); + Ok(()) + } else { + tracing::warn!(%name, error = %msg, "forge: user create unclear; trying token anyway"); + Ok(()) + } + } + } +} + +/// Set the forgejo password for an existing user. Used by the operator +/// path in `hivectl forge create-user --password` so re-running on an +/// already-created account still updates the password (covers the +/// "I forgot the password I set last week" case + the "argus retried +/// the verb to verify the fix" case — `forgejo admin user create` +/// silently skips a password change once the account exists). Idempotent +/// from the operator's point of view: same password input → same final +/// account state. +async fn change_user_password(name: &str, password: &str) -> Result<()> { + let args = [ + "user", + "change-password", + "--username", + name, + "--password", + password, + ]; + forge_admin(&args) + .await + .with_context(|| format!("forgejo admin user change-password {name}"))?; + tracing::info!(%name, "forge: changed user password"); + Ok(()) +} + +/// Idempotently align the Forgejo account email to `agent_email(name)`. +/// Existing agents were created with `{name}@hive.local`; this corrects +/// that so git commits (which use `{name}@hyperhive`) link to profiles. +/// Best-effort: failures are warned, not propagated. +/// +/// Marker-guarded: writes `EMAIL_ALIGNED_MARKER_PREFIX{name}` on first +/// success and skips the PATCH on all subsequent calls. This prevents +/// Forgejo's admin-user-edit endpoint from resetting `use_custom_avatar` +/// on every `sync_agent` tick. Delete the marker to force re-alignment. +/// +/// Uses the admin REST API (`PATCH /api/v1/admin/users/{name}`) rather +/// than `forgejo admin user edit` because the CLI dropped the `edit` +/// subcommand somewhere between forgejo 8 and current. Body includes +/// `login_name` (required by Forgejo's `EditUserOption` validator) and +/// `source_id = 0` (local auth, the default for users hive-c0re creates). +pub(super) async fn ensure_user_email(name: &str) { + let marker = crate::paths::forge_email_aligned_marker(name); + if marker.exists() { + return; + } + let Some(token) = core_token() else { + tracing::debug!(%name, "forge: skipping ensure_user_email — no core token yet"); + return; + }; + let email = agent_email(name); + // `login_name` is required by Forgejo's EditUserOption validator. + // Omitting it caused Forgejo to reset use_custom_avatar on each call. + let body = format!(r#"{{"email":"{email}","login_name":"{name}","source_id":0}}"#); + let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}"); + match forge_http(reqwest::Method::PATCH, &url, &token, &body).await { + Ok((status, _)) if status.is_success() => { + if let Some(parent) = marker.parent() { + std::fs::create_dir_all(parent).ok(); + } + std::fs::write(&marker, "").ok(); + tracing::info!(%name, %email, "forge: user email aligned"); + } + Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => { + // Core token missing admin scope — see + // `docs/forge.md::Token scopes` migration note. + tracing::warn!( + %name, %email, %status, + "forge: PATCH user email forbidden — core token likely missing admin scope. \ + Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes." + ); + } + Ok((status, _)) => { + tracing::warn!(%name, %email, %status, "forge: PATCH user email returned non-success"); + } + Err(e) => tracing::warn!(%name, error = %e, "forge: PATCH user email transport error"), + } +} + +/// Disable direct repo creation for agent `name` by setting +/// `max_repo_creation = 0` on its Forgejo account. Agents must +/// create repos *through hive-c0re* (which owns the perms), never with +/// their own token — a write-scoped token can otherwise create + own +/// repos and self-merge, bypassing the operator-only-merge policy. +/// +/// `max_repo_creation = 0` means `CanCreateRepo()` is false for any +/// count (Forgejo: `MaxRepoCreation >= 0 && NumRepos >= MaxRepoCreation`), +/// so creation is refused while push / PR / clone stay intact. **Existing +/// repos are untouched** — this only blocks *new* direct creation. +/// +/// Marker-guarded like [`ensure_user_email`]: the PATCH runs once per +/// agent (delete the marker to re-apply). Body carries `login_name` + +/// `source_id` for the same reason `ensure_user_email` does — omitting +/// `login_name` makes Forgejo's `EditUserOption` validator reset +/// `use_custom_avatar`. Best-effort: failures warn, don't propagate. +pub(super) async fn ensure_repo_creation_disabled(name: &str) { + let marker = crate::paths::forge_repo_creation_disabled_marker(name); + if marker.exists() { + return; + } + let Some(token) = core_token() else { + tracing::debug!(%name, "forge: skipping ensure_repo_creation_disabled — no core token yet"); + return; + }; + let body = format!(r#"{{"login_name":"{name}","source_id":0,"max_repo_creation":0}}"#); + let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}"); + match forge_http(reqwest::Method::PATCH, &url, &token, &body).await { + Ok((status, _)) if status.is_success() => { + if let Some(parent) = marker.parent() { + std::fs::create_dir_all(parent).ok(); + } + std::fs::write(&marker, "").ok(); + tracing::info!(%name, "forge: disabled direct repo creation (max_repo_creation=0)"); + } + Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => { + tracing::warn!( + %name, %status, + "forge: PATCH max_repo_creation forbidden — core token likely missing admin scope. \ + Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes." + ); + } + Ok((status, _)) => { + tracing::warn!(%name, %status, "forge: PATCH max_repo_creation returned non-success"); + } + Err(e) => { + tracing::warn!(%name, error = %e, "forge: PATCH max_repo_creation transport error"); + } + } +} + +/// Mint a fresh access token for `name`. Token name is suffixed with +/// a monotonic clock so re-issuing doesn't collide with an existing +/// token of the same name in the DB. `scopes` is the scope string +/// passed to `forgejo admin user generate-access-token --scopes`; +/// use `TOKEN_SCOPES` for agents, `CORE_TOKEN_SCOPES` for the +/// bootstrap `core` user. +async fn mint_token(name: &str, scopes: &str) -> Result { + let token_name = format!( + "{TOKEN_NAME_PREFIX}-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) + ); + let stdout = forge_admin(&[ + "user", + "generate-access-token", + "--username", + name, + "--token-name", + &token_name, + "--scopes", + scopes, + ]) + .await?; + let token = extract_token(&stdout) + .with_context(|| format!("parse token from forgejo output: {stdout:?}"))?; + tracing::debug!(%name, %token_name, "forge: minted access token"); + Ok(token) +} + +/// Mint a fresh Forgejo access token for an agent and write it to the +/// agent's state dir via hive-priv. hive-c0re runs unprivileged and +/// cannot write to agent-owned (0755) state directories directly. +async fn mint_and_persist_agent_token(name: &str) -> Result<()> { + let token = mint_token(name, TOKEN_SCOPES).await?; + crate::priv_client::write_agent_forge_token(name, &token) + .await + .with_context(|| format!("write forge-token for {name} via hive-priv")) +} + +/// Mint a fresh Forgejo access token for the `core` admin user and +/// write it directly to `path`. Unlike agent tokens this path is owned +/// by hive-c0re itself (under `/var/lib/hyperhive/`), so a direct +/// write is both correct and necessary (no priv round-trip). +async fn mint_and_persist_core_token(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let token = mint_token("core", CORE_TOKEN_SCOPES).await?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).ok(); + } + std::fs::write(path, format!("{token}\n")) + .with_context(|| format!("write core token to {}", path.display()))?; + let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)); + tracing::info!(path = %path.display(), "forge: persisted core access token"); + Ok(()) +} + +/// Ensure `name` has a forgejo user + token file. Always re-mints the +/// token so the on-disk file always reflects the current `TOKEN_SCOPES`. +/// Safe to call on every spawn and on every hive-c0re startup. +pub async fn ensure_user_for(name: &str) -> Result<()> { + if !is_present().await { + return Ok(()); + } + ensure_user_exists(name, false, None).await?; + ensure_user_email(name).await; + mint_and_persist_agent_token(name).await +} + +/// Provision a forgejo user for `name` and return the freshly-minted +/// token. Unlike [`ensure_user_for`], the token is **not** persisted to +/// disk — the caller is responsible for storing it. Used by `hivectl +/// forge create-user` for human (non-agent) accounts so we don't create +/// stray `/var/lib/hyperhive/agents//` directories for users that +/// aren't agents. +/// +/// `password` picks the account password. `None` keeps the existing +/// random-throwaway shape (caller doesn't need web UI access — token +/// alone is enough). `Some(pw)` sets `pw` as the password, including +/// running `forgejo admin user change-password` if the account already +/// exists, so the operator can log into the forge web UI afterwards. +/// Idempotent: re-running with the same `Some(pw)` lands on the same +/// final state. +pub async fn provision_user_token(name: &str, password: Option<&str>) -> Result { + if !is_present().await { + anyhow::bail!( + "hive-forge container not running — wait for hive-c0re to start it before provisioning forge users" + ); + } + ensure_user_exists(name, false, password).await?; + if let Some(pw) = password { + // `user create` silently no-ops on an existing account, so + // we run change-password unconditionally when the caller + // asked for a specific password — keeps the verb idempotent + // for "set or reset" use. + change_user_password(name, pw).await?; + } + ensure_user_email(name).await; + mint_token(name, TOKEN_SCOPES).await +} + +/// Set `core`'s Forgejo avatar to the hyperhive logo once, then +/// remember it so subsequent startups don't re-upload. Best-effort +/// — any non-2xx is logged at the caller; the project runs fine +/// with the default hash identicon. +pub(super) async fn ensure_core_avatar(token: &str) -> Result<()> { + let marker = crate::paths::forge_core_avatar_marker(); + if marker.exists() { + return Ok(()); + } + let png_path = hive_sh4re::assets::core_avatar_png(); + let png_bytes = tokio::fs::read(&png_path) + .await + .with_context(|| format!("read core avatar PNG from {}", png_path.display()))?; + let body = format!( + r#"{{"image":"{}"}}"#, + base64::engine::general_purpose::STANDARD.encode(&png_bytes), + ); + let url = format!("{FORGE_HTTP}/api/v1/admin/users/core/avatar"); + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; + if !status.is_success() { + anyhow::bail!("set core avatar: HTTP {status}"); + } + if let Some(parent) = marker.parent() { + std::fs::create_dir_all(parent).ok(); + } + std::fs::write(marker, "").ok(); + tracing::info!("forge: set core user avatar to hyperhive logo"); + Ok(()) +} + +/// Set the `agent-configs` org's Forgejo avatar to the +/// configs-stack glyph once. Sibling to `ensure_core_avatar`: +/// one-shot, marker-guarded, best-effort. Forgejo's per-org avatar +/// endpoint is `POST /api/v1/orgs/{org}/avatar` with a base64-PNG +/// JSON body — same shape as the admin user endpoint above. +pub(super) async fn ensure_config_org_avatar(token: &str) -> Result<()> { + let marker = crate::paths::forge_config_org_avatar_marker(); + if marker.exists() { + return Ok(()); + } + let png_path = hive_sh4re::assets::config_org_avatar_png(); + let png_bytes = tokio::fs::read(&png_path) + .await + .with_context(|| format!("read {CONFIG_ORG} avatar PNG from {}", png_path.display()))?; + let body = format!( + r#"{{"image":"{}"}}"#, + base64::engine::general_purpose::STANDARD.encode(&png_bytes), + ); + let url = format!("{FORGE_HTTP}/api/v1/orgs/{CONFIG_ORG}/avatar"); + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; + if !status.is_success() { + anyhow::bail!("set {CONFIG_ORG} avatar: HTTP {status}"); + } + if let Some(parent) = marker.parent() { + std::fs::create_dir_all(parent).ok(); + } + std::fs::write(marker, "").ok(); + tracing::info!( + org = CONFIG_ORG, + "forge: set org avatar to configs-stack logo" + ); + Ok(()) +} + +/// Outcome of probing whether the persisted core token still works +/// against the *current* forge. Existence on disk is not validity: a +/// token minted before a forge rebuild / re-provision is unknown to the +/// new forge's DB and 401s on every call — which silently breaks the +/// hive-ci runner-registration prefetch (it reads this same token to +/// fetch a runner registration token). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CoreTokenCheck { + /// Token authenticated successfully — keep using it. + Valid, + /// Forge explicitly rejected the token (401/403) — re-mint. + Invalid, + /// Couldn't determine (forge unreachable / 5xx). Don't re-mint on a + /// transient: keep the existing token and let a later ensure pass + /// re-check once the forge is responsive. Re-minting here would both + /// fail (mint needs the forge too) and churn tokens needlessly. + Indeterminate, +} + +/// Map the HTTP status of the token-probe call to a [`CoreTokenCheck`]. +/// Pure so the decision logic is unit-testable without a live forge. +fn classify_core_token_status(status: StatusCode) -> CoreTokenCheck { + if status.is_success() { + CoreTokenCheck::Valid + } else if status == StatusCode::UNAUTHORIZED || status == StatusCode::FORBIDDEN { + CoreTokenCheck::Invalid + } else { + CoreTokenCheck::Indeterminate + } +} + +/// Probe whether `token` is still accepted by the current forge with a +/// cheap authenticated `GET /api/v1/user` (covered by the core token's +/// `read:user` scope). See [`CoreTokenCheck`] for how the outcome is +/// interpreted. +async fn check_core_token(token: &str) -> CoreTokenCheck { + let url = format!("{FORGE_HTTP}/api/v1/user"); + match forge_http(reqwest::Method::GET, &url, token, "").await { + Ok((status, _)) => classify_core_token_status(status), + Err(e) => { + tracing::debug!( + error = %e, + "forge: core-token probe could not reach forge; treating as indeterminate" + ); + CoreTokenCheck::Indeterminate + } + } +} + +/// Ensure the bootstrap `core` admin user + a token at +/// `CORE_TOKEN_PATH`. The token is what hive-c0re uses for forgejo +/// API calls (org creation, meta-repo push, and the hive-ci +/// runner-registration prefetch). Returns the token. +/// +/// Idempotent, but validity-aware: when a token file is already present +/// it is **probed against the current forge** before being trusted. A +/// token persisted before a forge rebuild / re-provision is stale (the +/// new forge DB doesn't know it) and would 401 every caller — so on a +/// definitive rejection the token is re-minted. A merely-unreachable +/// forge leaves the existing token in place (a later ensure pass +/// re-checks) rather than churning tokens on a transient. +pub(super) async fn ensure_core_user_and_token() -> Result { + let path = std::path::Path::new(CORE_TOKEN_PATH); + if let Ok(existing) = std::fs::read_to_string(path) { + let trimmed = existing.trim().to_owned(); + if !trimmed.is_empty() { + match check_core_token(&trimmed).await { + CoreTokenCheck::Valid | CoreTokenCheck::Indeterminate => return Ok(trimmed), + CoreTokenCheck::Invalid => { + tracing::warn!( + path = %path.display(), + "forge: persisted core token rejected by forge (stale after rebuild?); \ + re-minting" + ); + } + } + } + } + ensure_user_exists("core", true, None).await?; + mint_and_persist_core_token(path).await?; + let raw = std::fs::read_to_string(path) + .with_context(|| format!("read {CORE_TOKEN_PATH} after mint"))?; + Ok(raw.trim().to_owned()) +} + +/// Read the persisted core token, or None when the forge isn't +/// seeded yet. Cheap — just a file read. +pub fn core_token() -> Option { + std::fs::read_to_string(CORE_TOKEN_PATH) + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::{CoreTokenCheck, classify_core_token_status}; + use reqwest::StatusCode; + + #[test] + fn success_statuses_are_valid() { + assert_eq!( + classify_core_token_status(StatusCode::OK), + CoreTokenCheck::Valid + ); + assert_eq!( + classify_core_token_status(StatusCode::NO_CONTENT), + CoreTokenCheck::Valid + ); + } + + #[test] + fn auth_rejection_statuses_are_invalid() { + // The whole point: a stale token (forge rebuilt out from under it) + // 401s, and 401/403 are the only outcomes that trigger a re-mint. + assert_eq!( + classify_core_token_status(StatusCode::UNAUTHORIZED), + CoreTokenCheck::Invalid + ); + assert_eq!( + classify_core_token_status(StatusCode::FORBIDDEN), + CoreTokenCheck::Invalid + ); + } + + #[test] + fn transient_and_unexpected_statuses_are_indeterminate() { + // Never re-mint on a transient — minting needs the forge too, and + // churning tokens on a blip is worse than keeping the existing one. + for s in [ + StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::BAD_GATEWAY, + StatusCode::SERVICE_UNAVAILABLE, + StatusCode::GATEWAY_TIMEOUT, + StatusCode::NOT_FOUND, + ] { + assert_eq!( + classify_core_token_status(s), + CoreTokenCheck::Indeterminate, + "status {s} should be indeterminate" + ); + } + } +} diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs new file mode 100644 index 00000000..93cb974f --- /dev/null +++ b/hive-c0re/src/job_queue/exec.rs @@ -0,0 +1,491 @@ +//! Node executors — one async fn per [`NodeKind`], each a thin wrapper +//! over existing `lifecycle.rs` / `meta.rs` / `actions.rs` code. Node +//! executors keep their own internal error handling where it exists +//! today (cold-start fallback inside `Reconcile`, non-fatal boot-time +//! lock bump inside the sweep `MetaLock`, warn-only forge sync in the +//! `Swap` tail); DAG-level failure handling is cancel-downstream in +//! the queue. + +use std::sync::Arc; + +use anyhow::{Context as _, Result}; + +use super::model::{NodeKind, State, Template}; +use super::{Claim, TerminalDag}; +use crate::coordinator::Coordinator; +use crate::power::{ReconcileAction, reconcile_action}; + +/// Max time `Drain` waits for the harness to run its stop-checkpoint +/// turn before falling back to the hard stop. Generous — a checkpoint +/// turn can take a while — but bounded so a wedged agent never blocks +/// the stop indefinitely. Drains hold no build slot, so a whole-hive +/// graceful stop overlaps every agent's drain instead of serialising +/// N × this timeout. +pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3); + +/// Extra signal an executor hands back to the scheduler alongside +/// success. +#[derive(Debug, Default)] +pub struct NodeOutput { + /// Agents to fan child `Rebuild` DAGs out for (`MetaLock` only). + pub fanout: Vec, +} + +/// Step-label + build-log sink for one claimed node. +struct Ctx<'a> { + coord: &'a Arc, + dag_id: u64, + node_id: super::NodeId, +} + +impl Ctx<'_> { + fn step(&self, step: &str) { + if self + .coord + .job_queue + .set_step(self.dag_id, self.node_id, step) + { + self.coord.emit_rebuild_queue_snapshot(); + } + } + + fn build_log(&self, log_id: i64) { + if self + .coord + .job_queue + .set_build_log_id(self.dag_id, self.node_id, log_id) + { + self.coord.emit_rebuild_queue_snapshot(); + } + } +} + +/// Run one claimed node to completion. Called from a task the +/// scheduler spawns per claim; the `Result` (stringified) becomes the +/// node's terminal state. +pub(super) async fn run_node(coord: &Arc, claim: &Claim) -> Result { + let ctx = Ctx { + coord, + dag_id: claim.dag_id, + node_id: claim.node_id, + }; + match &claim.kind { + NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await, + NodeKind::Swap => run_swap(coord, claim, &ctx).await, + NodeKind::Create => run_create(coord, claim, &ctx).await, + NodeKind::MetaLock { sweep, fanout } => { + run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await + } + NodeKind::Reconcile => run_reconcile(coord, claim, &ctx).await, + NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await, + NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)), + NodeKind::Drain => run_drain(coord, claim, &ctx).await, + NodeKind::WriteDropin => run_write_dropin(coord, claim).await, + NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await, + NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await, + } +} + +/// Out-of-band toplevel build while the container keeps serving: meta +/// sync + optional per-agent relock, then warm +/// `system.build.toplevel` so the later `Swap` hits cache and skips +/// straight to the profile-swap. +async fn run_prebuild( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, + relock: bool, +) -> Result { + let name = &claim.agent; + let agent_dir = coord + .ensure_runtime(name) + .with_context(|| format!("ensure_runtime {name}"))?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + crate::lifecycle::prepare_rebuild_dirs(name, &paths).await?; + // Idempotent meta sync so a manual rebuild can also recover from a + // divergent meta repo; then bump just this agent's input. `relock = + // false` only for meta-update cascade children, where re-locking + // would revert the bump the cascade just committed. Both run under + // the deploy-window gate so they can never land inside another + // node's staged prepare→finalize window; the gate drops before the + // (long) toplevel build, which only reads the store. + { + let _window = crate::meta::exclusive().await; + let agents = crate::lifecycle::agents_for_meta_listing().await?; + crate::meta::sync_agents(&hive, &agents).await?; + if relock { + crate::meta::lock_update_for_rebuild(name).await?; + } + } + ctx.step("nix build"); + let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display()); + crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)).await?; + Ok(NodeOutput::default()) +} + +/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), +/// `nixos-container update`, then the post-rebuild bookkeeping tail +/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan). +/// The recovery-start on failure is NOT here — the DAG's tail +/// `Reconcile` runs after this node terminal ok *or* fail. +async fn run_swap(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; + let agent_dir = coord.ensure_runtime(name)?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + let result = + crate::lifecycle::swap_update(name, &hive, &paths, &|step| ctx.step(step), &|log_id| { + ctx.build_log(log_id); + }) + .await; + match &result { + Ok(()) => { + if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) + && let Err(e) = std::fs::write(crate::auto_update::rev_marker_path(name), rev) + { + tracing::warn!(%name, error = ?e, "write rev marker failed"); + } + // The `Rebuilt` manager event fires exactly once per DAG + // from the terminal hook — emitting ok here and letting a + // failed tail `Reconcile` add a contradictory !ok would + // double-report the same rebuild. + ctx.step("forge sync"); + // Full forge + matrix sync on every successful rebuild so + // the rebuild path is equivalent to the startup sweep: + // tokens, config-repo mirror, meta access all recover + // without a hive-c0re restart. + crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; + crate::matrix::sync_agent_standalone(name).await; + // Wake the agent on its next turn so claude sees a "you + // were rebuilt" hint; rescan so dashboards drop the + // "needs update" chip; lock bump → meta-inputs re-render. + coord.kick_agent(name, "container rebuilt"); + coord.rescan_containers_and_emit().await; + crate::dashboard::emit_meta_inputs_snapshot(coord); + } + Err(_) => { + // The `Rebuilt { ok: false }` manager event fires once per + // DAG from the terminal hook (any node may be the one that + // failed); here only refresh the observed state. + coord.rescan_containers_and_emit().await; + } + } + result.map(|()| NodeOutput::default()) +} + +/// First-spawn provisioning + `nixos-container create` (atomic +/// build+create — no prebuild needed). +async fn run_create(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; + let agent_dir = coord.ensure_runtime(name)?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + ctx.step("nixos-container create"); + // create_container registers the new agent in the meta flake + // (sync_agents commit) before `nixos-container create` — hold the + // deploy-window gate so that commit can't land inside another + // node's staged deploy window. + let _window = crate::meta::exclusive().await; + crate::lifecycle::create_container(name, &hive, &paths).await?; + Ok(NodeOutput::default()) +} + +/// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed +/// bump must not cancel the fan-out rebuilds — they proceed against +/// the current lock, exactly like today's sweep); the meta-update +/// flavour propagates errors, and a failed bump fans out nothing. +async fn run_meta_lock( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, + sweep: bool, + fanout: Option>, +) -> Result { + if sweep { + ctx.step("nix flake update hyperhive"); + let _window = crate::meta::exclusive().await; + if let Err(e) = crate::meta::lock_update_hyperhive().await { + tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed"); + } + return Ok(NodeOutput { + fanout: fanout.unwrap_or_default(), + }); + } + let _progress = coord.meta_update_guard(); + ctx.step("nix flake update"); + { + let _window = crate::meta::exclusive().await; + crate::meta::lock_update(&claim.inputs).await?; + } + // Lock file changed — meta-inputs panel re-renders. + crate::dashboard::emit_meta_inputs_snapshot(coord); + let cascade = match fanout { + Some(list) => list, + None => meta_update_cascade_agents(&claim.inputs).await, + }; + Ok(NodeOutput { fanout: cascade }) +} + +/// Idempotent power converge: `wanted` (durable intent) vs observed. +async fn run_reconcile( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + let name = &claim.agent; + let running = crate::lifecycle::is_running(name).await; + let wanted = coord.power.get_or_seed(name, running)?; + match reconcile_action(wanted, running) { + ReconcileAction::Start => { + // Node-local transient only when the DAG holds none (the + // boot-reconcile template); lease-window guards otherwise + // already cover this node. + let _guard = claim + .transient + .is_none() + .then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting)); + ctx.step("nixos-container start"); + crate::lifecycle::start_with_fallback(name).await?; + coord.kick_agent(name, "container started"); + coord.rescan_containers_and_emit().await; + } + ReconcileAction::Stop => { + let _guard = claim + .transient + .is_none() + .then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping)); + ctx.step("nixos-container stop"); + crate::lifecycle::kill(name).await?; + coord.unregister_agent(name); + coord.notify_manager(&hive_sh4re::HelperEvent::Killed { + agent: name.clone(), + }); + coord.rescan_containers_and_emit().await; + } + ReconcileAction::Noop => { + tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop"); + } + } + Ok(NodeOutput::default()) +} + +/// Mechanical stop for the profile swap. Never *changes* `wanted`; +/// noop when already stopped. +async fn run_stop_for_update( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + let name = &claim.agent; + if crate::lifecycle::is_running(name).await { + // Seed a missing agent_power row from the PRE-stop observation + // — the DAG's tail `Reconcile` observes only the mechanically + // stopped state and would otherwise seed a running-but-unknown + // agent as `Offline`, stranding it down after its own rebuild. + if let Err(e) = coord.power.get_or_seed(name, true) { + tracing::warn!(%name, error = ?e, "agent_power: pre-stop seed failed"); + } + ctx.step("nixos-container stop"); + crate::lifecycle::kill(name).await?; + coord.rescan_containers_and_emit().await; + } + Ok(NodeOutput::default()) +} + +/// Set the graceful fence + kick so the harness sees it promptly and +/// runs its one stop-checkpoint turn. +fn run_signal(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> NodeOutput { + ctx.step("graceful stop: signalling agent"); + coord.mark_graceful_stop(&claim.agent); + coord.kick_agent(&claim.agent, "graceful stop requested"); + NodeOutput::default() +} + +/// Await the harness clearing the fence (`GracefulStopComplete`) or +/// the timeout — either way the downstream `Reconcile` proceeds with +/// the actual stop. +async fn run_drain(coord: &Arc, claim: &Claim, ctx: &Ctx<'_>) -> Result { + let name = &claim.agent; + ctx.step("graceful stop: draining"); + let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT; + while coord.is_graceful_stop_pending(name) { + if std::time::Instant::now() >= deadline { + tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping"); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + coord.clear_graceful_stop(name); + Ok(NodeOutput::default()) +} + +/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload. +async fn run_write_dropin(coord: &Arc, claim: &Claim) -> Result { + let name = &claim.agent; + let agent_dir = coord.ensure_runtime(name)?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name, agent_dir); + crate::lifecycle::write_dropins(name, &hive, &paths).await?; + Ok(NodeOutput::default()) +} + +/// Write + commit the perm file(s) (fused under `META_LOCK` so the +/// working tree is never left dirty), then emit the P3RM1SS10NS-tab +/// snapshots so the dashboard reflects the new assignment. +async fn run_write_perm_file( + coord: &Arc, + claim: &Claim, + ctx: &Ctx<'_>, +) -> Result { + use super::model::PermPayload; + let name = &claim.agent; + ctx.step("writing + committing perm file"); + // Deploy-window gate: a perm commit landing inside another node's + // staged prepare→finalize window would sweep the staged deploy + // lock into its commit (the commits are also path-limited in + // meta.rs — belt and braces). + let _window = crate::meta::exclusive().await; + match &claim.perm_payload { + Some(PermPayload::ToolGroups { groups }) => { + crate::meta::commit_tool_groups(name, groups) + .await + .with_context(|| format!("commit tool-groups for {name}"))?; + coord.emit_tool_groups_snapshot(); + } + Some(PermPayload::Capabilities { caps }) => { + crate::meta::commit_capabilities(name, caps) + .await + .with_context(|| format!("commit capabilities for {name}"))?; + coord.emit_capabilities_snapshot(); + } + Some(PermPayload::Combined { groups, caps }) => { + crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref()) + .await + .with_context(|| format!("commit perms for {name}"))?; + if groups.is_some() { + coord.emit_tool_groups_snapshot(); + } + if caps.is_some() { + coord.emit_capabilities_snapshot(); + } + } + None => anyhow::bail!( + "perm_change dag {} for {name} is missing perm_payload", + claim.dag_id + ), + } + Ok(NodeOutput::default()) +} + +/// Opaque approval deploy pipeline: `ApplyCommit` and `MergeConfigPr` +/// both end in a container rebuild; branch on the approval row's kind +/// (the authoritative source). The two-phase prepare/finalize/abort +/// meta deploy — and the approval resolution — stay inside +/// `actions.rs` in v1 (design doc §9). +async fn run_approval_deploy(coord: &Arc, claim: &Claim) -> Result { + let approval_id = claim + .approval_id + .with_context(|| format!("approval_deploy dag {} has no approval_id", claim.dag_id))?; + // Hold the deploy-window gate for the whole prepare→finalize span: + // `prepare_deploy` stages `flake.lock` uncommitted for the entire + // container build, and no other meta mutation may land inside that + // window (it would sweep the staged lock and neuter `abort_deploy`). + let _window = crate::meta::exclusive().await; + let kind = coord + .approvals + .get(approval_id) + .ok() + .flatten() + .map(|a| a.kind); + let result = if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) { + crate::actions::run_approval_merge_config_pr(coord, Some(claim.dag_id), approval_id).await + } else { + crate::actions::run_approval_apply_commit(coord, Some(claim.dag_id), approval_id).await + }; + result.map(|()| NodeOutput::default()) +} + +/// Terminal-roll-up hook, fired exactly once per DAG (node completion +/// and cancel paths alike — the queue buffers roll-ups and the +/// scheduler drains them). Three concerns: +/// - approval DAGs resolve their approval row (except the opaque +/// deploy pipeline, which resolves inside its node — unless it was +/// cancelled while still queued and the node never ran); +/// - non-approval rebuild-shaped DAGs emit exactly one `Rebuilt` +/// manager event: ok on `Done`, !ok on `Failed`, none on cancel; +/// - a cancelled power-op DAG reverts the `wanted` intent its submit +/// wrote: the operator's cancel means "don't do it", so intent +/// snaps back to the observed state instead of the flip executing +/// as a surprise side effect of some later reconcile. +pub(super) async fn on_dag_terminal(coord: &Arc, terminal: &TerminalDag) { + if terminal.state == State::Cancelled + && matches!( + terminal.template, + Template::Start | Template::Stop | Template::GracefulStop | Template::Restart + ) + { + let running = crate::lifecycle::is_running(&terminal.agent).await; + if let Err(e) = coord + .power + .set(&terminal.agent, crate::power::Wanted::from_running(running)) + { + tracing::warn!(agent = %terminal.agent, error = ?e, "agent_power: cancel revert failed"); + } + } + if terminal.approval_id.is_some() { + crate::actions::resolve_approval_dag(coord, terminal).await; + return; + } + if matches!(terminal.template, Template::Rebuild | Template::PermChange) { + match terminal.state { + State::Done => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: terminal.agent.clone(), + ok: true, + note: None, + sha: None, + tag: None, + }), + State::Failed => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { + agent: terminal.agent.clone(), + ok: false, + note: terminal.error.clone(), + sha: None, + tag: None, + }), + _ => {} + } + } +} + +/// Compute which agents a `nix flake update ` on the meta +/// flake affects — the fan-out set for `MetaUpdate` DAGs. Empty +/// `inputs` or any input under `hyperhive` → every container; +/// otherwise just the agents named by `agent-` inputs. +/// Topology-sorted so parents rebuild before their children. +pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec { + let touched_hyperhive = inputs + .iter() + .any(|i| i == "hyperhive" || i.starts_with("hyperhive/")); + let touched_agents: Vec = inputs + .iter() + .filter_map(|i| i.strip_prefix("agent-")) + .map(|rest| rest.split('/').next().unwrap_or(rest).to_owned()) + .collect(); + let mut names = if touched_hyperhive || inputs.is_empty() { + crate::lifecycle::list() + .await + .unwrap_or_default() + .into_iter() + .filter_map(|c| { + c.strip_prefix(crate::lifecycle::AGENT_PREFIX) + .map(str::to_owned) + }) + .collect() + } else { + touched_agents + }; + let topo = crate::topology::read(); + crate::auto_update::topology_sort(&mut names, &topo); + names +} diff --git a/hive-c0re/src/job_queue/mod.rs b/hive-c0re/src/job_queue/mod.rs new file mode 100644 index 00000000..bfb118b2 --- /dev/null +++ b/hive-c0re/src/job_queue/mod.rs @@ -0,0 +1,592 @@ +//! Generic job-DAG queue + desired-state reconciliation — replaces the +//! old flat `rebuild_queue`. Jobs are nodes in per-request DAGs (see +//! [`templates`]); the special cases (graceful-stop watcher thread, +//! deferred-start follow-up, meta-update cascade) collapse into DAG +//! *shapes* over a shared set of primitive nodes ([`model::NodeKind`]). +//! +//! Concurrency is gated by two resource classes: +//! 1. **Build slots** — N permits (`services.hyperhive.c0re.buildSlots`, +//! default 1) held by nix-heavy nodes for the node's duration. +//! 2. **Per-agent lifecycle lease** — DAG-scoped: acquired before the +//! DAG's first container-affecting node runs, held until the DAG is +//! terminal, so two lifecycle DAGs for one agent never interleave +//! their container ops. +//! +//! The meta *repo* is serialized by `meta::META_LOCK` inside the +//! executors themselves. Per-agent power *intent* (`wanted`) lives in +//! the durable [`crate::power`] store; the DAGs are the reconcile +//! mechanism. Design + rationale: `docs/coordinator.md::Job queue`. + +pub mod exec; +pub mod model; +pub mod scheduler; +pub mod submit; +pub mod templates; +#[cfg(test)] +mod tests; + +use std::collections::{HashMap, VecDeque}; +use std::sync::Mutex; + +use hive_sh4re::wire_time::now_unix; +use tokio::sync::Notify; + +pub use model::{ + Dag, DagSpec, DagView, DepWhen, Node, NodeId, NodeKind, PermPayload, Source, State, Template, +}; + +/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain +/// per template in the snapshot, matching the old per-kind history cap. +const MAX_HISTORY_PER_TEMPLATE: usize = 5; + +/// Terminal DAGs younger than this are exempt from the per-template +/// history cap. A broad `hivectl stop`/`start` submits many +/// same-template DAGs that can all settle within one poll interval — +/// without the grace, the cap would evict some before the ~1s +/// `QueueDag` poller ever observes their terminal state, silently +/// swallowing failures. +const HISTORY_GRACE_SECS: i64 = 300; + +/// Cap on stored node error strings. +const MAX_ERROR_LEN: usize = 2_000; + +/// A node claimed for execution — everything the executor needs, +/// snapshotted at claim time. +#[derive(Debug, Clone)] +pub struct Claim { + pub dag_id: u64, + pub node_id: NodeId, + pub kind: NodeKind, + pub agent: String, + pub template: Template, + pub source: Source, + pub approval_id: Option, + pub inputs: Vec, + pub perm_payload: Option, + /// True when claiming this node acquired the DAG's agent lease — + /// the scheduler creates the DAG-scoped transient guard on this + /// edge. + pub lease_acquired: bool, + /// Transient pill kind for the lease window (from the spec). + pub transient: Option, +} + +/// Summary of a DAG that just reached its terminal roll-up state — +/// input to the approval-resolution hook and the lease/transient +/// release. +#[derive(Debug, Clone)] +pub struct TerminalDag { + pub dag_id: u64, + pub template: Template, + pub agent: String, + pub approval_id: Option, + pub state: State, + /// First failed node's error when `state == Failed`. + pub error: Option, +} + +#[derive(Debug, Default)] +struct Inner { + dags: VecDeque, + next_id: u64, + build_slots: usize, + slots_used: usize, + /// agent → dag id currently holding that agent's lifecycle lease. + leases: HashMap, + /// Terminal roll-ups not yet consumed by the scheduler + /// ([`JobQueue::drain_terminal`]). Fed by every path that settles + /// state — node completion AND the cancel surfaces — so the + /// terminal hooks (approval resolution, intent revert, transient + /// release) fire exactly once per DAG no matter how it ended. + pending_terminal: Vec, +} + +/// The queue. Lives on `Coordinator` (one per hive-c0re process); a +/// single scheduler task ([`scheduler::run_worker`]) drives it — +/// concurrency comes from the build-slot count, not multiple workers. +#[derive(Debug)] +pub struct JobQueue { + inner: Mutex, + /// Wakes the scheduler when something new arrives or state changed. + pub(crate) notify: Notify, +} + +impl Default for JobQueue { + fn default() -> Self { + Self::new(1) + } +} + +impl JobQueue { + pub fn new(build_slots: usize) -> Self { + Self { + inner: Mutex::new(Inner { + build_slots: build_slots.max(1), + ..Inner::default() + }), + notify: Notify::new(), + } + } + + /// Submit a DAG. Validates the spec (cycle rejection) and dedups + /// against non-started DAGs; returns the DAG id (newly-allocated, + /// or the existing DAG's id with the new reason appended). + /// + /// Dedup: a DAG whose roll-up is still `Queued` (no node started) + /// with the same `(template, agent, parent_id, approval_id)` — plus + /// `inputs` for `MetaUpdate` and the perm-type discriminant for + /// `PermChange` — swallows the repeat. `parent_id` is part of the + /// key so a meta-update cascade rebuild never collapses into a + /// standalone or sweep rebuild. Running / terminal DAGs never + /// dedup — operators are free to re-queue. + pub fn submit(&self, spec: DagSpec) -> anyhow::Result { + templates::validate(&spec)?; + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + if let Some(existing) = Self::dedup_target(&mut inner, &spec) { + if !existing.reason.contains(&spec.reason) { + use std::fmt::Write as _; + let _ = write!(existing.reason, "\nalso requested by: {}", spec.reason); + } + return Ok(existing.id); + } + let id = Self::push_dag(&mut inner, spec); + drop(inner); + self.notify.notify_one(); + Ok(id) + } + + /// Append fan-out children under a parent DAG (meta-update / sweep + /// cascade). Applies the same dedup as [`Self::submit`]; returns + /// the child ids actually created or coalesced into. + pub fn append_children(&self, specs: Vec) -> Vec { + let mut ids = Vec::with_capacity(specs.len()); + for spec in specs { + match self.submit(spec) { + Ok(id) => ids.push(id), + Err(e) => tracing::error!(error = ?e, "job_queue: invalid fan-out child spec"), + } + } + ids + } + + fn dedup_target<'a>(inner: &'a mut Inner, spec: &DagSpec) -> Option<&'a mut Dag> { + inner.dags.iter_mut().find(|d| { + d.rollup() == State::Queued + && d.template == spec.template + && d.agent == spec.agent + && d.parent_id == spec.parent_id + && d.approval_id == spec.approval_id + && (d.template != Template::MetaUpdate || d.inputs == spec.inputs) + && model::perm_payload_same_type( + d.perm_payload.as_ref(), + spec.perm_payload.as_ref(), + ) + }) + } + + fn push_dag(inner: &mut Inner, spec: DagSpec) -> u64 { + inner.next_id += 1; + let id = inner.next_id; + let nodes = spec + .nodes + .into_iter() + .enumerate() + .map(|(i, n)| Node { + id: u32::try_from(i).unwrap_or(u32::MAX), + kind: n.kind, + deps: n.deps, + state: State::Queued, + step: None, + build_log_id: None, + started_at: None, + finished_at: None, + error: None, + }) + .collect(); + inner.dags.push_back(Dag { + id, + template: spec.template, + agent: spec.agent, + source: spec.source, + reason: spec.reason, + parent_id: spec.parent_id, + approval_id: spec.approval_id, + inputs: spec.inputs, + perm_payload: spec.perm_payload, + transient: spec.transient, + created_at: now_unix(), + nodes, + terminal_reported: false, + }); + id + } + + /// Claim every currently-ready node, acquiring resources, and mark + /// them `Running`. A node is ready when it's `Queued`, every dep is + /// satisfied (`AfterOk`: dep `Done`; `AfterAny`: dep terminal), and + /// its resources are free (build slot; agent lease free or already + /// held by this DAG). Iteration is in DAG-submit order, so + /// simultaneously-ready nodes compete FIFO — bulk operations drain + /// predictably. + pub fn claim_ready(&self) -> Vec { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + Self::propagate_cancellations(&mut inner); + let mut claims = Vec::new(); + let inner = &mut *inner; + for di in 0..inner.dags.len() { + // Split-borrow dance: deps are checked against the same + // DAG's other nodes, so snapshot the states first. + let dag = &inner.dags[di]; + let dag_id = dag.id; + let ready_ids: Vec = dag + .nodes + .iter() + .filter(|n| n.state == State::Queued && Self::deps_satisfied(dag, n)) + .map(|n| n.id) + .collect(); + for node_id in ready_ids { + let dag = &inner.dags[di]; + let node = dag.node(node_id).expect("node id from same dag"); + let needs_slot = node.kind.needs_build_slot(); + if needs_slot && inner.slots_used >= inner.build_slots { + continue; + } + let mut lease_acquired = false; + if node.kind.needs_lease() { + match inner.leases.get(dag.agent.as_str()) { + Some(&holder) if holder != dag_id => continue, + Some(_) => {} + None => { + inner.leases.insert(dag.agent.clone(), dag_id); + lease_acquired = true; + } + } + } + if needs_slot { + inner.slots_used += 1; + } + let dag = &mut inner.dags[di]; + let claim = Claim { + dag_id, + node_id, + kind: dag.node(node_id).expect("node").kind.clone(), + agent: dag.agent.clone(), + template: dag.template, + source: dag.source, + approval_id: dag.approval_id, + inputs: dag.inputs.clone(), + perm_payload: dag.perm_payload.clone(), + lease_acquired, + transient: dag.transient, + }; + let node = dag.node_mut(node_id).expect("node"); + node.state = State::Running; + node.started_at = Some(now_unix()); + claims.push(claim); + } + } + claims + } + + fn deps_satisfied(dag: &Dag, node: &Node) -> bool { + node.deps.iter().all(|dep| { + dag.node(dep.on).is_some_and(|d| match dep.when { + DepWhen::AfterOk => d.state == State::Done, + DepWhen::AfterAny => d.state.is_terminal(), + }) + }) + } + + /// Cancel-downstream: a `Queued` node with an `AfterOk` dep that + /// `Failed` / `Cancelled` becomes `Cancelled` itself. Loops to a + /// fixpoint so the cancellation cascades through chains. + fn propagate_cancellations(inner: &mut Inner) { + for dag in &mut inner.dags { + loop { + let doomed: Vec = dag + .nodes + .iter() + .filter(|n| { + n.state == State::Queued + && n.deps.iter().any(|dep| { + dep.when == DepWhen::AfterOk + && dag.node(dep.on).is_some_and(|d| { + matches!(d.state, State::Failed | State::Cancelled) + }) + }) + }) + .map(|n| n.id) + .collect(); + if doomed.is_empty() { + break; + } + let now = now_unix(); + for id in doomed { + if let Some(n) = dag.node_mut(id) { + n.state = State::Cancelled; + n.finished_at = Some(now); + } + } + } + } + } + + /// Mark a claimed node terminal, release its build slot, cascade + /// cancellations, and settle terminal DAGs (lease release + history + /// trim; the terminal roll-up lands in the [`Self::drain_terminal`] + /// buffer). `error` is stored (truncated) when `result` is `Err`. + pub fn complete_node(&self, dag_id: u64, node_id: NodeId, result: Result<(), String>) { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) + && let Some(node) = dag.node_mut(node_id) + && node.state == State::Running + { + let needs_slot = node.kind.needs_build_slot(); + node.finished_at = Some(now_unix()); + node.step = None; + match result { + Ok(()) => node.state = State::Done, + Err(e) => { + node.state = State::Failed; + let mut msg = e; + if msg.len() > MAX_ERROR_LEN { + msg.truncate( + (0..=MAX_ERROR_LEN) + .rev() + .find(|i| msg.is_char_boundary(*i)) + .unwrap_or(0), + ); + msg.push('…'); + } + node.error = Some(msg); + } + } + if needs_slot { + inner.slots_used = inner.slots_used.saturating_sub(1); + } + } + Self::settle(&mut inner); + drop(inner); + self.notify.notify_one(); + } + + /// Take the terminal roll-ups accumulated since the last drain. + /// The scheduler calls this after every wakeup and runs the + /// terminal hooks on each entry. + pub fn drain_terminal(&self) -> Vec { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + std::mem::take(&mut inner.pending_terminal) + } + + /// Propagate cancellations, release the leases of newly-terminal + /// DAGs, buffer each terminal roll-up exactly once (the + /// `terminal_reported` flag) for [`Self::drain_terminal`], and trim + /// history. + fn settle(inner: &mut Inner) { + Self::propagate_cancellations(inner); + let mut freed: Vec = Vec::new(); + let mut reports: Vec = Vec::new(); + for dag in &mut inner.dags { + if !dag.is_terminal() || dag.terminal_reported { + continue; + } + dag.terminal_reported = true; + if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) { + freed.push(dag.agent.clone()); + } + reports.push(TerminalDag { + dag_id: dag.id, + template: dag.template, + agent: dag.agent.clone(), + approval_id: dag.approval_id, + state: dag.rollup(), + error: dag.first_error().map(str::to_owned), + }); + } + inner.pending_terminal.append(&mut reports); + for agent in freed { + inner.leases.remove(&agent); + } + Self::trim_history(inner, now_unix() - HISTORY_GRACE_SECS); + } + + /// Cancel a DAG that hasn't started yet (roll-up `Queued`): every + /// node flips to `Cancelled`. No-op (returns `false`) once any node + /// is running or terminal — an in-flight nix build isn't + /// interruptible, matching the old queue's rule. + pub fn cancel(&self, dag_id: u64) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else { + return false; + }; + if dag.rollup() != State::Queued { + return false; + } + let now = now_unix(); + for n in &mut dag.nodes { + n.state = State::Cancelled; + n.finished_at = Some(now); + } + // Settle buffers the terminal roll-up; the notify wakes the + // scheduler, which drains it and fires the terminal hooks + // (approval resolution, power-intent revert). + Self::settle(&mut inner); + drop(inner); + self.notify.notify_one(); + true + } + + /// Cancel every still-fully-queued child DAG of `parent`. Running + /// children are left alone. Returns the count of cancelled DAGs. + pub fn cancel_children(&self, parent: u64) -> usize { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let now = now_unix(); + let mut count = 0; + for dag in &mut inner.dags { + if dag.parent_id == Some(parent) && dag.rollup() == State::Queued { + for n in &mut dag.nodes { + n.state = State::Cancelled; + n.finished_at = Some(now); + } + count += 1; + } + } + if count > 0 { + Self::settle(&mut inner); + drop(inner); + self.notify.notify_one(); + } + count + } + + /// Set the step label on a `Running` node. Returns `true` when the + /// label actually changed (callers emit a snapshot only then). + pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.node_mut(node_id)) + else { + return false; + }; + if node.state != State::Running || node.step.as_deref() == Some(step) { + return false; + } + node.step = Some(step.to_owned()); + true + } + + /// Set the step label on the DAG's currently-running node — + /// compatibility surface for the opaque approval pipeline, whose + /// callbacks only know the DAG id. Single-node approval DAGs make + /// this exact. + pub fn set_step_running(&self, dag_id: u64, step: &str) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running)) + else { + return false; + }; + if node.step.as_deref() == Some(step) { + return false; + } + node.step = Some(step.to_owned()); + true + } + + /// Link a `build_logs` row to a specific `Running` node. + pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.node_mut(node_id)) + else { + return false; + }; + if node.state != State::Running { + return false; + } + node.build_log_id = Some(log_id); + true + } + + /// Link a `build_logs` row to the DAG's currently-running node — + /// DAG-id-only compatibility surface (approval pipeline callbacks). + pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + let Some(node) = inner + .dags + .iter_mut() + .find(|d| d.id == dag_id) + .and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running)) + else { + return false; + }; + node.build_log_id = Some(log_id); + true + } + + /// Snapshot every DAG for `/api/state` + `RebuildQueueChanged`. + pub fn snapshot(&self) -> Vec { + let inner = self.inner.lock().expect("job_queue mutex poisoned"); + inner.dags.iter().map(Dag::view).collect() + } + + /// Number of live (non-terminal) DAGs — used by tests and + /// diagnostics. + #[cfg(test)] + pub fn live_count(&self) -> usize { + let inner = self.inner.lock().expect("job_queue mutex poisoned"); + inner.dags.iter().filter(|d| !d.is_terminal()).count() + } + + /// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs + /// per template. Never evicted: live DAGs; terminal parents with + /// live children (a fan-out parent is terminal the moment its + /// `MetaLock` completes — evicting it while cascade rebuilds run + /// would orphan their dashboard group); and terminal DAGs that + /// finished after `grace_cutoff` (see [`HISTORY_GRACE_SECS`]). + fn trim_history(inner: &mut Inner, grace_cutoff: i64) { + let live_parents: std::collections::HashSet = inner + .dags + .iter() + .filter(|d| !d.is_terminal()) + .filter_map(|d| d.parent_id) + .collect(); + let mut counts: HashMap = HashMap::new(); + let kept: Vec = inner + .dags + .iter() + .rev() + .filter(|d| { + if !d.is_terminal() || live_parents.contains(&d.id) { + return true; + } + let finished = d.nodes.iter().filter_map(|n| n.finished_at).max(); + if finished.is_none_or(|t| t > grace_cutoff) { + return true; + } + let n = counts.entry(d.template).or_insert(0); + *n += 1; + *n <= MAX_HISTORY_PER_TEMPLATE + }) + .cloned() + .collect(); + inner.dags = kept.into_iter().rev().collect(); + } + + /// Test hook: trim with the grace window disabled, so eviction + /// behavior is assertable without aging real timestamps. + #[cfg(test)] + pub(crate) fn trim_ignoring_grace(&self) { + let mut inner = self.inner.lock().expect("job_queue mutex poisoned"); + Self::trim_history(&mut inner, i64::MAX); + } +} diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs new file mode 100644 index 00000000..c572e7ae --- /dev/null +++ b/hive-c0re/src/job_queue/model.rs @@ -0,0 +1,332 @@ +//! Data model for the generic job-DAG queue: node kinds (the primitive +//! operations), dependency edges, and the runtime `Dag` / `Node` store. +//! The serialized *views* — `DagView` / `NodeView` plus the `Template` +//! / `Source` / `State` / `PermPayload` wire enums — live in +//! `hive_sh4re::jobs` (wire types belong to the shared crate) and are +//! re-exported here for the queue's internal use. +//! +//! Two levels: the **DAG** is the unit of dedup / cancel / +//! approval-resolution and the dashboard group; the **node** is the +//! unit of scheduling / execution / build-log / step label. See +//! `docs/coordinator.md::Job queue` for the full design. + +pub use hive_sh4re::jobs::{DagView, NodeId, NodeView, PermPayload, Source, State, Template}; +use serde::Serialize; + +/// Dedup compares the perm *type*, not the value — a tool-groups +/// change and a capabilities change for the same agent are distinct +/// operations that must not collapse. +pub(super) fn perm_payload_same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool { + matches!( + (a, b), + ( + Some(PermPayload::ToolGroups { .. }), + Some(PermPayload::ToolGroups { .. }) + ) | ( + Some(PermPayload::Capabilities { .. }), + Some(PermPayload::Capabilities { .. }) + ) | ( + Some(PermPayload::Combined { .. }), + Some(PermPayload::Combined { .. }) + ) | (None, None) + ) +} + +/// When a dependency edge is considered satisfied. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DepWhen { + /// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this + /// node (cancel-downstream). + AfterOk, + /// Dep must merely reach a terminal state (ok *or* fail). Used only + /// by `rebuild`'s tail `Reconcile` so the recovery-start runs even + /// when `Swap` failed. + AfterAny, +} + +/// A dependency edge (intra-DAG only — cross-DAG ordering comes from +/// the per-agent lease + dedup, never from edges between DAGs). +#[derive(Debug, Clone, Copy, Serialize)] +pub struct Dep { + pub on: NodeId, + pub when: DepWhen, +} + +/// The primitive operations — each kind maps to one executor fn in +/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs` +/// code. Concurrency is gated by two resource classes (see +/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the +/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped +/// functions themselves, which is why there is no `GitCommit` node — +/// a standalone commit node would open a dirty-working-tree window +/// between nodes that the fused `meta.rs` ops deliberately close. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum NodeKind { + /// Out-of-band toplevel build while the container keeps serving: + /// meta `sync_agents`, optional per-agent relock, then + /// `lifecycle::prebuild_toplevel`. `relock = false` only for + /// meta-update cascade rebuilds (re-locking would revert the bump + /// the cascade just committed). + Prebuild { relock: bool }, + /// `nixos-container update` profile-swap (requires the container + /// stopped). Re-applies nspawn flags + resource limits first — + /// rebuild is the reconcile verb — and carries the post-rebuild + /// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan). + Swap, + /// First-spawn `nixos-container create` plus the pre-create + /// provisioning (proposed/applied repos, state subvolume, meta + /// registration). + Create, + /// Meta flake lock bump. `sweep = false`: `meta::lock_update` + /// (commit fused, under `META_LOCK`) with the DAG's `inputs`; + /// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a + /// failed boot-time bump must not cancel the fan-out rebuilds). + /// On success the scheduler appends child `Rebuild` DAGs: the + /// precomputed `fanout` list when present (boot sweep), else the + /// post-bump affected set (`meta_update_cascade_agents`). + MetaLock { + sweep: bool, + fanout: Option>, + }, + /// Idempotent power converge: read `wanted` + observed state; + /// start if `Up` & down (with cold-start fallback), stop if + /// `Offline` & up, else noop. + Reconcile, + /// Mechanical `nixos-container stop` for the profile swap. Never + /// touches `wanted`. Noop if already stopped. + StopForUpdate, + /// Set the graceful-stop fence + kick the harness so it runs one + /// stop-checkpoint turn. + Signal, + /// Await the harness clearing the fence, bounded by + /// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the + /// downstream `Reconcile` performs the actual stop. + Drain, + /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload. + WriteDropin, + /// Commit `tool-groups.json` / `capabilities.json` per the DAG's + /// `perm_payload` (commit fused under `META_LOCK`). + WritePermFile, + /// Opaque approval deploy pipeline (`ApplyCommit` / + /// `MergeConfigPr`): the two-phase prepare/finalize/abort meta + /// deploy stays inside `actions.rs` in v1 — deliberately not + /// modeled as scheduler nodes (see the design doc §9). + ApprovalDeploy, +} + +impl NodeKind { + /// Wire string for `NodeView.kind`. + pub fn as_str(&self) -> &'static str { + match self { + NodeKind::Prebuild { .. } => "prebuild", + NodeKind::Swap => "swap", + NodeKind::Create => "create", + NodeKind::MetaLock { .. } => "meta_lock", + NodeKind::Reconcile => "reconcile", + NodeKind::StopForUpdate => "stop_for_update", + NodeKind::Signal => "signal", + NodeKind::Drain => "drain", + NodeKind::WriteDropin => "write_dropin", + NodeKind::WritePermFile => "write_perm_file", + NodeKind::ApprovalDeploy => "approval_deploy", + } + } + + /// Nix-heavy kinds hold one of the `buildSlots` semaphore permits + /// for the node's duration. + pub fn needs_build_slot(&self) -> bool { + matches!( + self, + NodeKind::Prebuild { .. } + | NodeKind::Swap + | NodeKind::Create + | NodeKind::MetaLock { .. } + | NodeKind::ApprovalDeploy + ) + } + + /// Container-affecting kinds require the DAG to hold the agent's + /// lifecycle lease (acquired at the first such node, held until the + /// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`, + /// `WritePermFile`) touch the store / meta repo, not the running + /// container — which is exactly why a `Prebuild` can overlap + /// another DAG's work on the same agent. + pub fn needs_lease(&self) -> bool { + matches!( + self, + NodeKind::Swap + | NodeKind::Create + | NodeKind::Reconcile + | NodeKind::StopForUpdate + | NodeKind::Signal + | NodeKind::Drain + | NodeKind::WriteDropin + | NodeKind::ApprovalDeploy + ) + } +} + +/// One schedulable unit inside a DAG. +#[derive(Debug, Clone)] +pub struct Node { + pub id: NodeId, + pub kind: NodeKind, + pub deps: Vec, + pub state: State, + /// Live sub-label while `Running` (kept for parity with the old + /// per-entry `step`). + pub step: Option, + /// Row id of the `build_logs` entry this node opened (`Prebuild` / + /// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link. + pub build_log_id: Option, + pub started_at: Option, + pub finished_at: Option, + /// Populated when `state == Failed` (truncated by the queue). + pub error: Option, +} + +/// Submit-time spec for one node. +#[derive(Debug, Clone)] +pub struct NodeSpec { + pub kind: NodeKind, + pub deps: Vec, +} + +/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated +/// (cycle rejection) and dedup'd by `JobQueue::submit`. +#[derive(Debug, Clone)] +pub struct DagSpec { + pub template: Template, + /// Primary target agent, or `"hyperhive"` for meta-level DAGs. + pub agent: String, + pub source: Source, + /// Free-form "why"; dedup appends "also requested by …" lines. + pub reason: String, + /// Cascade grouping (meta-update / sweep children). + pub parent_id: Option, + /// Fires the approval-resolution hook on DAG terminal. + pub approval_id: Option, + /// `MetaUpdate`-only: the inputs to bump (also part of the dedup + /// key for that template). Display copy lives on the DAG. + pub inputs: Vec, + /// `PermChange`-only payload. + pub perm_payload: Option, + /// Dashboard transient pill (and crash-watch suppression) held for + /// the lease window — from lease acquisition to DAG terminal. + pub transient: Option, + pub nodes: Vec, +} + +/// A live DAG in the queue. +#[derive(Debug, Clone)] +pub struct Dag { + pub id: u64, + pub template: Template, + pub agent: String, + pub source: Source, + pub reason: String, + pub parent_id: Option, + pub approval_id: Option, + pub inputs: Vec, + pub perm_payload: Option, + pub transient: Option, + pub created_at: i64, + pub nodes: Vec, + /// Terminal roll-up already reported to the scheduler's hooks + /// (approval resolution, transient release). Internal bookkeeping, + /// never serialized. + pub terminal_reported: bool, +} + +impl Dag { + /// Roll-up state: `Failed` if any node failed; else `Running` if + /// any running; else `Queued` if any queued; else `Cancelled` if + /// any cancelled; else `Done`. + pub fn rollup(&self) -> State { + let mut any_cancelled = false; + let mut any_queued = false; + let mut any_running = false; + for n in &self.nodes { + match n.state { + State::Failed => return State::Failed, + State::Running => any_running = true, + State::Queued => any_queued = true, + State::Cancelled => any_cancelled = true, + State::Done => {} + } + } + if any_running { + State::Running + } else if any_queued { + State::Queued + } else if any_cancelled { + State::Cancelled + } else { + State::Done + } + } + + /// True when every node is terminal. + pub fn is_terminal(&self) -> bool { + self.nodes.iter().all(|n| n.state.is_terminal()) + } + + /// First failed node's error, for the roll-up `error` field. + pub fn first_error(&self) -> Option<&str> { + self.nodes + .iter() + .find(|n| n.state == State::Failed) + .and_then(|n| n.error.as_deref()) + } + + pub fn node(&self, id: NodeId) -> Option<&Node> { + self.nodes.iter().find(|n| n.id == id) + } + + pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> { + self.nodes.iter_mut().find(|n| n.id == id) + } +} + +impl Dag { + pub fn view(&self) -> DagView { + let started_at = self.nodes.iter().filter_map(|n| n.started_at).min(); + let finished_at = if self.is_terminal() { + self.nodes.iter().filter_map(|n| n.finished_at).max() + } else { + None + }; + DagView { + id: self.id, + agent: self.agent.clone(), + kind: self.template, + state: self.rollup(), + source: self.source, + parent_id: self.parent_id, + reason: self.reason.clone(), + enqueued_at: self.created_at, + started_at, + finished_at, + inputs: self.inputs.clone(), + approval_id: self.approval_id, + perm_payload: self.perm_payload.clone(), + nodes: self + .nodes + .iter() + .map(|n| NodeView { + id: n.id, + kind: n.kind.as_str().to_owned(), + deps: n.deps.iter().map(|d| d.on).collect(), + state: n.state, + step: n.step.clone(), + build_log_id: n.build_log_id, + started_at: n.started_at, + finished_at: n.finished_at, + error: n.error.clone(), + }) + .collect(), + } + } +} diff --git a/hive-c0re/src/job_queue/scheduler.rs b/hive-c0re/src/job_queue/scheduler.rs new file mode 100644 index 00000000..fb7997a4 --- /dev/null +++ b/hive-c0re/src/job_queue/scheduler.rs @@ -0,0 +1,163 @@ +//! The single scheduler task that drives all DAGs: claim every ready +//! node (as many as the build slots / leases allow), spawn one +//! executor task per claim, and on any completion re-evaluate. +//! Concurrency comes from the build-slot count, not multiple workers. +//! +//! Also owns the two DAG-lifetime side channels the sync queue core +//! can't hold itself: +//! - the per-DAG transient guard (dashboard pill + crash-watch +//! suppression), created when a DAG acquires its agent lease and +//! dropped when the DAG settles terminal; +//! - the `MetaLock` fan-out: appending child `Rebuild` DAGs once the +//! lock bump lands, so children build against the post-bump lock +//! (and a failed bump fans out nothing — replacing the old +//! pre-enqueue + cancel-children dance). + +use std::collections::HashMap; +use std::sync::Arc; + +use super::exec::{self, NodeOutput}; +use super::{Claim, Source, Template, templates}; +use crate::coordinator::Coordinator; + +struct NodeDone { + claim: Claim, + result: anyhow::Result, +} + +/// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`. +/// +/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true +/// signal the loop exits immediately; already-running node tasks ride +/// the runtime down with the process, and pending `Queued` DAGs are +/// dropped — desired state is re-derived on next boot (boot sweep + +/// reconcile), so the in-memory queue is deliberately not durable. +pub async fn run_worker(coord: Arc) { + let mut shutdown = coord.shutdown_rx(); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + // DAG id → transient guard held for the lease window. + let mut transients: HashMap = HashMap::new(); + loop { + // Terminal roll-ups can appear without a node completion — + // the cancel surfaces settle DAGs directly and wake this loop + // via notify — so drain on every iteration, not just inside + // handle_completion. + process_terminals(&coord, &mut transients).await; + let claims = coord.job_queue.claim_ready(); + if !claims.is_empty() { + for claim in claims { + if claim.lease_acquired + && let Some(kind) = claim.transient + { + transients.insert(claim.dag_id, coord.transient_guard(&claim.agent, kind)); + } + tracing::info!( + dag = claim.dag_id, + node = claim.node_id, + kind = claim.kind.as_str(), + agent = %claim.agent, + template = claim.template.as_str(), + "job_queue: node running" + ); + let coord = Arc::clone(&coord); + let tx = tx.clone(); + tokio::spawn(async move { + let result = exec::run_node(&coord, &claim).await; + // Send failure = scheduler gone (shutdown); drop. + let _ = tx.send(NodeDone { claim, result }); + }); + } + coord.emit_rebuild_queue_snapshot(); + continue; + } + tokio::select! { + biased; + res = shutdown.changed() => { + if res.is_err() || *shutdown.borrow() { + tracing::info!("job_queue: scheduler exiting on shutdown"); + return; + } + } + Some(done) = rx.recv() => { + handle_completion(&coord, &mut transients, done).await; + } + () = coord.job_queue.notify.notified() => {} + } + } +} + +async fn handle_completion( + coord: &Arc, + transients: &mut HashMap, + done: NodeDone, +) { + let NodeDone { claim, result } = done; + let (queue_result, fanout) = match result { + Ok(output) => { + tracing::info!( + dag = claim.dag_id, + node = claim.node_id, + "job_queue: node done" + ); + (Ok(()), output.fanout) + } + Err(e) => { + let msg = format!("{e:#}"); + tracing::warn!( + dag = claim.dag_id, + node = claim.node_id, + kind = claim.kind.as_str(), + agent = %claim.agent, + error = %msg, + "job_queue: node failed" + ); + (Err(msg), Vec::new()) + } + }; + coord + .job_queue + .complete_node(claim.dag_id, claim.node_id, queue_result); + if !fanout.is_empty() { + let specs = fanout_specs(&claim, fanout); + coord.job_queue.append_children(specs); + } + process_terminals(coord, transients).await; + coord.emit_rebuild_queue_snapshot(); +} + +/// Drain buffered terminal roll-ups: drop each DAG's lease-window +/// transient guard, then run the terminal hook (approval resolution, +/// `Rebuilt` events, cancelled-power-op intent revert). +async fn process_terminals( + coord: &Arc, + transients: &mut HashMap, +) { + for terminal in coord.job_queue.drain_terminal() { + transients.remove(&terminal.dag_id); + exec::on_dag_terminal(coord, &terminal).await; + } +} + +/// Child `Rebuild` specs for a completed `MetaLock` fan-out, grouped +/// under the parent via `parent_id`. Meta-update children skip the +/// per-agent relock (it would revert the bump the parent just +/// committed); sweep children relock like a manual rebuild. +fn fanout_specs(claim: &Claim, agents: Vec) -> Vec { + let sweep = claim.template == Template::StartupSweep; + let (source, relock) = if sweep { + (Source::StartupSweep, true) + } else { + (Source::MetaUpdate, false) + }; + let reason = if sweep { + "startup sweep".to_owned() + } else if let Some(approval_id) = claim.approval_id { + format!("approval #{approval_id} meta input cascade") + } else { + "meta-update cascade".to_owned() + }; + agents + .into_iter() + .map(|agent| templates::rebuild(&agent, source, reason.clone(), Some(claim.dag_id), relock)) + .collect() +} diff --git a/hive-c0re/src/job_queue/submit.rs b/hive-c0re/src/job_queue/submit.rs new file mode 100644 index 00000000..fa889fe7 --- /dev/null +++ b/hive-c0re/src/job_queue/submit.rs @@ -0,0 +1,125 @@ +//! Request-level submit API — the surface the dashboard POST handlers, +//! the MCP socket handlers, and `hivectl` paths call. Owns the +//! submit-time side effects the DAG templates deliberately don't: +//! writing the durable `wanted` power intent (synchronously, +//! last-writer-wins) before the DAG whose `Reconcile` reads it, and +//! upgrading a stale start to a full rebuild. Every helper emits a +//! fresh queue snapshot so the dashboard shows the new DAG immediately. + +use std::sync::Arc; + +use super::{Source, Template, templates}; +use crate::coordinator::Coordinator; +use crate::power::Wanted; + +fn submit_and_emit(coord: &Arc, spec: super::DagSpec) -> u64 { + let id = coord + .job_queue + .submit(spec) + .expect("template-built dag specs are acyclic"); + coord.emit_rebuild_queue_snapshot(); + id +} + +fn set_wanted(coord: &Arc, agent: &str, wanted: Wanted) { + if let Err(e) = coord.power.set(agent, wanted) { + tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed"); + } +} + +/// Manual/approval-independent rebuild (always relocks the agent's +/// meta input — cascade children are built by the scheduler's fan-out +/// instead of this surface). +pub fn rebuild(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + submit_and_emit(coord, templates::rebuild(agent, source, reason, None, true)) +} + +/// Restart: mechanical stop + converge to `wanted = Up`. The intent +/// write matters when `wanted` drifted `Offline` under a running +/// agent — the old `kill + start` always ended up, and an operator +/// asking for a restart plainly wants it running, not a stop. +pub fn restart(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + set_wanted(coord, agent, Wanted::Up); + submit_and_emit(coord, templates::restart(agent, source, reason)) +} + +/// Start: persist `wanted = Up`, then reconcile. A stale rev marker +/// upgrades the start to a full rebuild (whose tail `Reconcile` does +/// the start) so the container always comes up on current derivations +/// — the old fast-lane `run_start` upgrade, moved to submit time. +pub fn start(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + set_wanted(coord, agent, Wanted::Up); + let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(agent)).ok(); + let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake) + .is_some_and(|rev| stored.as_deref() != Some(rev.as_str())); + if stale { + tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start"); + return submit_and_emit( + coord, + templates::rebuild( + agent, + source, + format!("{reason} (stale — rebuild+start)"), + None, + true, + ), + ); + } + submit_and_emit( + coord, + templates::reconcile_only( + Template::Start, + agent, + source, + reason, + Some(crate::coordinator::TransientKind::Starting), + ), + ) +} + +/// Hard stop: persist `wanted = Offline`, then reconcile (kill + +/// unregister + `Killed` event). +pub fn stop(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + set_wanted(coord, agent, Wanted::Offline); + submit_and_emit( + coord, + templates::reconcile_only( + Template::Stop, + agent, + source, + reason, + Some(crate::coordinator::TransientKind::Stopping), + ), + ) +} + +/// Graceful stop: persist `wanted = Offline`, then signal → drain → +/// reconcile (the actual stop). +pub fn graceful_stop(coord: &Arc, agent: &str, source: Source, reason: String) -> u64 { + set_wanted(coord, agent, Wanted::Offline); + submit_and_emit(coord, templates::graceful_stop(agent, source, reason)) +} + +/// Perm change: commit the JSON file(s) then rebuild. +pub fn perm_change( + coord: &Arc, + agent: &str, + source: Source, + reason: String, + payload: super::PermPayload, +) -> u64 { + submit_and_emit( + coord, + templates::perm_change(agent, source, reason, payload), + ) +} + +/// Meta-input lock bump; cascade rebuilds fan out on completion. +pub fn meta_update( + coord: &Arc, + inputs: Vec, + source: Source, + reason: String, +) -> u64 { + submit_and_emit(coord, templates::meta_update(inputs, source, reason, None)) +} diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs new file mode 100644 index 00000000..46f817c2 --- /dev/null +++ b/hive-c0re/src/job_queue/templates.rs @@ -0,0 +1,334 @@ +//! DAG shape builders — every operation as a template over the shared +//! node primitives — plus submit-time cycle validation (petgraph is +//! confined to this validation; the runtime store stays the plain +//! `Vec` + `deps`). +//! +//! ```text +//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a) +//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a) +//! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a) +//! start(a): [wanted=Up] Reconcile(a) +//! stop(a): [wanted=Offline] Reconcile(a) +//! spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a) +//! perm-change(a): WritePermFile(a) → «rebuild subgraph» +//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a» +//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)» +//! ``` + +use anyhow::{Result, bail}; + +use super::model::{DagSpec, Dep, DepWhen, NodeKind, NodeSpec, PermPayload, Source, Template}; +use crate::coordinator::TransientKind; + +/// After-ok edge on the previous node — the common chain link. +fn after_ok(on: u32) -> Vec { + vec![Dep { + on, + when: DepWhen::AfterOk, + }] +} + +/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`: +/// it must run even when the profile swap failed, so a previously-up +/// agent comes back on its old config (today's recovery-start). This +/// is the only `AfterAny` edge in v1. +fn rebuild_nodes(relock: bool, base: u32) -> Vec { + vec![ + NodeSpec { + kind: NodeKind::Prebuild { relock }, + deps: if base == 0 { + Vec::new() + } else { + after_ok(base - 1) + }, + }, + NodeSpec { + kind: NodeKind::StopForUpdate, + deps: after_ok(base), + }, + NodeSpec { + kind: NodeKind::Swap, + deps: after_ok(base + 1), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: vec![Dep { + on: base + 2, + when: DepWhen::AfterAny, + }], + }, + ] +} + +/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate` +/// noops when already down; the tail `Reconcile` auto-noops the start +/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent +/// leaves it stopped). `relock = false` only for meta-update cascade +/// children. +pub fn rebuild( + agent: &str, + source: Source, + reason: String, + parent_id: Option, + relock: bool, +) -> DagSpec { + DagSpec { + template: Template::Rebuild, + agent: agent.to_owned(), + source, + reason, + parent_id, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Rebuilding), + nodes: rebuild_nodes(relock, 0), + } +} + +/// Approval-driven deploy (`ApplyCommit` / `MergeConfigPr`): the whole +/// two-phase pipeline stays one opaque node in v1 (design doc §9) — +/// wire-visible as a `rebuild` card like today. +pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec { + DagSpec { + template: Template::Rebuild, + agent: agent.to_owned(), + source: Source::Approval, + reason, + parent_id: None, + approval_id: Some(approval_id), + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Rebuilding), + nodes: vec![NodeSpec { + kind: NodeKind::ApprovalDeploy, + deps: Vec::new(), + }], + } +} + +/// Graceful stop: cheap `Signal` fires immediately (no build slot), the +/// `Drain` awaits the harness checkpoint (bounded), and the tail +/// `Reconcile` performs the actual container stop — the caller sets +/// `wanted = Offline` at submit time. A whole-hive graceful stop +/// therefore signals every agent up front and overlaps every drain, +/// replacing the old detached-watcher thread structurally. +pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec { + DagSpec { + template: Template::GracefulStop, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Stopping), + nodes: vec![ + NodeSpec { + kind: NodeKind::Signal, + deps: Vec::new(), + }, + NodeSpec { + kind: NodeKind::Drain, + deps: after_ok(0), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: after_ok(1), + }, + ], + } +} + +/// Restart: mechanical stop, then converge to `wanted` — the submit +/// layer writes `wanted = Up` first, so this is a stop + start like +/// the old `lifecycle::restart` regardless of prior intent drift. +pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec { + DagSpec { + template: Template::Restart, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Restarting), + nodes: vec![ + NodeSpec { + kind: NodeKind::StopForUpdate, + deps: Vec::new(), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: after_ok(0), + }, + ], + } +} + +/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted` +/// first) and the boot-time `Reconcile` converge (wanted untouched). +pub fn reconcile_only( + template: Template, + agent: &str, + source: Source, + reason: String, + transient: Option, +) -> DagSpec { + DagSpec { + template, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient, + nodes: vec![NodeSpec { + kind: NodeKind::Reconcile, + deps: Vec::new(), + }], + } +} + +/// First-deploy spawn (approval-driven): pre-start provisioning + +/// `nixos-container create`, drop-in write, then `Reconcile` starts the +/// container (`wanted = Up` written at approve time). +pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec { + DagSpec { + template: Template::Spawn, + agent: agent.to_owned(), + source: Source::Approval, + reason, + parent_id: None, + approval_id: Some(approval_id), + inputs: Vec::new(), + perm_payload: None, + transient: Some(TransientKind::Spawning), + nodes: vec![ + NodeSpec { + kind: NodeKind::Create, + deps: Vec::new(), + }, + NodeSpec { + kind: NodeKind::WriteDropin, + deps: after_ok(0), + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: after_ok(1), + }, + ], + } +} + +/// Perm change: commit the JSON file(s), then the rebuild subgraph so +/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes +/// effect in the container. +pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec { + let mut nodes = vec![NodeSpec { + kind: NodeKind::WritePermFile, + deps: Vec::new(), + }]; + nodes.extend(rebuild_nodes(true, 1)); + DagSpec { + template: Template::PermChange, + agent: agent.to_owned(), + source, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: Some(payload), + transient: Some(TransientKind::Rebuilding), + nodes, + } +} + +/// Meta-input lock bump. Child `Rebuild` DAGs fan out on completion — +/// appended *after* the bump lands so their prebuilds run against the +/// post-bump lock (and so a failed bump simply fans out nothing, +/// replacing the old pre-enqueue + `cancel_children` dance). +pub fn meta_update( + inputs: Vec, + source: Source, + reason: String, + approval_id: Option, +) -> DagSpec { + DagSpec { + template: Template::MetaUpdate, + agent: "hyperhive".to_owned(), + source, + reason, + parent_id: None, + approval_id, + inputs, + perm_payload: None, + transient: None, + nodes: vec![NodeSpec { + kind: NodeKind::MetaLock { + sweep: false, + fanout: None, + }, + deps: Vec::new(), + }], + } +} + +/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal), +/// then fan out `Rebuild` children for the precomputed stale agent +/// list (topology-sorted by the caller). +pub fn startup_sweep(reason: String, stale_agents: Vec) -> DagSpec { + DagSpec { + template: Template::StartupSweep, + agent: "hyperhive".to_owned(), + source: Source::AutoUpdate, + reason, + parent_id: None, + approval_id: None, + inputs: Vec::new(), + perm_payload: None, + transient: None, + nodes: vec![NodeSpec { + kind: NodeKind::MetaLock { + sweep: true, + fanout: Some(stale_agents), + }, + deps: Vec::new(), + }], + } +} + +/// Validate a spec before it enters the queue: node ids are dense +/// (index = id), deps reference existing nodes, and the dep graph is +/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old +/// queue's documented "circular dep silently deadlocks forever" caveat. +pub fn validate(spec: &DagSpec) -> Result<()> { + if spec.nodes.is_empty() { + bail!("dag spec {:?} has no nodes", spec.template); + } + let n = spec.nodes.len(); + let mut graph = petgraph::graph::DiGraph::::new(); + let idx: Vec<_> = (0..n) + .map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX))) + .collect(); + for (i, node) in spec.nodes.iter().enumerate() { + for dep in &node.deps { + let Some(&dep_idx) = idx.get(dep.on as usize) else { + bail!( + "dag spec {:?} node {i} depends on unknown node {}", + spec.template, + dep.on + ); + }; + graph.add_edge(dep_idx, idx[i], ()); + } + } + if petgraph::algo::toposort(&graph, None).is_err() { + bail!("dag spec {:?} contains a dependency cycle", spec.template); + } + Ok(()) +} diff --git a/hive-c0re/src/job_queue/tests.rs b/hive-c0re/src/job_queue/tests.rs new file mode 100644 index 00000000..321a6419 --- /dev/null +++ b/hive-c0re/src/job_queue/tests.rs @@ -0,0 +1,920 @@ +//! Queue-core unit tests: dedup, cycle rejection, resource +//! serialization (build slots / per-agent leases), lease-exempt +//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure +//! routing, fan-out, and history retention. All synchronous — the +//! scheduler's async loop is a thin claim/complete pump over the same +//! methods exercised here. + +use super::model::{Dep, DepWhen, NodeKind, NodeSpec}; +use super::*; + +fn submit(q: &JobQueue, spec: DagSpec) -> u64 { + q.submit(spec).expect("valid spec") +} + +fn rebuild(agent: &str, reason: &str) -> DagSpec { + templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true) +} + +/// Claim helper asserting exactly one node comes back. +fn claim_one(q: &JobQueue) -> Claim { + let mut claims = q.claim_ready(); + assert_eq!( + claims.len(), + 1, + "expected exactly one claim, got {claims:?}" + ); + claims.pop().expect("one claim") +} + +fn state_of(q: &JobQueue, dag_id: u64) -> State { + q.snapshot() + .iter() + .find(|d| d.id == dag_id) + .expect("dag present") + .state +} + +// ---- submit / dedup ---- + +#[test] +fn submit_assigns_distinct_ids() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "first")); + let b = submit(&q, rebuild("agent-b", "second")); + assert_ne!(a, b); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn dedup_pending_same_template_and_agent() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "first")); + let b = submit(&q, rebuild("agent-a", "auto sweep")); + assert_eq!(a, b, "dedup should return existing id"); + let snap = q.snapshot(); + assert_eq!(snap.len(), 1); + assert!(snap[0].reason.contains("first")); + assert!(snap[0].reason.contains("auto sweep")); +} + +#[test] +fn dedup_does_not_apply_across_templates_or_agents() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + let c = submit( + &q, + templates::restart("agent-a", Source::Manual, "r".to_owned()), + ); + assert_ne!(a, b); + assert_ne!(a, c); + assert_eq!(q.snapshot().len(), 3); +} + +#[test] +fn dedup_skips_running_dags() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "first")); + let claim = claim_one(&q); // Prebuild running + assert_eq!(claim.dag_id, a); + // While the original runs, re-submit is legitimate new work. + let again = submit(&q, rebuild("agent-a", "config bumped during build")); + assert_ne!(a, again); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn meta_update_dedup_matches_inputs() { + let q = JobQueue::new(1); + let a = submit( + &q, + templates::meta_update( + vec!["nixpkgs".to_owned()], + Source::Manual, + "first".to_owned(), + None, + ), + ); + let b = submit( + &q, + templates::meta_update( + vec!["nixpkgs".to_owned()], + Source::Manual, + "duplicate click".to_owned(), + None, + ), + ); + assert_eq!(a, b, "identical-inputs meta-updates should dedup"); + let c = submit( + &q, + templates::meta_update( + vec!["agent-bitburner/bitburner-agent".to_owned()], + Source::Manual, + "bump agent".to_owned(), + None, + ), + ); + assert_ne!(a, c, "different-inputs meta-updates must NOT dedup"); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn approval_dags_dedup_only_on_matching_id() { + let q = JobQueue::new(1); + let a = submit( + &q, + templates::approval_deploy("agent-a", 1, "approval #1".to_owned()), + ); + let b = submit( + &q, + templates::approval_deploy("agent-a", 2, "approval #2".to_owned()), + ); + assert_ne!(a, b, "distinct approvals must not collapse"); + // Rapid double-click on the same approval IS a single op. + let c = submit( + &q, + templates::approval_deploy("agent-a", 1, "approval #1 (dup)".to_owned()), + ); + assert_eq!(a, c); + assert_eq!(q.snapshot().len(), 2); +} + +#[test] +fn perm_change_dedup_respects_perm_type() { + let q = JobQueue::new(1); + let groups = templates::perm_change( + "agent-a", + Source::Manual, + "groups".to_owned(), + PermPayload::ToolGroups { groups: vec![] }, + ); + let caps = templates::perm_change( + "agent-a", + Source::Manual, + "caps".to_owned(), + PermPayload::Capabilities { caps: vec![] }, + ); + let a = submit(&q, groups.clone()); + let b = submit(&q, caps); + assert_ne!(a, b, "tool-groups vs capabilities must not collapse"); + let c = submit(&q, groups); + assert_eq!(a, c, "same perm type dedups"); +} + +/// A `MetaUpdate` cascade `Rebuild` (with `parent_id = Some(meta_id)`) +/// must NOT dedup into a queued `Rebuild` with a different +/// `parent_id` (e.g. from a startup sweep) — without the guard the +/// cascade child would be swallowed and the agent never rebuilt +/// against the post-bump meta. +#[test] +fn dedup_respects_parent_id() { + let q = JobQueue::new(1); + let sweep = submit(&q, templates::startup_sweep("boot".to_owned(), vec![])); + let sweep_child = submit( + &q, + templates::rebuild( + "alice", + Source::StartupSweep, + "startup sweep".to_owned(), + Some(sweep), + true, + ), + ); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let cascade_child = submit( + &q, + templates::rebuild( + "alice", + Source::MetaUpdate, + "meta-update cascade".to_owned(), + Some(meta), + false, + ), + ); + assert_ne!(sweep_child, cascade_child); + let rebuilds = q + .snapshot() + .iter() + .filter(|d| d.kind == Template::Rebuild && d.agent == "alice") + .count(); + assert_eq!(rebuilds, 2, "both rebuilds must be present"); +} + +// ---- cycle rejection ---- + +#[test] +fn cyclic_dag_is_rejected_at_submit() { + let q = JobQueue::new(1); + let mut spec = rebuild("agent-a", "cyclic"); + // 0 → 1 → 0 cycle. + spec.nodes = vec![ + NodeSpec { + kind: NodeKind::StopForUpdate, + deps: vec![Dep { + on: 1, + when: DepWhen::AfterOk, + }], + }, + NodeSpec { + kind: NodeKind::Reconcile, + deps: vec![Dep { + on: 0, + when: DepWhen::AfterOk, + }], + }, + ]; + assert!(q.submit(spec).is_err(), "cyclic spec must be refused"); + assert!(q.snapshot().is_empty()); +} + +#[test] +fn unknown_dep_is_rejected_at_submit() { + let q = JobQueue::new(1); + let mut spec = rebuild("agent-a", "bad dep"); + spec.nodes = vec![NodeSpec { + kind: NodeKind::Reconcile, + deps: vec![Dep { + on: 9, + when: DepWhen::AfterOk, + }], + }]; + assert!(q.submit(spec).is_err()); +} + +// ---- dependency order within a DAG ---- + +#[test] +fn rebuild_chain_claims_in_dep_order() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + for expected in ["prebuild", "stop_for_update", "swap", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.dag_id, id); + assert_eq!(c.kind.as_str(), expected); + assert!( + q.claim_ready().is_empty(), + "chain must serialize: nothing ready while {expected} runs" + ); + q.complete_node(id, c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); +} + +// ---- build slots ---- + +#[test] +fn build_slot_serializes_nix_heavy_nodes() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + let first = claim_one(&q); // a's Prebuild takes the only slot + assert_eq!(first.dag_id, a); + assert_eq!(first.kind.as_str(), "prebuild"); + q.complete_node(a, first.node_id, Ok(())); + // With the slot free again, FIFO gives... a's StopForUpdate is + // slot-free (lease) and b's Prebuild takes the slot — both run. + let claims = q.claim_ready(); + let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect(); + assert!(kinds.contains(&(a, "stop_for_update"))); + assert!(kinds.contains(&(b, "prebuild"))); + assert_eq!(claims.len(), 2); +} + +#[test] +fn two_build_slots_run_two_prebuilds() { + let q = JobQueue::new(2); + submit(&q, rebuild("agent-a", "r")); + submit(&q, rebuild("agent-b", "r")); + let claims = q.claim_ready(); + assert_eq!(claims.len(), 2, "two slots → two concurrent prebuilds"); + assert!(claims.iter().all(|c| c.kind.as_str() == "prebuild")); +} + +#[test] +fn fifo_fairness_for_the_slot() { + let q = JobQueue::new(1); + let a = submit(&q, rebuild("agent-a", "r")); + let b = submit(&q, rebuild("agent-b", "r")); + let c = submit(&q, rebuild("agent-c", "r")); + let first = claim_one(&q); + assert_eq!(first.dag_id, a, "submit order wins the slot"); + q.complete_node(a, first.node_id, Ok(())); + let next: Vec = q.claim_ready().iter().map(|cl| cl.dag_id).collect(); + assert!(next.contains(&b), "b's prebuild before c's"); + assert!(!next.contains(&c)); +} + +// ---- per-agent lease ---- + +#[test] +fn lease_serializes_two_lifecycle_dags_for_same_agent() { + let q = JobQueue::new(4); + let restart = submit( + &q, + templates::restart("agent-a", Source::Manual, "restart".to_owned()), + ); + let stop = submit( + &q, + templates::reconcile_only( + Template::Stop, + "agent-a", + Source::Manual, + "stop".to_owned(), + None, + ), + ); + // Restart's StopForUpdate acquires the lease; stop's Reconcile + // must wait even though slots are free. + let first = claim_one(&q); + assert_eq!(first.dag_id, restart); + assert!(first.lease_acquired); + q.complete_node(restart, first.node_id, Ok(())); + // Same DAG keeps the lease for its Reconcile. + let second = claim_one(&q); + assert_eq!(second.dag_id, restart); + assert!(!second.lease_acquired, "lease already held by this DAG"); + q.complete_node(restart, second.node_id, Ok(())); + // Restart terminal → lease released → stop's Reconcile runs. + let third = claim_one(&q); + assert_eq!(third.dag_id, stop); + q.complete_node(stop, third.node_id, Ok(())); + assert_eq!(state_of(&q, restart), State::Done); + assert_eq!(state_of(&q, stop), State::Done); +} + +#[test] +fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() { + let q = JobQueue::new(2); + submit(&q, rebuild("agent-a", "rebuild")); + let stop = submit( + &q, + templates::reconcile_only( + Template::Stop, + "agent-a", + Source::Manual, + "stop".to_owned(), + None, + ), + ); + // Prebuild is lease-exempt: the stop's Reconcile takes the lease + // and runs concurrently with the rebuild's out-of-band nix build. + let claims = q.claim_ready(); + let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect(); + assert!(kinds.contains(&"prebuild")); + assert!(kinds.contains(&"reconcile")); + // But the rebuild's StopForUpdate must then wait for the stop DAG + // to finish (lease). + let prebuild = claims + .iter() + .find(|c| c.kind.as_str() == "prebuild") + .expect("prebuild claim") + .clone(); + q.complete_node(prebuild.dag_id, prebuild.node_id, Ok(())); + assert!( + q.claim_ready().is_empty(), + "StopForUpdate blocked while stop DAG holds the lease" + ); + let reconcile = claims + .iter() + .find(|c| c.kind.as_str() == "reconcile") + .expect("reconcile claim") + .clone(); + q.complete_node(stop, reconcile.node_id, Ok(())); + let next = claim_one(&q); + assert_eq!(next.kind.as_str(), "stop_for_update"); +} + +#[test] +fn agents_do_not_contend_on_each_others_leases() { + let q = JobQueue::new(4); + submit( + &q, + templates::restart("agent-a", Source::Manual, "r".to_owned()), + ); + submit( + &q, + templates::restart("agent-b", Source::Manual, "r".to_owned()), + ); + let claims = q.claim_ready(); + assert_eq!(claims.len(), 2, "different agents run concurrently"); +} + +// ---- failure: cancel-downstream + AfterAny ---- + +#[test] +fn failed_node_cancels_downstream_but_afterany_reconcile_runs() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let prebuild = claim_one(&q); + q.complete_node(id, prebuild.node_id, Err("nix build exploded".to_owned())); + // StopForUpdate + Swap are cancelled (AfterOk on a failed chain); + // the AfterAny Reconcile still runs once Swap is terminal. + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(id, reconcile.node_id, Ok(())); + let snap = q.snapshot(); + let dag = snap.iter().find(|d| d.id == id).expect("dag"); + assert_eq!(dag.state, State::Failed, "roll-up failed"); + let by_kind = |k: &str| { + dag.nodes + .iter() + .find(|n| n.kind == k) + .expect("node present") + .state + }; + assert_eq!(by_kind("prebuild"), State::Failed); + assert_eq!(by_kind("stop_for_update"), State::Cancelled); + assert_eq!(by_kind("swap"), State::Cancelled); + assert_eq!(by_kind("reconcile"), State::Done); + assert_eq!( + dag.nodes + .iter() + .find(|n| n.kind == "prebuild") + .and_then(|n| n.error.as_deref()), + Some("nix build exploded") + ); +} + +/// The swap-failure recovery: `Swap` fails → the `AfterAny` edge still +/// runs `Reconcile`, which brings a wanted-up agent back on its old +/// config. +#[test] +fn swap_failure_still_runs_reconcile() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + for _ in 0..2 { + let c = claim_one(&q); + q.complete_node(id, c.node_id, Ok(())); + } + let swap = claim_one(&q); + assert_eq!(swap.kind.as_str(), "swap"); + q.complete_node(id, swap.node_id, Err("update failed".to_owned())); + let reconcile = claim_one(&q); + assert_eq!(reconcile.kind.as_str(), "reconcile"); + q.complete_node(id, reconcile.node_id, Ok(())); + assert_eq!(state_of(&q, id), State::Failed); +} + +#[test] +fn failed_reconcile_marks_dag_failed() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::reconcile_only( + Template::Start, + "agent-a", + Source::Manual, + "start".to_owned(), + None, + ), + ); + let c = claim_one(&q); + q.complete_node(id, c.node_id, Err("start failed".to_owned())); + assert_eq!(state_of(&q, id), State::Failed); +} + +// ---- cancel ---- + +#[test] +fn cancel_clears_queued_dag() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + assert!(q.cancel(id)); + assert_eq!(state_of(&q, id), State::Cancelled); + assert!(q.claim_ready().is_empty()); +} + +#[test] +fn cancel_refuses_running_dag() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let _ = claim_one(&q); + assert!(!q.cancel(id)); + assert_eq!(state_of(&q, id), State::Running); +} + +#[test] +fn cancel_children_marks_queued_children_only() { + let q = JobQueue::new(1); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + // Parent's MetaLock is running while children exist. + let lock = claim_one(&q); + assert_eq!(lock.dag_id, meta); + let child_a = submit( + &q, + templates::rebuild( + "agent-a", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + let child_b = submit( + &q, + templates::rebuild( + "agent-b", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + let unrelated = submit(&q, rebuild("agent-c", "operator queued")); + // MetaLock holds the single build slot, so both children (and the + // unrelated rebuild) are still fully queued here. + let cancelled = q.cancel_children(meta); + assert_eq!(cancelled, 2); + assert_eq!(state_of(&q, child_a), State::Cancelled); + assert_eq!(state_of(&q, child_b), State::Cancelled); + assert_eq!(state_of(&q, unrelated), State::Queued); +} + +#[test] +fn cancel_children_skips_running_child() { + let q = JobQueue::new(2); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let lock = claim_one(&q); + let running_child = submit( + &q, + templates::rebuild( + "agent-a", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + let queued_child = submit( + &q, + templates::rebuild( + "agent-b", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + // Second slot lets running_child's prebuild start. + let child_claim = claim_one(&q); + assert_eq!(child_claim.dag_id, running_child); + let n = q.cancel_children(meta); + assert_eq!(n, 1); + assert_eq!(state_of(&q, running_child), State::Running); + assert_eq!(state_of(&q, queued_child), State::Cancelled); + q.complete_node(meta, lock.node_id, Ok(())); +} + +// ---- fan-out ---- + +#[test] +fn append_children_sets_parent_and_dedups() { + let q = JobQueue::new(1); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let specs = vec![ + templates::rebuild( + "alice", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + templates::rebuild( + "bob", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + // Duplicate — must coalesce into the first alice child. + templates::rebuild( + "alice", + Source::MetaUpdate, + "cascade again".to_owned(), + Some(meta), + false, + ), + ]; + let ids = q.append_children(specs); + assert_eq!(ids.len(), 3); + assert_eq!(ids[0], ids[2], "duplicate child dedups"); + let snap = q.snapshot(); + let children: Vec<_> = snap.iter().filter(|d| d.parent_id == Some(meta)).collect(); + assert_eq!(children.len(), 2); +} + +// ---- terminal reporting + lease release ---- + +#[test] +fn terminal_dag_reported_exactly_once_and_lease_released() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::restart("agent-a", Source::Manual, "r".to_owned()), + ); + let stop = claim_one(&q); + q.complete_node(id, stop.node_id, Ok(())); + assert!(q.drain_terminal().is_empty(), "dag not terminal yet"); + let rec = claim_one(&q); + q.complete_node(id, rec.node_id, Ok(())); + let reports = q.drain_terminal(); + assert_eq!(reports.len(), 1); + assert_eq!(reports[0].dag_id, id); + assert_eq!(reports[0].state, State::Done); + assert!(q.drain_terminal().is_empty(), "reported exactly once"); + // Lease released: a new DAG for the agent can claim immediately. + let next = submit( + &q, + templates::reconcile_only( + Template::Stop, + "agent-a", + Source::Manual, + "stop".to_owned(), + None, + ), + ); + let c = claim_one(&q); + assert_eq!(c.dag_id, next); + assert!(c.lease_acquired); +} + +/// A DAG cancelled while fully queued must still surface a terminal +/// roll-up for the scheduler's hooks — otherwise a queued approval +/// DAG cancelled by the operator would dangle its approval forever. +#[test] +fn cancelled_dag_reports_terminal_once() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::approval_deploy("agent-a", 7, "approval #7".to_owned()), + ); + assert!(q.cancel(id)); + let reports = q.drain_terminal(); + assert_eq!(reports.len(), 1); + assert_eq!(reports[0].dag_id, id); + assert_eq!(reports[0].state, State::Cancelled); + assert_eq!(reports[0].approval_id, Some(7)); + // Never re-reported by later activity. + let other = submit(&q, rebuild("agent-b", "r")); + let c = claim_one(&q); + assert_eq!(c.dag_id, other); + q.complete_node(other, c.node_id, Err("boom".to_owned())); + assert!(q.drain_terminal().iter().all(|t| t.dag_id != id)); +} + +#[test] +fn cancel_children_reports_terminals() { + let q = JobQueue::new(1); + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let _lock = claim_one(&q); + let child = submit( + &q, + templates::rebuild( + "agent-a", + Source::MetaUpdate, + "cascade".to_owned(), + Some(meta), + false, + ), + ); + assert_eq!(q.cancel_children(meta), 1); + let reports = q.drain_terminal(); + assert_eq!(reports.len(), 1); + assert_eq!(reports[0].dag_id, child); + assert_eq!(reports[0].state, State::Cancelled); +} + +/// History trim must not evict a terminal fan-out parent while its +/// children are still live — the dashboard groups children under it. +#[test] +fn trim_keeps_terminal_parent_with_live_children() { + let q = JobQueue::new(1); + // Pin agent-x's lease with a running stop DAG so the child below + // stays fully queued while we churn history. + let pin = submit( + &q, + templates::reconcile_only( + Template::Stop, + "agent-x", + Source::Manual, + "lease pin".to_owned(), + None, + ), + ); + let pin_claim = claim_one(&q); + assert_eq!(pin_claim.dag_id, pin); + // Terminal fan-out parent + a lease-blocked child under it. + let meta = submit( + &q, + templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None), + ); + let lock = claim_one(&q); + q.complete_node(meta, lock.node_id, Ok(())); + let mut child_spec = templates::restart("agent-x", Source::MetaUpdate, "cascade".to_owned()); + child_spec.parent_id = Some(meta); + let child = submit(&q, child_spec); + // Churn > MAX_HISTORY_PER_TEMPLATE terminal meta_update DAGs. + for i in 0..7 { + let id = submit( + &q, + templates::meta_update( + vec![format!("input-{i}")], + Source::Manual, + "churn".to_owned(), + None, + ), + ); + let c = claim_one(&q); + assert_eq!(c.dag_id, id, "child is lease-blocked; churn claims freely"); + q.complete_node(id, c.node_id, Ok(())); + } + let snap = q.snapshot(); + assert!( + snap.iter().any(|d| d.id == meta), + "terminal parent with live child must survive trim" + ); + assert!(snap.iter().any(|d| d.id == child)); +} + +// ---- steps, build logs, history ---- + +#[test] +fn set_step_only_on_running_and_signals_change() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + assert!(!q.set_step(id, 0, "too early"), "queued node refuses step"); + let c = claim_one(&q); + assert!(q.set_step(id, c.node_id, "nix build")); + assert!( + !q.set_step(id, c.node_id, "nix build"), + "same label → false" + ); + assert!(q.set_step(id, c.node_id, "next phase")); + assert!(q.set_step_running(id, "via running lookup")); + q.complete_node(id, c.node_id, Ok(())); + let snap = q.snapshot(); + let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0]; + assert_eq!(node.step, None, "step cleared on completion"); +} + +#[test] +fn set_build_log_id_links_running_node() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id"); + let c = claim_one(&q); + assert!(q.set_build_log_id(id, c.node_id, 42)); + assert!(q.set_build_log_id_running(id, 43)); + q.complete_node(id, c.node_id, Ok(())); + let snap = q.snapshot(); + let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0]; + assert_eq!(node.build_log_id, Some(43), "log id survives completion"); +} + +#[test] +fn history_evicts_old_terminals_per_template() { + let q = JobQueue::new(1); + for i in 0..8 { + let id = submit( + &q, + templates::reconcile_only( + Template::Start, + &format!("agent-{i}"), + Source::Manual, + "start".to_owned(), + None, + ), + ); + let c = claim_one(&q); + q.complete_node(id, c.node_id, Ok(())); + } + // Fresh terminals are inside the grace window: nothing evicts yet, + // so a ~1s QueueDag poller can still observe every terminal state + // (a broad stop/start settles many same-template DAGs at once). + assert_eq!( + q.snapshot().len(), + 8, + "grace window protects fresh terminals" + ); + // Past the grace window the per-template cap applies. + q.trim_ignoring_grace(); + assert_eq!(q.snapshot().len(), 5, "per-template history cap"); + assert_eq!(q.live_count(), 0); +} + +#[test] +fn error_is_truncated() { + let q = JobQueue::new(1); + let id = submit(&q, rebuild("agent-a", "r")); + let c = claim_one(&q); + q.complete_node(id, c.node_id, Err("x".repeat(5000))); + let snap = q.snapshot(); + let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0] + .error + .clone() + .expect("error stored"); + assert!(err.chars().count() <= 2001, "truncated + ellipsis"); + assert!(err.ends_with('…')); +} + +// ---- template shapes ---- + +#[test] +fn graceful_stop_shape_signal_drain_reconcile() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()), + ); + for expected in ["signal", "drain", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(id, c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); +} + +#[test] +fn graceful_signal_and_drain_hold_no_build_slot() { + // A whole-hive graceful stop overlaps every drain even at + // buildSlots = 1 while a rebuild hogs the slot. + let q = JobQueue::new(1); + submit(&q, rebuild("builder", "slot hog")); + submit( + &q, + templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()), + ); + submit( + &q, + templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()), + ); + let claims = q.claim_ready(); + let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect(); + assert_eq!( + kinds, + vec!["prebuild", "signal", "signal"], + "both agents' signals fire while the slot is held" + ); +} + +#[test] +fn spawn_shape_create_dropin_reconcile() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::spawn("newbie", 7, "approval #7 spawn".to_owned()), + ); + for expected in ["create", "write_dropin", "reconcile"] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + assert_eq!(c.approval_id, Some(7)); + q.complete_node(id, c.node_id, Ok(())); + } + let report_terminal = state_of(&q, id); + assert_eq!(report_terminal, State::Done); +} + +#[test] +fn perm_change_shape_prefixes_rebuild_chain() { + let q = JobQueue::new(1); + let id = submit( + &q, + templates::perm_change( + "agent-a", + Source::Manual, + "perm".to_owned(), + PermPayload::Combined { + groups: Some(vec![]), + caps: None, + }, + ), + ); + for expected in [ + "write_perm_file", + "prebuild", + "stop_for_update", + "swap", + "reconcile", + ] { + let c = claim_one(&q); + assert_eq!(c.kind.as_str(), expected); + q.complete_node(id, c.node_id, Ok(())); + } + assert_eq!(state_of(&q, id), State::Done); +} diff --git a/hive-c0re/src/lib.rs b/hive-c0re/src/lib.rs index e58f2d0f..3b20bd5e 100644 --- a/hive-c0re/src/lib.rs +++ b/hive-c0re/src/lib.rs @@ -11,43 +11,45 @@ //! Every module is re-exported `pub` so anything in the crate is //! addressable from either binary; the lib doesn't have a curated //! surface beyond "this is where the modules live". +//! +//! Cohesive clusters live in directory submodules (`stores`, `stats`, +//! `agent_config`, `workers`); each of their children is re-exported +//! at the crate root so pre-existing `crate::broker::…` / +//! `hive_c0re::broker::…` paths keep compiling unchanged. pub mod actions; -pub mod agent_sockets; -pub mod approvals; -pub mod audit_log; -pub mod auto_update; -pub mod broker; -pub mod build_logs; -pub mod capabilities; +pub mod agent_config; pub mod client; -pub mod container_stats; pub mod container_view; pub mod coordinator; -pub mod crash_watch; pub mod dashboard; pub mod dashboard_events; pub mod flake_check; pub mod forge; pub mod gateway_nginx; -pub mod hive_stats; -pub mod host_stats; -pub mod knowledge; +pub mod job_queue; pub mod lifecycle; -pub mod limits; pub mod loose_ends; pub mod matrix; pub mod meta; pub mod migrate; -pub mod operator_questions; pub mod paths; pub mod priv_client; pub mod questions; -pub mod rebuild_queue; -pub mod reminder_scheduler; -pub mod scheduled_prompts; -pub mod scheduled_prompts_worker; pub mod server; pub mod socket_server; -pub mod tool_groups; -pub mod topology; +pub mod stats; +pub mod stores; +pub mod workers; + +// Root re-exports: keep every pre-grouping `crate::` / +// `hive_c0re::` path compiling without touching consumers. +pub use agent_config::{capabilities, limits, tool_groups, topology}; +pub use stats::{container_stats, hive_stats, host_stats}; +pub use stores::{ + approvals, audit_log, broker, build_logs, db, operator_questions, power, scheduled_prompts, +}; +pub use workers::{ + agent_sockets, auto_update, crash_watch, knowledge, reminder_scheduler, + scheduled_prompts_worker, +}; diff --git a/hive-c0re/src/lifecycle.rs b/hive-c0re/src/lifecycle.rs deleted file mode 100644 index 3296866a..00000000 --- a/hive-c0re/src/lifecycle.rs +++ /dev/null @@ -1,1811 +0,0 @@ -//! `nixos-container` lifecycle + per-agent config flake generation. - -use std::path::Path; - -use anyhow::{Context, Result, bail}; -use hive_sh4re::priv_proto::{BindMount, CredentialMount}; -use tokio::process::Command; - -use crate::coordinator::{AgentPaths, HiveEnv}; - -/// Sub-agent container prefix. `nixos-container` caps the total container name -/// at 11 chars (it gets encoded into network interface names), so the agent -/// name itself can be at most `MAX_AGENT_NAME` chars. -pub const AGENT_PREFIX: &str = "h-"; -pub const MAX_AGENT_NAME: usize = 9; -/// Logical name of the manager agent (broker recipient, state-dir key, -/// meta flake attribute). All persistent state lives under `ruth/`. -pub const MANAGER_NAME: &str = "ruth"; -/// Container name of the manager. Uses the same `h-` prefix as sub-agents -/// so `nixos-container list` output is uniform and the list filter is -/// a single `starts_with(AGENT_PREFIX)` check. Logical name → container -/// name: `ruth` → `h-ruth`. -pub const MANAGER_CONTAINER: &str = "h-ruth"; - -/// Mount point of the per-agent runtime directory inside the container. -pub const CONTAINER_RUNTIME_MOUNT: &str = "/run/hive"; - -/// Where the per-agent Claude credentials dir mounts inside the -/// container. The harness service runs as a non-root unix user -/// whose home is `/home//`, so the mount path varies per -/// agent — `container_claude_mount(name)` returns -/// `/home//.claude` for every agent including the manager. -/// `claude` inside the container reads -/// `$HOME/.claude` and the service environment sets `HOME` to the -/// same path, so the OAuth session survives container restarts. -#[must_use] -pub fn container_claude_mount(name: &str) -> String { - format!("/home/{name}/.claude") -} - -/// Mount point of the shared directory accessible to all agents. -/// All agents can read/write here; agents should only put things they're -/// willing to lose (other agents may delete them). -pub const CONTAINER_SHARED_MOUNT: &str = "/shared"; - -const GIT_NAME: &str = "c0re"; -const GIT_EMAIL: &str = "c0re@hyperhive.local"; - -/// Sub-agent web UI port range. Deterministic from the agent's name (FNV-1a -/// hash mod range size), so the dashboard can compute the same port without -/// asking hive-c0re. -const WEB_PORT_BASE: u16 = 8100; -const WEB_PORT_RANGE: u16 = 900; - -/// FNV-1a hash of a string — shared by `agent_web_port` and -/// `agent_network_ip` so the derivation rule is identical. -fn fnv1a(s: &str) -> u32 { - let mut hash: u32 = 2_166_136_261; - for b in s.bytes() { - hash ^= u32::from(b); - hash = hash.wrapping_mul(16_777_619); - } - hash -} - -/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) % -/// WEB_PORT_RANGE` for every agent including the manager. The port -/// allocation rule reads the same for every name; collisions are -/// possible (birthday paradox at ~30 agents) and the operator -/// resolves them by renaming an agent (different hash → different -/// port). Stable across hosts, restarts, and dashboard renders — -/// no state-file dance. -#[must_use] -pub fn agent_web_port(name: &str) -> u16 { - // Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails. - WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0) -} - -/// Deterministic IPv4 address for an agent inside an isolated subnet. -/// -/// Parses `subnet_cidr` as `/` (e.g. -/// `"10.42.0.0/24"`), then computes: -/// -/// ```text -/// host_count = 2^(32 - prefix_len) -/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast) -/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot -/// agent_ip = network_base_u32 + offset -/// ``` -/// -/// Returns `None` when `subnet_cidr` can't be parsed (invalid format, -/// prefix out of range, etc.) so callers can fall back gracefully. -/// Collisions are possible (birthday paradox) and the operator resolves -/// them by renaming an agent, same as for port collisions. -#[must_use] -pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option { - let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; - let prefix_len: u32 = prefix_str.parse().ok()?; - if prefix_len > 30 { - // /31 and /32 have no room for agents; /30 has 1 usable slot. - // /0 (the other extreme) is handled further down: host_count - // overflows checked_shl(32) → 0 → usable = 0 → None. - return None; - } - // Parse dotted-decimal IPv4. - let octets: Vec = ip_str - .split('.') - .map(|o| o.parse::().ok()) - .collect::>>()?; - if octets.len() != 4 { - return None; - } - let base_u32 = u32::from_be_bytes([octets[0], octets[1], octets[2], octets[3]]); - // Mask off host bits to get the true network address. - let mask = if prefix_len == 0 { - 0u32 - } else { - !0u32 << (32 - prefix_len) - }; - let network_base = base_u32 & mask; - let host_count: u32 = 1u32.checked_shl(32 - prefix_len).unwrap_or(0); - // `.0` = network, `.1` = bridge gateway, last = broadcast → 3 reserved. - let usable = host_count.saturating_sub(3); - if usable == 0 { - return None; - } - let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1 - let ip_u32 = network_base + offset; - let [a, b, c, d] = ip_u32.to_be_bytes(); - Some(format!("{a}.{b}.{c}.{d}")) -} - -/// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`. -/// -/// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim -/// (e.g. `10.42.0.1/24`), **not** the canonical network address — see -/// the note in `set_nspawn_flags` + `docs/network.md`. The IP part is -/// therefore the bridge IP itself: the host end of the bridge, the -/// default-route target for isolated containers, and the address the -/// hive dnsmasq resolver binds. Returns the dotted-decimal IP with the -/// `/` stripped, or `None` if the input isn't a valid -/// `/` pair. -/// -/// Deliberately returns the operator-configured address verbatim rather -/// than deriving `network + 1`: an operator who sets `bridgeIp` to a -/// non-`.1` host address (e.g. `10.42.0.254`) runs the bridge + resolver -/// there, so that — not `.1` — is the real gateway. -#[must_use] -pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option { - let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; - // Validate the prefix is a sane IPv4 CIDR length and the address is - // dotted-decimal IPv4 — same shape `agent_network_ip` accepts — so a - // malformed `HIVE_NETWORK_SUBNET` can't smuggle a bogus HOST_ADDRESS - // into the nspawn conf. - let prefix_len: u32 = prefix_str.parse().ok()?; - if prefix_len > 32 { - return None; - } - let octets: Vec = ip_str - .split('.') - .map(|o| o.parse::().ok()) - .collect::>>()?; - if octets.len() != 4 { - return None; - } - Some(ip_str.to_owned()) -} - -#[must_use] -pub fn container_name(name: &str) -> String { - format!("{AGENT_PREFIX}{name}") -} - -/// Read the agent user's `(uid, gid)` from the container's nixos-managed -/// `/etc/passwd`. Returns `None` when the container hasn't been built -/// yet, the passwd file is unparseable, or the agent user is missing -/// (e.g. legacy container that still runs as root). -/// -/// Used by `forge` + `matrix` after writing per-agent state files so -/// the bind-mounted host file ends up readable by the agent user -/// without waiting for the next container activation to run the chown -/// fixup. -/// -/// Notes: -/// - Reads the *container-local* passwd at -/// `/var/lib/nixos-containers//etc/passwd`, not the host's. -/// The container's user-namespace shares uids with the host (no -/// `PrivateUsers`), so the uid is directly usable in host-side -/// `chown(2)`. -/// - Best-effort: caller treats `None` as "skip the chown". -#[must_use] -pub fn agent_uid_gid(agent_name: &str) -> Option<(u32, u32)> { - let container = container_name(agent_name); - let passwd_path = format!("/var/lib/nixos-containers/{container}/etc/passwd"); - let content = std::fs::read_to_string(&passwd_path).ok()?; - for line in content.lines() { - let mut parts = line.split(':'); - let user = parts.next()?; - if user != agent_name { - continue; - } - let _ = parts.next()?; // x (password placeholder) - let uid: u32 = parts.next()?.parse().ok()?; - let gid: u32 = parts.next()?.parse().ok()?; - return Some((uid, gid)); - } - None -} - -/// Best-effort `chown(path, agent_uid, agent_gid)`. Resolves the agent's -/// uid/gid via [`agent_uid_gid`] and shells out to `std::os::unix::fs::chown`. -/// Silently no-ops when the container isn't built yet (`None` from -/// [`agent_uid_gid`]) and logs at debug on chown syscall failure — the -/// activation script in `harness-base.nix` is the steady-state safety -/// net. Used by per-agent state writers in `forge` + `matrix` so the -/// agent can read the file without waiting for the next container -/// rebuild. -pub fn chown_to_agent(name: &str, path: &Path, subsystem: &str) { - let Some((uid, gid)) = agent_uid_gid(name) else { - return; - }; - if let Err(e) = std::os::unix::fs::chown(path, Some(uid), Some(gid)) { - tracing::debug!(%name, %subsystem, path = %path.display(), error = %e, "chown to agent failed"); - } -} - -fn validate(name: &str) -> Result<()> { - if name.is_empty() { - bail!("agent name must not be empty"); - } - if name.len() > MAX_AGENT_NAME { - bail!( - "agent name '{name}' is too long ({} chars); max {MAX_AGENT_NAME}", - name.len() - ); - } - Ok(()) -} - -/// First name (≠ `self_name`) currently running whose hashed port -/// matches this agent's. The harness inside the colliding container -/// would otherwise loop on `AddrInUse` forever; we surface the -/// conflict here so spawn / rebuild fails loudly with an actionable -/// message instead. -async fn port_collision(self_name: &str) -> Option { - let port = agent_web_port(self_name); - let raw = list().await.unwrap_or_default(); - for c in raw { - let Some(other) = c.strip_prefix(AGENT_PREFIX) else { - continue; - }; - if other == self_name { - continue; - } - if agent_web_port(other) == port && is_running(other).await { - return Some(other.to_owned()); - } - } - None -} - -pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { - validate(name)?; - if let Some(other) = port_collision(name).await { - bail!( - "port {} is already taken by '{other}' — rename one of them and retry", - agent_web_port(name) - ); - } - setup_proposed(&paths.proposed_dir, name).await?; - setup_applied(&paths.applied_dir, Some(&paths.proposed_dir), name).await?; - ensure_agent_state_subvolume(name).await?; - ensure_claude_dir(&paths.claude_dir)?; - ensure_state_dir(&paths.notes_dir)?; - // Meta flake gets the new agent's input + nixosConfiguration - // before `nixos-container create` so the `--flake meta#` - // ref resolves. - let agents = agents_after_spawn(name).await?; - crate::meta::sync_agents(hive, &agents).await?; - let container = container_name(name); - priv_run("create", name).await?; - set_nspawn_flags( - &container, - &paths.agent_dir, - &paths.claude_dir, - &paths.notes_dir, - ) - .await?; - set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; - systemd_daemon_reload().await?; - priv_run("start", name).await -} - -/// Build the `AgentSpec` list for the meta flake from `nixos-container -/// list` + a hypothetical extra name not yet in the list (for spawn -/// where the new agent's container doesn't exist yet). Pass empty -/// `name_to_add` from rebuild paths where the agent is already in the -/// container list. -/// -/// Propagates errors from `list()` rather than swallowing them. -/// Using `.unwrap_or_default()` here would silently produce an empty -/// agent list when `nixos-container list` fails (priv helper down, race), -/// which `sync_agents` would then commit to meta — dropping every agent -/// from `flake.nix`. Callers that can tolerate failures (e.g. migration) -/// handle the `Err` themselves with `.unwrap_or_default()`. -async fn agents_for_meta(name_to_add: Option<&str>) -> Result> { - let containers = list().await?; - let mut out: Vec = containers - .into_iter() - .filter_map(|c| { - let name = c.strip_prefix(AGENT_PREFIX)?.to_owned(); - Some(crate::meta::AgentSpec { - is_manager: name == MANAGER_NAME, - port: agent_web_port(&name), - name, - }) - }) - .collect(); - if let Some(extra) = name_to_add - && !out.iter().any(|a| a.name == extra) - { - out.push(crate::meta::AgentSpec { - is_manager: extra == MANAGER_NAME, - port: agent_web_port(extra), - name: extra.to_owned(), - }); - } - out.sort_by(|a, b| a.name.cmp(&b.name)); - Ok(out) -} - -async fn agents_after_spawn(name: &str) -> Result> { - agents_for_meta(Some(name)).await -} - -/// Like `agents_for_meta_listing` but with an extra agent added (for a -/// container that doesn't exist yet). Used by the first-spawn path in -/// `actions::run_apply_commit` to register the new agent in meta before -/// `prepare_deploy` tries to update its input lock. -pub async fn agents_for_meta_listing_with(extra: &str) -> Result> { - agents_for_meta(Some(extra)).await -} - -/// Public enumeration of currently-existing agents (whatever -/// `nixos-container list` says), sorted, no extras. For callers -/// outside this module that need to reseed meta after lifecycle -/// changes — destroy, startup reconciliation, etc. -pub async fn agents_for_meta_listing() -> Result> { - agents_for_meta(None).await -} - -/// True when the named container already exists (appears in -/// `nixos-container list`). Used by the apply-commit path to decide -/// between first-spawn (`nixos-container create`) and normal rebuild -/// (`nixos-container update`). -pub async fn container_exists(name: &str) -> bool { - let container = container_name(name); - list() - .await - .unwrap_or_default() - .iter() - .any(|c| c == &container) -} - -pub async fn kill(name: &str) -> Result<()> { - validate(name)?; - priv_run("stop", name).await -} - -pub async fn start(name: &str) -> Result<()> { - validate(name)?; - priv_run("start", name).await -} - -/// Start with the cold-start fallback: when a plain start fails (the -/// activation-error shape), retry once via stop + kill + start before -/// giving up. Used by the queue's fast-lane `Start` handler and the -/// inline start-after-rebuild path. -/// See `docs/coordinator.md::Cold-start fallback`. -/// -/// # Errors -/// -/// Propagates the retry's start error (annotated with the original -/// failure) when the fallback also fails. -pub async fn start_with_fallback(name: &str) -> Result<()> { - validate(name)?; - if let Err(start_err) = priv_run("start", name).await { - let container = container_name(name); - tracing::warn!( - container = %container, - error = %start_err, - "start failed (possible activation error); retrying via stop + kill + start" - ); - priv_run("stop", name).await.unwrap_or_else(|e| { - tracing::warn!( - container = %container, - error = %e, - "stop before cold-start retry failed (ignored)" - ); - }); - priv_run("kill", name).await.unwrap_or_else(|e| { - tracing::warn!( - container = %container, - error = %e, - "kill before cold-start retry failed (ignored)" - ); - }); - priv_run("start", name).await.map_err(|e| { - anyhow::anyhow!( - "cold-start fallback also failed: {e:#} \ - (original start error: {start_err:#})" - ) - }) - } else { - Ok(()) - } -} - -/// Stop + start without regenerating any config. For "kick the container" -/// without touching the flake or nspawn flags. -pub async fn restart(name: &str) -> Result<()> { - kill(name).await?; - start(name).await -} - -/// True when the container's systemd unit is active. Used by the dashboard -/// to gate stop/restart buttons. -pub async fn is_running(name: &str) -> bool { - let container = container_name(name); - let unit = format!("container@{container}.service"); - Command::new("systemctl") - .args(["is-active", "--quiet", &unit]) - .status() - .await - .is_ok_and(|s| s.success()) -} - -/// Fully tear down a sub-agent's container: stop + remove via `nixos-container -/// destroy`, then clean our own systemd drop-in. Leaves it to the caller to -/// wipe `/var/lib/hyperhive/...` state and the per-agent runtime dir. -pub async fn destroy(name: &str) -> Result<()> { - validate(name)?; - let container = container_name(name); - // nixos-container destroy handles stop + removal of /var/lib/nixos-containers/ - // and /etc/nixos-containers/.conf. Tolerate "no such container". - if let Err(e) = priv_run("destroy", name).await { - tracing::warn!(error = ?e, "nixos-container destroy returned an error; continuing cleanup"); - } - // Remove the systemd resource-limits drop-in via hive-priv. - if let Err(e) = crate::priv_client::remove_service_dropin(&container).await { - tracing::warn!(error = ?e, "remove service drop-in failed (non-fatal)"); - } - Ok(()) -} - -/// Rebuild `name`'s container: sync the meta flake, optionally re-lock -/// the agent's input, then re-apply + restart via `nixos-container`. -/// -/// When `relock` is `true` the agent's meta input is bumped to whatever -/// `applied//main` points at before the build. Pass `false` for -/// meta-update cascade rebuilds, where re-locking would revert the bump -/// the cascade just committed (see the inline note below). -/// -/// # Errors -/// -/// Propagates errors from meta-flake sync / lock-update and the -/// `nixos-container` apply + restart shellouts. -/// -/// Returns `true` when `defer_start` suppressed the start-after-update — -/// the caller owns bringing the container back up (see -/// [`rebuild_no_meta`]). -pub async fn rebuild( - name: &str, - hive: &HiveEnv, - paths: &AgentPaths, - relock: bool, - defer_start: bool, - on_step: &(dyn Fn(&str) + Send + Sync), - on_build_log_id: &(dyn Fn(i64) + Send + Sync), -) -> Result { - // Sync the meta flake (idempotent — no-op when the rendered - // flake matches disk) so a manual rebuild from the dashboard - // can also recover from a divergent meta repo (e.g. an agent - // got added directly via `nixos-container create` outside - // hive-c0re). - let agents = agents_for_meta(None).await?; - crate::meta::sync_agents(hive, &agents).await?; - // Then bump just this agent's input — picks up whatever - // `applied//main` currently points at (deployed/). - // Commits the lock if it changed. - // - // `relock = false` skips this: a meta-update cascade has *just* set - // the meta lock deliberately, and `lock_update_for_rebuild` re-runs - // `nix flake update agent-`, which re-resolves the agent's - // transitive inputs back to the agent's own flake.lock — reverting - // the input the meta-update just bumped. Cascade rebuilds therefore - // build against the freshly-set on-disk lock as-is. - if relock { - crate::meta::lock_update_for_rebuild(name).await?; - } - rebuild_no_meta(name, hive, paths, defer_start, on_step, on_build_log_id).await -} - -/// Container-level rebuild without touching the meta repo. Callers -/// that own the meta side themselves (`actions::run_apply_commit` -/// drives meta through the two-phase prepare/finalize/abort flow) -/// use this directly. Public `rebuild` wraps it with idempotent meta -/// sync + lock-bump-and-commit. -/// -/// `on_step` is called at each phase boundary with a short human-readable -/// label so callers can surface progress (e.g. update the rebuild-queue -/// step shown in the dashboard). Pass `&|_| ()` when progress reporting -/// is not needed. -/// -/// `on_build_log_id` is called with the build-log row id immediately after -/// the `nixos-container update` log row opens, before the actual update -/// command starts. Callers can use this to link the queue entry to the log -/// for live streaming. Pass `&|_| ()` when not needed. -/// -/// `defer_start` skips the start-after-update for a previously-running -/// container and returns `true` instead, so a queue-side caller can hand -/// the (potentially slow) container boot to the fast lane rather than -/// holding the serialized build lane through it. With `defer_start = -/// false` the start (with cold-start fallback) runs inline as before and -/// the return value is always `false`. The spawn path always starts -/// inline — a freshly-created container boots as part of provisioning. -pub async fn rebuild_no_meta( - name: &str, - hive: &HiveEnv, - paths: &AgentPaths, - defer_start: bool, - on_step: &(dyn Fn(&str) + Send + Sync), - on_build_log_id: &(dyn Fn(i64) + Send + Sync), -) -> Result { - validate(name)?; - if let Some(other) = port_collision(name).await { - bail!( - "port {} is already taken by '{other}' — rename one of them and retry", - agent_web_port(name) - ); - } - setup_applied(&paths.applied_dir, None, name).await?; - ensure_agent_state_subvolume(name).await?; - ensure_claude_dir(&paths.claude_dir)?; - ensure_state_dir(&paths.notes_dir)?; - let container = container_name(name); - let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display()); - if container_exists(name).await { - // Rebuild strategy: stop-before-update + pre-build. - // See `docs/coordinator.md::Container lifecycle`. - let was_running = is_running(name).await; - set_nspawn_flags( - &container, - &paths.agent_dir, - &paths.claude_dir, - &paths.notes_dir, - ) - .await?; - set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; - systemd_daemon_reload().await?; - if was_running { - on_step("nix build"); - prebuild_toplevel(name, &flake_ref).await?; - on_step("nixos-container stop"); - priv_run("stop", name).await?; - } - on_step("nixos-container update"); - let update_result = priv_run_inner("update", name, Some(on_build_log_id)).await; - if let Err(ref update_err) = update_result { - // The update failed (e.g. nix build error). If the agent was - // running before we stopped it, try to bring it back up on the - // previous successful configuration so it doesn't stay dead. - // The start failure is logged but not promoted to an error — - // we always propagate the original update error (below). - if was_running { - tracing::warn!( - %name, - error = %update_err, - "nixos-container update failed; attempting restart on old config" - ); - on_step("nixos-container start (recovery)"); - if let Err(e) = priv_run("start", name).await { - tracing::warn!(%name, error = %e, "recovery start after failed update also failed"); - } - } - } - update_result?; - if was_running { - if defer_start { - // The caller re-queues the start on the fast lane so the - // build lane is freed for the next entry instead of - // waiting out the container boot here. - return Ok(true); - } - on_step("nixos-container start"); - start_with_fallback(name).await?; - } - Ok(false) - } else { - // Spawn path: create is atomic, no prebuild needed. - // See `docs/coordinator.md::Spawn path`. - on_step("nixos-container create"); - priv_run("create", name).await?; - set_nspawn_flags( - &container, - &paths.agent_dir, - &paths.claude_dir, - &paths.notes_dir, - ) - .await?; - set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; - systemd_daemon_reload().await?; - on_step("nixos-container start"); - priv_run("start", name).await?; - Ok(false) - } -} - -/// Pre-build `system.build.toplevel` against `meta#` so the -/// subsequent `nixos-container update` finds the result cached and -/// skips straight to the profile-swap. Store-warming only — container -/// is untouched. See `docs/coordinator.md::Rebuild path` for why -/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild -/// attr path` for why the explicit nixosConfigurations attr is required. -async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> { - use tokio::io::{AsyncBufReadExt, BufReader}; - // Split `#` so we can re-emit with the explicit - // `nixosConfigurations.` segment. The flake_ref shape is - // constructed by `rebuild_no_meta` and always contains exactly one - // `#`; `split_once` returning None here would be a programmer - // error we'd want to surface loudly rather than paper over. - let (flake_root, fragment) = flake_ref - .split_once('#') - .with_context(|| format!("flake_ref {flake_ref:?} missing '#' fragment"))?; - // Sanity-check the fragment matches the agent name we were - // passed — guards against future calls that pass a divergent - // pair (no current callsite does, but the pair is redundant - // and worth checking once). - if fragment != name { - anyhow::bail!("prebuild_toplevel: flake_ref fragment '{fragment}' ≠ agent name '{name}'"); - } - let attr = format!("{flake_root}#nixosConfigurations.{name}.config.system.build.toplevel"); - let args = vec![ - "--extra-experimental-features", - "nix-command flakes", - "build", - "--no-link", - "--print-out-paths", - &attr, - ]; - let cmdline = format!("nix {}", args.join(" ")); - tracing::info!(%name, %cmdline, "prebuild: warming system toplevel"); - - // Open a build_logs row for this attempt (best-effort — None when - // the global handle hasn't been installed, e.g. early startup - // or standalone tests). Lines pumped from stdout/stderr append - // into the row; `finish` lands the terminal status before we bail. - let logs = crate::build_logs::global(); - let log_id = logs.as_ref().and_then(|h| { - h.start(name, "prebuild", &cmdline) - .map_err(|e| { - tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)"); - }) - .ok() - }); - - let mut child = Command::new("nix") - .args(&args) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .with_context(|| format!("spawn {cmdline}"))?; - - let stdout = child.stdout.take().expect("piped stdout"); - let stderr = child.stderr.take().expect("piped stderr"); - - let stdout_cmdline = cmdline.clone(); - let stdout_logs = logs.clone(); - let pump_stdout = tokio::spawn(async move { - let mut lines = BufReader::new(stdout).lines(); - while let Ok(Some(line)) = lines.next_line().await { - tracing::info!(target: "nix-prebuild", cmdline = %stdout_cmdline, "{line}"); - if let (Some(h), Some(id)) = (&stdout_logs, log_id) { - h.append_stdout(id, &line); - } - } - }); - - let stderr_cmdline = cmdline.clone(); - let stderr_logs = logs.clone(); - let pump_stderr = tokio::spawn(async move { - let mut lines = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = lines.next_line().await { - tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}"); - if let (Some(h), Some(id)) = (&stderr_logs, log_id) { - h.append_stderr(id, &line); - } - } - }); - - let status = child - .wait() - .await - .with_context(|| format!("wait {cmdline}"))?; - let _ = pump_stdout.await; - let _ = pump_stderr.await; - - let ok = status.success(); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.finish( - id, - if ok { - crate::build_logs::BuildStatus::Ok - } else { - crate::build_logs::BuildStatus::Fail - }, - ); - } - if !ok { - match log_id { - Some(id) => bail!("prebuild {cmdline} failed ({status}); see build log #{id}"), - None => bail!("prebuild {cmdline} failed ({status})"), - } - } - Ok(()) -} - -pub async fn list() -> Result> { - let stdout = crate::priv_client::list_containers().await?; - Ok(stdout - .lines() - .map(str::trim) - .filter(|line| line.starts_with(AGENT_PREFIX)) - .map(str::to_owned) - .collect()) -} - -/// Initialize the manager-editable proposed repo. Seeds two tracked -/// files: `agent.nix` (the module the manager edits) and `flake.nix` -/// (the boilerplate that lets the meta flake import this repo as an -/// input — meta locks at a specific sha and reads -/// `nixosModules.default`, so `flake.nix` must be in the commit). The -/// manager shouldn't edit `flake.nix` (the prompt says so) but it's -/// visible so they can introspect. -/// -/// Touched by hive-c0re only on first spawn — never again — so the -/// manager can't be surprised by hive-c0re commits or working-tree -/// resets. -pub async fn setup_proposed(proposed_dir: &Path, name: &str) -> Result<()> { - let fresh = !proposed_dir.join(".git").exists(); - if fresh { - std::fs::create_dir_all(proposed_dir) - .with_context(|| format!("create {}", proposed_dir.display()))?; - let agent_path = proposed_dir.join("agent.nix"); - if !agent_path.exists() { - std::fs::write(&agent_path, initial_agent_nix(name)) - .with_context(|| format!("write {}", agent_path.display()))?; - } - let flake_path = proposed_dir.join("flake.nix"); - if !flake_path.exists() { - std::fs::write(&flake_path, initial_flake_nix()) - .with_context(|| format!("write {}", flake_path.display()))?; - } - git(proposed_dir, &["init", "--initial-branch=main"]).await?; - git(proposed_dir, &["add", "agent.nix", "flake.nix"]).await?; - git_commit(proposed_dir, "hive-c0re init").await?; - } - // Idempotently wire the `applied` remote — purely for the - // manager's ergonomics. The URL is the path inside the manager - // container (`/applied//.git`), where the RO bind in - // `set_nspawn_flags` makes it real. hive-c0re itself never - // dereferences this remote; the host-side fetch in - // `request_apply_commit` uses absolute host paths. - ensure_applied_remote(proposed_dir, name).await -} - -async fn ensure_applied_remote(proposed_dir: &Path, name: &str) -> Result<()> { - let want = format!("/applied/{name}/.git"); - let existing = git_command() - .current_dir(proposed_dir) - .args(["remote", "get-url", "applied"]) - .output() - .await - .with_context(|| format!("git remote get-url applied in {}", proposed_dir.display()))?; - if existing.status.success() { - let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned(); - if current == want { - return Ok(()); - } - // URL drifted (path scheme changed, etc.) — re-point it. - return git(proposed_dir, &["remote", "set-url", "applied", &want]).await; - } - git(proposed_dir, &["remote", "add", "applied", &want]).await -} - -/// Set up the applied repo. First-spawn only: init the repo, pull -/// proposed's initial commit in via `git fetch`, tag it `deployed/0`. -/// This is the *only* time hive-c0re reads from `proposed` for an -/// agent — subsequent proposals are fetched at `request_apply_commit` -/// time and tagged `proposal/` (see `actions::approve` for the -/// tag state machine). -/// -/// `proposed_dir` is `None` on rebuild paths where the repo already -/// exists — we just verify it's the right shape and bail otherwise. -/// Unlike the pre-overhaul code path, `flake.nix` is no longer -/// regenerated at the host level: it's tracked in proposed (seeded by -/// `setup_proposed`) and rides along on every fetch. -pub async fn setup_applied( - applied_dir: &Path, - proposed_dir: Option<&Path>, - name: &str, -) -> Result<()> { - std::fs::create_dir_all(applied_dir) - .with_context(|| format!("create {}", applied_dir.display()))?; - - if !applied_dir.join(".git").exists() { - let Some(proposed) = proposed_dir else { - bail!( - "applied repo at {} is missing its .git directory; \ - cannot rebuild without a proposed source to seed from. \ - destroy --purge and re-spawn this agent.", - applied_dir.display() - ); - }; - git(applied_dir, &["init", "--initial-branch=main"]).await?; - let proposed_str = proposed.display().to_string(); - // Seed the applied repo at the root (template) commit of proposed, - // not at `main`. This ensures `deployed/0` is the template baseline - // so the first ApplyCommit diff shows the manager's real changes - // rather than an empty diff (which happens when the manager has - // already committed their config and proposed/main == proposal/). - let root_sha = git_root_commit(proposed).await?; - git( - applied_dir, - // --update-head-ok lets us fetch into refs/heads/main while - // HEAD still points there. git's default safeguard refuses - // to avoid index/working-tree desync, but the working tree - // is empty (we just `init`'d) and we read-tree-reset right - // after, so the safeguard is moot here. - &[ - "fetch", - "--no-tags", - "--update-head-ok", - &proposed_str, - &format!("{root_sha}:refs/heads/main"), - ], - ) - .await?; - git_read_tree_reset(applied_dir, "refs/heads/main").await?; - git_tag(applied_dir, "deployed/0", "refs/heads/main").await?; - } else if git_rev_parse(applied_dir, "refs/tags/deployed/0") - .await - .is_err() - { - // Pre-overhaul applied repo — no deployed/* tag scheme, - // flake.nix may be untracked, agent.nix possibly authored by - // hive-c0re directly. The startup auto-migration fixes this - // in place; if it didn't run (or got skipped), surface a - // clear error. - bail!( - "applied repo at {} predates the meta-flake layout. \ - Restart hive-c0re to let the auto-migration run, or \ - destroy --purge {name} and re-spawn.", - applied_dir.display() - ); - } - Ok(()) -} - -/// Create the per-agent Claude credentials dir if missing. Mode 0755 — hive-core -/// needs read+execute to list the directory so `claude_has_session` can detect a -/// valid session; credential files inside (`.credentials.json` etc.) are 0600 so -/// secrets stay private regardless of the directory mode. Idempotent: existing -/// dirs are left untouched (an agent's OAuth tokens survive `destroy`/recreate). -/// Public for the `InitConfig` approval path in `actions.rs` which seeds -/// dirs without calling the full `spawn`. -pub fn ensure_claude_dir(claude_dir: &Path) -> Result<()> { - use std::io; - if !claude_dir.exists() { - std::fs::create_dir_all(claude_dir) - .with_context(|| format!("create {}", claude_dir.display()))?; - } - // 0755: hive-core (different user from the agent) needs read+execute to - // list the directory so `claude_has_session` can detect a valid session. - // The credential files inside (`.credentials.json` etc.) are 0600 so the - // secrets themselves stay private regardless of the directory mode. - // - // Best-effort: on the first container boot, `hive-agent-user-migrate` - // chowns this dir to the agent user. After that, hive-core (a different - // user) cannot chmod it (EPERM) — that's fine because the mode set during - // initial creation (0755) is preserved through the chown. Any other error - // (ENOENT, I/O error) is unexpected and propagated. - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - match std::fs::set_permissions(claude_dir, std::fs::Permissions::from_mode(0o755)) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { - tracing::debug!( - path = %claude_dir.display(), - "ensure_claude_dir: chmod 755 skipped (dir likely owned by agent user after migration)" - ); - } - Err(e) => { - return Err(e).with_context(|| format!("chmod 755 {}", claude_dir.display())); - } - } - } - Ok(()) -} - -/// Public for the `InitConfig` approval path in `actions.rs` which seeds -/// dirs without calling the full `spawn`. Also creates the sibling `harness/` -/// dir so the first harness startup can write its sqlite files immediately. -pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> { - if !notes_dir.exists() { - std::fs::create_dir_all(notes_dir) - .with_context(|| format!("create {}", notes_dir.display()))?; - } - // Harness dir is a sibling of the agent-visible state dir. - if let Some(parent) = notes_dir.parent() { - let harness_dir = parent.join("harness"); - if !harness_dir.exists() { - std::fs::create_dir_all(&harness_dir) - .with_context(|| format!("create {}", harness_dir.display()))?; - } - } - Ok(()) -} - -/// Ensure agent `name`'s persistent state root -/// (`/var/lib/hyperhive/agents/`) is a btrfs subvolume — when the host -/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`, -/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`. -/// -/// Progressive enhancement: if the root already exists -/// (any agent provisioned before this landed, plain dir or subvol) it's left -/// exactly as-is — no auto-migration — and the priv round-trip is skipped. On -/// a non-btrfs host the priv op no-ops and the root is later created as a -/// plain dir by `ensure_*_dir`, identical to the old behaviour. Only a -/// brand-new agent on a btrfs host gets a real subvolume. Subvolume creation -/// is privileged, so it's delegated to hive-priv. -pub async fn ensure_agent_state_subvolume(name: &str) -> Result<()> { - let root = Path::new(HOST_AGENTS_ROOT).join(name); - if root.exists() { - return Ok(()); - } - crate::priv_client::ensure_agent_subvolume(name) - .await - .with_context(|| format!("ensure btrfs subvolume for agent {name}")) -} - -fn initial_agent_nix(name: &str) -> String { - format!( - "{{ config, pkgs, lib, ... }}:\n{{\n # Per-agent overrides for {name}. This is a regular NixOS module\n # — add packages, services, modules, imports as needed.\n #\n # imports = [ ./extra-module.nix ];\n # environment.systemPackages = with pkgs; [ ];\n}}\n", - ) -} - -/// Module-only flake exposed by every agent's repo. Consumed by the -/// hive-c0re-owned meta flake at `/var/lib/hyperhive/meta/` as a flake -/// input. The wrapper is intentionally permissive: -/// -/// - Manager edits `inputs.* = …` to add other flakes (e.g. an MCP -/// server's own flake) — the lock for those lands in the agent's -/// own `flake.lock` and rolls up into meta's lock transitively. -/// - The outputs block forwards every input (minus `self`) into -/// `agent.nix` as the `flakeInputs` module argument, so the -/// manager just references `flakeInputs..packages.${pkgs.system}.default` -/// without further plumbing. -/// -/// Identity injection (`HIVE_PORT` / `HIVE_LABEL` / dashboard port / -/// git committer) still lives in the meta flake's wrapper. -pub fn initial_flake_nix() -> &'static str { - "{\n description = \"hyperhive agent\";\n inputs = { };\n outputs =\n { self, ... }@inputs:\n {\n nixosModules.default = {\n imports = [ ./agent.nix ];\n _module.args.flakeInputs = builtins.removeAttrs inputs [ \"self\" ];\n };\n };\n}\n" -} - -/// Return the SHA of the root (oldest, no-parent) commit in a repo. -/// Used to seed the applied repo at the template baseline rather than at -/// `main`, so the first `ApplyCommit` diff shows the manager's real changes. -async fn git_root_commit(dir: &Path) -> Result { - let out = git_command() - .current_dir(dir) - .args(["rev-list", "--max-parents=0", "HEAD"]) - .output() - .await - .with_context(|| format!("git rev-list --max-parents=0 HEAD in {}", dir.display()))?; - if !out.status.success() { - anyhow::bail!( - "git rev-list --max-parents=0 failed: {}", - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) -} - -async fn git_commit(dir: &Path, message: &str) -> Result<()> { - git( - dir, - &[ - "-c", - &format!("user.name={GIT_NAME}"), - "-c", - &format!("user.email={GIT_EMAIL}"), - "commit", - "-m", - message, - ], - ) - .await -} - -/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in -/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise. -#[must_use] -pub fn git_command() -> Command { - let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into()); - Command::new(exe) -} - -pub async fn git(dir: &Path, args: &[&str]) -> Result<()> { - let out = git_command() - .current_dir(dir) - .args(args) - .output() - .await - .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?; - if !out.status.success() { - bail!( - "git {} failed ({}): {}", - args.join(" "), - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok(()) -} - -/// Fetch the commit `sha` from the `src` git repo into `dst` and pin -/// it as `refs/tags/`. Used at `request_apply_commit` time so -/// hive-c0re captures an immutable handle on the manager's commit; -/// subsequent amendments / force-pushes in `src` no longer affect -/// what gets built. Returns the resolved full sha. -/// -/// `sha` must be a commit sha (short or full) — the caller -/// (`submit_apply_commit`) shape-checks it first. We resolve it -/// LOCALLY against `src` rather than asking the remote to resolve -/// it: `git fetch :` treats the left side as a -/// remote *ref name*, and a bare sha is not one ("couldn't find -/// remote ref ..."). Fetching by sha would need a full 40-hex sha -/// plus `uploadpack.allow*SHA1InWant` on the remote, which the -/// proposed repos don't set. hive-c0re has direct read access to -/// `src`, so a local `rev-parse` + a branch-glob fetch sidesteps -/// the whole sha-want negotiation. -pub async fn git_fetch_to_tag(dst: &Path, src: &Path, sha: &str, tag: &str) -> Result { - let src_str = src.display().to_string(); - // Resolve the (short-or-full) sha to a full sha against the - // source repo. The `^{commit}` peel + non-zero exit on a missing - // object means a typo'd / stale sha fails loudly right here. - let full = git_rev_parse(src, &format!("{sha}^{{commit}}")) - .await - .with_context(|| format!("commit '{sha}' not found in proposed repo {src_str}"))?; - // Bring src's objects into dst. Fetching every head pulls the - // wanted commit's history (always reachable from a branch in the - // manager's flow) into dst's object db without sha-want. - git( - dst, - &[ - "fetch", - "--no-tags", - &src_str, - "+refs/heads/*:refs/remotes/proposal-src/*", - ], - ) - .await?; - // Pin the exact commit as the proposal tag. The objects are now - // local so this resolves without touching the remote. - git(dst, &["tag", tag, &full]).await.with_context(|| { - format!("tag {tag} at {full}: commit not reachable from any branch in proposed repo") - })?; - Ok(full) -} - -/// Resolve `refname` (a tag, branch, or sha) in `dir` to its full sha. -pub async fn git_rev_parse(dir: &Path, refname: &str) -> Result { - let out = git_command() - .current_dir(dir) - .args(["rev-parse", refname]) - .output() - .await - .with_context(|| format!("git rev-parse {refname} in {}", dir.display()))?; - if !out.status.success() { - bail!( - "git rev-parse {refname} failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) -} - -/// Plant a lightweight tag at `target`. Errors if the tag already -/// exists — we want loud failures on id reuse, not silent -/// overwrites. -pub async fn git_tag(dir: &Path, name: &str, target: &str) -> Result<()> { - git(dir, &["tag", name, target]).await -} - -/// Plant an annotated tag with `body` as the message. Used for -/// `failed/` (body = build error) and `denied/` (body = -/// operator note). Multi-line bodies handled via stdin so we don't -/// have to escape anything. -pub async fn git_tag_annotated(dir: &Path, name: &str, target: &str, body: &str) -> Result<()> { - use tokio::io::AsyncWriteExt; - // Annotated tags are git objects, so they need a tagger identity - // (same constraint as a commit). Pass the hive-c0re identity - // inline rather than relying on a global git config — applied - // repos are hive-c0re-owned and the host's user might not have - // user.email set. - let mut child = git_command() - .current_dir(dir) - .args([ - "-c", - &format!("user.name={GIT_NAME}"), - "-c", - &format!("user.email={GIT_EMAIL}"), - "tag", - "-a", - name, - target, - "-F", - "-", - ]) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .with_context(|| format!("spawn git tag -a {name} in {}", dir.display()))?; - if let Some(mut stdin) = child.stdin.take() { - stdin - .write_all(body.as_bytes()) - .await - .context("write tag body to git stdin")?; - // Drop closes stdin so git can finish reading. - drop(stdin); - } - let out = child.wait_with_output().await.context("wait git tag -a")?; - if !out.status.success() { - bail!( - "git tag -a {name} failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok(()) -} - -/// Replace working tree + index with the tree at `target` without -/// moving HEAD. `applied/main` stays pointing at the last known-good -/// `deployed/*` while we let `nixos-container update` evaluate the -/// candidate. On build failure callers reset back to HEAD; on -/// success they fast-forward main to `target`. -pub async fn git_read_tree_reset(dir: &Path, target: &str) -> Result<()> { - git(dir, &["read-tree", "--reset", "-u", target]).await -} - -/// Hard-set a ref to `target`. Used to fast-forward `refs/heads/main` -/// to the just-deployed proposal commit. Uses `update-ref`, not -/// `branch -f`, so it works regardless of where HEAD currently sits. -pub async fn git_update_ref(dir: &Path, refname: &str, target: &str) -> Result<()> { - git(dir, &["update-ref", refname, target]).await -} - -/// Write a systemd drop-in for `container@.service` that applies -/// our default resource caps. Goes under `/run/systemd/system/...` so it's -/// ephemeral (regenerated on every spawn / rebuild). -async fn set_resource_limits(container: &str, cpu_quota: &str, memory_max: &str) -> Result<()> { - crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await -} - -async fn systemd_daemon_reload() -> Result<()> { - crate::priv_client::daemon_reload().await -} - -/// Idempotently rewrite the lines in `/etc/nixos-containers/.conf` -/// that hive-c0re owns: `PRIVATE_NETWORK` (forced 0 so the agent's web UI port -/// is reachable on the host) and `EXTRA_NSPAWN_FLAGS` (the runtime-dir bind). -/// The start script expands `$EXTRA_NSPAWN_FLAGS` unquoted into the -/// `systemd-nspawn` command. -/// Where in the container's filesystem the manager sees its agents tree. -/// Matches the `/agents` path that pre-Phase-8 hosts declared via -/// `containers.root.bindMounts."/agents"`. -pub const CONTAINER_MANAGER_AGENTS_MOUNT: &str = "/agents"; - -/// Where the manager sees the applied trees of every agent, read-only. -/// Manager runs `git fetch /applied//.git refs/tags/*:refs/tags/applied/*` -/// to learn what hive-c0re deployed (or rejected, or failed to -/// build); the RO bind makes accidental writes impossible from -/// inside the container. -pub const CONTAINER_MANAGER_APPLIED_MOUNT: &str = "/applied"; - -/// The on-host root that gets bind-mounted to `/agents` inside the manager. -/// Hard-coded to match `AGENT_STATE_ROOT` in coordinator.rs (kept duplicated -/// here so lifecycle stays usable as a leaf module). -const HOST_AGENTS_ROOT: &str = "/var/lib/hyperhive/agents"; - -/// On-host applied repo root, mirrored RO into the manager. Matches -/// `APPLIED_STATE_ROOT` in coordinator.rs. -const HOST_APPLIED_ROOT: &str = "/var/lib/hyperhive/applied"; - -/// On-host meta repo root, mirrored RO into the manager. Matches -/// `meta::meta_dir()` but duplicated here so lifecycle stays a leaf. -const HOST_META_ROOT: &str = "/var/lib/hyperhive/meta"; - -/// Shared directory accessible to all agents. All agents bind-mount this RW. -const HOST_SHARED_ROOT: &str = "/var/lib/hyperhive/shared"; - -/// Append bind flags for `child`'s state, harness, and config dirs into -/// `binds`, all read-write. The RW on `state` is deliberate (recovery), -/// not an oversight; see docs/persistence.md ("Parent access to child -/// state") for the rationale. Creates missing host-side directories so -/// nspawn doesn't refuse to start; missing dirs are non-fatal. -fn bind_child_agent_dirs(child: &str, binds: &mut Vec) { - let state_dir = format!("{HOST_AGENTS_ROOT}/{child}/state"); - let harness_dir = format!("{HOST_AGENTS_ROOT}/{child}/harness"); - let config_dir = format!("{HOST_AGENTS_ROOT}/{child}/config"); - for dir in [&state_dir, &harness_dir, &config_dir] { - let _ = std::fs::create_dir_all(dir); - } - binds.push(BindMount { - host_path: state_dir, - container_path: format!("/agents/{child}/state"), - read_only: false, - }); - binds.push(BindMount { - host_path: harness_dir, - container_path: format!("/agents/{child}/harness"), - read_only: false, - }); - binds.push(BindMount { - host_path: config_dir, - container_path: format!("/agents/{child}/config"), - read_only: false, - }); -} - -/// Hive-wide secrets forwarded into every agent container via nspawn -/// `--load-credential=:`. Currently just the OTEL -/// auth-header secret, when `services.hyperhive.otel.headersCredential` -/// is set (surfaced as `HYPERHIVE_OTEL_HEADERS_CREDENTIAL` on hive-c0re's -/// unit env — the same host option meta.rs reads to inject -/// `hyperhive.otel.headersCredential`). The inner harness unit reads it -/// via `LoadCredential=otel-headers` (inherit). The secret never lands in -/// a bind mount, the nix store, or the generated config. -/// -/// A configured-but-missing file is skipped with a warning rather than -/// forwarded (nspawn would refuse to start the container otherwise): a -/// host-level secret typo shouldn't take down every agent's start; OTEL -/// just exports without the auth header until the file appears. -fn hive_load_credentials() -> Vec { - let mut out = Vec::new(); - let Ok(path) = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL") else { - return out; - }; - if path.is_empty() { - return out; - } - if std::path::Path::new(&path).is_file() { - out.push(CredentialMount { - name: "otel-headers".to_owned(), - host_path: path, - }); - } else { - tracing::warn!( - %path, - "HYPERHIVE_OTEL_HEADERS_CREDENTIAL is set but the file is missing; \ - skipping --load-credential (OTEL will export without the auth header)" - ); - } - out -} - -#[allow( - clippy::too_many_lines, - reason = "one contiguous nspawn-flag assembly block; the length is the flag \ - surface itself, splitting it would just hide the shape" -)] -async fn set_nspawn_flags( - container: &str, - runtime_dir: &Path, - claude_dir: &Path, - notes_dir: &Path, -) -> Result<()> { - // Ensure /shared directory exists before binding. systemd-nspawn requires the bind source to exist. - std::fs::create_dir_all(HOST_SHARED_ROOT) - .with_context(|| format!("create {HOST_SHARED_ROOT}"))?; - // Make /shared writable by every agent. Containers share host uids (no - // PrivateUsers), but each agent is a distinct unix user, so a root-owned - // 0755 dir leaves them unable to write — the documented "read/write for - // all agents" contract was broken. A setgid group would need a - // pinned GID declared in every container plus all agent users joined to - // it (cross-container coordination + a rebuild cascade); instead we use - // the /tmp model — sticky world-writable (1777). The sticky bit lets any - // agent create files while protecting each agent's entries from deletion - // by the others, and matches /shared's documented "free-for-all, may be - // deleted/lost" semantics without touching any per-agent config. - { - use std::os::unix::fs::PermissionsExt as _; - let perms = std::fs::Permissions::from_mode(0o1777); - std::fs::set_permissions(HOST_SHARED_ROOT, perms) - .with_context(|| format!("chmod 1777 {HOST_SHARED_ROOT}"))?; - } - // Ensure /knowledge dir exists. It may be empty until forge seeds it; - // nspawn refuses to start if the bind source is missing entirely. - std::fs::create_dir_all(crate::knowledge::LOCAL_DIR) - .with_context(|| format!("create {}", crate::knowledge::LOCAL_DIR))?; - - // Logical agent name — strip the `h-` prefix. - // For the manager: `h-ruth` → `ruth`. For sub-agents: `h-iris` → `iris`. - let agent_name = container.strip_prefix(AGENT_PREFIX).unwrap_or(container); - - // Claude credentials land at `/home//.claude` so the - // `claude` CLI (which reads `$HOME/.claude`) finds them. The - // harness service's environment sets `HOME` to the same path - // (`agent-base.nix` / `manager.nix`), so no `--setenv` plumbing - // is needed here — the bind alone is enough. - let claude_mount = container_claude_mount(agent_name); - - // Hive-wide secrets forwarded into the container's credential store - // (currently just the OTEL auth-header). Same for every agent. - let load_creds = hive_load_credentials(); - - let mut binds: Vec = vec![ - BindMount { - host_path: runtime_dir.to_string_lossy().into_owned(), - container_path: CONTAINER_RUNTIME_MOUNT.to_owned(), - read_only: false, - }, - BindMount { - host_path: claude_dir.to_string_lossy().into_owned(), - container_path: claude_mount, - read_only: false, - }, - BindMount { - host_path: HOST_SHARED_ROOT.to_owned(), - container_path: CONTAINER_SHARED_MOUNT.to_owned(), - read_only: false, - }, - BindMount { - host_path: crate::knowledge::LOCAL_DIR.to_owned(), - container_path: crate::knowledge::CONTAINER_MOUNT.to_owned(), - read_only: true, - }, - ]; - - // Own state, harness, and config dirs — same for every agent including - // the manager. Config is RO: an agent must not edit its own config; changes - // only ever flow through the approval queue. - binds.push(BindMount { - host_path: notes_dir.to_string_lossy().into_owned(), - container_path: format!("/agents/{agent_name}/state"), - read_only: false, - }); - if let Some(state_parent) = notes_dir.parent() { - let harness_dir = state_parent.join("harness"); - if !harness_dir.exists() { - let _ = std::fs::create_dir_all(&harness_dir); - } - binds.push(BindMount { - host_path: harness_dir.to_string_lossy().into_owned(), - container_path: format!("/agents/{agent_name}/harness"), - read_only: false, - }); - } - let own_config = format!("{HOST_AGENTS_ROOT}/{agent_name}/config"); - std::fs::create_dir_all(&own_config).with_context(|| format!("create {own_config}"))?; - binds.push(BindMount { - host_path: own_config, - container_path: format!("/agents/{agent_name}/config"), - read_only: true, - }); - - // Topology-driven child mounts: every direct child of this agent gets - // its state, harness, and config dirs bind-mounted RW (parent reads + - // writes child state for recovery, and manages config). See - // `bind_child_agent_dirs`. - let direct_children = crate::topology::children_of(agent_name); - for child in &direct_children { - bind_child_agent_dirs(child, &mut binds); - } - - // `can_manage_top_level_agents` role: additionally mount every - // parentless agent in the topology as a virtual child. Enables - // recovery — a role holder can update those agents' configs even - // when they are down. Also grants RO access to /applied and /meta. - if crate::topology::has_role( - agent_name, - crate::topology::ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS, - ) { - let top_level = crate::topology::top_level_agents(); - for tl in &top_level { - if !direct_children.contains(tl) { - bind_child_agent_dirs(tl, &mut binds); - } - } - // systemd-nspawn refuses to start a container whose bind - // source doesn't exist. The meta repo is created by the - // startup migration, but make sure the directory is there - // before the role holder comes up in case set_nspawn_flags - // fires first (e.g. cold start with no agents). - std::fs::create_dir_all(HOST_META_ROOT) - .with_context(|| format!("create {HOST_META_ROOT}"))?; - binds.push(BindMount { - host_path: HOST_APPLIED_ROOT.to_owned(), - container_path: CONTAINER_MANAGER_APPLIED_MOUNT.to_owned(), - read_only: true, - }); - binds.push(BindMount { - host_path: HOST_META_ROOT.to_owned(), - container_path: crate::meta::CONTAINER_MANAGER_META_MOUNT.to_owned(), - read_only: true, - }); - } - - // Web-socket subdir: bind-mount `/run/hive-agent//` into the - // container so the harness can bind `web.sock` there and the host-side - // gateway sees it. Subdir bind (not socket file) keeps the inode - // visible after the harness unlinks a stale socket on rebind. - // Applies to manager and sub-agents alike. - let socket_dir = crate::agent_sockets::agent_dir_for(agent_name); - std::fs::create_dir_all(&socket_dir) - .with_context(|| format!("create {}", socket_dir.display()))?; - // Chown to the agent user so the non-root harness can bind(2) here. - // Falls back to 0777 on first spawn when uid lookup returns None - // (container /etc/passwd not yet rendered). - if let Some((uid, gid)) = agent_uid_gid(agent_name) { - if let Err(e) = crate::priv_client::chown_socket_dir(agent_name, uid, gid).await { - tracing::warn!(%agent_name, error = ?e, "chown socket dir failed"); - } - } else if let Err(e) = crate::priv_client::chmod_socket_dir(agent_name, 0o777).await { - tracing::warn!(%agent_name, error = ?e, "chmod socket dir failed"); - } - binds.push(BindMount { - host_path: socket_dir.to_string_lossy().into_owned(), - container_path: socket_dir.to_string_lossy().into_owned(), - read_only: false, - }); - - // Network isolation: when HIVE_NETWORK_ISOLATION=1 is set (by the - // hive-network.nix module's `isolateContainers` option), flip the - // container to a private network namespace with a veth pair attached - // to the host bridge. Applies to all containers including the manager - // (all hive-c0re<->agent comms go through bind-mounted UDS, not TCP). - let isolation = { - let isolate = std::env::var("HIVE_NETWORK_ISOLATION").ok().as_deref() == Some("1"); - let bridge = std::env::var("HIVE_NETWORK_BRIDGE").unwrap_or_default(); - let subnet = std::env::var("HIVE_NETWORK_SUBNET").unwrap_or_default(); - if isolate && !bridge.is_empty() && !subnet.is_empty() { - let Some(agent_ip) = agent_network_ip(agent_name, &subnet) else { - tracing::warn!( - %agent_name, %subnet, - "HIVE_NETWORK_SUBNET is set but could not derive a valid IP for agent \ - (bad CIDR? prefix too narrow?); skipping PRIVATE_NETWORK write to \ - avoid misconfigured isolation" - ); - return crate::priv_client::write_nspawn_flags( - container, - &binds, - None, - &load_creds, - ) - .await; - }; - let Some(gateway_ip) = bridge_gateway_ip(&subnet) else { - tracing::warn!( - %agent_name, %subnet, - "HIVE_NETWORK_SUBNET is set but the bridge gateway IP is unparseable; \ - skipping PRIVATE_NETWORK write to avoid an isolated container with no \ - default route or resolver" - ); - return crate::priv_client::write_nspawn_flags( - container, - &binds, - None, - &load_creds, - ) - .await; - }; - tracing::info!( - %agent_name, %agent_ip, %gateway_ip, %bridge, - "network isolation: PRIVATE_NETWORK=1" - ); - Some(hive_sh4re::priv_proto::NetworkIsolation { - agent_ip, - bridge, - gateway_ip, - }) - } else { - None - } - }; - - // Delegate the actual conf-file rewrite to hive-priv (runs as root). - crate::priv_client::write_nspawn_flags(container, &binds, isolation, &load_creds).await -} - -/// Build the per-line callback for `create_container_streaming` / -/// `update_container_streaming`. Both ops share identical dispatch logic -/// (stdout → info + `append_stdout`, stderr → warn + `append_stderr`); this -/// helper avoids duplicating that match body across the two call sites. -fn make_log_callback( - logs: Option>, - log_id: Option, - cmdline: String, -) -> impl FnMut(hive_sh4re::priv_proto::PrivStream, &str) { - use hive_sh4re::priv_proto::PrivStream; - move |stream, line| match stream { - PrivStream::Stdout => { - tracing::info!(target: "nixos-container", cmdline = %cmdline, "{line}"); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.append_stdout(id, line); - } - } - PrivStream::Stderr => { - tracing::warn!(target: "nixos-container", cmdline = %cmdline, "{line}"); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.append_stderr(id, line); - } - } - } -} - -/// Execute a container operation via hive-priv and integrate with -/// `build_logs.sqlite`. hive-priv runs as root and forwards output lines -/// to hive-c0re in real time via the streaming priv protocol. Each line -/// is appended to the build-log row as it arrives, so the dashboard -/// shows live progress during long `nixos-container create` / `update` runs. -async fn priv_run(kind: &str, name: &str) -> Result<()> { - priv_run_inner(kind, name, None).await -} - -/// Like `priv_run` but calls `on_log_id(log_id)` immediately after the -/// build-log row is opened — before the actual container op starts. -/// This lets callers surface the row id for live streaming (e.g. the -/// rebuild-queue worker sets `build_log_id` on the queue entry so the -/// dashboard can link to `/api/build-logs/id/{id}/stream`). -/// -/// The callback fires only when a build-log row is successfully opened -/// (i.e. the global `BuildLogs` handle is installed AND `h.start()` -/// succeeds). No-op when `on_log_id` is `None` — that's the path for -/// all callers that don't need the id. -async fn priv_run_inner( - kind: &str, - name: &str, - on_log_id: Option<&(dyn Fn(i64) + Send + Sync)>, -) -> Result<()> { - let container = container_name(name); - let cmdline = format!("nixos-container {kind} {container}"); - - let logs = crate::build_logs::global(); - let log_id = logs.as_ref().and_then(|h| { - h.start(name, kind, &cmdline) - .map_err(|e| { - tracing::warn!(error = ?e, "build_logs: start failed (priv_run log dropped)"); - }) - .ok() - }); - // Notify the caller as soon as the log row exists so it can surface - // the id for live streaming before the container op even starts. - if let (Some(id), Some(cb)) = (log_id, on_log_id) { - cb(id); - } - - // For long-running ops use the streaming protocol so build_logs - // receives lines in real time rather than as a batch at completion. - let result: Result<()> = match kind { - "create" => { - crate::priv_client::create_container_streaming( - name, - make_log_callback(logs.clone(), log_id, cmdline.clone()), - ) - .await - } - "update" => { - crate::priv_client::update_container_streaming( - name, - make_log_callback(logs.clone(), log_id, cmdline.clone()), - ) - .await - } - "start" => crate::priv_client::start_container(name).await, - "stop" => crate::priv_client::stop_container(name).await, - "kill" => crate::priv_client::kill_container(name).await, - "destroy" => crate::priv_client::destroy_container(name).await, - other => Err(anyhow::anyhow!("unknown container op: {other}")), - }; - - let succeeded = result.is_ok(); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.finish( - id, - if succeeded { - crate::build_logs::BuildStatus::Ok - } else { - crate::build_logs::BuildStatus::Fail - }, - ); - } - - match result { - Ok(()) => Ok(()), - Err(e) => { - let journal = if kind == "update" { - container_journal_tail(&container).await - } else { - String::new() - }; - match log_id { - Some(id) => bail!("{e:#}; see build log #{id}{journal}"), - None => bail!("{e:#}{journal}"), - } - } - } -} - -/// On a failed `nixos-container update`, the stderr nixos-container -/// itself prints is often terse ("failed to reload container") — the -/// real reason (which unit failed `switch-to-configuration` during -/// the reload phase) lands in the *container's* own journal, not on -/// the host. Fetch the tail of it so a failed rebuild self-documents -/// the failing unit in the error string, no second round-trip. -/// -/// Scoped to `update`: that's the reload-phase case, and the -/// container is still up (running the old generation) so -/// `journalctl -M` works. Best-effort — returns "" for other verbs -/// or when the journal can't be read (machine gone, journalctl -/// missing); it never produces an error of its own. -async fn container_journal_tail(container: &str) -> String { - // `-M` enters the container namespace and needs root, so the read - // is delegated to hive-priv (hive-c0re itself runs unprivileged). - let res = crate::priv_client::read_container_journal( - container, - hive_sh4re::priv_proto::JournalQuery { - lines: 40, - ..Default::default() - }, - ) - .await; - match res { - Ok((stdout, _)) if !stdout.is_empty() => format!( - "\n--- last 40 journal lines from container '{container}' ---\n{}", - stdout.trim_end() - ), - _ => String::new(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Regression test: `setup_proposed` must seed both agent.nix and flake.nix - /// in the initial commit. Before commit 5b5a93e flake.nix was missing from - /// the scaffold, requiring manual creation (seen with the damocles agent). - #[tokio::test] - async fn setup_proposed_seeds_flake_nix() { - let dir = tempfile::tempdir().expect("tempdir"); - let proposed = dir.path().join("proposed"); - setup_proposed(&proposed, "test-agent") - .await - .expect("setup_proposed"); - - // Both files must exist on disk. - assert!(proposed.join("agent.nix").exists(), "agent.nix missing"); - assert!(proposed.join("flake.nix").exists(), "flake.nix missing"); - - // flake.nix must export nixosModules.default (the meta-flake contract). - let flake = std::fs::read_to_string(proposed.join("flake.nix")).unwrap(); - assert!( - flake.contains("nixosModules.default"), - "flake.nix does not export nixosModules.default" - ); - - // Both files must be tracked in the initial git commit. - let out = git_command() - .current_dir(&proposed) - .args(["show", "--name-only", "--format=", "HEAD"]) - .output() - .await - .expect("git show"); - let tracked = String::from_utf8_lossy(&out.stdout); - assert!(tracked.contains("agent.nix"), "agent.nix not committed"); - assert!(tracked.contains("flake.nix"), "flake.nix not committed"); - } - - #[test] - fn agent_network_ip_is_in_subnet() { - // Default subnet 10.42.0.0/24 — agents get .2 to .254. - let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP"); - let octets: Vec = ip.split('.').map(|o| o.parse().unwrap()).collect(); - assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix"); - assert!( - octets[3] >= 2 && octets[3] <= 254, - "host byte {}", - octets[3] - ); - } - - #[test] - fn agent_network_ip_stable() { - // Same name + subnet must always produce the same IP. - let a = agent_network_ip("damocles", "10.42.0.0/24"); - let b = agent_network_ip("damocles", "10.42.0.0/24"); - assert_eq!(a, b); - } - - #[test] - fn agent_network_ip_different_agents() { - // Different agent names very likely produce different IPs (not guaranteed, - // but for these two names the hashes don't collide). - let alice = agent_network_ip("alice", "10.42.0.0/24").unwrap(); - let bob = agent_network_ip("bob", "10.42.0.0/24").unwrap(); - assert_ne!(alice, bob, "alice and bob collide — rename one"); - } - - #[test] - fn agent_network_ip_different_subnet() { - let ip = agent_network_ip("alice", "192.168.5.0/24").expect("should produce an IP"); - let octets: Vec = ip.split('.').map(|o| o.parse().unwrap()).collect(); - assert_eq!(&octets[..3], &[192, 168, 5]); - } - - #[test] - fn bridge_gateway_ip_extracts_verbatim_address() { - // HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the - // canonical network — the gateway is the address before the `/`. - assert_eq!( - bridge_gateway_ip("10.42.0.1/24").as_deref(), - Some("10.42.0.1") - ); - // Non-`.1` operator override: the gateway is wherever the bridge is. - assert_eq!( - bridge_gateway_ip("10.42.0.254/24").as_deref(), - Some("10.42.0.254") - ); - assert_eq!( - bridge_gateway_ip("172.30.0.1/16").as_deref(), - Some("172.30.0.1") - ); - } - - #[test] - fn bridge_gateway_ip_rejects_bad_input() { - assert!(bridge_gateway_ip("notanip/24").is_none()); - assert!(bridge_gateway_ip("10.42.0.1").is_none()); // no prefix - assert!(bridge_gateway_ip("10.42.0.1/33").is_none()); // prefix > 32 - assert!(bridge_gateway_ip("10.42.0.999/24").is_none()); // octet > 255 - assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets - } - - #[test] - fn agent_network_ip_rejects_bad_input() { - assert!(agent_network_ip("alice", "notanip/24").is_none()); - assert!(agent_network_ip("alice", "10.0.0.0/33").is_none()); // prefix > 32 - assert!(agent_network_ip("alice", "10.0.0.0/31").is_none()); // too small - assert!(agent_network_ip("alice", "10.0.0.0").is_none()); // no prefix - } - - #[test] - fn agent_network_ip_normalizes_bridge_ip_subnet() { - // HIVE_NETWORK_SUBNET carries the bridge IP (10.42.0.1/24), not - // canonical network (10.42.0.0/24). Both must produce the same result - // after host-bit masking. - let from_bridge = agent_network_ip("alice", "10.42.0.1/24"); - let from_canonical = agent_network_ip("alice", "10.42.0.0/24"); - assert_eq!( - from_bridge, from_canonical, - "bridge-IP and canonical-network form should normalize to the same result" - ); - // Result must still be in .2-.254. - let ip = from_bridge.unwrap(); - let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap(); - assert!((2..=254).contains(&last), "host byte {last}"); - } - - /// `setup_proposed` is idempotent: calling it on an existing repo is a - /// no-op (the fresh guard skips all writes). - #[tokio::test] - async fn setup_proposed_idempotent() { - let dir = tempfile::tempdir().expect("tempdir"); - let proposed = dir.path().join("proposed"); - setup_proposed(&proposed, "test-agent") - .await - .expect("first call"); - // Second call must not error even though .git already exists. - setup_proposed(&proposed, "test-agent") - .await - .expect("second call"); - // Still one commit. - let out = git_command() - .current_dir(&proposed) - .args(["rev-list", "--count", "HEAD"]) - .output() - .await - .expect("git rev-list"); - let count = String::from_utf8_lossy(&out.stdout).trim().to_owned(); - assert_eq!( - count, "1", - "expected exactly one commit after idempotent call" - ); - } -} diff --git a/hive-c0re/src/lifecycle/git.rs b/hive-c0re/src/lifecycle/git.rs new file mode 100644 index 00000000..66567779 --- /dev/null +++ b/hive-c0re/src/lifecycle/git.rs @@ -0,0 +1,207 @@ +//! Git shellout helpers for the per-agent proposed/applied repos: run +//! `git` with the hive-c0re identity, resolve/plant refs and tags, and +//! fetch proposal commits into the applied repo. + +use std::path::Path; + +use anyhow::{Context, Result, bail}; +use tokio::process::Command; + +const GIT_NAME: &str = "c0re"; +const GIT_EMAIL: &str = "c0re@hyperhive.local"; + +/// Return the SHA of the root (oldest, no-parent) commit in a repo. +/// Used to seed the applied repo at the template baseline rather than at +/// `main`, so the first `ApplyCommit` diff shows the manager's real changes. +pub(super) async fn git_root_commit(dir: &Path) -> Result { + let out = git_command() + .current_dir(dir) + .args(["rev-list", "--max-parents=0", "HEAD"]) + .output() + .await + .with_context(|| format!("git rev-list --max-parents=0 HEAD in {}", dir.display()))?; + if !out.status.success() { + anyhow::bail!( + "git rev-list --max-parents=0 failed: {}", + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) +} + +pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> { + git( + dir, + &[ + "-c", + &format!("user.name={GIT_NAME}"), + "-c", + &format!("user.email={GIT_EMAIL}"), + "commit", + "-m", + message, + ], + ) + .await +} + +/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in +/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise. +#[must_use] +pub fn git_command() -> Command { + let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into()); + Command::new(exe) +} + +pub async fn git(dir: &Path, args: &[&str]) -> Result<()> { + let out = git_command() + .current_dir(dir) + .args(args) + .output() + .await + .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?; + if !out.status.success() { + bail!( + "git {} failed ({}): {}", + args.join(" "), + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(()) +} + +/// Fetch the commit `sha` from the `src` git repo into `dst` and pin +/// it as `refs/tags/`. Used at `request_apply_commit` time so +/// hive-c0re captures an immutable handle on the manager's commit; +/// subsequent amendments / force-pushes in `src` no longer affect +/// what gets built. Returns the resolved full sha. +/// +/// `sha` must be a commit sha (short or full) — the caller +/// (`submit_apply_commit`) shape-checks it first. We resolve it +/// LOCALLY against `src` rather than asking the remote to resolve +/// it: `git fetch :` treats the left side as a +/// remote *ref name*, and a bare sha is not one ("couldn't find +/// remote ref ..."). Fetching by sha would need a full 40-hex sha +/// plus `uploadpack.allow*SHA1InWant` on the remote, which the +/// proposed repos don't set. hive-c0re has direct read access to +/// `src`, so a local `rev-parse` + a branch-glob fetch sidesteps +/// the whole sha-want negotiation. +pub async fn git_fetch_to_tag(dst: &Path, src: &Path, sha: &str, tag: &str) -> Result { + let src_str = src.display().to_string(); + // Resolve the (short-or-full) sha to a full sha against the + // source repo. The `^{commit}` peel + non-zero exit on a missing + // object means a typo'd / stale sha fails loudly right here. + let full = git_rev_parse(src, &format!("{sha}^{{commit}}")) + .await + .with_context(|| format!("commit '{sha}' not found in proposed repo {src_str}"))?; + // Bring src's objects into dst. Fetching every head pulls the + // wanted commit's history (always reachable from a branch in the + // manager's flow) into dst's object db without sha-want. + git( + dst, + &[ + "fetch", + "--no-tags", + &src_str, + "+refs/heads/*:refs/remotes/proposal-src/*", + ], + ) + .await?; + // Pin the exact commit as the proposal tag. The objects are now + // local so this resolves without touching the remote. + git(dst, &["tag", tag, &full]).await.with_context(|| { + format!("tag {tag} at {full}: commit not reachable from any branch in proposed repo") + })?; + Ok(full) +} + +/// Resolve `refname` (a tag, branch, or sha) in `dir` to its full sha. +pub async fn git_rev_parse(dir: &Path, refname: &str) -> Result { + let out = git_command() + .current_dir(dir) + .args(["rev-parse", refname]) + .output() + .await + .with_context(|| format!("git rev-parse {refname} in {}", dir.display()))?; + if !out.status.success() { + bail!( + "git rev-parse {refname} failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) +} + +/// Plant a lightweight tag at `target`. Errors if the tag already +/// exists — we want loud failures on id reuse, not silent +/// overwrites. +pub async fn git_tag(dir: &Path, name: &str, target: &str) -> Result<()> { + git(dir, &["tag", name, target]).await +} + +/// Plant an annotated tag with `body` as the message. Used for +/// `failed/` (body = build error) and `denied/` (body = +/// operator note). Multi-line bodies handled via stdin so we don't +/// have to escape anything. +pub async fn git_tag_annotated(dir: &Path, name: &str, target: &str, body: &str) -> Result<()> { + use tokio::io::AsyncWriteExt; + // Annotated tags are git objects, so they need a tagger identity + // (same constraint as a commit). Pass the hive-c0re identity + // inline rather than relying on a global git config — applied + // repos are hive-c0re-owned and the host's user might not have + // user.email set. + let mut child = git_command() + .current_dir(dir) + .args([ + "-c", + &format!("user.name={GIT_NAME}"), + "-c", + &format!("user.email={GIT_EMAIL}"), + "tag", + "-a", + name, + target, + "-F", + "-", + ]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .with_context(|| format!("spawn git tag -a {name} in {}", dir.display()))?; + if let Some(mut stdin) = child.stdin.take() { + stdin + .write_all(body.as_bytes()) + .await + .context("write tag body to git stdin")?; + // Drop closes stdin so git can finish reading. + drop(stdin); + } + let out = child.wait_with_output().await.context("wait git tag -a")?; + if !out.status.success() { + bail!( + "git tag -a {name} failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(()) +} + +/// Replace working tree + index with the tree at `target` without +/// moving HEAD. `applied/main` stays pointing at the last known-good +/// `deployed/*` while we let `nixos-container update` evaluate the +/// candidate. On build failure callers reset back to HEAD; on +/// success they fast-forward main to `target`. +pub async fn git_read_tree_reset(dir: &Path, target: &str) -> Result<()> { + git(dir, &["read-tree", "--reset", "-u", target]).await +} + +/// Hard-set a ref to `target`. Used to fast-forward `refs/heads/main` +/// to the just-deployed proposal commit. Uses `update-ref`, not +/// `branch -f`, so it works regardless of where HEAD currently sits. +pub async fn git_update_ref(dir: &Path, refname: &str, target: &str) -> Result<()> { + git(dir, &["update-ref", refname, target]).await +} diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs new file mode 100644 index 00000000..f7ae69b9 --- /dev/null +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -0,0 +1,367 @@ +//! Per-container host-side config: the nspawn conf rewrite (bind mounts, +//! network isolation, forwarded credentials), the systemd resource-limits +//! drop-in, and the `write_dropins` verb that re-applies both. + +use std::path::Path; + +use anyhow::{Context, Result}; +use hive_sh4re::priv_proto::{BindMount, CredentialMount}; + +use crate::coordinator::{AgentPaths, HiveEnv}; + +use super::{ + AGENT_PREFIX, CONTAINER_RUNTIME_MOUNT, CONTAINER_SHARED_MOUNT, agent_network_ip, agent_uid_gid, + bridge_gateway_ip, container_claude_mount, container_name, validate, +}; + +/// Re-apply the per-container host-side config: nspawn flags (bind +/// mounts etc.), the systemd resource-limits drop-in, and a daemon +/// reload so both take effect on the next unit (re)start. Idempotent — +/// the job queue's `WriteDropin` node, also folded into every `Swap` +/// (rebuild is the reconcile verb). +pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { + validate(name)?; + let container = container_name(name); + set_nspawn_flags( + &container, + &paths.agent_dir, + &paths.claude_dir, + &paths.notes_dir, + ) + .await?; + set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; + systemd_daemon_reload().await +} + +/// Write a systemd drop-in for `container@.service` that applies +/// our default resource caps. Goes under `/run/systemd/system/...` so it's +/// ephemeral (regenerated on every spawn / rebuild). +async fn set_resource_limits(container: &str, cpu_quota: &str, memory_max: &str) -> Result<()> { + crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await +} + +async fn systemd_daemon_reload() -> Result<()> { + crate::priv_client::daemon_reload().await +} + +/// Idempotently rewrite the lines in `/etc/nixos-containers/.conf` +/// that hive-c0re owns: `PRIVATE_NETWORK` (forced 0 so the agent's web UI port +/// is reachable on the host) and `EXTRA_NSPAWN_FLAGS` (the runtime-dir bind). +/// The start script expands `$EXTRA_NSPAWN_FLAGS` unquoted into the +/// `systemd-nspawn` command. +/// Where in the container's filesystem the manager sees its agents tree. +/// Matches the `/agents` path that pre-Phase-8 hosts declared via +/// `containers.root.bindMounts."/agents"`. +pub const CONTAINER_MANAGER_AGENTS_MOUNT: &str = "/agents"; + +/// Where the manager sees the applied trees of every agent, read-only. +/// Manager runs `git fetch /applied//.git refs/tags/*:refs/tags/applied/*` +/// to learn what hive-c0re deployed (or rejected, or failed to +/// build); the RO bind makes accidental writes impossible from +/// inside the container. +pub const CONTAINER_MANAGER_APPLIED_MOUNT: &str = "/applied"; + +/// The on-host root that gets bind-mounted to `/agents` inside the manager. +/// Hard-coded to match `AGENT_STATE_ROOT` in coordinator.rs (kept duplicated +/// here so lifecycle stays usable as a leaf module). +pub(super) const HOST_AGENTS_ROOT: &str = "/var/lib/hyperhive/agents"; + +/// On-host applied repo root, mirrored RO into the manager. Matches +/// `APPLIED_STATE_ROOT` in coordinator.rs. +const HOST_APPLIED_ROOT: &str = "/var/lib/hyperhive/applied"; + +/// On-host meta repo root, mirrored RO into the manager. Matches +/// `meta::meta_dir()` but duplicated here so lifecycle stays a leaf. +const HOST_META_ROOT: &str = "/var/lib/hyperhive/meta"; + +/// Shared directory accessible to all agents. All agents bind-mount this RW. +const HOST_SHARED_ROOT: &str = "/var/lib/hyperhive/shared"; + +/// Append bind flags for `child`'s state, harness, and config dirs into +/// `binds`, all read-write. The RW on `state` is deliberate (recovery), +/// not an oversight; see docs/persistence.md ("Parent access to child +/// state") for the rationale. Creates missing host-side directories so +/// nspawn doesn't refuse to start; missing dirs are non-fatal. +fn bind_child_agent_dirs(child: &str, binds: &mut Vec) { + let state_dir = format!("{HOST_AGENTS_ROOT}/{child}/state"); + let harness_dir = format!("{HOST_AGENTS_ROOT}/{child}/harness"); + let config_dir = format!("{HOST_AGENTS_ROOT}/{child}/config"); + for dir in [&state_dir, &harness_dir, &config_dir] { + let _ = std::fs::create_dir_all(dir); + } + binds.push(BindMount { + host_path: state_dir, + container_path: format!("/agents/{child}/state"), + read_only: false, + }); + binds.push(BindMount { + host_path: harness_dir, + container_path: format!("/agents/{child}/harness"), + read_only: false, + }); + binds.push(BindMount { + host_path: config_dir, + container_path: format!("/agents/{child}/config"), + read_only: false, + }); +} + +/// Hive-wide secrets forwarded into every agent container via nspawn +/// `--load-credential=:`. Currently just the OTEL +/// auth-header secret, when `services.hyperhive.otel.headersCredential` +/// is set (surfaced as `HYPERHIVE_OTEL_HEADERS_CREDENTIAL` on hive-c0re's +/// unit env — the same host option meta.rs reads to inject +/// `hyperhive.otel.headersCredential`). The inner harness unit reads it +/// via `LoadCredential=otel-headers` (inherit). The secret never lands in +/// a bind mount, the nix store, or the generated config. +/// +/// A configured-but-missing file is skipped with a warning rather than +/// forwarded (nspawn would refuse to start the container otherwise): a +/// host-level secret typo shouldn't take down every agent's start; OTEL +/// just exports without the auth header until the file appears. +fn hive_load_credentials() -> Vec { + let mut out = Vec::new(); + let Ok(path) = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL") else { + return out; + }; + if path.is_empty() { + return out; + } + if std::path::Path::new(&path).is_file() { + out.push(CredentialMount { + name: "otel-headers".to_owned(), + host_path: path, + }); + } else { + tracing::warn!( + %path, + "HYPERHIVE_OTEL_HEADERS_CREDENTIAL is set but the file is missing; \ + skipping --load-credential (OTEL will export without the auth header)" + ); + } + out +} + +#[allow( + clippy::too_many_lines, + reason = "one contiguous nspawn-flag assembly block; the length is the flag \ + surface itself, splitting it would just hide the shape" +)] +async fn set_nspawn_flags( + container: &str, + runtime_dir: &Path, + claude_dir: &Path, + notes_dir: &Path, +) -> Result<()> { + // Ensure /shared directory exists before binding. systemd-nspawn requires the bind source to exist. + std::fs::create_dir_all(HOST_SHARED_ROOT) + .with_context(|| format!("create {HOST_SHARED_ROOT}"))?; + // Make /shared writable by every agent. Containers share host uids (no + // PrivateUsers), but each agent is a distinct unix user, so a root-owned + // 0755 dir leaves them unable to write — the documented "read/write for + // all agents" contract was broken. A setgid group would need a + // pinned GID declared in every container plus all agent users joined to + // it (cross-container coordination + a rebuild cascade); instead we use + // the /tmp model — sticky world-writable (1777). The sticky bit lets any + // agent create files while protecting each agent's entries from deletion + // by the others, and matches /shared's documented "free-for-all, may be + // deleted/lost" semantics without touching any per-agent config. + { + use std::os::unix::fs::PermissionsExt as _; + let perms = std::fs::Permissions::from_mode(0o1777); + std::fs::set_permissions(HOST_SHARED_ROOT, perms) + .with_context(|| format!("chmod 1777 {HOST_SHARED_ROOT}"))?; + } + // Ensure /knowledge dir exists. It may be empty until forge seeds it; + // nspawn refuses to start if the bind source is missing entirely. + std::fs::create_dir_all(crate::knowledge::LOCAL_DIR) + .with_context(|| format!("create {}", crate::knowledge::LOCAL_DIR))?; + + // Logical agent name — strip the `h-` prefix. + // For the manager: `h-ruth` → `ruth`. For sub-agents: `h-iris` → `iris`. + let agent_name = container.strip_prefix(AGENT_PREFIX).unwrap_or(container); + + // Claude credentials land at `/home//.claude` so the + // `claude` CLI (which reads `$HOME/.claude`) finds them. The + // harness service's environment sets `HOME` to the same path + // (`agent-base.nix` / `manager.nix`), so no `--setenv` plumbing + // is needed here — the bind alone is enough. + let claude_mount = container_claude_mount(agent_name); + + // Hive-wide secrets forwarded into the container's credential store + // (currently just the OTEL auth-header). Same for every agent. + let load_creds = hive_load_credentials(); + + let mut binds: Vec = vec![ + BindMount { + host_path: runtime_dir.to_string_lossy().into_owned(), + container_path: CONTAINER_RUNTIME_MOUNT.to_owned(), + read_only: false, + }, + BindMount { + host_path: claude_dir.to_string_lossy().into_owned(), + container_path: claude_mount, + read_only: false, + }, + BindMount { + host_path: HOST_SHARED_ROOT.to_owned(), + container_path: CONTAINER_SHARED_MOUNT.to_owned(), + read_only: false, + }, + BindMount { + host_path: crate::knowledge::LOCAL_DIR.to_owned(), + container_path: crate::knowledge::CONTAINER_MOUNT.to_owned(), + read_only: true, + }, + ]; + + // Own state, harness, and config dirs — same for every agent including + // the manager. Config is RO: an agent must not edit its own config; changes + // only ever flow through the approval queue. + binds.push(BindMount { + host_path: notes_dir.to_string_lossy().into_owned(), + container_path: format!("/agents/{agent_name}/state"), + read_only: false, + }); + if let Some(state_parent) = notes_dir.parent() { + let harness_dir = state_parent.join("harness"); + if !harness_dir.exists() { + let _ = std::fs::create_dir_all(&harness_dir); + } + binds.push(BindMount { + host_path: harness_dir.to_string_lossy().into_owned(), + container_path: format!("/agents/{agent_name}/harness"), + read_only: false, + }); + } + let own_config = format!("{HOST_AGENTS_ROOT}/{agent_name}/config"); + std::fs::create_dir_all(&own_config).with_context(|| format!("create {own_config}"))?; + binds.push(BindMount { + host_path: own_config, + container_path: format!("/agents/{agent_name}/config"), + read_only: true, + }); + + // Topology-driven child mounts: every direct child of this agent gets + // its state, harness, and config dirs bind-mounted RW (parent reads + + // writes child state for recovery, and manages config). See + // `bind_child_agent_dirs`. + let direct_children = crate::topology::children_of(agent_name); + for child in &direct_children { + bind_child_agent_dirs(child, &mut binds); + } + + // `can_manage_top_level_agents` role: additionally mount every + // parentless agent in the topology as a virtual child. Enables + // recovery — a role holder can update those agents' configs even + // when they are down. Also grants RO access to /applied and /meta. + if crate::topology::has_role( + agent_name, + crate::topology::ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS, + ) { + let top_level = crate::topology::top_level_agents(); + for tl in &top_level { + if !direct_children.contains(tl) { + bind_child_agent_dirs(tl, &mut binds); + } + } + // systemd-nspawn refuses to start a container whose bind + // source doesn't exist. The meta repo is created by the + // startup migration, but make sure the directory is there + // before the role holder comes up in case set_nspawn_flags + // fires first (e.g. cold start with no agents). + std::fs::create_dir_all(HOST_META_ROOT) + .with_context(|| format!("create {HOST_META_ROOT}"))?; + binds.push(BindMount { + host_path: HOST_APPLIED_ROOT.to_owned(), + container_path: CONTAINER_MANAGER_APPLIED_MOUNT.to_owned(), + read_only: true, + }); + binds.push(BindMount { + host_path: HOST_META_ROOT.to_owned(), + container_path: crate::meta::CONTAINER_MANAGER_META_MOUNT.to_owned(), + read_only: true, + }); + } + + // Web-socket subdir: bind-mount `/run/hive-agent//` into the + // container so the harness can bind `web.sock` there and the host-side + // gateway sees it. Subdir bind (not socket file) keeps the inode + // visible after the harness unlinks a stale socket on rebind. + // Applies to manager and sub-agents alike. + let socket_dir = crate::agent_sockets::agent_dir_for(agent_name); + std::fs::create_dir_all(&socket_dir) + .with_context(|| format!("create {}", socket_dir.display()))?; + // Chown to the agent user so the non-root harness can bind(2) here. + // Falls back to 0777 on first spawn when uid lookup returns None + // (container /etc/passwd not yet rendered). + if let Some((uid, gid)) = agent_uid_gid(agent_name) { + if let Err(e) = crate::priv_client::chown_socket_dir(agent_name, uid, gid).await { + tracing::warn!(%agent_name, error = ?e, "chown socket dir failed"); + } + } else if let Err(e) = crate::priv_client::chmod_socket_dir(agent_name, 0o777).await { + tracing::warn!(%agent_name, error = ?e, "chmod socket dir failed"); + } + binds.push(BindMount { + host_path: socket_dir.to_string_lossy().into_owned(), + container_path: socket_dir.to_string_lossy().into_owned(), + read_only: false, + }); + + // Network isolation: when HIVE_NETWORK_ISOLATION=1 is set (by the + // hive-network.nix module's `isolateContainers` option), flip the + // container to a private network namespace with a veth pair attached + // to the host bridge. Applies to all containers including the manager + // (all hive-c0re<->agent comms go through bind-mounted UDS, not TCP). + let isolation = { + let isolate = std::env::var("HIVE_NETWORK_ISOLATION").ok().as_deref() == Some("1"); + let bridge = std::env::var("HIVE_NETWORK_BRIDGE").unwrap_or_default(); + let subnet = std::env::var("HIVE_NETWORK_SUBNET").unwrap_or_default(); + if isolate && !bridge.is_empty() && !subnet.is_empty() { + let Some(agent_ip) = agent_network_ip(agent_name, &subnet) else { + tracing::warn!( + %agent_name, %subnet, + "HIVE_NETWORK_SUBNET is set but could not derive a valid IP for agent \ + (bad CIDR? prefix too narrow?); skipping PRIVATE_NETWORK write to \ + avoid misconfigured isolation" + ); + return crate::priv_client::write_nspawn_flags( + container, + &binds, + None, + &load_creds, + ) + .await; + }; + let Some(gateway_ip) = bridge_gateway_ip(&subnet) else { + tracing::warn!( + %agent_name, %subnet, + "HIVE_NETWORK_SUBNET is set but the bridge gateway IP is unparseable; \ + skipping PRIVATE_NETWORK write to avoid an isolated container with no \ + default route or resolver" + ); + return crate::priv_client::write_nspawn_flags( + container, + &binds, + None, + &load_creds, + ) + .await; + }; + tracing::info!( + %agent_name, %agent_ip, %gateway_ip, %bridge, + "network isolation: PRIVATE_NETWORK=1" + ); + Some(hive_sh4re::priv_proto::NetworkIsolation { + agent_ip, + bridge, + gateway_ip, + }) + } else { + None + } + }; + + // Delegate the actual conf-file rewrite to hive-priv (runs as root). + crate::priv_client::write_nspawn_flags(container, &binds, isolation, &load_creds).await +} diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs new file mode 100644 index 00000000..b2c90733 --- /dev/null +++ b/hive-c0re/src/lifecycle/mod.rs @@ -0,0 +1,874 @@ +//! `nixos-container` lifecycle + per-agent config flake generation. + +mod git; +mod host_config; +mod setup; +#[cfg(test)] +mod tests; + +pub use git::{ + git, git_command, git_fetch_to_tag, git_read_tree_reset, git_rev_parse, git_tag, + git_tag_annotated, git_update_ref, +}; +pub use host_config::{ + CONTAINER_MANAGER_AGENTS_MOUNT, CONTAINER_MANAGER_APPLIED_MOUNT, write_dropins, +}; +pub use setup::{ + ensure_agent_state_subvolume, ensure_claude_dir, ensure_state_dir, initial_flake_nix, + setup_applied, setup_proposed, +}; + +use std::path::Path; + +use anyhow::{Context, Result, bail}; +use tokio::process::Command; + +use crate::coordinator::{AgentPaths, HiveEnv}; + +/// Sub-agent container prefix. `nixos-container` caps the total container name +/// at 11 chars (it gets encoded into network interface names), so the agent +/// name itself can be at most `MAX_AGENT_NAME` chars. +pub const AGENT_PREFIX: &str = "h-"; +pub const MAX_AGENT_NAME: usize = 9; +/// Logical name of the manager agent (broker recipient, state-dir key, +/// meta flake attribute). All persistent state lives under `ruth/`. +pub const MANAGER_NAME: &str = "ruth"; +/// Container name of the manager. Uses the same `h-` prefix as sub-agents +/// so `nixos-container list` output is uniform and the list filter is +/// a single `starts_with(AGENT_PREFIX)` check. Logical name → container +/// name: `ruth` → `h-ruth`. +pub const MANAGER_CONTAINER: &str = "h-ruth"; + +/// Mount point of the per-agent runtime directory inside the container. +pub const CONTAINER_RUNTIME_MOUNT: &str = "/run/hive"; + +/// Where the per-agent Claude credentials dir mounts inside the +/// container. The harness service runs as a non-root unix user +/// whose home is `/home//`, so the mount path varies per +/// agent — `container_claude_mount(name)` returns +/// `/home//.claude` for every agent including the manager. +/// `claude` inside the container reads +/// `$HOME/.claude` and the service environment sets `HOME` to the +/// same path, so the OAuth session survives container restarts. +#[must_use] +pub fn container_claude_mount(name: &str) -> String { + format!("/home/{name}/.claude") +} + +/// Mount point of the shared directory accessible to all agents. +/// All agents can read/write here; agents should only put things they're +/// willing to lose (other agents may delete them). +pub const CONTAINER_SHARED_MOUNT: &str = "/shared"; + +/// Sub-agent web UI port range. Deterministic from the agent's name (FNV-1a +/// hash mod range size), so the dashboard can compute the same port without +/// asking hive-c0re. +const WEB_PORT_BASE: u16 = 8100; +const WEB_PORT_RANGE: u16 = 900; + +/// FNV-1a hash of a string — shared by `agent_web_port` and +/// `agent_network_ip` so the derivation rule is identical. +fn fnv1a(s: &str) -> u32 { + let mut hash: u32 = 2_166_136_261; + for b in s.bytes() { + hash ^= u32::from(b); + hash = hash.wrapping_mul(16_777_619); + } + hash +} + +/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) % +/// WEB_PORT_RANGE` for every agent including the manager. The port +/// allocation rule reads the same for every name; collisions are +/// possible (birthday paradox at ~30 agents) and the operator +/// resolves them by renaming an agent (different hash → different +/// port). Stable across hosts, restarts, and dashboard renders — +/// no state-file dance. +#[must_use] +pub fn agent_web_port(name: &str) -> u16 { + // Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails. + WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0) +} + +/// Deterministic IPv4 address for an agent inside an isolated subnet. +/// +/// Parses `subnet_cidr` as `/` (e.g. +/// `"10.42.0.0/24"`), then computes: +/// +/// ```text +/// host_count = 2^(32 - prefix_len) +/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast) +/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot +/// agent_ip = network_base_u32 + offset +/// ``` +/// +/// Returns `None` when `subnet_cidr` can't be parsed (invalid format, +/// prefix out of range, etc.) so callers can fall back gracefully. +/// Collisions are possible (birthday paradox) and the operator resolves +/// them by renaming an agent, same as for port collisions. +#[must_use] +pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option { + let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; + let prefix_len: u32 = prefix_str.parse().ok()?; + if prefix_len > 30 { + // /31 and /32 have no room for agents; /30 has 1 usable slot. + // /0 (the other extreme) is handled further down: host_count + // overflows checked_shl(32) → 0 → usable = 0 → None. + return None; + } + // Parse dotted-decimal IPv4. + let octets: Vec = ip_str + .split('.') + .map(|o| o.parse::().ok()) + .collect::>>()?; + if octets.len() != 4 { + return None; + } + let base_u32 = u32::from_be_bytes([octets[0], octets[1], octets[2], octets[3]]); + // Mask off host bits to get the true network address. + let mask = if prefix_len == 0 { + 0u32 + } else { + !0u32 << (32 - prefix_len) + }; + let network_base = base_u32 & mask; + let host_count: u32 = 1u32.checked_shl(32 - prefix_len).unwrap_or(0); + // `.0` = network, `.1` = bridge gateway, last = broadcast → 3 reserved. + let usable = host_count.saturating_sub(3); + if usable == 0 { + return None; + } + let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1 + let ip_u32 = network_base + offset; + let [a, b, c, d] = ip_u32.to_be_bytes(); + Some(format!("{a}.{b}.{c}.{d}")) +} + +/// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`. +/// +/// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim +/// (e.g. `10.42.0.1/24`), **not** the canonical network address — see +/// the note in `set_nspawn_flags` + `docs/network.md`. The IP part is +/// therefore the bridge IP itself: the host end of the bridge, the +/// default-route target for isolated containers, and the address the +/// hive dnsmasq resolver binds. Returns the dotted-decimal IP with the +/// `/` stripped, or `None` if the input isn't a valid +/// `/` pair. +/// +/// Deliberately returns the operator-configured address verbatim rather +/// than deriving `network + 1`: an operator who sets `bridgeIp` to a +/// non-`.1` host address (e.g. `10.42.0.254`) runs the bridge + resolver +/// there, so that — not `.1` — is the real gateway. +#[must_use] +pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option { + let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; + // Validate the prefix is a sane IPv4 CIDR length and the address is + // dotted-decimal IPv4 — same shape `agent_network_ip` accepts — so a + // malformed `HIVE_NETWORK_SUBNET` can't smuggle a bogus HOST_ADDRESS + // into the nspawn conf. + let prefix_len: u32 = prefix_str.parse().ok()?; + if prefix_len > 32 { + return None; + } + let octets: Vec = ip_str + .split('.') + .map(|o| o.parse::().ok()) + .collect::>>()?; + if octets.len() != 4 { + return None; + } + Some(ip_str.to_owned()) +} + +#[must_use] +pub fn container_name(name: &str) -> String { + format!("{AGENT_PREFIX}{name}") +} + +/// Read the agent user's `(uid, gid)` from the container's nixos-managed +/// `/etc/passwd`. Returns `None` when the container hasn't been built +/// yet, the passwd file is unparseable, or the agent user is missing +/// (e.g. legacy container that still runs as root). +/// +/// Used by `forge` + `matrix` after writing per-agent state files so +/// the bind-mounted host file ends up readable by the agent user +/// without waiting for the next container activation to run the chown +/// fixup. +/// +/// Notes: +/// - Reads the *container-local* passwd at +/// `/var/lib/nixos-containers//etc/passwd`, not the host's. +/// The container's user-namespace shares uids with the host (no +/// `PrivateUsers`), so the uid is directly usable in host-side +/// `chown(2)`. +/// - Best-effort: caller treats `None` as "skip the chown". +#[must_use] +pub fn agent_uid_gid(agent_name: &str) -> Option<(u32, u32)> { + let container = container_name(agent_name); + let passwd_path = format!("/var/lib/nixos-containers/{container}/etc/passwd"); + let content = std::fs::read_to_string(&passwd_path).ok()?; + for line in content.lines() { + let mut parts = line.split(':'); + let user = parts.next()?; + if user != agent_name { + continue; + } + let _ = parts.next()?; // x (password placeholder) + let uid: u32 = parts.next()?.parse().ok()?; + let gid: u32 = parts.next()?.parse().ok()?; + return Some((uid, gid)); + } + None +} + +/// Best-effort `chown(path, agent_uid, agent_gid)`. Resolves the agent's +/// uid/gid via [`agent_uid_gid`] and shells out to `std::os::unix::fs::chown`. +/// Silently no-ops when the container isn't built yet (`None` from +/// [`agent_uid_gid`]) and logs at debug on chown syscall failure — the +/// activation script in `harness-base.nix` is the steady-state safety +/// net. Used by per-agent state writers in `forge` + `matrix` so the +/// agent can read the file without waiting for the next container +/// rebuild. +pub fn chown_to_agent(name: &str, path: &Path, subsystem: &str) { + let Some((uid, gid)) = agent_uid_gid(name) else { + return; + }; + if let Err(e) = std::os::unix::fs::chown(path, Some(uid), Some(gid)) { + tracing::debug!(%name, %subsystem, path = %path.display(), error = %e, "chown to agent failed"); + } +} + +fn validate(name: &str) -> Result<()> { + if name.is_empty() { + bail!("agent name must not be empty"); + } + if name.len() > MAX_AGENT_NAME { + bail!( + "agent name '{name}' is too long ({} chars); max {MAX_AGENT_NAME}", + name.len() + ); + } + Ok(()) +} + +/// First name (≠ `self_name`) currently running whose hashed port +/// matches this agent's. The harness inside the colliding container +/// would otherwise loop on `AddrInUse` forever; we surface the +/// conflict here so spawn / rebuild fails loudly with an actionable +/// message instead. +async fn port_collision(self_name: &str) -> Option { + let port = agent_web_port(self_name); + let raw = list().await.unwrap_or_default(); + for c in raw { + let Some(other) = c.strip_prefix(AGENT_PREFIX) else { + continue; + }; + if other == self_name { + continue; + } + if agent_web_port(other) == port && is_running(other).await { + return Some(other.to_owned()); + } + } + None +} + +pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { + create_container(name, hive, paths).await?; + write_dropins(name, hive, paths).await?; + priv_run("start", name).await +} + +/// First-spawn provisioning + `nixos-container create`, without the +/// drop-in write or the start — the job queue's `Create` node. +/// `spawn` composes this with `write_dropins` + start for direct +/// callers (root-agent bootstrap). +pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { + validate(name)?; + if let Some(other) = port_collision(name).await { + bail!( + "port {} is already taken by '{other}' — rename one of them and retry", + agent_web_port(name) + ); + } + setup_proposed(&paths.proposed_dir, name).await?; + setup_applied(&paths.applied_dir, Some(&paths.proposed_dir), name).await?; + ensure_agent_state_subvolume(name).await?; + ensure_claude_dir(&paths.claude_dir)?; + ensure_state_dir(&paths.notes_dir)?; + // Meta flake gets the new agent's input + nixosConfiguration + // before `nixos-container create` so the `--flake meta#` + // ref resolves. + let agents = agents_after_spawn(name).await?; + crate::meta::sync_agents(hive, &agents).await?; + priv_run("create", name).await +} + +/// Rebuild-path preamble shared by the job queue's `Prebuild` node and +/// `rebuild_no_meta`: fail fast on a port collision, then make sure +/// the applied repo + state dirs exist. Container untouched. +pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> { + validate(name)?; + if let Some(other) = port_collision(name).await { + bail!( + "port {} is already taken by '{other}' — rename one of them and retry", + agent_web_port(name) + ); + } + setup_applied(&paths.applied_dir, None, name).await?; + ensure_agent_state_subvolume(name).await?; + ensure_claude_dir(&paths.claude_dir)?; + ensure_state_dir(&paths.notes_dir)?; + Ok(()) +} + +/// Profile-swap for an existing, stopped container: re-apply the +/// drop-ins, then `nixos-container update`. The job queue's `Swap` +/// node. Requires the container stopped (the queue's `StopForUpdate` +/// upstream); does NOT start it — the DAG's tail `Reconcile` owns +/// bringing the agent back to its wanted power state. +pub async fn swap_update( + name: &str, + hive: &HiveEnv, + paths: &AgentPaths, + on_step: &(dyn Fn(&str) + Send + Sync), + on_build_log_id: &(dyn Fn(i64) + Send + Sync), +) -> Result<()> { + write_dropins(name, hive, paths).await?; + on_step("nixos-container update"); + priv_run_inner("update", name, Some(on_build_log_id)).await +} + +/// Build the `AgentSpec` list for the meta flake from `nixos-container +/// list` + a hypothetical extra name not yet in the list (for spawn +/// where the new agent's container doesn't exist yet). Pass empty +/// `name_to_add` from rebuild paths where the agent is already in the +/// container list. +/// +/// Propagates errors from `list()` rather than swallowing them. +/// Using `.unwrap_or_default()` here would silently produce an empty +/// agent list when `nixos-container list` fails (priv helper down, race), +/// which `sync_agents` would then commit to meta — dropping every agent +/// from `flake.nix`. Callers that can tolerate failures (e.g. migration) +/// handle the `Err` themselves with `.unwrap_or_default()`. +async fn agents_for_meta(name_to_add: Option<&str>) -> Result> { + let containers = list().await?; + let mut out: Vec = containers + .into_iter() + .filter_map(|c| { + let name = c.strip_prefix(AGENT_PREFIX)?.to_owned(); + Some(crate::meta::AgentSpec { + is_manager: name == MANAGER_NAME, + port: agent_web_port(&name), + name, + }) + }) + .collect(); + if let Some(extra) = name_to_add + && !out.iter().any(|a| a.name == extra) + { + out.push(crate::meta::AgentSpec { + is_manager: extra == MANAGER_NAME, + port: agent_web_port(extra), + name: extra.to_owned(), + }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) +} + +async fn agents_after_spawn(name: &str) -> Result> { + agents_for_meta(Some(name)).await +} + +/// Like `agents_for_meta_listing` but with an extra agent added (for a +/// container that doesn't exist yet). Used by the first-spawn path in +/// `actions::run_apply_commit` to register the new agent in meta before +/// `prepare_deploy` tries to update its input lock. +pub async fn agents_for_meta_listing_with(extra: &str) -> Result> { + agents_for_meta(Some(extra)).await +} + +/// Public enumeration of currently-existing agents (whatever +/// `nixos-container list` says), sorted, no extras. For callers +/// outside this module that need to reseed meta after lifecycle +/// changes — destroy, startup reconciliation, etc. +pub async fn agents_for_meta_listing() -> Result> { + agents_for_meta(None).await +} + +/// True when the named container already exists (appears in +/// `nixos-container list`). Used by the apply-commit path to decide +/// between first-spawn (`nixos-container create`) and normal rebuild +/// (`nixos-container update`). +pub async fn container_exists(name: &str) -> bool { + let container = container_name(name); + list() + .await + .unwrap_or_default() + .iter() + .any(|c| c == &container) +} + +pub async fn kill(name: &str) -> Result<()> { + validate(name)?; + priv_run("stop", name).await +} + +pub async fn start(name: &str) -> Result<()> { + validate(name)?; + priv_run("start", name).await +} + +/// Start with the cold-start fallback: when a plain start fails (the +/// activation-error shape), retry once via stop + kill + start before +/// giving up. Used by the queue's fast-lane `Start` handler and the +/// inline start-after-rebuild path. +/// See `docs/coordinator.md::Cold-start fallback`. +/// +/// # Errors +/// +/// Propagates the retry's start error (annotated with the original +/// failure) when the fallback also fails. +pub async fn start_with_fallback(name: &str) -> Result<()> { + validate(name)?; + if let Err(start_err) = priv_run("start", name).await { + let container = container_name(name); + tracing::warn!( + container = %container, + error = %start_err, + "start failed (possible activation error); retrying via stop + kill + start" + ); + priv_run("stop", name).await.unwrap_or_else(|e| { + tracing::warn!( + container = %container, + error = %e, + "stop before cold-start retry failed (ignored)" + ); + }); + priv_run("kill", name).await.unwrap_or_else(|e| { + tracing::warn!( + container = %container, + error = %e, + "kill before cold-start retry failed (ignored)" + ); + }); + priv_run("start", name).await.map_err(|e| { + anyhow::anyhow!( + "cold-start fallback also failed: {e:#} \ + (original start error: {start_err:#})" + ) + }) + } else { + Ok(()) + } +} + +/// Stop + start without regenerating any config. For "kick the container" +/// without touching the flake or nspawn flags. +pub async fn restart(name: &str) -> Result<()> { + kill(name).await?; + start(name).await +} + +/// True when the container's systemd unit is active. Used by the dashboard +/// to gate stop/restart buttons. +pub async fn is_running(name: &str) -> bool { + let container = container_name(name); + let unit = format!("container@{container}.service"); + Command::new("systemctl") + .args(["is-active", "--quiet", &unit]) + .status() + .await + .is_ok_and(|s| s.success()) +} + +/// Fully tear down a sub-agent's container: stop + remove via `nixos-container +/// destroy`, then clean our own systemd drop-in. Leaves it to the caller to +/// wipe `/var/lib/hyperhive/...` state and the per-agent runtime dir. +pub async fn destroy(name: &str) -> Result<()> { + validate(name)?; + let container = container_name(name); + // nixos-container destroy handles stop + removal of /var/lib/nixos-containers/ + // and /etc/nixos-containers/.conf. Tolerate "no such container". + if let Err(e) = priv_run("destroy", name).await { + tracing::warn!(error = ?e, "nixos-container destroy returned an error; continuing cleanup"); + } + // Remove the systemd resource-limits drop-in via hive-priv. + if let Err(e) = crate::priv_client::remove_service_dropin(&container).await { + tracing::warn!(error = ?e, "remove service drop-in failed (non-fatal)"); + } + Ok(()) +} + +/// Container-level rebuild without touching the meta repo. The one +/// remaining fused stop/update/start pipeline: the approval deploy +/// (`actions::deploy_applied_target`) drives meta through the +/// two-phase prepare/finalize/abort flow itself and needs the inline +/// start to verify the agent comes back up before finalizing. Every +/// other rebuild is a job-queue DAG (`Prebuild → StopForUpdate → Swap +/// → Reconcile`) whose `Prebuild` executor owns the meta sync + +/// relock this path's deleted `rebuild` wrapper used to do. +/// +/// `on_step` is called at each phase boundary with a short human-readable +/// label so callers can surface progress (e.g. update the rebuild-queue +/// step shown in the dashboard). Pass `&|_| ()` when progress reporting +/// is not needed. +/// +/// `on_build_log_id` is called with the build-log row id immediately after +/// the `nixos-container update` log row opens, before the actual update +/// command starts. Callers can use this to link the queue entry to the log +/// for live streaming. Pass `&|_| ()` when not needed. +/// +/// `defer_start` skips the start-after-update for a previously-running +/// container and returns `true` instead, so a queue-side caller can hand +/// the (potentially slow) container boot to the fast lane rather than +/// holding the serialized build lane through it. With `defer_start = +/// false` the start (with cold-start fallback) runs inline as before and +/// the return value is always `false`. The spawn path always starts +/// inline — a freshly-created container boots as part of provisioning. +pub async fn rebuild_no_meta( + name: &str, + hive: &HiveEnv, + paths: &AgentPaths, + defer_start: bool, + on_step: &(dyn Fn(&str) + Send + Sync), + on_build_log_id: &(dyn Fn(i64) + Send + Sync), +) -> Result { + prepare_rebuild_dirs(name, paths).await?; + let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display()); + if container_exists(name).await { + // Rebuild strategy: stop-before-update + pre-build. + // See `docs/coordinator.md::Container lifecycle`. + let was_running = is_running(name).await; + write_dropins(name, hive, paths).await?; + if was_running { + on_step("nix build"); + prebuild_toplevel(name, &flake_ref, &|_| ()).await?; + on_step("nixos-container stop"); + priv_run("stop", name).await?; + } + on_step("nixos-container update"); + let update_result = priv_run_inner("update", name, Some(on_build_log_id)).await; + if let Err(ref update_err) = update_result { + // The update failed (e.g. nix build error). If the agent was + // running before we stopped it, try to bring it back up on the + // previous successful configuration so it doesn't stay dead. + // The start failure is logged but not promoted to an error — + // we always propagate the original update error (below). + if was_running { + tracing::warn!( + %name, + error = %update_err, + "nixos-container update failed; attempting restart on old config" + ); + on_step("nixos-container start (recovery)"); + if let Err(e) = priv_run("start", name).await { + tracing::warn!(%name, error = %e, "recovery start after failed update also failed"); + } + } + } + update_result?; + if was_running { + if defer_start { + // The caller re-queues the start on the fast lane so the + // build lane is freed for the next entry instead of + // waiting out the container boot here. + return Ok(true); + } + on_step("nixos-container start"); + start_with_fallback(name).await?; + } + Ok(false) + } else { + // Spawn path: create is atomic, no prebuild needed. + // See `docs/coordinator.md::Spawn path`. + on_step("nixos-container create"); + priv_run("create", name).await?; + write_dropins(name, hive, paths).await?; + on_step("nixos-container start"); + priv_run("start", name).await?; + Ok(false) + } +} + +/// Pre-build `system.build.toplevel` against `meta#` so the +/// subsequent `nixos-container update` finds the result cached and +/// skips straight to the profile-swap. Store-warming only — container +/// is untouched. See `docs/coordinator.md::Rebuild path` for why +/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild +/// attr path` for why the explicit nixosConfigurations attr is required. +/// +/// `on_build_log_id` fires with the `build_logs` row id as soon as the +/// row opens, so queue-side callers can link their node to the live +/// stream. Pass `&|_| ()` when not needed. +pub async fn prebuild_toplevel( + name: &str, + flake_ref: &str, + on_build_log_id: &(dyn Fn(i64) + Send + Sync), +) -> Result<()> { + use tokio::io::{AsyncBufReadExt, BufReader}; + // Split `#` so we can re-emit with the explicit + // `nixosConfigurations.` segment. The flake_ref shape is + // constructed by `rebuild_no_meta` and always contains exactly one + // `#`; `split_once` returning None here would be a programmer + // error we'd want to surface loudly rather than paper over. + let (flake_root, fragment) = flake_ref + .split_once('#') + .with_context(|| format!("flake_ref {flake_ref:?} missing '#' fragment"))?; + // Sanity-check the fragment matches the agent name we were + // passed — guards against future calls that pass a divergent + // pair (no current callsite does, but the pair is redundant + // and worth checking once). + if fragment != name { + anyhow::bail!("prebuild_toplevel: flake_ref fragment '{fragment}' ≠ agent name '{name}'"); + } + let attr = format!("{flake_root}#nixosConfigurations.{name}.config.system.build.toplevel"); + let args = vec![ + "--extra-experimental-features", + "nix-command flakes", + "build", + "--no-link", + "--print-out-paths", + &attr, + ]; + let cmdline = format!("nix {}", args.join(" ")); + tracing::info!(%name, %cmdline, "prebuild: warming system toplevel"); + + // Open a build_logs row for this attempt (best-effort — None when + // the global handle hasn't been installed, e.g. early startup + // or standalone tests). Lines pumped from stdout/stderr append + // into the row; `finish` lands the terminal status before we bail. + let logs = crate::build_logs::global(); + let log_id = logs.as_ref().and_then(|h| { + h.start(name, "prebuild", &cmdline) + .map_err(|e| { + tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)"); + }) + .ok() + }); + if let Some(id) = log_id { + on_build_log_id(id); + } + + let mut child = Command::new("nix") + .args(&args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .with_context(|| format!("spawn {cmdline}"))?; + + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + + let stdout_cmdline = cmdline.clone(); + let stdout_logs = logs.clone(); + let pump_stdout = tokio::spawn(async move { + let mut lines = BufReader::new(stdout).lines(); + while let Ok(Some(line)) = lines.next_line().await { + tracing::info!(target: "nix-prebuild", cmdline = %stdout_cmdline, "{line}"); + if let (Some(h), Some(id)) = (&stdout_logs, log_id) { + h.append_stdout(id, &line); + } + } + }); + + let stderr_cmdline = cmdline.clone(); + let stderr_logs = logs.clone(); + let pump_stderr = tokio::spawn(async move { + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}"); + if let (Some(h), Some(id)) = (&stderr_logs, log_id) { + h.append_stderr(id, &line); + } + } + }); + + let status = child + .wait() + .await + .with_context(|| format!("wait {cmdline}"))?; + let _ = pump_stdout.await; + let _ = pump_stderr.await; + + let ok = status.success(); + if let (Some(h), Some(id)) = (&logs, log_id) { + h.finish( + id, + if ok { + crate::build_logs::BuildStatus::Ok + } else { + crate::build_logs::BuildStatus::Fail + }, + ); + } + if !ok { + match log_id { + Some(id) => bail!("prebuild {cmdline} failed ({status}); see build log #{id}"), + None => bail!("prebuild {cmdline} failed ({status})"), + } + } + Ok(()) +} + +pub async fn list() -> Result> { + let stdout = crate::priv_client::list_containers().await?; + Ok(stdout + .lines() + .map(str::trim) + .filter(|line| line.starts_with(AGENT_PREFIX)) + .map(str::to_owned) + .collect()) +} + +/// Build the per-line callback for `create_container_streaming` / +/// `update_container_streaming`. Both ops share identical dispatch logic +/// (stdout → info + `append_stdout`, stderr → warn + `append_stderr`); this +/// helper avoids duplicating that match body across the two call sites. +fn make_log_callback( + logs: Option>, + log_id: Option, + cmdline: String, +) -> impl FnMut(hive_sh4re::priv_proto::PrivStream, &str) { + use hive_sh4re::priv_proto::PrivStream; + move |stream, line| match stream { + PrivStream::Stdout => { + tracing::info!(target: "nixos-container", cmdline = %cmdline, "{line}"); + if let (Some(h), Some(id)) = (&logs, log_id) { + h.append_stdout(id, line); + } + } + PrivStream::Stderr => { + tracing::warn!(target: "nixos-container", cmdline = %cmdline, "{line}"); + if let (Some(h), Some(id)) = (&logs, log_id) { + h.append_stderr(id, line); + } + } + } +} + +/// Execute a container operation via hive-priv and integrate with +/// `build_logs.sqlite`. hive-priv runs as root and forwards output lines +/// to hive-c0re in real time via the streaming priv protocol. Each line +/// is appended to the build-log row as it arrives, so the dashboard +/// shows live progress during long `nixos-container create` / `update` runs. +async fn priv_run(kind: &str, name: &str) -> Result<()> { + priv_run_inner(kind, name, None).await +} + +/// Like `priv_run` but calls `on_log_id(log_id)` immediately after the +/// build-log row is opened — before the actual container op starts. +/// This lets callers surface the row id for live streaming (e.g. the +/// rebuild-queue worker sets `build_log_id` on the queue entry so the +/// dashboard can link to `/api/build-logs/id/{id}/stream`). +/// +/// The callback fires only when a build-log row is successfully opened +/// (i.e. the global `BuildLogs` handle is installed AND `h.start()` +/// succeeds). No-op when `on_log_id` is `None` — that's the path for +/// all callers that don't need the id. +async fn priv_run_inner( + kind: &str, + name: &str, + on_log_id: Option<&(dyn Fn(i64) + Send + Sync)>, +) -> Result<()> { + let container = container_name(name); + let cmdline = format!("nixos-container {kind} {container}"); + + let logs = crate::build_logs::global(); + let log_id = logs.as_ref().and_then(|h| { + h.start(name, kind, &cmdline) + .map_err(|e| { + tracing::warn!(error = ?e, "build_logs: start failed (priv_run log dropped)"); + }) + .ok() + }); + // Notify the caller as soon as the log row exists so it can surface + // the id for live streaming before the container op even starts. + if let (Some(id), Some(cb)) = (log_id, on_log_id) { + cb(id); + } + + // For long-running ops use the streaming protocol so build_logs + // receives lines in real time rather than as a batch at completion. + let result: Result<()> = match kind { + "create" => { + crate::priv_client::create_container_streaming( + name, + make_log_callback(logs.clone(), log_id, cmdline.clone()), + ) + .await + } + "update" => { + crate::priv_client::update_container_streaming( + name, + make_log_callback(logs.clone(), log_id, cmdline.clone()), + ) + .await + } + "start" => crate::priv_client::start_container(name).await, + "stop" => crate::priv_client::stop_container(name).await, + "kill" => crate::priv_client::kill_container(name).await, + "destroy" => crate::priv_client::destroy_container(name).await, + other => Err(anyhow::anyhow!("unknown container op: {other}")), + }; + + let succeeded = result.is_ok(); + if let (Some(h), Some(id)) = (&logs, log_id) { + h.finish( + id, + if succeeded { + crate::build_logs::BuildStatus::Ok + } else { + crate::build_logs::BuildStatus::Fail + }, + ); + } + + match result { + Ok(()) => Ok(()), + Err(e) => { + let journal = if kind == "update" { + container_journal_tail(&container).await + } else { + String::new() + }; + match log_id { + Some(id) => bail!("{e:#}; see build log #{id}{journal}"), + None => bail!("{e:#}{journal}"), + } + } + } +} + +/// On a failed `nixos-container update`, the stderr nixos-container +/// itself prints is often terse ("failed to reload container") — the +/// real reason (which unit failed `switch-to-configuration` during +/// the reload phase) lands in the *container's* own journal, not on +/// the host. Fetch the tail of it so a failed rebuild self-documents +/// the failing unit in the error string, no second round-trip. +/// +/// Scoped to `update`: that's the reload-phase case, and the +/// container is still up (running the old generation) so +/// `journalctl -M` works. Best-effort — returns "" for other verbs +/// or when the journal can't be read (machine gone, journalctl +/// missing); it never produces an error of its own. +async fn container_journal_tail(container: &str) -> String { + // `-M` enters the container namespace and needs root, so the read + // is delegated to hive-priv (hive-c0re itself runs unprivileged). + let res = crate::priv_client::read_container_journal( + container, + hive_sh4re::priv_proto::JournalQuery { + lines: 40, + ..Default::default() + }, + ) + .await; + match res { + Ok((stdout, _)) if !stdout.is_empty() => format!( + "\n--- last 40 journal lines from container '{container}' ---\n{}", + stdout.trim_end() + ), + _ => String::new(), + } +} diff --git a/hive-c0re/src/lifecycle/setup.rs b/hive-c0re/src/lifecycle/setup.rs new file mode 100644 index 00000000..a97898ba --- /dev/null +++ b/hive-c0re/src/lifecycle/setup.rs @@ -0,0 +1,251 @@ +//! First-spawn provisioning: seed the manager-editable proposed repo and +//! the hive-c0re-owned applied repo, and ensure the per-agent state / +//! claude-credentials dirs (btrfs subvolume when available) exist. + +use std::path::Path; + +use anyhow::{Context, Result, bail}; + +use super::git::{ + git, git_command, git_commit, git_read_tree_reset, git_rev_parse, git_root_commit, git_tag, +}; +use super::host_config::HOST_AGENTS_ROOT; + +/// Initialize the manager-editable proposed repo. Seeds two tracked +/// files: `agent.nix` (the module the manager edits) and `flake.nix` +/// (the boilerplate that lets the meta flake import this repo as an +/// input — meta locks at a specific sha and reads +/// `nixosModules.default`, so `flake.nix` must be in the commit). The +/// manager shouldn't edit `flake.nix` (the prompt says so) but it's +/// visible so they can introspect. +/// +/// Touched by hive-c0re only on first spawn — never again — so the +/// manager can't be surprised by hive-c0re commits or working-tree +/// resets. +pub async fn setup_proposed(proposed_dir: &Path, name: &str) -> Result<()> { + let fresh = !proposed_dir.join(".git").exists(); + if fresh { + std::fs::create_dir_all(proposed_dir) + .with_context(|| format!("create {}", proposed_dir.display()))?; + let agent_path = proposed_dir.join("agent.nix"); + if !agent_path.exists() { + std::fs::write(&agent_path, initial_agent_nix(name)) + .with_context(|| format!("write {}", agent_path.display()))?; + } + let flake_path = proposed_dir.join("flake.nix"); + if !flake_path.exists() { + std::fs::write(&flake_path, initial_flake_nix()) + .with_context(|| format!("write {}", flake_path.display()))?; + } + git(proposed_dir, &["init", "--initial-branch=main"]).await?; + git(proposed_dir, &["add", "agent.nix", "flake.nix"]).await?; + git_commit(proposed_dir, "hive-c0re init").await?; + } + // Idempotently wire the `applied` remote — purely for the + // manager's ergonomics. The URL is the path inside the manager + // container (`/applied//.git`), where the RO bind in + // `set_nspawn_flags` makes it real. hive-c0re itself never + // dereferences this remote; the host-side fetch in + // `request_apply_commit` uses absolute host paths. + ensure_applied_remote(proposed_dir, name).await +} + +async fn ensure_applied_remote(proposed_dir: &Path, name: &str) -> Result<()> { + let want = format!("/applied/{name}/.git"); + let existing = git_command() + .current_dir(proposed_dir) + .args(["remote", "get-url", "applied"]) + .output() + .await + .with_context(|| format!("git remote get-url applied in {}", proposed_dir.display()))?; + if existing.status.success() { + let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned(); + if current == want { + return Ok(()); + } + // URL drifted (path scheme changed, etc.) — re-point it. + return git(proposed_dir, &["remote", "set-url", "applied", &want]).await; + } + git(proposed_dir, &["remote", "add", "applied", &want]).await +} + +/// Set up the applied repo. First-spawn only: init the repo, pull +/// proposed's initial commit in via `git fetch`, tag it `deployed/0`. +/// This is the *only* time hive-c0re reads from `proposed` for an +/// agent — subsequent proposals are fetched at `request_apply_commit` +/// time and tagged `proposal/` (see `actions::approve` for the +/// tag state machine). +/// +/// `proposed_dir` is `None` on rebuild paths where the repo already +/// exists — we just verify it's the right shape and bail otherwise. +/// Unlike the pre-overhaul code path, `flake.nix` is no longer +/// regenerated at the host level: it's tracked in proposed (seeded by +/// `setup_proposed`) and rides along on every fetch. +pub async fn setup_applied( + applied_dir: &Path, + proposed_dir: Option<&Path>, + name: &str, +) -> Result<()> { + std::fs::create_dir_all(applied_dir) + .with_context(|| format!("create {}", applied_dir.display()))?; + + if !applied_dir.join(".git").exists() { + let Some(proposed) = proposed_dir else { + bail!( + "applied repo at {} is missing its .git directory; \ + cannot rebuild without a proposed source to seed from. \ + destroy --purge and re-spawn this agent.", + applied_dir.display() + ); + }; + git(applied_dir, &["init", "--initial-branch=main"]).await?; + let proposed_str = proposed.display().to_string(); + // Seed the applied repo at the root (template) commit of proposed, + // not at `main`. This ensures `deployed/0` is the template baseline + // so the first ApplyCommit diff shows the manager's real changes + // rather than an empty diff (which happens when the manager has + // already committed their config and proposed/main == proposal/). + let root_sha = git_root_commit(proposed).await?; + git( + applied_dir, + // --update-head-ok lets us fetch into refs/heads/main while + // HEAD still points there. git's default safeguard refuses + // to avoid index/working-tree desync, but the working tree + // is empty (we just `init`'d) and we read-tree-reset right + // after, so the safeguard is moot here. + &[ + "fetch", + "--no-tags", + "--update-head-ok", + &proposed_str, + &format!("{root_sha}:refs/heads/main"), + ], + ) + .await?; + git_read_tree_reset(applied_dir, "refs/heads/main").await?; + git_tag(applied_dir, "deployed/0", "refs/heads/main").await?; + } else if git_rev_parse(applied_dir, "refs/tags/deployed/0") + .await + .is_err() + { + // Pre-overhaul applied repo — no deployed/* tag scheme, + // flake.nix may be untracked, agent.nix possibly authored by + // hive-c0re directly. The startup auto-migration fixes this + // in place; if it didn't run (or got skipped), surface a + // clear error. + bail!( + "applied repo at {} predates the meta-flake layout. \ + Restart hive-c0re to let the auto-migration run, or \ + destroy --purge {name} and re-spawn.", + applied_dir.display() + ); + } + Ok(()) +} + +/// Create the per-agent Claude credentials dir if missing. Mode 0755 — hive-core +/// needs read+execute to list the directory so `claude_has_session` can detect a +/// valid session; credential files inside (`.credentials.json` etc.) are 0600 so +/// secrets stay private regardless of the directory mode. Idempotent: existing +/// dirs are left untouched (an agent's OAuth tokens survive `destroy`/recreate). +/// Public for the `InitConfig` approval path in `actions.rs` which seeds +/// dirs without calling the full `spawn`. +pub fn ensure_claude_dir(claude_dir: &Path) -> Result<()> { + use std::io; + if !claude_dir.exists() { + std::fs::create_dir_all(claude_dir) + .with_context(|| format!("create {}", claude_dir.display()))?; + } + // 0755: hive-core (different user from the agent) needs read+execute to + // list the directory so `claude_has_session` can detect a valid session. + // The credential files inside (`.credentials.json` etc.) are 0600 so the + // secrets themselves stay private regardless of the directory mode. + // + // Best-effort: on the first container boot, `hive-agent-user-migrate` + // chowns this dir to the agent user. After that, hive-core (a different + // user) cannot chmod it (EPERM) — that's fine because the mode set during + // initial creation (0755) is preserved through the chown. Any other error + // (ENOENT, I/O error) is unexpected and propagated. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + match std::fs::set_permissions(claude_dir, std::fs::Permissions::from_mode(0o755)) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { + tracing::debug!( + path = %claude_dir.display(), + "ensure_claude_dir: chmod 755 skipped (dir likely owned by agent user after migration)" + ); + } + Err(e) => { + return Err(e).with_context(|| format!("chmod 755 {}", claude_dir.display())); + } + } + } + Ok(()) +} + +/// Public for the `InitConfig` approval path in `actions.rs` which seeds +/// dirs without calling the full `spawn`. Also creates the sibling `harness/` +/// dir so the first harness startup can write its sqlite files immediately. +pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> { + if !notes_dir.exists() { + std::fs::create_dir_all(notes_dir) + .with_context(|| format!("create {}", notes_dir.display()))?; + } + // Harness dir is a sibling of the agent-visible state dir. + if let Some(parent) = notes_dir.parent() { + let harness_dir = parent.join("harness"); + if !harness_dir.exists() { + std::fs::create_dir_all(&harness_dir) + .with_context(|| format!("create {}", harness_dir.display()))?; + } + } + Ok(()) +} + +/// Ensure agent `name`'s persistent state root +/// (`/var/lib/hyperhive/agents/`) is a btrfs subvolume — when the host +/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`, +/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`. +/// +/// Progressive enhancement: if the root already exists +/// (any agent provisioned before this landed, plain dir or subvol) it's left +/// exactly as-is — no auto-migration — and the priv round-trip is skipped. On +/// a non-btrfs host the priv op no-ops and the root is later created as a +/// plain dir by `ensure_*_dir`, identical to the old behaviour. Only a +/// brand-new agent on a btrfs host gets a real subvolume. Subvolume creation +/// is privileged, so it's delegated to hive-priv. +pub async fn ensure_agent_state_subvolume(name: &str) -> Result<()> { + let root = Path::new(HOST_AGENTS_ROOT).join(name); + if root.exists() { + return Ok(()); + } + crate::priv_client::ensure_agent_subvolume(name) + .await + .with_context(|| format!("ensure btrfs subvolume for agent {name}")) +} + +fn initial_agent_nix(name: &str) -> String { + format!( + "{{ config, pkgs, lib, ... }}:\n{{\n # Per-agent overrides for {name}. This is a regular NixOS module\n # — add packages, services, modules, imports as needed.\n #\n # imports = [ ./extra-module.nix ];\n # environment.systemPackages = with pkgs; [ ];\n}}\n", + ) +} + +/// Module-only flake exposed by every agent's repo. Consumed by the +/// hive-c0re-owned meta flake at `/var/lib/hyperhive/meta/` as a flake +/// input. The wrapper is intentionally permissive: +/// +/// - Manager edits `inputs.* = …` to add other flakes (e.g. an MCP +/// server's own flake) — the lock for those lands in the agent's +/// own `flake.lock` and rolls up into meta's lock transitively. +/// - The outputs block forwards every input (minus `self`) into +/// `agent.nix` as the `flakeInputs` module argument, so the +/// manager just references `flakeInputs..packages.${pkgs.system}.default` +/// without further plumbing. +/// +/// Identity injection (`HIVE_PORT` / `HIVE_LABEL` / dashboard port / +/// git committer) still lives in the meta flake's wrapper. +pub fn initial_flake_nix() -> &'static str { + "{\n description = \"hyperhive agent\";\n inputs = { };\n outputs =\n { self, ... }@inputs:\n {\n nixosModules.default = {\n imports = [ ./agent.nix ];\n _module.args.flakeInputs = builtins.removeAttrs inputs [ \"self\" ];\n };\n };\n}\n" +} diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs new file mode 100644 index 00000000..af77f5bd --- /dev/null +++ b/hive-c0re/src/lifecycle/tests.rs @@ -0,0 +1,155 @@ +//! Unit tests for the lifecycle module (moved verbatim from the old +//! single-file `lifecycle.rs` `#[cfg(test)]` block). + +use super::*; + +/// Regression test: `setup_proposed` must seed both agent.nix and flake.nix +/// in the initial commit. Before commit 5b5a93e flake.nix was missing from +/// the scaffold, requiring manual creation (seen with the damocles agent). +#[tokio::test] +async fn setup_proposed_seeds_flake_nix() { + let dir = tempfile::tempdir().expect("tempdir"); + let proposed = dir.path().join("proposed"); + setup_proposed(&proposed, "test-agent") + .await + .expect("setup_proposed"); + + // Both files must exist on disk. + assert!(proposed.join("agent.nix").exists(), "agent.nix missing"); + assert!(proposed.join("flake.nix").exists(), "flake.nix missing"); + + // flake.nix must export nixosModules.default (the meta-flake contract). + let flake = std::fs::read_to_string(proposed.join("flake.nix")).unwrap(); + assert!( + flake.contains("nixosModules.default"), + "flake.nix does not export nixosModules.default" + ); + + // Both files must be tracked in the initial git commit. + let out = git_command() + .current_dir(&proposed) + .args(["show", "--name-only", "--format=", "HEAD"]) + .output() + .await + .expect("git show"); + let tracked = String::from_utf8_lossy(&out.stdout); + assert!(tracked.contains("agent.nix"), "agent.nix not committed"); + assert!(tracked.contains("flake.nix"), "flake.nix not committed"); +} + +#[test] +fn agent_network_ip_is_in_subnet() { + // Default subnet 10.42.0.0/24 — agents get .2 to .254. + let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP"); + let octets: Vec = ip.split('.').map(|o| o.parse().unwrap()).collect(); + assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix"); + assert!( + octets[3] >= 2 && octets[3] <= 254, + "host byte {}", + octets[3] + ); +} + +#[test] +fn agent_network_ip_stable() { + // Same name + subnet must always produce the same IP. + let a = agent_network_ip("damocles", "10.42.0.0/24"); + let b = agent_network_ip("damocles", "10.42.0.0/24"); + assert_eq!(a, b); +} + +#[test] +fn agent_network_ip_different_agents() { + // Different agent names very likely produce different IPs (not guaranteed, + // but for these two names the hashes don't collide). + let alice = agent_network_ip("alice", "10.42.0.0/24").unwrap(); + let bob = agent_network_ip("bob", "10.42.0.0/24").unwrap(); + assert_ne!(alice, bob, "alice and bob collide — rename one"); +} + +#[test] +fn agent_network_ip_different_subnet() { + let ip = agent_network_ip("alice", "192.168.5.0/24").expect("should produce an IP"); + let octets: Vec = ip.split('.').map(|o| o.parse().unwrap()).collect(); + assert_eq!(&octets[..3], &[192, 168, 5]); +} + +#[test] +fn bridge_gateway_ip_extracts_verbatim_address() { + // HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the + // canonical network — the gateway is the address before the `/`. + assert_eq!( + bridge_gateway_ip("10.42.0.1/24").as_deref(), + Some("10.42.0.1") + ); + // Non-`.1` operator override: the gateway is wherever the bridge is. + assert_eq!( + bridge_gateway_ip("10.42.0.254/24").as_deref(), + Some("10.42.0.254") + ); + assert_eq!( + bridge_gateway_ip("172.30.0.1/16").as_deref(), + Some("172.30.0.1") + ); +} + +#[test] +fn bridge_gateway_ip_rejects_bad_input() { + assert!(bridge_gateway_ip("notanip/24").is_none()); + assert!(bridge_gateway_ip("10.42.0.1").is_none()); // no prefix + assert!(bridge_gateway_ip("10.42.0.1/33").is_none()); // prefix > 32 + assert!(bridge_gateway_ip("10.42.0.999/24").is_none()); // octet > 255 + assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets +} + +#[test] +fn agent_network_ip_rejects_bad_input() { + assert!(agent_network_ip("alice", "notanip/24").is_none()); + assert!(agent_network_ip("alice", "10.0.0.0/33").is_none()); // prefix > 32 + assert!(agent_network_ip("alice", "10.0.0.0/31").is_none()); // too small + assert!(agent_network_ip("alice", "10.0.0.0").is_none()); // no prefix +} + +#[test] +fn agent_network_ip_normalizes_bridge_ip_subnet() { + // HIVE_NETWORK_SUBNET carries the bridge IP (10.42.0.1/24), not + // canonical network (10.42.0.0/24). Both must produce the same result + // after host-bit masking. + let from_bridge = agent_network_ip("alice", "10.42.0.1/24"); + let from_canonical = agent_network_ip("alice", "10.42.0.0/24"); + assert_eq!( + from_bridge, from_canonical, + "bridge-IP and canonical-network form should normalize to the same result" + ); + // Result must still be in .2-.254. + let ip = from_bridge.unwrap(); + let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap(); + assert!((2..=254).contains(&last), "host byte {last}"); +} + +/// `setup_proposed` is idempotent: calling it on an existing repo is a +/// no-op (the fresh guard skips all writes). +#[tokio::test] +async fn setup_proposed_idempotent() { + let dir = tempfile::tempdir().expect("tempdir"); + let proposed = dir.path().join("proposed"); + setup_proposed(&proposed, "test-agent") + .await + .expect("first call"); + // Second call must not error even though .git already exists. + setup_proposed(&proposed, "test-agent") + .await + .expect("second call"); + // Still one commit. + let out = git_command() + .current_dir(&proposed) + .args(["rev-list", "--count", "HEAD"]) + .output() + .await + .expect("git rev-list"); + let count = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + assert_eq!( + count, "1", + "expected exactly one commit after idempotent call" + ); +} diff --git a/hive-c0re/src/loose_ends.rs b/hive-c0re/src/loose_ends.rs index bdecc232..741d0789 100644 --- a/hive-c0re/src/loose_ends.rs +++ b/hive-c0re/src/loose_ends.rs @@ -13,12 +13,11 @@ //! the dashboard uses, so the bottleneck would be json //! (de)serialisation, not the read. -use std::time::{SystemTime, UNIX_EPOCH}; - use anyhow::Result; use hive_sh4re::LooseEnd; use crate::coordinator::Coordinator; +use hive_sh4re::wire_time::now_unix; /// Open threads pending against `agent`: /// - undelivered inbox messages this agent still owes itself a `recv` @@ -143,14 +142,6 @@ fn saturating_age(now: i64, then: i64) -> u64 { u64::try_from(delta).unwrap_or(0) } -fn now_unix() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - #[cfg(test)] mod tests { use super::*; diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index f39fb1e4..2b61082f 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -13,8 +13,8 @@ use hive_sh4re::{HostRequest, HostResponse}; use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig}; use hive_c0re::{ agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge, - knowledge, matrix, migrate, rebuild_queue, reminder_scheduler, scheduled_prompts_worker, - server, socket_server, + job_queue, knowledge, matrix, migrate, reminder_scheduler, scheduled_prompts_worker, server, + socket_server, }; #[derive(Parser)] @@ -85,6 +85,11 @@ enum Cmd { /// option. #[arg(long)] model_prices: Option, + /// Override: number of concurrent nix-heavy job-queue nodes + /// (prebuild / profile-swap / create / meta lock). Set via the + /// `services.hyperhive.c0re.buildSlots` NixOS option. + #[arg(long)] + build_slots: Option, }, /// Spawn a new agent container directly (`hive-agent-`). Bypasses /// the approval queue — use only as an operator on the host. For @@ -156,6 +161,7 @@ async fn main() -> Result<()> { agent_cpu_quota, agent_memory_max, model_prices, + build_slots, } => { // Base config from the --config file (or the built-in // defaults), then apply any per-flag overrides — config @@ -195,7 +201,10 @@ async fn main() -> Result<()> { sc.model_prices = serde_json::from_str(&v).context("--model-prices: invalid JSON")?; } - cmd_serve(sc.env, sc.model_prices, db, &cli.socket).await + if let Some(v) = build_slots { + sc.build_slots = v; + } + cmd_serve(sc.env, sc.model_prices, sc.build_slots, db, &cli.socket).await } Cmd::Spawn { name } => { render(client::request(&cli.socket, HostRequest::Spawn { name }).await?) @@ -244,6 +253,7 @@ async fn main() -> Result<()> { async fn cmd_serve( env: HiveEnv, model_prices: hive_c0re::hive_stats::PriceTable, + build_slots: usize, db: std::path::PathBuf, socket: &std::path::Path, ) -> Result<()> { @@ -255,7 +265,7 @@ async fn cmd_serve( // `dashboard_port` is consumed into the Coordinator below; capture the // Copy value first for the dashboard + knowledge-webhook tasks. let dashboard_port = env.dashboard_port; - let coord = Arc::new(Coordinator::open(&db, env, model_prices)?); + let coord = Arc::new(Coordinator::open(&db, env, model_prices, build_slots)?); socket_server::start_manager(coord.clone())?; // Idempotent pre-flight: rewrite pre-meta-layout applied // repos, ensure proposed repos carry the `applied` @@ -413,25 +423,17 @@ async fn cmd_serve( // and fans the body out to each active target's inbox. See // scheduled_prompts_worker.rs. scheduled_prompts_worker::spawn(coord.clone()); - // Rebuild-queue worker: drains the global rebuild/meta-update/ - // spawn queue FIFO so hive-c0re never runs two heavyweight - // container ops concurrently. Existing rebuild call sites - // (auto_update, dashboard, manager, approval handler) enqueue - // here instead of awaiting `rebuild_agent` inline. See - // `rebuild_queue.rs`. + // Job-queue scheduler: drives the global DAG queue (rebuild / + // meta-update / spawn / power ops). Concurrency comes from the + // build-slot count + per-agent leases inside the queue, not from + // multiple workers — cheap nodes (graceful signals, drains, + // reconciles) overlap nix-heavy ones structurally. Call sites + // (auto_update, dashboard, manager, approval handler) submit DAGs + // instead of awaiting lifecycle work inline. See `job_queue/`. { let q_coord = coord.clone(); tokio::spawn(async move { - rebuild_queue::run_worker(q_coord).await; - }); - // Fast lane: a second serial worker for hard Start/Stop, running - // concurrently with the build worker above so a stop/start never - // waits behind a slow build for another container. Per-agent - // ordering vs that agent's own build is enforced in the queue's - // claim logic (a fast op defers behind its agent's running build). - let fast_coord = coord.clone(); - tokio::spawn(async move { - rebuild_queue::run_fast_worker(fast_coord).await; + job_queue::scheduler::run_worker(q_coord).await; }); } // Forward every broker event onto the unified dashboard diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index c5764faa..ad9034bf 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -29,6 +29,25 @@ const GIT_EMAIL: &str = "c0re@hyperhive.local"; /// take turns instead of colliding. static META_LOCK: Mutex<()> = Mutex::const_new(()); +/// Coarse exclusivity for meta-repo *windows* that span multiple +/// `META_LOCK` acquisitions — above all the two-phase deploy +/// (`prepare_deploy` stages `flake.lock` uncommitted for the whole +/// container build; `finalize_deploy` / `abort_deploy` resolve it). +/// `META_LOCK` serializes individual git ops but cannot keep another +/// op out of that staged window: a perm-file or lock-bump commit +/// landing mid-window would sweep the staged deploy lock into its own +/// commit and neuter `abort_deploy`. Job-queue executors that mutate +/// the meta repo hold this gate for their mutation span; the opaque +/// approval-deploy node holds it across its whole prepare→finalize +/// span. Never acquired inside this module's functions (they run +/// *under* a caller's window — nesting would deadlock). +static DEPLOY_GATE: Mutex<()> = Mutex::const_new(()); + +/// Acquire the deploy/meta-mutation window gate. See [`DEPLOY_GATE`]. +pub async fn exclusive() -> tokio::sync::MutexGuard<'static, ()> { + DEPLOY_GATE.lock().await +} + /// Where the manager sees this directory inside its container (RO bind). pub const CONTAINER_MANAGER_META_MOUNT: &str = "/meta"; @@ -239,11 +258,16 @@ pub async fn prepare_deploy(name: &str) -> Result<()> { pub async fn finalize_deploy(name: &str, sha: &str, tag: &str) -> Result<()> { let _guard = META_LOCK.lock().await; let dir = meta_dir(); - if !has_staged_changes(&dir).await? { + if !paths_dirty(&dir, &["flake.lock"]).await? { return Ok(()); } let short = &sha[..sha.len().min(12)]; - git_commit(&dir, &format!("deploy {name} {tag} {short}")).await + git_commit_paths( + &dir, + &format!("deploy {name} {tag} {short}"), + &["flake.lock"], + ) + .await } /// Phase 2-failure. Unstage + restore the lock so meta returns to @@ -256,21 +280,6 @@ pub async fn abort_deploy() -> Result<()> { git(&dir, &["restore", "flake.lock"]).await } -async fn has_staged_changes(dir: &Path) -> Result { - let st = lifecycle::git_command() - .current_dir(dir) - .args(["diff", "--cached", "--quiet"]) - .status() - .await - .with_context(|| format!("git diff --cached in {}", dir.display()))?; - // exit 1 = differences present, 0 = no diff, other = error - match st.code() { - Some(0) => Ok(false), - Some(1) => Ok(true), - _ => bail!("git diff --cached exited unexpectedly"), - } -} - /// One-shot used by the manual-rebuild path: relock just one /// agent's input and commit the lock change if any. Single-phase /// (no separate finalize) because rebuild has no failure-revert @@ -280,11 +289,16 @@ pub async fn lock_update_for_rebuild(name: &str) -> Result<()> { let dir = meta_dir(); let input = format!("agent-{name}"); nix(&dir, &["flake", "update", &input]).await?; - if git_is_clean(&dir).await? { + if !paths_dirty(&dir, &["flake.lock"]).await? { return Ok(()); } git(&dir, &["add", "flake.lock"]).await?; - git_commit(&dir, &format!("rebuild {name}: lock update")).await + git_commit_paths( + &dir, + &format!("rebuild {name}: lock update"), + &["flake.lock"], + ) + .await } /// Build the `--override-input` value pinning an agent's config repo to @@ -349,7 +363,7 @@ pub async fn lock_update(inputs: &[String]) -> Result<()> { args.push(i.as_str()); } nix(&dir, &args).await?; - if git_is_clean(&dir).await? { + if !paths_dirty(&dir, &["flake.lock"]).await? { return Ok(()); } git(&dir, &["add", "flake.lock"]).await?; @@ -360,7 +374,7 @@ pub async fn lock_update(inputs: &[String]) -> Result<()> { } else { format!("lock update: {}", inputs.join(", ")) }; - git_commit(&dir, &msg).await + git_commit_paths(&dir, &msg, &["flake.lock"]).await } /// One-shot used by the auto-update path: pin the latest hyperhive @@ -370,11 +384,11 @@ pub async fn lock_update_hyperhive() -> Result<()> { let _guard = META_LOCK.lock().await; let dir = meta_dir(); nix(&dir, &["flake", "update", "hyperhive"]).await?; - if git_is_clean(&dir).await? { + if !paths_dirty(&dir, &["flake.lock"]).await? { return Ok(()); } git(&dir, &["add", "flake.lock"]).await?; - git_commit(&dir, "bump hyperhive").await + git_commit_paths(&dir, "bump hyperhive", &["flake.lock"]).await } /// Write the tool-groups file for `agent` and commit it atomically @@ -388,8 +402,13 @@ pub async fn commit_tool_groups(agent: &str, groups: &[String]) -> Result<()> { if crate::tool_groups::tool_groups_path().exists() { git(&dir, &["add", "tool-groups.json"]).await?; } - if has_staged_changes(&dir).await? { - git_commit(&dir, &format!("set tool-groups for {agent}")).await?; + if paths_dirty(&dir, &["tool-groups.json"]).await? { + git_commit_paths( + &dir, + &format!("set tool-groups for {agent}"), + &["tool-groups.json"], + ) + .await?; } Ok(()) } @@ -404,8 +423,13 @@ pub async fn commit_capabilities(agent: &str, caps: &[String]) -> Result<()> { if crate::capabilities::capabilities_path().exists() { git(&dir, &["add", "capabilities.json"]).await?; } - if has_staged_changes(&dir).await? { - git_commit(&dir, &format!("set capabilities for {agent}")).await?; + if paths_dirty(&dir, &["capabilities.json"]).await? { + git_commit_paths( + &dir, + &format!("set capabilities for {agent}"), + &["capabilities.json"], + ) + .await?; } Ok(()) } @@ -445,8 +469,14 @@ pub async fn commit_perms( } parts.push("capabilities"); } - if has_staged_changes(&dir).await? { - git_commit(&dir, &format!("set {} for {agent}", parts.join(" + "))).await?; + let paths = ["tool-groups.json", "capabilities.json"]; + if paths_dirty(&dir, &paths).await? { + git_commit_paths( + &dir, + &format!("set {} for {agent}", parts.join(" + ")), + &paths, + ) + .await?; } Ok(()) } @@ -467,10 +497,11 @@ pub async fn commit_topology( let dir = meta_dir(); let stage = async { git(&dir, &["add", "topology.json"]).await?; - if has_staged_changes(&dir).await? { - git_commit( + if paths_dirty(&dir, &["topology.json"]).await? { + git_commit_paths( &dir, &format!("topology: {} → {}", child, new_parent.unwrap_or("")), + &["topology.json"], ) .await?; } @@ -541,8 +572,8 @@ pub async fn bulk_commit_topology( }; let stage = async { git(&dir, &["add", "topology.json"]).await?; - if has_staged_changes(&dir).await? { - git_commit(&dir, &commit_msg).await?; + if paths_dirty(&dir, &["topology.json"]).await? { + git_commit_paths(&dir, &commit_msg, &["topology.json"]).await?; } Ok::<_, anyhow::Error>(()) }; @@ -1158,16 +1189,6 @@ where out } -async fn git_is_clean(dir: &Path) -> Result { - let out = lifecycle::git_command() - .current_dir(dir) - .args(["status", "--porcelain"]) - .output() - .await - .with_context(|| format!("git status in {}", dir.display()))?; - Ok(out.stdout.iter().all(u8::is_ascii_whitespace)) -} - /// Return the list of file names that are currently staged (index differs /// from HEAD). On the initial commit (`HEAD` doesn't exist yet) falls back /// to `git diff --cached --name-only HEAD` failing gracefully by using @@ -1248,6 +1269,43 @@ async fn git_commit(dir: &Path, message: &str) -> Result<()> { Ok(()) } +/// Path-limited commit: commits ONLY the given paths, so unrelated +/// staged content — above all a `prepare_deploy`-staged `flake.lock` +/// — can never be swept into someone else's commit. Every targeted +/// meta commit (perm files, topology, lock bumps) goes through this; +/// only `sync_agents` uses the bare [`git_commit`], because its +/// staged set *is* its intentional commit set. +async fn git_commit_paths(dir: &Path, message: &str, paths: &[&str]) -> Result<()> { + let name = format!("user.name={GIT_NAME}"); + let email = format!("user.email={GIT_EMAIL}"); + let mut args = vec!["-c", &name, "-c", &email, "commit", "-m", message, "--"]; + args.extend_from_slice(paths); + git(dir, &args).await?; + if let Err(e) = crate::forge::push_meta(dir).await { + tracing::warn!(error = ?e, "forge: meta push after commit failed (non-fatal)"); + } + Ok(()) +} + +/// True when any of `paths` differs between HEAD and the index or +/// working tree — the path-scoped replacement for whole-tree +/// `git_is_clean` / `has_staged_changes` guards, which a concurrently +/// staged deploy lock would otherwise trip. +async fn paths_dirty(dir: &Path, paths: &[&str]) -> Result { + let mut args = vec!["diff", "--quiet", "HEAD", "--"]; + args.extend_from_slice(paths); + let out = lifecycle::git_command() + .current_dir(dir) + .args(&args) + .output() + .await + .with_context(|| format!("git diff --quiet in {}", dir.display()))?; + // Exit 0 = no differences; 1 = differences; anything else (e.g. + // no HEAD yet on a fresh repo) → treat as dirty so the commit + // path runs and surfaces real errors loudly. + Ok(!out.status.success()) +} + async fn nix(dir: &Path, args: &[&str]) -> Result<()> { // `--extra-experimental-features` belt-and-suspenders for hosts // that haven't set this in nix.conf. The hyperhive module's @@ -1276,6 +1334,43 @@ async fn nix(dir: &Path, args: &[&str]) -> Result<()> { mod tests { use super::*; + /// The regression the deploy-window bug review surfaced: a + /// path-limited commit must leave an unrelated staged file (the + /// prepare_deploy-staged `flake.lock`) untouched, so a later + /// `abort_deploy` still has something to restore. + #[tokio::test] + async fn path_limited_commit_leaves_unrelated_staged_file_alone() { + let tmp = tempfile::tempdir().expect("tempdir"); + let dir = tmp.path(); + git(dir, &["init", "--initial-branch=main"]) + .await + .expect("git init"); + std::fs::write(dir.join("tool-groups.json"), "{}").expect("write"); + std::fs::write(dir.join("flake.lock"), "v1").expect("write"); + git(dir, &["add", "-A"]).await.expect("add"); + git_commit(dir, "seed").await.expect("seed commit"); + // A deploy stages a new lock (uncommitted)… + std::fs::write(dir.join("flake.lock"), "v2-staged-by-deploy").expect("write"); + git(dir, &["add", "flake.lock"]).await.expect("stage lock"); + // …and a perm change commits, path-limited. + std::fs::write(dir.join("tool-groups.json"), r#"{"alice":[]}"#).expect("write"); + git(dir, &["add", "tool-groups.json"]).await.expect("add"); + git_commit_paths(dir, "set tool-groups for alice", &["tool-groups.json"]) + .await + .expect("path-limited commit"); + // The perm file is committed; the deploy's staged lock is not. + assert!( + !paths_dirty(dir, &["tool-groups.json"]) + .await + .expect("check"), + "perm file must be committed" + ); + assert!( + paths_dirty(dir, &["flake.lock"]).await.expect("check"), + "staged deploy lock must survive the perm commit" + ); + } + fn sample_spec(name: &str, is_manager: bool, port: u16) -> AgentSpec { AgentSpec { name: name.to_owned(), diff --git a/hive-c0re/src/questions.rs b/hive-c0re/src/questions.rs index ca388839..f6e878a3 100644 --- a/hive-c0re/src/questions.rs +++ b/hive-c0re/src/questions.rs @@ -18,7 +18,6 @@ use std::sync::Arc; -use crate::approvals::kind_to_str; use crate::coordinator::Coordinator; use crate::limits; use crate::socket_server::spawn_question_watchdog; @@ -229,7 +228,7 @@ pub fn handle_cancel_loose_end( coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { id: approval.id, agent: &approval.agent, - approval_kind: kind_to_str(approval.kind), + approval_kind: approval.kind.as_str(), sha_short, status: "cancelled", note: approval.note, diff --git a/hive-c0re/src/rebuild_queue.rs b/hive-c0re/src/rebuild_queue.rs deleted file mode 100644 index b6853698..00000000 --- a/hive-c0re/src/rebuild_queue.rs +++ /dev/null @@ -1,2158 +0,0 @@ -//! Global rebuild queue — serialises all long-running container/meta -//! operations (rebuild, meta-update, first-spawn) through a single -//! background worker. Design rationale, kind taxonomy, dedup rules, -//! cascade parent tracking, and step labels: -//! `docs/coordinator.md::Rebuild queue`. - -use std::collections::VecDeque; -use std::sync::Mutex; - -use anyhow::Context as _; -use serde::{Deserialize, Serialize}; -use tokio::sync::Notify; - -/// What the queue can run. Each variant maps to a specific worker -/// execution path; `agent` (in `QueueEntry`) names the target where -/// relevant. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum QueueKind { - /// Rebuild a single agent's container (`auto_update::rebuild_agent`). - Rebuild, - /// Run `nix flake update` on the meta flake. Triggers cascade - /// `Rebuild` entries (with `parent_id`) once the lock bump lands. - MetaUpdate, - /// First-deploy spawn of a new agent (approval-driven). - Spawn, - /// Destroy with `--purge` (real fs work). Not yet routed here; the - /// variant exists so the wire shape doesn't need to change later. - #[allow(dead_code, reason = "wire shape — routed by a future PR")] - Destroy, - /// hive-c0re boot-time sweep: bumps the meta hyperhive lock then - /// enqueues a `Rebuild` child for every managed container. Completes - /// after the lock bump; children run as independent queue entries - /// grouped under this parent's `id`. `agent` = `"hyperhive"`. - StartupSweep, - /// Stop + start a container without touching config. Fast op (~5-10s). - /// Queued so it serialises against in-flight rebuilds for the same - /// agent — prevents a restart racing a rebuild mid-flight. - Restart, - /// Write a tool-group or capability change to the shared JSON file, - /// then rebuild the agent so the new env var takes effect. - /// Serialised through the queue so concurrent dashboard batch-apply - /// actions for different agents never race on the shared JSON file. - PermChange, - /// Gracefully stop a container: signal the harness to run one - /// stop-checkpoint turn (flush durable `/state`), then hand the drain-wait - /// to a detached watcher (freeing the build lane) which, once the agent - /// drains or `GRACEFUL_STOP_TIMEOUT` elapses, enqueues a fast-lane `Stop` - /// for the actual `nixos-container stop`. The build worker only does the - /// cheap signal, so whole-hive graceful stops overlap every agent's drain. - GracefulStop, - /// Start a stopped container (`lifecycle::start`). Routed through the - /// queue so the dashboard shows a visible queued→running transient — a - /// direct sub-second start only flashes the badge — and bulk starts - /// serialise legibly on the queue. Fast op. - Start, - /// Hard-stop a container (`lifecycle::kill`), no quiesce. Routed through - /// the queue for the same visible-progress reason as `Start`; the - /// quiescing variant is `GracefulStop`. Fast op. - Stop, -} - -impl QueueKind { - pub fn as_str(self) -> &'static str { - match self { - QueueKind::Rebuild => "rebuild", - QueueKind::MetaUpdate => "meta_update", - QueueKind::Spawn => "spawn", - QueueKind::Destroy => "destroy", - QueueKind::StartupSweep => "startup_sweep", - QueueKind::Restart => "restart", - QueueKind::PermChange => "perm_change", - QueueKind::GracefulStop => "graceful_stop", - QueueKind::Start => "start", - QueueKind::Stop => "stop", - } - } - - /// Fast-lane kinds: hard `Start` / `Stop`. These run on a separate - /// serial fast worker concurrently with the build lane (so a stop/start - /// never waits behind another container's slow build). `GracefulStop` - /// and `Restart` are deliberately NOT fast — they go through the build - /// lane (`GracefulStop` does the cheap harness signal then detaches the - /// drain-wait, enqueueing a fast-lane `Stop` for the real container stop; - /// `Restart` is a stop+start). - pub fn is_fast(self) -> bool { - matches!(self, QueueKind::Start | QueueKind::Stop) - } -} - -/// Kind-specific payload for `QueueKind::PermChange` entries. -/// Carries the desired new value so the worker can apply the file -/// write (serialised, in FIFO order) without racing concurrent HTTP -/// handlers writing to the same shared JSON file. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum PermPayload { - /// Set the tool groups for one agent (`tool-groups.json`). - ToolGroups { groups: Vec }, - /// Set the capabilities for one agent (`capabilities.json`). - Capabilities { caps: Vec }, - /// Set both perm-types for one agent in a single entry — the batch - /// `POST /api/permissions` path. Either field `None` leaves that - /// file untouched (no write, no commit); the worker commits whichever - /// are present in one git commit, then rebuilds once. Collapses the - /// dedup key to `(kind, agent)` so caps + groups for one agent - /// produce a single rebuild rather than two. - Combined { - groups: Option>, - caps: Option>, - }, -} - -/// Where the enqueue request originated. Drives the "why" chip on the -/// dashboard and lets the UI group cascade entries under their parent -/// without parsing the reason text. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum QueueSource { - /// Operator clicked rebuild / update-all / meta-update on the - /// dashboard, or any other direct human action (CLI, manager tool). - Manual, - /// Spawned as a cascade from a `MetaUpdate` entry's lock-bump - /// fan-out. The `parent_id` on the `QueueEntry` points back at - /// the originating meta-update. - MetaUpdate, - /// `auto_update::run` startup sweep — rebuild every container on - /// hive-c0re boot. Legacy flat source (no parent); replaced by - /// `StartupSweep` for the parent entry and child rebuilds once the - /// queue introduced `parent_id` grouping. Kept for wire compatibility - /// with entries logged before the migration. - AutoUpdate, - /// Direct child of a `StartupSweep` queue entry — one per agent in - /// the boot-time rebuild sweep. Carries `parent_id` back-link so - /// the dashboard renders the sweep's per-agent rebuilds nested under - /// the parent header. The parent entry itself uses `QueueSource::AutoUpdate` - /// (automated boot action, not operator-driven). - StartupSweep, - /// Crash recovery path (future use — currently no auto-rebuild on - /// crash, but the variant exists for the imminent feature). - #[allow(dead_code, reason = "wire shape — used by a future feature")] - CrashRecover, - /// Operator approved a pending `Approval` row on the dashboard. - /// `QueueEntry.approval_id` points back at the source row so the - /// worker can fetch the kind-specific payload (`commit_ref`, inputs, - /// description) before dispatching. - Approval, -} - -impl QueueSource { - pub fn as_str(self) -> &'static str { - match self { - QueueSource::Manual => "manual", - QueueSource::MetaUpdate => "meta_update", - QueueSource::AutoUpdate => "auto_update", - QueueSource::StartupSweep => "startup_sweep", - QueueSource::CrashRecover => "crash_recover", - QueueSource::Approval => "approval", - } - } -} - -/// Lifecycle state of an entry. `Done` / `Failed` / `Cancelled` are -/// retained in the queue snapshot for a short tail (`MAX_HISTORY_PER_KIND`) -/// so the dashboard can show "last few" runs alongside live state. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum QueueState { - Queued, - Running, - Done, - Failed, - Cancelled, -} - -impl QueueState { - pub fn is_terminal(self) -> bool { - matches!( - self, - QueueState::Done | QueueState::Failed | QueueState::Cancelled - ) - } -} - -/// A single queue entry — what's pending, running, or recently finished. -/// Serialised verbatim onto the dashboard event channel and the -/// `/api/state` snapshot. -#[derive(Debug, Clone, Serialize)] -pub struct QueueEntry { - /// Monotonic per-process id. Stable for the lifetime of the entry - /// so SSE upserts land in place rather than churning the list. - pub id: u64, - /// Target agent name, or the literal `"hyperhive"` for entries - /// (`MetaUpdate`) that affect the meta flake rather than a single - /// agent. - pub agent: String, - pub kind: QueueKind, - pub state: QueueState, - pub source: QueueSource, - /// Groups cascade entries under their originating parent. For a - /// `MetaUpdate` entry this is `None`; for the per-agent rebuilds - /// the worker enqueues after the lock bump it's `Some(meta_id)`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_id: Option, - /// Human-readable "why" — populated by the enqueuer (`"manual via - /// dashboard"`, `"meta-update cascade (hyperhive bumped)"`, - /// `"startup sweep"`). Free-form; dedup appends `(also requested - /// by …)` lines on repeated enqueues. - pub reason: String, - pub enqueued_at: i64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub started_at: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finished_at: Option, - /// Populated when `state == Failed`. Carries the worker's error - /// string (already truncated to a reasonable length by the caller). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, - /// `MetaUpdate`-only payload: the list of meta flake inputs to run - /// through `nix flake update`. Empty / absent on `Rebuild` / - /// `Spawn` / `Destroy` entries; absent on the wire (never - /// serialised) when the entry kind doesn't have meaningful inputs. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub inputs: Vec, - /// Source approval row id when this entry was created by an - /// operator-approve POST (`source == Approval`). The worker uses - /// it to re-fetch the kind-specific payload (`commit_ref` / inputs / - /// description / `fetched_sha`) and to fire `ApprovalResolved` on - /// completion. `None` for non-approval entries — preserved on - /// the wire that way too. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub approval_id: Option, - /// Current sub-step inside the running entry. Worker mutates this - /// as the kind-specific pipeline - /// advances through phases (e.g. `"plant tags"` → - /// `"nixos-container update"` → `"finalize deploy"`). `None` while - /// `Queued` and after terminal — only meaningful with - /// `state == Running`. Each transition fires a fresh - /// `RebuildQueueChanged` snapshot so the dashboard can render - /// the label as a sub-line on the queue card. Free-form per - /// pipeline; the kind-specific worker is the source of truth. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub step: Option, - /// `PermChange`-only payload: the desired new permission value to - /// apply. Absent (`None`) on all other entry kinds — omitted from - /// the wire in those cases. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub perm_payload: Option, - /// Entries this entry must wait for before it can run. The worker - /// skips this entry until every id in the list has reached a - /// terminal state (`Done` / `Failed` / `Cancelled`) — or no longer - /// exists in the queue (evicted terminal entries are treated as - /// resolved, since `trim_history` only evicts terminals). Empty on - /// most entries; serialised only when non-empty. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub depends_on: Vec, - /// Row id of the associated `build_logs` entry (opened by the - /// lifecycle worker when `nixos-container update` starts). Set - /// shortly after `state` transitions to `Running`; `None` while - /// `Queued` or for entries that don't open a build log (`Restart`, - /// `PermChange` file-write phase, etc.). Links the queue card to - /// the live-streaming `/api/build-logs/id/{id}/stream` endpoint so - /// the operator can follow the nix build output in real time. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub build_log_id: Option, -} - -/// How many terminal-state entries (`Done` / `Failed` / `Cancelled`) -/// to retain per kind in the snapshot. Older entries get evicted to -/// keep `/api/state` tight; the live event channel is unaffected. -const MAX_HISTORY_PER_KIND: usize = 5; - -/// Inner state guarded by a single mutex. Held briefly — every -/// operation is constant-time relative to the queue's depth, and -/// the depths in practice are tiny (single-digit). -#[derive(Debug, Default)] -struct Inner { - entries: VecDeque, - next_id: u64, -} - -/// Global rebuild queue. Lives on `Coordinator` (one per hive-c0re -/// process). The associated `Notify` wakes the worker when something -/// new arrives. -#[derive(Debug)] -pub struct RebuildQueue { - inner: Mutex, - /// Build-lane worker wakes on this signal. The worker checks the queue - /// and loops back to `notified().await` when there's nothing to run. - /// Also nudged by the fast worker when a fast op finishes (a build may - /// have been deferred behind a `Running` fast op for the same agent). - pub(crate) notify: Notify, - /// Fast-lane worker wakes on this signal. Nudged on a fast `Start` / - /// `Stop` enqueue and by the build worker when a build finishes (a - /// deferred `Start` may now be runnable). - pub(crate) fast_notify: Notify, -} - -impl Default for RebuildQueue { - fn default() -> Self { - Self { - inner: Mutex::new(Inner::default()), - notify: Notify::new(), - fast_notify: Notify::new(), - } - } -} - -/// Full-shape submit spec for [`RebuildQueue::enqueue_full`] — every -/// `QueueEntry` field settable at submit time. The thinner `enqueue` -/// / `enqueue_with_inputs` / `enqueue_with_perm` wrappers build this -/// for the common cases. -pub struct FullEnqueue { - pub kind: QueueKind, - pub agent: String, - pub source: QueueSource, - pub reason: String, - pub parent_id: Option, - pub inputs: Vec, - pub approval_id: Option, - pub perm_payload: Option, - pub depends_on: Vec, -} - -impl RebuildQueue { - pub fn new() -> Self { - Self::default() - } - - /// Add an entry to the queue. Returns the entry's id (newly-allocated - /// or — on dedup — the existing entry's id with the new reason - /// appended). - /// - /// Dedup rule: - /// - `Rebuild` / `Spawn` / `Destroy`: a `Queued` entry with the same - /// `(kind, agent, parent_id)` swallows the new request. `parent_id` - /// is part of the key so that a `MetaUpdate` cascade rebuild (with a - /// specific `parent_id`) never collapses into a standalone rebuild or - /// a cascade from a different `MetaUpdate`. Without this guard a - /// cascade rebuild pre-enqueued before the lock bump would be swallowed - /// by an existing `Queued` startup-sweep rebuild, causing the agent to - /// never rebuild against the post-bump meta. - /// - `MetaUpdate`: dedup ALSO requires the `inputs` field to match — - /// two meta-updates with different input lists are distinct work - /// and must queue separately, otherwise the second meta-update - /// would silently collapse into the first whenever it was still - /// `Queued`, losing the second's input set. - /// - /// Running and terminal entries never dedup — operators are free - /// to re-queue a rebuild that's currently running (something - /// changed since it started) or re-run one that just finished. - pub fn enqueue( - &self, - kind: QueueKind, - agent: String, - source: QueueSource, - reason: String, - parent_id: Option, - ) -> u64 { - self.enqueue_full(FullEnqueue { - kind, - agent, - source, - reason, - parent_id, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }) - } - - /// Same as `enqueue` but carries an `inputs` payload — used by - /// `MetaUpdate` enqueues to tell the worker which meta-flake - /// inputs to bump. For `MetaUpdate` the `inputs` value is part of - /// the dedup key (two meta-updates with different inputs are - /// distinct operations). - pub fn enqueue_with_inputs( - &self, - kind: QueueKind, - agent: String, - source: QueueSource, - reason: String, - parent_id: Option, - inputs: Vec, - ) -> u64 { - self.enqueue_full(FullEnqueue { - kind, - agent, - source, - reason, - parent_id, - inputs, - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }) - } - - /// Enqueue a `PermChange` entry for `agent`. The worker applies the - /// JSON file write (serialised through FIFO) then rebuilds the - /// container so the updated env var takes effect. - pub fn enqueue_with_perm( - &self, - agent: String, - source: QueueSource, - reason: String, - payload: PermPayload, - ) -> u64 { - self.enqueue_full(FullEnqueue { - kind: QueueKind::PermChange, - agent, - source, - reason, - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: Some(payload), - depends_on: Vec::new(), - }) - } - - /// Full-shape enqueue — every `QueueEntry` field that's settable - /// at submit time. Existing `enqueue` / `enqueue_with_inputs` / - /// `enqueue_with_perm` delegate to this; the approval-driven POST - /// handlers call it directly with the source row's id so the - /// worker can re-fetch the kind-specific payload. - pub fn enqueue_full(&self, spec: FullEnqueue) -> u64 { - let FullEnqueue { - kind, - agent, - source, - reason, - parent_id, - inputs, - approval_id, - perm_payload, - depends_on, - } = spec; - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - // Dedup against a pending entry with the same (kind, agent) — - // and, for MetaUpdate, the same `inputs` list (see method - // docstring for why). Approval-driven entries also require the - // approval_id to match so two distinct approvals for the same - // agent never collapse into one queue slot. Rebuild (and Spawn / - // Destroy) entries also require parent_id to match so a - // MetaUpdate cascade rebuild is never swallowed by an unrelated - // queued rebuild (e.g. from the startup sweep). PermChange - // entries additionally check the perm type discriminant — a - // tool-groups change and a capabilities change for the same - // agent are distinct operations and must not collapse into one. - for entry in &mut inner.entries { - let perm_type_matches = matches!( - (&entry.perm_payload, &perm_payload), - ( - Some(PermPayload::ToolGroups { .. }), - Some(PermPayload::ToolGroups { .. }) - ) | ( - Some(PermPayload::Capabilities { .. }), - Some(PermPayload::Capabilities { .. }) - ) | ( - Some(PermPayload::Combined { .. }), - Some(PermPayload::Combined { .. }) - ) | (None, None) - ); - if entry.state == QueueState::Queued - && entry.kind == kind - && entry.agent == agent - && (kind != QueueKind::MetaUpdate || entry.inputs == inputs) - && entry.approval_id == approval_id - && entry.parent_id == parent_id - && perm_type_matches - && entry.depends_on == depends_on - { - if !entry.reason.contains(&reason) { - use std::fmt::Write as _; - let _ = write!(entry.reason, "\nalso requested by: {reason}"); - } - return entry.id; - } - } - inner.next_id += 1; - let id = inner.next_id; - let entry = QueueEntry { - id, - agent, - kind, - state: QueueState::Queued, - source, - parent_id, - reason, - enqueued_at: now_unix(), - started_at: None, - finished_at: None, - error: None, - inputs, - approval_id, - step: None, - perm_payload, - depends_on, - build_log_id: None, - }; - inner.entries.push_back(entry); - // Wake the worker for this entry's lane (fast = Start/Stop, build = - // everything else). `notify_one` is a no-op when there's no waiter; - // the next `notified().await` returns immediately. - if kind.is_fast() { - self.fast_notify.notify_one(); - } else { - self.notify.notify_one(); - } - id - } - - /// Claim the next runnable `Queued` entry for the **build** lane (every - /// kind except the fast `Start` / `Stop`) and mark it `Running`. See - /// [`Self::claim`] for the dependency + per-agent rules. - pub fn take_next_build(&self) -> Option { - self.claim(false) - } - - /// Claim the next runnable `Queued` entry for the **fast** lane (hard - /// `Start` / `Stop`) and mark it `Running`. Runs on its own serial - /// worker concurrently with the build lane. See [`Self::claim`]. - pub fn take_next_fast(&self) -> Option { - self.claim(true) - } - - /// Pop the next `Queued` entry for one lane whose dependencies are - /// resolved and which doesn't race the same agent's other-lane work, - /// and mark it `Running`. Returns the entry (a clone — the original - /// stays in the queue so live state reflects "this is currently - /// running"). Returns `None` when nothing in this lane is runnable. - /// - /// `want_fast` selects the lane: `true` = fast (`Start` / `Stop`), - /// `false` = build (everything else). The two lanes run on separate - /// serial workers, so this is called from both — the lane filter keeps - /// each worker to its own kinds. - /// - /// A dependency is "resolved" when the dep's id is either: - /// - still in the queue AND in a terminal state (`Done` / `Failed` - /// / `Cancelled`), OR - /// - no longer in the queue (evicted by `trim_history` — only - /// terminal entries are ever evicted, so missing == completed). - /// - /// Per-agent cross-lane guard (so a fast op never races that agent's - /// own build, and vice versa): - /// - a fast op waits while its agent has a build entry `Running`; - /// a `Start` additionally waits while its agent has a build entry - /// `Queued` (a start of a soon-to-be-rebuilt container is pointless); - /// - a build op waits while its agent has a fast op `Running`. - fn claim(&self, want_fast: bool) -> Option { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - // Collect ids that are still in the queue and terminal. Entries - // absent from the queue are also considered resolved (see above). - let terminal_ids: std::collections::HashSet = inner - .entries - .iter() - .filter(|e| e.state.is_terminal()) - .map(|e| e.id) - .collect(); - // Active (non-terminal) ids: Queued + Running. Named `active_ids` - // rather than `queued_ids` because Running entries are included; - // used to distinguish "still in flight" from "evicted (= resolved)". - let active_ids: std::collections::HashSet = inner - .entries - .iter() - .filter(|e| !e.state.is_terminal()) - .map(|e| e.id) - .collect(); - let pos = { - let entries = &inner.entries; - entries.iter().position(|e| { - e.state == QueueState::Queued - && e.kind.is_fast() == want_fast - && e.depends_on.iter().all(|dep_id| { - // Resolved if terminal in queue OR not in queue at all. - // Note: circular deps (A depends on B, B depends on A) - // silently deadlock — neither entry ever becomes - // runnable. Callers must ensure acyclic dep graphs. - terminal_ids.contains(dep_id) || !active_ids.contains(dep_id) - }) - && lane_clear(entries, e) - }) - }?; - let entry = &mut inner.entries[pos]; - entry.state = QueueState::Running; - entry.started_at = Some(now_unix()); - Some(entry.clone()) - } - - /// Mark an entry terminal. `error` is populated for `Failed`; - /// `Done` / `Cancelled` ignore it. Trims the history tail. - /// Clears `step` — the field is only meaningful while `Running`, - /// and leaving a stale "in flight" label after a terminal - /// transition would mislead the dashboard render. - pub fn finish(&self, id: u64, state: QueueState, error: Option) { - debug_assert!( - state.is_terminal(), - "finish() called with non-terminal {state:?}" - ); - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - if let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) { - entry.state = state; - entry.finished_at = Some(now_unix()); - entry.error = error.filter(|_| state == QueueState::Failed); - entry.step = None; - } - Self::trim_history(&mut inner); - } - - /// Set the current sub-step label on a `Running` entry. - /// Returns `true` when the row was found AND the label changed - /// (caller should emit a `RebuildQueueChanged` snapshot only on - /// `true` to avoid noisy duplicate frames). No-op for entries not - /// in `Running` — the field is conceptually undefined outside - /// that state. - pub fn set_step(&self, id: u64, step: impl Into) -> bool { - let new_step = step.into(); - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) else { - return false; - }; - if entry.state != QueueState::Running { - return false; - } - if entry.step.as_deref() == Some(new_step.as_str()) { - return false; - } - entry.step = Some(new_step); - true - } - - /// Link a `build_logs` row to a `Running` entry. Called by the - /// lifecycle worker when `nixos-container update` opens a build log - /// row so the dashboard can surface a "view logs" link while the - /// build is in flight. Returns `true` when the row was found and - /// the id was stored; `false` when the entry is no longer in the - /// queue or is not `Running`. - pub fn set_build_log_id(&self, id: u64, log_id: i64) -> bool { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) else { - return false; - }; - if entry.state != QueueState::Running { - return false; - } - entry.build_log_id = Some(log_id); - true - } - - /// Snapshot the queue for `/api/state` and `RebuildQueueChanged`. - /// Cheap clone — entries are small (~hundreds of bytes each). - pub fn snapshot(&self) -> Vec { - let inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - inner.entries.iter().cloned().collect() - } - - /// Cancel every `Queued` entry whose `parent_id` matches `parent`. - /// Used when a `MetaUpdate` parent fails its lock bump — the - /// cascade rebuilds the enqueuer pre-queued no longer apply - /// (nothing actually changed, so they'd be wasted work). Running - /// children are left alone — they were started under the parent's - /// assumption and can't be cleanly aborted from the queue side. - /// Returns the count of cancelled entries. - pub fn cancel_children(&self, parent: u64) -> usize { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - let mut count = 0; - for entry in &mut inner.entries { - if entry.parent_id == Some(parent) && entry.state == QueueState::Queued { - entry.state = QueueState::Cancelled; - entry.finished_at = Some(now_unix()); - count += 1; - } - } - if count > 0 { - Self::trim_history(&mut inner); - } - count - } - - /// Cancel a `Queued` entry (no-op for `Running` / terminal — the - /// in-flight rebuild owns the agent's nix store and can't be - /// safely interrupted). Returns true when an entry was cancelled. - pub fn cancel(&self, id: u64) -> bool { - let mut inner = self.inner.lock().expect("rebuild_queue mutex poisoned"); - if let Some(entry) = inner.entries.iter_mut().find(|e| e.id == id) - && entry.state == QueueState::Queued - { - entry.state = QueueState::Cancelled; - entry.finished_at = Some(now_unix()); - Self::trim_history(&mut inner); - return true; - } - false - } - - /// Keep only the most recent `MAX_HISTORY_PER_KIND` terminal entries - /// per kind. Pending + running entries are never evicted. - fn trim_history(inner: &mut Inner) { - let mut counts: std::collections::HashMap = - std::collections::HashMap::new(); - // Walk newest-first; keep the first MAX_HISTORY_PER_KIND - // terminals per kind, evict the rest. - let entries: Vec = inner - .entries - .iter() - .rev() - .filter(|e| { - if !e.state.is_terminal() { - return true; - } - let n = counts.entry(e.kind).or_insert(0); - *n += 1; - *n <= MAX_HISTORY_PER_KIND - }) - .cloned() - .collect(); - inner.entries = entries.into_iter().rev().collect(); - } -} - -/// Per-agent cross-lane guard for [`RebuildQueue::claim`]: returns true when -/// entry `e` is safe to start given the same agent's other-lane work in -/// `entries`. A fast op waits for the agent's `Running` build (and a `Start` -/// also for a `Queued` build); a build op waits for the agent's `Running` -/// fast op. Keeps a stop/start from racing that container's own rebuild. -fn lane_clear(entries: &VecDeque, e: &QueueEntry) -> bool { - let agent = e.agent.as_str(); - if e.kind.is_fast() { - let build_blocking = entries.iter().any(|b| { - !b.kind.is_fast() - && b.agent == agent - && (b.state == QueueState::Running - || (e.kind == QueueKind::Start && b.state == QueueState::Queued)) - }); - !build_blocking - } else { - !entries - .iter() - .any(|f| f.kind.is_fast() && f.agent == agent && f.state == QueueState::Running) - } -} - -/// Background worker that drains the queue. Spawned once at hive-c0re -/// startup from `main.rs`. Loops forever: -/// 1. Pop the next `Queued` entry (`take_next` marks it `Running` and -/// fires a `RebuildQueueChanged` snapshot via the caller). -/// 2. Dispatch by kind — single-agent rebuild, meta-update + cascade, -/// or first-spawn. -/// 3. Mark the entry terminal (`finish`) and emit another snapshot. -/// 4. When the queue is empty, `await` on `notify` until something -/// new lands. -/// -/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true -/// signal the worker exits after its current entry finishes; pending -/// `Queued` entries are dropped (they'll either be replayed by the -/// startup sweep on next boot or left for an operator to re-queue). -/// Max time the `GracefulStop` drain watcher waits for the harness to run its -/// stop-checkpoint turn + drain before falling back to a hard container stop. -/// Generous — a checkpoint turn can take a while — but bounded so a wedged -/// agent never blocks the stop indefinitely. The wait runs in a detached -/// watcher task (not the build worker), so a whole-hive graceful stop overlaps -/// every agent's drain instead of serialising N × this timeout. -const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3); - -/// Run one claimed queue entry to completion: snapshot, dispatch, mark -/// terminal, snapshot. Shared by both lane workers. -async fn run_one(coord: &std::sync::Arc, entry: &QueueEntry) { - coord.emit_rebuild_queue_snapshot(); - tracing::info!( - id = entry.id, - kind = entry.kind.as_str(), - agent = %entry.agent, - source = entry.source.as_str(), - "rebuild_queue: running" - ); - match dispatch(coord, entry).await { - Ok(()) => { - coord.rebuild_queue.finish(entry.id, QueueState::Done, None); - tracing::info!(id = entry.id, "rebuild_queue: done"); - } - Err(e) => { - let msg = format!("{e:#}"); - let truncated = if msg.len() > 2_000 { - format!("{}…", &msg[..2_000]) - } else { - msg.clone() - }; - coord - .rebuild_queue - .finish(entry.id, QueueState::Failed, Some(truncated)); - tracing::warn!(id = entry.id, error = %msg, "rebuild_queue: failed"); - } - } - coord.emit_rebuild_queue_snapshot(); -} - -/// Build-lane worker: drains every non-fast kind serially. Spawned once at -/// hive-c0re startup from `main.rs`, alongside [`run_fast_worker`] which -/// drains the fast `Start` / `Stop` lane concurrently. -/// -/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true -/// signal the worker exits after its current entry finishes; pending -/// `Queued` entries are dropped (replayed by the startup sweep on next boot -/// or left for an operator to re-queue). -pub async fn run_worker(coord: std::sync::Arc) { - let mut shutdown = coord.shutdown_rx(); - loop { - while let Some(entry) = coord.rebuild_queue.take_next_build() { - run_one(&coord, &entry).await; - // A finished build may unblock a fast op that was deferred behind - // this agent's build — nudge the fast lane to re-check. - coord.rebuild_queue.fast_notify.notify_one(); - } - tokio::select! { - biased; - res = shutdown.changed() => { - if res.is_err() || *shutdown.borrow() { - tracing::info!("rebuild_queue: build worker exiting on shutdown"); - return; - } - } - () = coord.rebuild_queue.notify.notified() => {} - } - } -} - -/// Fast-lane worker: drains hard `Start` / `Stop` serially, concurrently -/// with [`run_worker`], so a stop/start never waits behind another -/// container's slow build. Per-agent ordering vs that agent's own build is -/// enforced in [`RebuildQueue::claim`]. -pub async fn run_fast_worker(coord: std::sync::Arc) { - let mut shutdown = coord.shutdown_rx(); - loop { - while let Some(entry) = coord.rebuild_queue.take_next_fast() { - run_one(&coord, &entry).await; - // A finished fast op may unblock a build deferred behind it. - coord.rebuild_queue.notify.notify_one(); - } - tokio::select! { - biased; - res = shutdown.changed() => { - if res.is_err() || *shutdown.borrow() { - tracing::info!("rebuild_queue: fast worker exiting on shutdown"); - return; - } - } - () = coord.rebuild_queue.fast_notify.notified() => {} - } - } -} - -/// Run a single queue entry to completion. Kind-dispatched; failures -/// bubble up to the worker which marks the entry `Failed`. -/// -/// Approval-driven entries (`approval_id.is_some()`) route through -/// `actions::run_approval_*` which carry the kind-specific commit -/// pipeline + the `ApprovalResolved` event fan-out. Non-approval -/// entries hit the original auto/manual rebuild paths. -/// Pick the right approval pipeline for a `Rebuild` queue entry. Both -/// `ApplyCommit` and `MergeConfigPr` approvals enqueue a `Rebuild` entry -/// (both end in a container rebuild); branch on the approval kind. Falls -/// back to the apply-commit path if the row can't be read — it re-fetches -/// + surfaces a clean error itself. -async fn dispatch_rebuild_approval( - coord: &std::sync::Arc, - entry: &QueueEntry, - approval_id: i64, -) -> anyhow::Result<()> { - let kind = coord - .approvals - .get(approval_id) - .ok() - .flatten() - .map(|a| a.kind); - if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) { - crate::actions::run_approval_merge_config_pr(coord, Some(entry.id), approval_id).await - } else { - crate::actions::run_approval_apply_commit(coord, Some(entry.id), approval_id).await - } -} - -async fn dispatch( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - match (entry.kind, entry.approval_id) { - (QueueKind::Rebuild, Some(approval_id)) => { - dispatch_rebuild_approval(coord, entry, approval_id).await - } - (QueueKind::Rebuild, None) => { - // A meta-update cascade has just set the meta lock; re-locking - // in the per-agent rebuild would revert it (the agent's own - // flake.lock wins). Every other source wants the relock so it - // advances to applied//main. - let relock = entry.source != QueueSource::MetaUpdate; - rebuild_for_entry(coord, entry, relock).await - } - (QueueKind::MetaUpdate, Some(approval_id)) => { - crate::actions::run_approval_update_meta_inputs(coord, Some(entry.id), approval_id) - .await - } - (QueueKind::MetaUpdate, None) => run_meta_update(coord, entry).await, - (QueueKind::Spawn, Some(approval_id)) => { - crate::actions::run_approval_spawn(coord, Some(entry.id), approval_id).await - } - (QueueKind::Spawn, None) => { - // Unreachable today: every Spawn entry is born from an - // approval (HostRequest::RequestSpawn → submit_kind → - // approve → enqueue with approval_id). The manager-side - // `RequestSpawn` surface that used to bypass approvals - // was removed; if a future direct-spawn admin path needs - // to skip the approval ride it should wire its own action - // call rather than route through here. - anyhow::bail!( - "rebuild_queue: Spawn entry id={} agent={} arrived without an approval_id — \ - nothing should enqueue this shape today", - entry.id, - entry.agent, - ) - } - (QueueKind::Destroy, _) => { - // Reserved for future `destroy --purge` integration. - anyhow::bail!("Destroy kind not yet implemented in rebuild_queue worker"); - } - (QueueKind::StartupSweep, _) => { - // Bump meta's hyperhive input before per-agent child rebuilds - // run so they build against the latest base. Non-fatal on - // failure — child rebuilds proceed regardless. After the bump - // (or failure) this entry transitions to Done and the worker - // drains the pre-enqueued child Rebuild entries. - coord.set_queue_step(Some(entry.id), "nix flake update hyperhive"); - if let Err(e) = crate::meta::lock_update_hyperhive().await { - tracing::warn!(error = ?e, "startup_sweep: meta lock_update_hyperhive failed"); - } - // `finish` clears the step label; no explicit clear needed here. - Ok(()) - } - (QueueKind::Restart, _) => { - let name = &entry.agent; - let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Restarting); - coord.set_queue_step(Some(entry.id), "nixos-container restart"); - crate::lifecycle::restart(name).await?; - coord.kick_agent(name, "container restarted"); - coord.rescan_containers_and_emit().await; - Ok(()) - } - (QueueKind::PermChange, _) => { - let name = &entry.agent; - // Write + commit the perm file under META_LOCK so the - // working tree is never left dirty between the file write - // and the subsequent prepare_deploy git operations. - coord.set_queue_step(Some(entry.id), "writing + committing perm file"); - match &entry.perm_payload { - Some(PermPayload::ToolGroups { groups }) => { - crate::meta::commit_tool_groups(name, groups) - .await - .with_context(|| format!("commit tool-groups for {name}"))?; - // Emit after the commit so the P3RM1SS10NS tab - // reflects the new assignment without the operator - // needing to navigate away and back. - coord.emit_tool_groups_snapshot(); - } - Some(PermPayload::Capabilities { caps }) => { - crate::meta::commit_capabilities(name, caps) - .await - .with_context(|| format!("commit capabilities for {name}"))?; - coord.emit_capabilities_snapshot(); - } - Some(PermPayload::Combined { groups, caps }) => { - // Batch perm change: commit whichever file(s) are - // present in a single git commit, then the rebuild - // below runs once — no double-rebuild for an agent - // whose caps AND groups both changed. - crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref()) - .await - .with_context(|| format!("commit perms for {name}"))?; - if groups.is_some() { - coord.emit_tool_groups_snapshot(); - } - if caps.is_some() { - coord.emit_capabilities_snapshot(); - } - } - None => { - anyhow::bail!( - "PermChange entry id={} agent={} is missing perm_payload", - entry.id, - entry.agent, - ); - } - } - // Now rebuild so the updated HIVE_TOOL_GROUPS / HIVE_CAPABILITIES - // env var takes effect in the container. - rebuild_for_entry(coord, entry, true).await - } - (QueueKind::GracefulStop, _) => { - run_graceful_stop(coord, entry); - Ok(()) - } - (QueueKind::Start, _) => run_start(coord, entry).await, - (QueueKind::Stop, _) => run_stop(coord, entry).await, - } -} - -/// Queue-side container rebuild for `entry.agent`: resolves the current -/// flake rev and hands off to `rebuild_agent` with the entry's id + -/// source. Passing the source defers the start-after-rebuild to a -/// fast-lane `Start` follow-up (grouped under this entry via -/// `parent_id`), so the build lane is freed for the next entry instead -/// of waiting out the container boot. -async fn rebuild_for_entry( - coord: &std::sync::Arc, - entry: &QueueEntry, - relock: bool, -) -> anyhow::Result<()> { - let current_rev = - crate::auto_update::current_flake_rev(&coord.hyperhive_flake).unwrap_or_default(); - crate::auto_update::rebuild_agent( - coord, - &entry.agent, - ¤t_rev, - Some(entry.id), - relock, - Some(entry.source), - ) - .await -} - -/// Start a stopped container off the queue (`QueueKind::Start`), with a -/// `Starting` transient so the dashboard shows a visible queued→running -/// progression rather than the sub-second flash of a direct start. -/// Uses the cold-start fallback (stop + kill + start retry) so the -/// deferred start-after-rebuild keeps the same activation-error recovery -/// it had when it ran inline on the build lane. -/// -/// If the hyperhive flake rev has changed since the container was last built -/// (i.e. the rev marker is stale or missing), the start is upgraded to a full -/// rebuild so the container runs current nix derivations. This is the -/// "deferred stopped container" path from `auto_update::run`. -async fn run_start( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - let name = &entry.agent; - // Upgrade to rebuild+start if the rev marker is stale. - let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); - if let Some(ref rev) = current_rev { - let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok(); - if stored.as_deref() != Some(rev.as_str()) { - tracing::info!(%name, "start: rev stale — upgrading to rebuild+start"); - return crate::auto_update::rebuild_agent( - coord, - name, - rev, - Some(entry.id), - true, - Some(entry.source), - ) - .await; - } - } - let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Starting); - coord.set_queue_step(Some(entry.id), "nixos-container start"); - crate::lifecycle::start_with_fallback(name).await?; - coord.kick_agent(name, "container started"); - coord.rescan_containers_and_emit().await; - Ok(()) -} - -/// Hard-stop a container off the queue (`QueueKind::Stop`) — same teardown as -/// a direct kill (unregister + `Killed` event), but with a `Stopping` -/// transient for visible queue progress. The quiescing variant is -/// `run_graceful_stop`. -async fn run_stop( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - let name = &entry.agent; - let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Stopping); - coord.set_queue_step(Some(entry.id), "nixos-container stop"); - crate::lifecycle::kill(name).await?; - coord.unregister_agent(name); - coord.notify_manager(&hive_sh4re::HelperEvent::Killed { - agent: name.clone(), - }); - coord.rescan_containers_and_emit().await; - Ok(()) -} - -/// Run one `GracefulStop` entry: signal the harness to quiesce (it returns -/// `GracefulStop` on its next `Recv`, runs one stop-checkpoint turn to flush -/// durable `/state`, then exits), then hand the drain-wait + container stop to -/// a detached watcher and return — freeing the build lane immediately. -/// -/// This is the concurrency split: the build worker only does the cheap signal, -/// so a whole-hive graceful stop signals every agent up front and their -/// checkpoint drains overlap. The watcher waits for this agent's drain -/// (bounded by `GRACEFUL_STOP_TIMEOUT` so a wedged agent can't block forever), -/// then enqueues a fast-lane `Stop` for the actual `nixos-container stop`. -/// Routing the real stop through the fast lane means the container stops -/// serialise there (one stop at a time) while the drains ran in parallel. -fn run_graceful_stop(coord: &std::sync::Arc, entry: &QueueEntry) { - let name = entry.agent.clone(); - let parent_id = entry.id; - let source = entry.source; - // Signal the harness; the kick breaks an idle long-poll so it's seen promptly. - coord.set_queue_step(Some(entry.id), "graceful stop: signalling agent"); - coord.mark_graceful_stop(&name); - coord.kick_agent(&name, "graceful stop requested"); - // Detached watcher: wait for the drain (or timeout), then enqueue the - // container stop on the fast lane. The build entry itself is now Done — - // the dashboard groups the follow-up `Stop` under it via `parent_id`. - let coord = std::sync::Arc::clone(coord); - tokio::spawn(async move { - // Hold the `Stopping` transient across the drain so the dashboard keeps - // showing the agent quiescing; dropped before the fast `Stop` is - // enqueued (its `run_stop` re-establishes the transient) so the two - // never clobber each other's clear-on-drop. - let guard = coord.transient_guard(&name, crate::coordinator::TransientKind::Stopping); - // Wait for the harness to drain (it clears the flag via - // `GracefulStopComplete`) or fall back to a hard stop after the timeout. - let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT; - while coord.is_graceful_stop_pending(&name) { - if std::time::Instant::now() >= deadline { - tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping"); - break; - } - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - } - coord.clear_graceful_stop(&name); - drop(guard); - // Enqueue the actual container stop on the fast lane (same teardown as a - // plain kill — `run_stop`). `parent_id` links it to the graceful entry - // for dashboard grouping. `enqueue_full` nudges the fast worker itself. - coord.rebuild_queue.enqueue_full(FullEnqueue { - kind: QueueKind::Stop, - agent: name.clone(), - source, - reason: format!("container stop after graceful drain of {name}"), - parent_id: Some(parent_id), - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: Vec::new(), - }); - coord.emit_rebuild_queue_snapshot(); - }); -} - -/// Run one `MetaUpdate` entry: bump the meta flake's locks for the -/// requested inputs, then enqueue a cascade of `Rebuild` entries -/// (with `parent_id` set to this entry's id) for every agent affected -/// by the bump. Mirrors the previous `dashboard::run_meta_update` -/// semantics; that path now enqueues into this queue rather than -/// running the bump + rebuild loop inline. -async fn run_meta_update( - coord: &std::sync::Arc, - entry: &QueueEntry, -) -> anyhow::Result<()> { - let _progress = coord.meta_update_guard(); - let inputs = entry.inputs.clone(); - tracing::info!( - ?inputs, - parent = entry.id, - "rebuild_queue: meta-update starting" - ); - coord.set_queue_step(Some(entry.id), "nix flake update"); - let result = if inputs.is_empty() { - crate::meta::lock_update(&[]).await - } else { - crate::meta::lock_update(&inputs).await - }; - if let Err(e) = result { - // Lock bump failed — cancel any pending cascade rebuilds the - // enqueuer pre-queued for this MetaUpdate. Their parent_id - // matches this entry; the children no longer make sense (we - // never bumped the lock that justified them). - let cancelled = coord.rebuild_queue.cancel_children(entry.id); - if cancelled > 0 { - tracing::warn!( - cancelled, - parent = entry.id, - "rebuild_queue: meta-update failed; cancelled cascade rebuilds" - ); - coord.emit_rebuild_queue_snapshot(); - } - return Err(e); - } - // Lock file changed — meta-inputs panel re-renders. The cascade - // rebuilds were already enqueued at MetaUpdate submission time, - // so no further enqueue is needed here. - crate::dashboard::emit_meta_inputs_snapshot(coord.as_ref()); - Ok(()) -} - -/// Compute which agents a `nix flake update ` on the meta -/// flake would affect. Used by callers that pre-enqueue cascade -/// `Rebuild` entries at `MetaUpdate` submission time so the dashboard -/// can render the dependent work alongside its parent before the lock -/// bump actually runs. -/// -/// Mirrors `run_meta_update`'s post-bump fan-out logic. Empty `inputs` -/// or any input under `hyperhive` → every container; otherwise just -/// the agents named by `agent-` inputs. -pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec { - let touched_hyperhive = inputs - .iter() - .any(|i| i == "hyperhive" || i.starts_with("hyperhive/")); - let touched_agents: Vec = inputs - .iter() - .filter_map(|i| i.strip_prefix("agent-")) - .map(|rest| rest.split('/').next().unwrap_or(rest).to_owned()) - .collect(); - let mut names = if touched_hyperhive || inputs.is_empty() { - crate::lifecycle::list() - .await - .unwrap_or_default() - .into_iter() - .filter_map(|c| { - c.strip_prefix(crate::lifecycle::AGENT_PREFIX) - .map(str::to_owned) - }) - .collect() - } else { - touched_agents - }; - // Sort parents before children so the sequential queue worker - // always rebuilds a parent before any of its dependents. - let topo = crate::topology::read(); - crate::auto_update::topology_sort(&mut names, &topo); - names -} - -/// Current unix timestamp in seconds. `now()` calls are pulled into a -/// helper so tests can swap them out later. -fn now_unix() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn enqueue_and_take_in_order() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::Manual, - "second".to_owned(), - None, - ); - assert_ne!(a, b); - let next = q.take_next_build().expect("queued"); - assert_eq!(next.id, a); - assert_eq!(next.state, QueueState::Running); - let next = q.take_next_build().expect("queued"); - assert_eq!(next.id, b); - assert!(q.take_next_build().is_none()); - } - - #[test] - fn dedup_pending_same_kind_and_agent() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::AutoUpdate, - "auto sweep".to_owned(), - None, - ); - assert_eq!(a, b, "dedup should return existing id"); - let snap = q.snapshot(); - assert_eq!(snap.len(), 1); - assert!(snap[0].reason.contains("first")); - assert!(snap[0].reason.contains("auto sweep")); - } - - #[test] - fn meta_update_dedup_matches_inputs() { - // Two MetaUpdate enqueues with identical inputs → dedup. - let q = RebuildQueue::new(); - let a = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - vec!["nixpkgs".to_owned()], - ); - let b = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "duplicate click".to_owned(), - None, - vec!["nixpkgs".to_owned()], - ); - assert_eq!(a, b, "identical-inputs meta-updates should dedup"); - assert_eq!(q.snapshot().len(), 1); - } - - #[test] - fn meta_update_dedup_separates_distinct_inputs() { - // Two MetaUpdate enqueues with DIFFERENT inputs → distinct - // entries, not deduped. - let q = RebuildQueue::new(); - let a = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "bump nixpkgs".to_owned(), - None, - vec!["nixpkgs".to_owned()], - ); - let b = q.enqueue_with_inputs( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "bump bitburner-agent".to_owned(), - None, - vec!["agent-bitburner/bitburner-agent".to_owned()], - ); - assert_ne!(a, b, "different-inputs meta-updates must NOT dedup"); - let snap = q.snapshot(); - assert_eq!(snap.len(), 2); - // Both inputs lists are preserved. - let inputs: Vec<&[String]> = snap.iter().map(|e| e.inputs.as_slice()).collect(); - assert!(inputs.iter().any(|i| *i == ["nixpkgs"])); - assert!( - inputs - .iter() - .any(|i| *i == ["agent-bitburner/bitburner-agent"]) - ); - } - - #[test] - fn dedup_does_not_apply_across_kinds_or_agents() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - let c = q.enqueue( - QueueKind::Spawn, - "agent-a".to_owned(), - QueueSource::Manual, - "s".to_owned(), - None, - ); - assert_ne!(a, b); - assert_ne!(a, c); - assert_eq!(q.snapshot().len(), 3); - } - - #[test] - fn dedup_skips_running_entries() { - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let running = q.take_next_build().expect("queued"); - assert_eq!(running.state, QueueState::Running); - // While the original is running, re-enqueue is legitimate. - let again = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "config bumped during build".to_owned(), - None, - ); - assert_ne!(running.id, again); - let snap = q.snapshot(); - assert_eq!(snap.len(), 2); - } - - #[test] - fn finish_marks_state_and_keeps_history() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Done, None); - let snap = q.snapshot(); - assert_eq!(snap.len(), 1); - assert_eq!(snap[0].state, QueueState::Done); - assert!(snap[0].finished_at.is_some()); - assert!(snap[0].error.is_none()); - } - - #[test] - fn finish_with_failure_records_error() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Failed, Some("nix build failed".to_owned())); - let snap = q.snapshot(); - assert_eq!(snap[0].state, QueueState::Failed); - assert_eq!(snap[0].error.as_deref(), Some("nix build failed")); - } - - #[test] - fn history_evicts_old_terminals_per_kind() { - let q = RebuildQueue::new(); - for i in 0..(MAX_HISTORY_PER_KIND + 3) { - let id = q.enqueue( - QueueKind::Rebuild, - format!("agent-{i}"), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Done, None); - } - let snap = q.snapshot(); - assert_eq!(snap.len(), MAX_HISTORY_PER_KIND); - } - - #[test] - fn cancel_clears_queued_entry() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - assert!(q.cancel(id)); - let snap = q.snapshot(); - assert_eq!(snap[0].state, QueueState::Cancelled); - assert!(q.take_next_build().is_none()); - } - - #[test] - fn cancel_refuses_running_entry() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "r".to_owned(), - None, - ); - q.take_next_build(); - assert!(!q.cancel(id)); - let snap = q.snapshot(); - assert_eq!(snap[0].state, QueueState::Running); - } - - #[test] - fn parent_id_groups_cascade() { - let q = RebuildQueue::new(); - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "lock bump".to_owned(), - None, - ); - let child = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - let snap = q.snapshot(); - let child_entry = snap.iter().find(|e| e.id == child).expect("child queued"); - assert_eq!(child_entry.parent_id, Some(meta)); - } - - #[test] - fn cancel_children_marks_queued_descendants() { - let q = RebuildQueue::new(); - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "lock bump".to_owned(), - None, - ); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - let b = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - // An unrelated queued entry must not be cancelled. - let c = q.enqueue( - QueueKind::Rebuild, - "agent-c".to_owned(), - QueueSource::Manual, - "operator queued".to_owned(), - None, - ); - let cancelled = q.cancel_children(meta); - assert_eq!(cancelled, 2); - let snap = q.snapshot(); - let find = |id: u64| snap.iter().find(|e| e.id == id).expect("present"); - assert_eq!(find(a).state, QueueState::Cancelled); - assert_eq!(find(b).state, QueueState::Cancelled); - assert_eq!(find(c).state, QueueState::Queued); - } - - #[test] - fn approval_entries_keep_approval_id() { - let q = RebuildQueue::new(); - let id = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval 42 apply commit".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(42), - perm_payload: None, - depends_on: Vec::new(), - }); - let snap = q.snapshot(); - let entry = snap.iter().find(|e| e.id == id).expect("entry present"); - assert_eq!(entry.approval_id, Some(42)); - assert_eq!(entry.source, QueueSource::Approval); - } - - #[test] - fn approval_entries_dedup_only_on_matching_id() { - // Two pending approval-driven entries for the same agent but - // DIFFERENT approval ids must NOT collapse — each operator - // approve click is a separate piece of work even when the - // (kind, agent) pair matches. - let q = RebuildQueue::new(); - let a = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval #1".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(1), - perm_payload: None, - depends_on: Vec::new(), - }); - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval #2".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(2), - perm_payload: None, - depends_on: Vec::new(), - }); - assert_ne!(a, b); - assert_eq!(q.snapshot().len(), 2); - // Same approval_id submitted twice DOES dedup (rapid double- - // click on the dashboard's approve button is a single op). - let c = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-a".to_owned(), - source: QueueSource::Approval, - reason: "approval #1 (duplicate)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: Some(1), - perm_payload: None, - depends_on: Vec::new(), - }); - assert_eq!(a, c); - assert_eq!(q.snapshot().len(), 2); - } - - #[test] - fn cancel_children_skips_running_and_terminal() { - let q = RebuildQueue::new(); - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "lock bump".to_owned(), - None, - ); - // Running child — must NOT be cancelled. - let running = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - q.take_next_build(); // pops meta, marks it Running - q.take_next_build(); // pops `running`, marks it Running - // Terminal child — must NOT be re-cancelled (its state stays Done). - let done = q.enqueue( - QueueKind::Rebuild, - "agent-b".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - q.take_next_build(); - q.finish(done, QueueState::Done, None); - // Queued child that should be cancelled. - let queued = q.enqueue( - QueueKind::Rebuild, - "agent-c".to_owned(), - QueueSource::MetaUpdate, - "cascade".to_owned(), - Some(meta), - ); - let n = q.cancel_children(meta); - assert_eq!(n, 1); - let snap = q.snapshot(); - let find = |id: u64| snap.iter().find(|e| e.id == id).expect("present"); - assert_eq!(find(running).state, QueueState::Running); - assert_eq!(find(done).state, QueueState::Done); - assert_eq!(find(queued).state, QueueState::Cancelled); - } - - #[test] - fn set_step_updates_running_entry_and_signals_change() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - "test".to_owned(), - None, - ); - // Queued — set_step should refuse (returns false). - assert!(!q.set_step(id, "plant tags")); - // Promote to Running. - let entry = q.take_next_build().expect("queued entry"); - assert_eq!(entry.id, id); - // First label transition — true. - assert!(q.set_step(id, "plant tags")); - assert_eq!( - q.snapshot() - .iter() - .find(|e| e.id == id) - .and_then(|e| e.step.as_deref()), - Some("plant tags") - ); - // Same label again — false (caller can skip the snapshot emit). - assert!(!q.set_step(id, "plant tags")); - // Different label — true. - assert!(q.set_step(id, "nixos-container update")); - assert_eq!( - q.snapshot() - .iter() - .find(|e| e.id == id) - .and_then(|e| e.step.as_deref()), - Some("nixos-container update") - ); - } - - #[test] - fn set_step_no_op_on_unknown_id() { - let q = RebuildQueue::new(); - assert!(!q.set_step(999, "anything")); - } - - /// A `MetaUpdate` cascade `Rebuild` (with `parent_id` = `Some(meta_id)`) must - /// NOT dedup into a pre-existing `Queued` `Rebuild` with a different `parent_id` - /// (e.g. from a startup sweep). Without the `parent_id` dedup guard the - /// cascade rebuild would be swallowed and the agent would never rebuild - /// against the post-lock-bump meta. - #[test] - fn meta_update_cascade_does_not_dedup_into_startup_sweep_rebuild() { - let q = RebuildQueue::new(); - // Startup sweep enqueues a Rebuild for alice with its own parent_id. - let sweep = q.enqueue( - QueueKind::StartupSweep, - "hyperhive".to_owned(), - QueueSource::AutoUpdate, - "boot sweep".to_owned(), - None, - ); - let sweep_rebuild = q.enqueue( - QueueKind::Rebuild, - "alice".to_owned(), - QueueSource::StartupSweep, - "startup sweep".to_owned(), - Some(sweep), - ); - // MetaUpdate cascade pre-enqueues another Rebuild for alice. - let meta = q.enqueue( - QueueKind::MetaUpdate, - "hyperhive".to_owned(), - QueueSource::Manual, - "bump nixpkgs".to_owned(), - None, - ); - let cascade_rebuild = q.enqueue( - QueueKind::Rebuild, - "alice".to_owned(), - QueueSource::MetaUpdate, - "meta-update cascade".to_owned(), - Some(meta), - ); - // The two Rebuilds have different parent_ids — must NOT dedup. - assert_ne!( - sweep_rebuild, cascade_rebuild, - "cascade rebuild must be distinct from startup-sweep rebuild" - ); - let snap = q.snapshot(); - let rebuilds: Vec<_> = snap - .iter() - .filter(|e| e.kind == QueueKind::Rebuild && e.agent == "alice") - .collect(); - assert_eq!( - rebuilds.len(), - 2, - "both rebuilds must be present in the queue" - ); - } - - #[test] - fn finish_clears_step() { - let q = RebuildQueue::new(); - let id = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - "test".to_owned(), - None, - ); - q.take_next_build(); - assert!(q.set_step(id, "running phase")); - q.finish(id, QueueState::Done, None); - assert_eq!( - q.snapshot() - .iter() - .find(|e| e.id == id) - .and_then(|e| e.step.as_deref()), - None - ); - } - - // --- depends_on tests --- - - /// An entry whose dep is not yet terminal must be skipped by - /// `take_next`; it runs only after the dep finishes. - #[test] - fn depends_on_blocks_until_dep_is_terminal() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "agent-a".to_owned(), - QueueSource::Manual, - "first".to_owned(), - None, - ); - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "agent-b".to_owned(), - source: QueueSource::Manual, - reason: "second (blocked on a)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![a], - }); - // B depends on A — take_next should give A first. - let first = q.take_next_build().expect("a is ready"); - assert_eq!(first.id, a); - // A is Running, not terminal — B must still be blocked. - assert!( - q.take_next_build().is_none(), - "b must be blocked while a runs" - ); - // Finish A → B should now be available. - q.finish(a, QueueState::Done, None); - let second = q.take_next_build().expect("b unblocked after a done"); - assert_eq!(second.id, b); - } - - /// An entry whose dep finished and was evicted from history is - /// treated as resolved (eviction only happens to terminal entries). - #[test] - fn depends_on_evicted_dep_counts_as_resolved() { - let q = RebuildQueue::new(); - // Fill the history cap for Rebuild so old terminals get evicted. - for i in 0..MAX_HISTORY_PER_KIND { - let id = q.enqueue( - QueueKind::Rebuild, - format!("filler-{i}"), - QueueSource::Manual, - "filler".to_owned(), - None, - ); - q.take_next_build(); - q.finish(id, QueueState::Done, None); - } - // `dep` gets enqueued, run, finished, and evicted by the - // next history-trimming call. - let dep = q.enqueue( - QueueKind::Rebuild, - "dep-agent".to_owned(), - QueueSource::Manual, - "dep".to_owned(), - None, - ); - q.take_next_build(); - q.finish(dep, QueueState::Done, None); - // Push `dep` out of the per-kind history window: `trim_history` - // keeps the newest MAX_HISTORY_PER_KIND terminals per kind, so it - // takes that many newer terminals to evict `dep`. - for i in 0..MAX_HISTORY_PER_KIND { - let extra = q.enqueue( - QueueKind::Rebuild, - format!("extra-{i}"), - QueueSource::Manual, - format!("extra-{i}"), - None, - ); - q.take_next_build(); - q.finish(extra, QueueState::Done, None); - } - // `dep` should now be evicted. - assert!( - q.snapshot().iter().all(|e| e.id != dep), - "dep must be evicted from history" - ); - // An entry that depends on the (evicted) dep must be immediately runnable. - let downstream = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "downstream".to_owned(), - source: QueueSource::Manual, - reason: "downstream (dep evicted = resolved)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep], - }); - let got = q - .take_next_build() - .expect("downstream runnable when dep evicted"); - assert_eq!(got.id, downstream); - } - - /// Dedup respects `depends_on`: two otherwise-identical entries with - /// different dep sets are distinct and must NOT collapse. - #[test] - fn depends_on_is_part_of_dedup_key() { - let q = RebuildQueue::new(); - let dep1 = q.enqueue( - QueueKind::Rebuild, - "dep1".to_owned(), - QueueSource::Manual, - "d1".to_owned(), - None, - ); - let dep2 = q.enqueue( - QueueKind::Rebuild, - "dep2".to_owned(), - QueueSource::Manual, - "d2".to_owned(), - None, - ); - let a = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "target".to_owned(), - source: QueueSource::Manual, - reason: "r".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep1], - }); - // Same kind+agent but different depends_on — must NOT dedup. - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "target".to_owned(), - source: QueueSource::Manual, - reason: "r".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep2], - }); - assert_ne!(a, b, "different depends_on must produce distinct entries"); - // Same depends_on as a — must dedup. - let c = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "target".to_owned(), - source: QueueSource::Manual, - reason: "r again".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![dep1], - }); - assert_eq!(a, c, "identical depends_on must dedup"); - } - - /// An entry with Failed dep is still resolved — the dependent runs - /// regardless of whether its upstream succeeded or not. Callers that - /// need to abort on dep failure should cancel the downstream manually. - #[test] - fn depends_on_failed_dep_counts_as_resolved() { - let q = RebuildQueue::new(); - let a = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - "a".to_owned(), - None, - ); - let b = q.enqueue_full(FullEnqueue { - kind: QueueKind::Rebuild, - agent: "b".to_owned(), - source: QueueSource::Manual, - reason: "b (blocked on a)".to_owned(), - parent_id: None, - inputs: Vec::new(), - approval_id: None, - perm_payload: None, - depends_on: vec![a], - }); - q.take_next_build(); // pop a, mark Running - q.finish(a, QueueState::Failed, Some("nix build exploded".to_owned())); - let got = q.take_next_build().expect("b runnable after a failed"); - assert_eq!(got.id, b); - } - - // ---- fast lane (Start / Stop run on a separate concurrent worker) ---- - - #[test] - fn lanes_claim_only_their_own_kinds() { - let q = RebuildQueue::new(); - let r = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let s = q.enqueue( - QueueKind::Stop, - "b".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let build = q.take_next_build().expect("build entry"); - assert_eq!(build.id, r); - let fast = q.take_next_fast().expect("fast entry"); - assert_eq!(fast.id, s); - assert!(q.take_next_build().is_none()); - assert!(q.take_next_fast().is_none()); - } - - #[test] - fn start_defers_behind_same_agent_queued_build() { - let q = RebuildQueue::new(); - let b = q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Start, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - assert!( - q.take_next_fast().is_none(), - "start blocked while same agent has a queued build" - ); - q.take_next_build().expect("build runs"); - q.finish(b, QueueState::Done, None); - let started = q - .take_next_fast() - .expect("start unblocked after build done"); - assert_eq!(started.kind, QueueKind::Start); - } - - #[test] - fn start_for_other_agent_runs_concurrently_with_a_build() { - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Start, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Start, - "b".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.take_next_build().expect("a's build running"); - let got = q - .take_next_fast() - .expect("start for b runs while a's build runs"); - assert_eq!(got.agent, "b"); - assert!( - q.take_next_fast().is_none(), - "start for a still blocked by a's running build" - ); - } - - #[test] - fn stop_jumps_queued_build_but_waits_running_build() { - // Stop jumps ahead of a *queued* build for the same agent. - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Stop, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let got = q - .take_next_fast() - .expect("stop jumps ahead of a's queued build"); - assert_eq!(got.kind, QueueKind::Stop); - - // But a stop waits for a *running* build of the same agent. - let q2 = RebuildQueue::new(); - let b = q2.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q2.enqueue( - QueueKind::Stop, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q2.take_next_build().expect("a's build running"); - assert!( - q2.take_next_fast().is_none(), - "stop waits for a's running build (no kill mid-rebuild)" - ); - q2.finish(b, QueueState::Done, None); - assert!( - q2.take_next_fast().is_some(), - "stop runs once a's build is done" - ); - } - - #[test] - fn build_defers_behind_same_agent_running_fast_op() { - let q = RebuildQueue::new(); - q.enqueue( - QueueKind::Stop, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - q.enqueue( - QueueKind::Rebuild, - "a".to_owned(), - QueueSource::Manual, - String::new(), - None, - ); - let s = q.take_next_fast().expect("stop running"); - assert!( - q.take_next_build().is_none(), - "build waits while a's fast op is running" - ); - q.finish(s.id, QueueState::Done, None); - assert!( - q.take_next_build().is_some(), - "build runs once the fast op is done" - ); - } -} diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index a711bf6b..63c5f885 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -90,13 +90,9 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { tracing::info!(%id, %name, "spawn approval queued"); HostResponse::success() } - HostRequest::Kill { name } => handle_kill(&coord, name).await?, - HostRequest::Restart { name } => { - tracing::info!(%name, "restart"); - lifecycle::restart(name).await?; - HostResponse::success() - } - HostRequest::RestartAll => handle_restart_all().await?, + HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill), + HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart), + HostRequest::RestartAll => handle_restart_all(&coord).await?, HostRequest::Stop { scope, graceful } => { // Resolve the scope to explicit container names at the entry // point, then operate on names — never pass the bare "all @@ -131,13 +127,23 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { agents.retain(|a| prev.contains(a)); } let infra = scoped_infra(scope); - handle_start(&agents, &infra).await? + handle_start(&coord, &agents, &infra).await? } HostRequest::Destroy { name, purge } => { actions::destroy(&coord, name, *purge).await?; HostResponse::success() } - HostRequest::Rebuild { name } => handle_rebuild(&coord, name).await?, + HostRequest::Rebuild { name } => submit_single(&coord, name, Verb::Rebuild), + HostRequest::QueueDag { id } => { + // The polled DAG first, then its live fan-out children. + let dags = coord + .job_queue + .snapshot() + .into_iter() + .filter(|d| d.id == *id || d.parent_id == Some(*id)) + .collect(); + HostResponse::dags(dags) + } HostRequest::List => HostResponse::list(lifecycle::list().await?), HostRequest::AgentStatus => { let rows = crate::container_view::build_all(&coord) @@ -201,6 +207,9 @@ async fn handle_spawn(coord: &Arc, name: &str) -> Result { + if let Err(e) = coord.power.set(name, crate::power::Wanted::Up) { + tracing::warn!(%name, error = ?e, "agent_power: set wanted=up failed"); + } coord.notify_manager(&hive_sh4re::HelperEvent::Spawned { agent: name.to_owned(), ok: true, @@ -223,44 +232,81 @@ async fn handle_spawn(coord: &Arc, name: &str) -> Result, name: &str) -> Result { - tracing::info!(%name, "kill"); - lifecycle::kill(name).await?; - coord.unregister_agent(name); - coord.notify_manager(&hive_sh4re::HelperEvent::Killed { - agent: name.to_owned(), - }); - Ok(HostResponse::success()) +/// Single-agent queue verbs the admin socket exposes. Each submits the +/// matching DAG (persisting the `wanted` intent, serializing on the +/// agent's lease, with the transient/crash-watch suppression the old +/// direct lifecycle calls lacked) and returns the DAG id for the +/// client's wait loop. +#[derive(Clone, Copy)] +enum Verb { + /// Stop DAG (`wanted = Offline`; Reconcile kills + unregisters + + /// fires `Killed`). + Kill, + /// Restart DAG (`wanted = Up`; mechanical stop + reconcile-start). + Restart, + /// Rebuild DAG — the Swap tail owns the manager `Rebuilt` events + + /// kick, so the CLI path can't drift from the dashboard's. + Rebuild, } -/// Restart every container, aggregating per-agent failures into one -/// response rather than aborting on the first error. -async fn handle_restart_all() -> Result { +fn submit_single(coord: &Arc, name: &str, verb: Verb) -> HostResponse { + use crate::job_queue::{Source, submit}; + let id = match verb { + Verb::Kill => { + tracing::info!(%name, "kill"); + submit::stop( + coord, + name, + Source::Manual, + "manual kill via hivectl".to_owned(), + ) + } + Verb::Restart => { + tracing::info!(%name, "restart"); + submit::restart( + coord, + name, + Source::Manual, + "manual restart via hivectl".to_owned(), + ) + } + Verb::Rebuild => { + tracing::info!(%name, "rebuild"); + submit::rebuild( + coord, + name, + Source::Manual, + "manual rebuild via hivectl".to_owned(), + ) + } + }; + HostResponse::queued(vec![id]) +} + +/// Restart every container by submitting one restart DAG per agent — +/// each serializes on its own lease, so unrelated agents' restarts +/// overlap while nothing races an in-flight rebuild. Returns once all +/// are queued; per-agent results surface on the queue. +async fn handle_restart_all(coord: &Arc) -> Result { tracing::info!("restart-all"); let agents = lifecycle::list().await?; let mut ok_agents: Vec = Vec::new(); - let mut errors: Vec = Vec::new(); + let mut queued: Vec = Vec::new(); for agent in &agents { - if let Err(e) = lifecycle::restart(agent).await { - tracing::warn!(%agent, error = ?e, "restart-all: failed to restart agent"); - errors.push(format!("{agent}: {e:#}")); - } else { - ok_agents.push(agent.clone()); - } - } - if errors.is_empty() { - Ok(HostResponse::list(ok_agents)) - } else { - Ok(HostResponse { - ok: false, - error: Some(errors.join("; ")), - agents: Some(ok_agents), - approvals: None, - urls: None, - agent_statuses: None, - }) + let Some(logical) = agent.strip_prefix(lifecycle::AGENT_PREFIX) else { + continue; + }; + queued.push(crate::job_queue::submit::restart( + coord, + logical, + crate::job_queue::Source::Manual, + "manual restart via hivectl restart-all".to_owned(), + )); + ok_agents.push(logical.to_owned()); } + let mut resp = HostResponse::list(ok_agents); + resp.queued_dags = Some(queued); + Ok(resp) } /// Stop the given `agents` (resolved logical names) then `infra` containers @@ -270,11 +316,13 @@ async fn handle_restart_all() -> Result { /// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these /// explicit name lists up front — this never sees the "all" flag. /// -/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the -/// harness, run one stop-checkpoint turn, drain, then container stop, with a -/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1` -/// path. `graceful` applies to agents only - infra containers have no harness -/// turn loop, so they're always hard-stopped. +/// Every agent rides the job queue: a `graceful` stop submits the +/// quiesce DAG (signal → drain → reconcile-stop; all drains overlap), +/// a hard stop a plain stop DAG — both persist `wanted = Offline` and +/// serialize on the agent's lease so nothing races an in-flight +/// rebuild. The response carries the DAG ids so `hivectl` can wait +/// with per-node progress. Infra containers have no harness / lease +/// and stay direct + synchronous. async fn handle_stop( coord: &Arc, agents: &[String], @@ -284,37 +332,41 @@ async fn handle_stop( tracing::info!(?agents, ?infra, graceful, "stop"); let mut ok_items: Vec = Vec::new(); let mut errors: Vec = Vec::new(); - let mut enqueued_graceful = false; + let mut queued: Vec = Vec::new(); for agent in agents { - if graceful { - // Graceful stop: enqueue the quiesce orchestration rather than a - // hard kill. Serialised through the rebuild queue so it can't race - // an in-flight rebuild for the same agent, and its per-step - // progress surfaces on the queue snapshot + build log. - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::GracefulStop, - agent.clone(), - crate::rebuild_queue::QueueSource::Manual, - "manual via hivectl graceful stop".to_owned(), - None, - ); - ok_items.push(agent.clone()); - enqueued_graceful = true; - continue; - } - match lifecycle::kill(agent).await { - Ok(()) => ok_items.push(agent.clone()), - Err(e) => { - tracing::warn!(%agent, error = ?e, "stop: agent kill failed"); - errors.push(format!("{agent}: {e:#}")); - } - } - } - if enqueued_graceful { - coord.emit_rebuild_queue_snapshot(); + let reason = if graceful { + "manual via hivectl graceful stop" + } else { + "manual via hivectl stop" + }; + let id = if graceful { + crate::job_queue::submit::graceful_stop( + coord, + agent, + crate::job_queue::Source::Manual, + reason.to_owned(), + ) + } else { + crate::job_queue::submit::stop( + coord, + agent, + crate::job_queue::Source::Manual, + reason.to_owned(), + ) + }; + queued.push(id); + ok_items.push(agent.clone()); } + // Agents go down before infra so they're not mid-request against a + // forge/matrix that's already gone. Hard stops are quick kills — + // await their DAGs (bounded) before touching infra. Graceful stops + // keep the immediate return (drains take minutes and the + // agents-then-infra race pre-existed there). + if !graceful && !infra.is_empty() { + await_dags(coord, &queued, std::time::Duration::from_mins(2)).await; + } for &container in infra { let name = container.unit_name(); match crate::priv_client::control_infra_container(container, InfraAction::Stop).await { @@ -326,14 +378,42 @@ async fn handle_stop( } } - Ok(finish_lifecycle(ok_items, &errors)) + let mut resp = finish_lifecycle(ok_items, &errors); + resp.queued_dags = Some(queued); + Ok(resp) +} + +/// Best-effort server-side wait for a set of DAGs to settle terminal, +/// bounded by `timeout` — used to preserve ordering invariants inside +/// one request (agent stops before infra stops) without trusting the +/// client to wait. +async fn await_dags(coord: &Arc, ids: &[u64], timeout: std::time::Duration) { + let deadline = std::time::Instant::now() + timeout; + loop { + let snap = coord.job_queue.snapshot(); + let pending = ids + .iter() + .any(|id| snap.iter().any(|d| d.id == *id && !d.state.is_terminal())); + if !pending { + return; + } + if std::time::Instant::now() >= deadline { + tracing::warn!(?ids, "await_dags: timed out; proceeding"); + return; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } } /// Start the given `infra` containers then `agents` (`hivectl start`) — the /// inverse of [`handle_stop`]. Infra comes up before agents so the agents /// find forge/matrix/gateway ready. Per-target failures aggregated. Callers /// resolve the [`LifecycleScope`] to these explicit name lists up front. -async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result { +async fn handle_start( + coord: &Arc, + agents: &[String], + infra: &[InfraContainer], +) -> Result { tracing::info!(?agents, ?infra, "start"); let mut ok_items: Vec = Vec::new(); let mut errors: Vec = Vec::new(); @@ -349,17 +429,23 @@ async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result = Vec::new(); for agent in agents { - match lifecycle::start(agent).await { - Ok(()) => ok_items.push(agent.clone()), - Err(e) => { - tracing::warn!(%agent, error = ?e, "start: agent start failed"); - errors.push(format!("{agent}: {e:#}")); - } - } + // Through the queue: persists `wanted = Up`, upgrades a + // stale-rev start to a full rebuild, and serializes on the + // agent's lease. Ids ride back for hivectl's wait loop. + queued.push(crate::job_queue::submit::start( + coord, + agent, + crate::job_queue::Source::Manual, + "manual via hivectl start".to_owned(), + )); + ok_items.push(agent.clone()); } - Ok(finish_lifecycle(ok_items, &errors)) + let mut resp = finish_lifecycle(ok_items, &errors); + resp.queued_dags = Some(queued); + Ok(resp) } /// Resolve which sub-agent logical names a scope targets: every live @@ -444,48 +530,7 @@ fn finish_lifecycle(ok_items: Vec, errors: &[String]) -> HostResponse { ok: false, error: Some(errors.join("; ")), agents: Some(ok_items), - approvals: None, - urls: None, - agent_statuses: None, + ..HostResponse::default() } } } - -/// Rebuild `name`'s container, notifying the manager of the outcome -/// (success or failure) and kicking the agent's next turn on success. -async fn handle_rebuild(coord: &Arc, name: &str) -> Result { - tracing::info!(%name, "rebuild"); - let agent_dir = coord.ensure_runtime(name)?; - let hive = coord.hive_env(); - let paths = Coordinator::agent_paths(name, agent_dir); - let result = lifecycle::rebuild(name, &hive, &paths, true, false, &|_| (), &|_| ()).await; - // Mirror auto_update::rebuild_agent — the manager wants to know - // about every rebuild attempt regardless of which surface triggered - // it, especially failures (build error → manager can adjust the - // agent's agent.nix). Without this the admin-socket CLI was a - // notify-gap. - match &result { - Ok(_) => { - coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: name.to_owned(), - ok: true, - note: None, - sha: None, - tag: None, - }); - // Wake the agent's next turn with the "you were rebuilt" - // hint. Same pattern as auto_update::rebuild_agent and the - // dashboard rebuild path — this is the CLI's equivalent. - coord.kick_agent(name, "container rebuilt"); - } - Err(e) => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { - agent: name.to_owned(), - ok: false, - note: Some(format!("{e:#}")), - sha: None, - tag: None, - }), - } - result?; - Ok(HostResponse::success()) -} diff --git a/hive-c0re/src/socket_server.rs b/hive-c0re/src/socket_server.rs deleted file mode 100644 index 8e192239..00000000 --- a/hive-c0re/src/socket_server.rs +++ /dev/null @@ -1,2212 +0,0 @@ -//! Unix-socket request server, shared by the per-agent sockets and the -//! (pure-transport) manager socket. The socket file's existence on disk -//! authenticates the caller: connecting to `<.../agents/foo/mcp.sock>` means -//! you are `foo`; the manager socket simply serves as `ruth`. There is no -//! privilege flag — both transports run the same [`serve`] / [`dispatch`] -//! code, and authority derives uniformly from the caller's identity: -//! topology (`is_descendant_of`) for subtree-relational verbs, capabilities -//! for hive-wide queries, and tool-group membership for the orchestration -//! verbs. `ruth` reaches every agent only as a consequence of being the -//! topology root, not via any hardcoded name match. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use hive_sh4re::{AgentRequest, AgentResponse, MANAGER_AGENT, Message}; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; -use tokio::net::{UnixListener, UnixStream}; -use tokio::task::JoinHandle; - -use crate::coordinator::Coordinator; - -pub struct AgentSocket { - pub path: PathBuf, - pub handle: JoinHandle<()>, -} - -pub fn start(agent: &str, socket_path: &Path, coord: Arc) -> Result { - use std::os::unix::fs::PermissionsExt as _; - let agent = agent.to_owned(); - if let Some(parent) = socket_path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("create agent socket dir {}", parent.display()))?; - } - if socket_path.exists() { - std::fs::remove_file(socket_path).context("remove stale agent socket")?; - } - let listener = UnixListener::bind(socket_path) - .with_context(|| format!("bind agent socket {}", socket_path.display()))?; - // The socket is bind-mounted into exactly one container as - // `/run/hive/mcp.sock` (`lifecycle::set_nspawn_flags`); the - // in-container harness connects as the per-agent unix user, - // not root, so the default `tokio::net::UnixListener::bind` - // perms (0755) lock it out. 0666 lets the agent user connect; - // the bind source dir is per-agent on host so blast radius is - // unchanged. - std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o666)) - .with_context(|| format!("chmod agent socket {}", socket_path.display()))?; - tracing::info!(%agent, socket = %socket_path.display(), "agent socket listening"); - - let path = socket_path.to_path_buf(); - let handle = tokio::spawn(async move { - loop { - match listener.accept().await { - Ok((stream, _)) => { - let agent = agent.clone(); - let coord = coord.clone(); - tokio::spawn(async move { - if let Err(e) = serve(stream, agent, coord).await { - tracing::warn!(error = ?e, "agent connection failed"); - } - }); - } - Err(e) => { - tracing::warn!(error = ?e, "agent listener accept failed; exiting"); - return; - } - } - } - }); - Ok(AgentSocket { path, handle }) -} - -/// Bind + serve the manager socket. This is now **pure transport**: it grants -/// no authority of its own — it just serves requests as `agent = MANAGER_AGENT` -/// ("ruth"), and ruth's reach comes entirely from being the topology root -/// (`is_descendant_of` covers every agent) plus the capabilities / tool-groups -/// it holds, identical to connecting on a per-agent socket — ruth uses the -/// standard per-agent runtime dir + socket, with no dedicated helpers. -pub fn start_manager(coord: Arc) -> Result<()> { - use std::os::unix::fs::PermissionsExt as _; - let dir = Coordinator::agent_dir(crate::lifecycle::MANAGER_NAME); - std::fs::create_dir_all(&dir) - .with_context(|| format!("create manager dir {}", dir.display()))?; - let socket = Coordinator::socket_path(crate::lifecycle::MANAGER_NAME); - if socket.exists() { - std::fs::remove_file(&socket).context("remove stale manager socket")?; - } - let listener = UnixListener::bind(&socket) - .with_context(|| format!("bind manager socket {}", socket.display()))?; - // 0666 so the in-container root user (non-root) can connect; the bind - // source dir is manager-only on host (see the per-agent socket above). - std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666)) - .with_context(|| format!("chmod manager socket {}", socket.display()))?; - tracing::info!(socket = %socket.display(), "manager socket listening"); - - tokio::spawn(async move { - loop { - match listener.accept().await { - Ok((stream, _)) => { - let coord = coord.clone(); - tokio::spawn(async move { - // Pure transport: serve as `ruth`, no privilege grant. - if let Err(e) = serve(stream, MANAGER_AGENT.to_owned(), coord).await { - tracing::warn!(error = ?e, "manager connection failed"); - } - }); - } - Err(e) => { - tracing::warn!(error = ?e, "manager listener accept failed"); - return; - } - } - } - }); - Ok(()) -} - -async fn serve(stream: UnixStream, agent: String, coord: Arc) -> Result<()> { - let (read, mut write) = stream.into_split(); - let mut reader = BufReader::new(read); - let mut line = String::new(); - loop { - line.clear(); - let n = reader.read_line(&mut line).await?; - if n == 0 { - return Ok(()); - } - let resp = match serde_json::from_str::(line.trim()) { - Ok(req) => dispatch(&req, &agent, &coord).await, - Err(e) => AgentResponse::Err { - message: format!("parse error: {e}"), - }, - }; - let mut payload = serde_json::to_string(&resp)?; - payload.push('\n'); - write.write_all(payload.as_bytes()).await?; - write.flush().await?; - } -} - -/// Max long-poll window the caller can ask for; values above the -/// cap are clamped. 180s keeps us under typical TCP/proxy idle -/// limits while still letting agents park their turn until a -/// message arrives. Omitting `wait_seconds` (or passing `0`) means -/// "peek, don't wait" — claude can call recv whenever it wants a -/// cheap "is there anything pending?" check without blocking the -/// turn for 30 seconds. To actually park, the caller passes a -/// positive `wait_seconds`. -pub(crate) const RECV_LONG_POLL_MAX: std::time::Duration = std::time::Duration::from_mins(3); - -/// Server-side hard cap on `Recv.max` — canonical value lives in -/// `hive_sh4re::RECV_BATCH_MAX` so the harness's wake-prompt hint and -/// this enforcement site can't drift apart. -pub(crate) const RECV_BATCH_MAX: u32 = hive_sh4re::RECV_BATCH_MAX; - -pub(crate) fn recv_timeout(wait_seconds: Option) -> std::time::Duration { - match wait_seconds { - Some(s) => std::time::Duration::from_secs(s).min(RECV_LONG_POLL_MAX), - None => std::time::Duration::ZERO, - } -} - -/// Handle the subset of `Request` variants that are identical on both -/// the agent socket and the manager socket. Returns `Some(response)` for -/// every variant it handles; returns `None` for variants with socket-specific -/// semantics (e.g. `GetLooseEnds` / `CountPendingReminders` / `ReminderRollup` -/// where the manager can target other agents) or for manager-only variants. -/// -/// The unified `dispatch` calls this first; the remaining arms (which gate -/// on topology / capabilities / tool-groups) are handled there. -pub(crate) async fn dispatch_shared( - req: &hive_sh4re::Request, - agent: &str, - coord: &Arc, -) -> Option { - Some(match req { - hive_sh4re::Request::Send { - to, - body, - in_reply_to, - } => handle_send(coord, agent, to, body, *in_reply_to), - hive_sh4re::Request::Recv { wait_seconds, max } => { - handle_recv(coord, agent, *wait_seconds, *max).await - } - hive_sh4re::Request::Status => handle_status(coord, agent), - hive_sh4re::Request::OperatorMsg { body } => handle_operator_msg(coord, agent, body), - hive_sh4re::Request::Wake { from, body } => handle_wake(coord, agent, from, body), - hive_sh4re::Request::Recent { limit } => handle_recent(coord, agent, *limit), - hive_sh4re::Request::Ask { - question, - options, - multi, - ttl_seconds, - to, - } => crate::questions::handle_ask( - coord, - agent, - question, - options, - *multi, - *ttl_seconds, - to.as_deref(), - ) - .map_or_else( - |message| hive_sh4re::Response::Err { message }, - |id| hive_sh4re::Response::QuestionQueued { id }, - ), - hive_sh4re::Request::Answer { id, answer } => { - crate::questions::handle_answer(coord, agent, *id, answer).map_or_else( - |message| hive_sh4re::Response::Err { message }, - |()| hive_sh4re::Response::Ok, - ) - } - hive_sh4re::Request::Remind { - message, - timing, - file_path, - } => handle_remind(coord, agent, message, timing, file_path.as_deref()), - hive_sh4re::Request::SetStatus { text } => handle_set_status(coord, text), - hive_sh4re::Request::GetAgentMeta { name } => { - handle_get_agent_meta(coord, agent, name.as_deref()).await - } - hive_sh4re::Request::CancelLooseEnd { kind, id } => { - crate::questions::handle_cancel_loose_end(coord, agent, *kind, *id).map_or_else( - |message| hive_sh4re::Response::Err { message }, - |()| hive_sh4re::Response::Ok, - ) - } - hive_sh4re::Request::CreateRepo { repo } => handle_create_repo(agent, repo).await, - hive_sh4re::Request::AckTurn => handle_ack_turn(coord, agent), - hive_sh4re::Request::AckUntil { up_to } => handle_ack_until(coord, agent, *up_to), - hive_sh4re::Request::RequeueInflight => handle_requeue_inflight(coord, agent), - hive_sh4re::Request::GracefulStopComplete => { - // Harness drained + is exiting: clear the fence so the - // `GracefulStop` orchestration (which polls this flag) proceeds - // to stop the container without waiting out its timeout. - coord.clear_graceful_stop(agent); - hive_sh4re::Response::Ok - } - hive_sh4re::Request::GetHostJournal { - unit, - container, - lines, - priority, - grep, - since, - until, - } => { - dispatch_host_journal( - agent, - HostJournalArgs { - unit, - container, - lines, - priority, - grep, - since, - until, - }, - ) - .await - } - // Not a shared variant. - _ => return None, - }) -} - -/// `Recv` — long-poll the broker for up to `max` messages (capped at -/// `RECV_BATCH_MAX`), mapping deliveries onto the wire response. -async fn handle_recv( - coord: &Arc, - agent: &str, - wait_seconds: Option, - max: Option, -) -> hive_sh4re::Response { - // Graceful-stop fence: while a graceful stop is pending for this agent, - // return `GracefulStop` instead of polling the broker. The harness runs - // one stop-checkpoint turn then exits; new sends keep queueing in the - // broker for the agent's next start. Checked before the (blocking) poll - // so a flag set between polls is seen on the next Recv — the orchestration - // also fires a transient wake to break an in-flight long-poll. - if coord.is_graceful_stop_pending(agent) { - return hive_sh4re::Response::GracefulStop; - } - let cap = max.unwrap_or(1).min(RECV_BATCH_MAX) as usize; - match coord - .broker - .recv_blocking_batch(agent, recv_timeout(wait_seconds), cap) - .await - { - Ok(deliveries) => hive_sh4re::Response::Messages { - messages: deliveries - .into_iter() - .map(|d| hive_sh4re::DeliveredMessage { - from: d.message.from, - body: d.message.body, - id: d.id, - redelivered: d.redelivered, - in_reply_to: d.message.in_reply_to, - }) - .collect(), - }, - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// `Wake` — inject a wake into `agent`'s own inbox. Persisted through -/// the sqlite broker like any other message so the agent can ack it -/// via `AckUntil` and it appears in message history for post-mortem. -fn handle_wake( - coord: &Arc, - agent: &str, - from: &str, - body: &str, -) -> hive_sh4re::Response { - match coord.broker.send(&Message { - from: from.to_owned(), - to: agent.to_owned(), - body: body.to_owned(), - in_reply_to: None, - }) { - Ok(()) => hive_sh4re::Response::Ok, - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// `SetStatus` — validate the status text, then trigger a dashboard -/// rescan. The harness has already written the status file to its own -/// `state/` dir (it runs as the agent user), so this only refreshes the -/// dashboard's view. -fn handle_set_status(coord: &Arc, text: &str) -> hive_sh4re::Response { - if let Err(message) = crate::limits::check_status_text(text) { - return hive_sh4re::Response::Err { message }; - } - let coord2 = Arc::clone(coord); - tokio::spawn(async move { coord2.rescan_containers_and_emit().await }); - hive_sh4re::Response::Ok -} - -/// Validate an agent-supplied repo name: a single safe slug segment, no -/// path traversal. Forgejo validates server-side too, but rejecting early -/// gives a clear message and avoids building odd API paths. -fn valid_repo_name(name: &str) -> bool { - !name.is_empty() - && name.len() <= 100 - && !name.starts_with(['-', '.']) - && name - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) -} - -/// `CreateRepo` — create a repo for `agent` *through hive-c0re* in the -/// c0re-owned `agents` org with operator-team branch protection. -/// The sanctioned create path now that agents can't create repos directly. -async fn handle_create_repo(agent: &str, repo: &str) -> hive_sh4re::Response { - if !valid_repo_name(repo) { - return hive_sh4re::Response::Err { - message: format!( - "invalid repo name {repo:?} — single segment of letters, digits, '-', '_', '.' \ - (no leading '-'/'.', max 100 chars)" - ), - }; - } - let Some(core_token) = crate::forge::core_token() else { - return hive_sh4re::Response::Err { - message: "forge unavailable (no core token) — cannot create repo".to_owned(), - }; - }; - match crate::forge::create_agent_repo(agent, repo, &core_token).await { - Ok(full_name) => hive_sh4re::Response::RepoCreated { - clone_url: format!("{}/{full_name}.git", crate::forge::FORGE_HTTP), - full_name, - }, - Err(e) => hive_sh4re::Response::Err { - message: format!("create repo {repo:?} failed: {e:#}"), - }, - } -} - -/// `GetAgentMeta` — identity + live status for `name` (defaults to the -/// caller). Reads the live container-view status and the hive/swarm -/// display names. -async fn handle_get_agent_meta( - coord: &Arc, - agent: &str, - name: Option<&str>, -) -> hive_sh4re::Response { - let target = name.unwrap_or(agent); - let (status_text, status_set_at, running) = - crate::container_view::read_agent_status_live(target).await; - let (hive_name, swarm_name) = crate::container_view::hive_swarm_names(); - hive_sh4re::Response::AgentMeta { - name: target.to_owned(), - running, - hyperhive_rev: crate::auto_update::current_flake_rev(&coord.hyperhive_flake), - status_text, - status_set_at, - hive_name, - swarm_name, - matrix_accounts: read_agent_matrix_identities(target), - } -} - -/// Read the target agent's matrix identities from the daemon's -/// `matrix-accounts.json` snapshot (under the agent's state dir). -/// Best-effort: an absent / unparseable snapshot (no matrix provisioning, -/// or the daemon not up yet) yields an empty list. The `MatrixIdentity` -/// serde shape matches the snapshot entries; the snapshot's `live` field is -/// ignored (only live accounts are written). -fn read_agent_matrix_identities(agent: &str) -> Vec { - let path = Coordinator::agent_notes_dir(agent).join("matrix-accounts.json"); - std::fs::read_to_string(&path) - .ok() - .and_then(|s| serde_json::from_str::>(&s).ok()) - .unwrap_or_default() -} - -/// `Status` — count of pending (unread) inbox messages for `agent`. -fn handle_status(coord: &Arc, agent: &str) -> hive_sh4re::Response { - match coord.broker.count_pending(agent) { - Ok(unread) => hive_sh4re::Response::Status { unread }, - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// `OperatorMsg` — deliver an operator-authored message into `agent`'s -/// inbox (from the `operator` recipient). -fn handle_operator_msg(coord: &Arc, agent: &str, body: &str) -> hive_sh4re::Response { - match coord.broker.send(&Message { - from: hive_sh4re::OPERATOR_RECIPIENT.to_owned(), - to: agent.to_owned(), - body: body.to_owned(), - in_reply_to: None, - }) { - Ok(()) => hive_sh4re::Response::Ok, - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// `Recent` — the last `limit` inbox rows for `agent` (read-only, -/// doesn't consume). -fn handle_recent(coord: &Arc, agent: &str, limit: u64) -> hive_sh4re::Response { - match coord.broker.recent_for(agent, limit) { - Ok(rows) => hive_sh4re::Response::Recent { rows }, - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// `AckTurn` — mark `agent`'s in-flight delivered messages acked so -/// they don't redeliver on the next turn. -fn handle_ack_turn(coord: &Arc, agent: &str) -> hive_sh4re::Response { - match coord.broker.ack_turn(agent) { - Ok(_n) => hive_sh4re::Response::Ok, - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// `AckUntil` — bulk-ack every message addressed to `agent` with row -/// id `<= up_to` (the agent-side backlog-triage escape hatch). -fn handle_ack_until(coord: &Arc, agent: &str, up_to: i64) -> hive_sh4re::Response { - match coord.broker.ack_until(agent, up_to) { - Ok(count) => hive_sh4re::Response::Acked { count }, - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// `RequeueInflight` — resurface `agent`'s unacked in-flight messages -/// (crash recovery on harness boot). -fn handle_requeue_inflight(coord: &Arc, agent: &str) -> hive_sh4re::Response { - match coord.broker.requeue_inflight(agent) { - Ok(n) => { - if n > 0 { - tracing::info!(%agent, requeued = %n, "requeued in-flight messages"); - } - hive_sh4re::Response::Ok - } - Err(e) => hive_sh4re::Response::Err { - message: format!("{e:#}"), - }, - } -} - -/// Unified dispatch for every socket connection — per-agent sockets and the -/// (now pure-transport) manager socket alike. There is no privilege bit; -/// authority derives uniformly from the caller's identity: subtree-relational -/// verbs (lifecycle/config/logs) require the caller to be an ancestor of the -/// target (`is_descendant_of`, so the root covers all); hive-wide agent-state -/// queries require the `QueryAgentState` capability; hive-wide orchestration -/// verbs (schedules / meta-inputs) require the matching tool-group (the -/// grantable capability). -async fn dispatch(req: &AgentRequest, agent: &str, coord: &Arc) -> AgentResponse { - if let Some(resp) = dispatch_shared(req, agent, coord).await { - return resp; - } - match req { - // Lifecycle + config: caller must be an ancestor of the target - // (a parent owns its whole subtree; the root covers every agent). - AgentRequest::Start { name } => handle_start(coord, agent, name).await, - AgentRequest::Restart { name } => handle_restart(coord, agent, name).await, - AgentRequest::Kill { name } => handle_kill(coord, agent, name).await, - AgentRequest::Update { name } => handle_update(coord, agent, name), - AgentRequest::ListDescendants => handle_list_descendants(agent).await, - AgentRequest::RequestInitConfig { name, description } => { - handle_request_init_config(coord, agent, name, description.clone()) - } - AgentRequest::RequestApplyCommit { - agent: target_agent, - commit_ref, - description, - } => { - handle_request_apply_commit( - coord, - agent, - target_agent, - commit_ref, - description.as_deref(), - ) - .await - } - // Agent-state queries: own subtree is free; other agents + the - // hive-wide `"*"` sweep require `QueryAgentState`. - AgentRequest::GetLooseEnds { agent: target } => { - handle_get_loose_ends(coord, agent, target.as_deref()) - } - AgentRequest::CountPendingReminders { agent: target } => { - handle_count_pending_reminders(coord, agent, target.as_deref()) - } - AgentRequest::ReminderRollup { - since_secs, - agent: target, - } => handle_reminder_rollup(coord, agent, target.as_deref(), *since_secs), - // Orchestration / diagnostics verbs — gated per-verb on tool-group - // membership or topology (see `dispatch_orchestration`). - _ => dispatch_orchestration(req, agent, coord).await, - } -} - -/// Handle the hive-wide orchestration verbs (scheduling, meta-input updates) -/// plus container-log reads. No blanket socket gate: each verb gates on the -/// grantable capability that authorises it — the matching tool-group -/// (`scheduling` / `approvals`), or `is_descendant_of` for `get_logs` -/// (a parent reads its subtree's logs). Any other variant is a host-admin / -/// unknown request invalid on either socket. -async fn dispatch_orchestration( - req: &AgentRequest, - agent: &str, - coord: &Arc, -) -> AgentResponse { - match req { - AgentRequest::RequestUpdateMetaInputs { - inputs, - description, - } => { - if let Some(err) = require_group(agent, "approvals", "request update_meta_inputs") { - return err; - } - handle_request_update_meta_inputs(coord, agent, inputs, description.as_deref()) - } - AgentRequest::RequestSchedulePrompt(payload) => { - if let Some(err) = require_group(agent, "scheduling", "schedule a prompt") { - return err; - } - handle_request_schedule_prompt(coord, agent, payload) - } - AgentRequest::CancelSchedule { id, targets } => { - if let Some(err) = require_group(agent, "scheduling", "cancel a schedule") { - return err; - } - handle_cancel_schedule(coord, agent, *id, targets.as_deref()) - } - AgentRequest::EditSchedule { - id, - body, - description, - interval_seconds, - next_fire_at_unix, - targets_add, - targets_remove, - } => { - if let Some(err) = require_group(agent, "scheduling", "edit a schedule") { - return err; - } - handle_edit_schedule( - coord, - agent, - *id, - EditSchedulePatch { - body: body.clone(), - description: description.clone(), - interval_seconds: *interval_seconds, - next_fire_at_unix: *next_fire_at_unix, - targets_add: targets_add.clone(), - targets_remove: targets_remove.clone(), - }, - ) - } - AgentRequest::ListSchedules => { - if let Some(err) = require_group(agent, "scheduling", "list schedules") { - return err; - } - handle_list_schedules(coord) - } - AgentRequest::FireScheduleNow { id } => { - if let Some(err) = require_group(agent, "scheduling", "fire a schedule") { - return err; - } - handle_fire_schedule_now(coord, agent, *id).await - } - AgentRequest::GetLogs { - agent: target, - lines, - } => { - if let Some(err) = require_descendant(agent, target, "read logs of") { - return err; - } - handle_get_logs(target, *lines).await - } - // Host-admin-only / unknown variants: never valid on either socket. - _ => AgentResponse::Err { - message: "request not handled on this socket".to_owned(), - }, - } -} - -/// Topology guard for the subtree-relational lifecycle/config/log tools: the -/// `target` must be the caller itself or one of its topology descendants — a -/// parent owns its whole subtree, and the root (`ruth`) covers every agent as -/// a consequence, with no positional/hardcoded privilege. Returns `Some(Err)` -/// to short-circuit the dispatch arm when it isn't, `None` when authorised. -/// `action` is the verb phrase for the message (e.g. `"start"`). -fn require_descendant(agent: &str, target: &str, action: &str) -> Option { - if crate::topology::is_descendant_of(target, agent) { - None - } else { - Some(AgentResponse::Err { - message: format!( - "agent `{agent}` cannot {action} `{target}`: \ - not in its subtree (topology)" - ), - }) - } -} - -/// Capability guard for the hive-wide orchestration verbs: the caller must -/// hold the given tool-group. The tool-group (c0re-owned `tool_groups.json`, -/// read server-side via [`crate::tool_groups::groups_for`]) is the grantable -/// capability — granting it to an orchestrator (e.g. the root) authorises -/// these verbs without any positional/hardcoded privilege. `action` is the -/// verb phrase for the message. -fn require_group(agent: &str, group: &str, action: &str) -> Option { - if crate::tool_groups::groups_for(agent) - .iter() - .any(|g| g == group) - { - None - } else { - Some(AgentResponse::Err { - message: format!("agent `{agent}` cannot {action}: requires the `{group}` tool group"), - }) - } -} - -/// Topology guard for `request_init_config` / `request_apply_commit`, -/// which may legitimately target a child that does not exist *yet* -/// (spawning a brand-new sub-agent). The caller may act on a -/// `target` that is EITHER already its direct child (re-init / config -/// update of an existing child) OR brand-new (absent from the topology -/// tree — the requester becomes its parent). A name that already -/// belongs to a *different* parent (or is a root agent) is refused so -/// one agent can't hijack another's sub-tree. -/// -/// Also re-runs the agent-name format check (a traversal / malformed name -/// could never be a descendant): a brand-new name now flows straight to -/// `submit_init_config`, which builds filesystem paths from it, so validate -/// before that. -fn require_new_child(agent: &str, target: &str, action: &str) -> Option { - if let Some(reason) = crate::dashboard::validate_agent_name(target) { - return Some(AgentResponse::Err { - message: format!("agent `{agent}` cannot {action} `{target}`: {reason}"), - }); - } - // brand-new name (absent from topology) — requester becomes the parent on - // approval; allowed for any caller. - if !crate::topology::read().contains_key(target) { - return None; - } - // existing agent — allowed only if it's in the caller's subtree - // (re-init / config update of an agent the caller owns; the root owns - // every existing agent). Refuses an agent outside the caller's subtree - // so one agent can't hijack another's config. - if crate::topology::is_descendant_of(target, agent) { - None - } else { - Some(AgentResponse::Err { - message: format!( - "agent `{agent}` cannot {action} `{target}`: it already exists \ - outside its subtree in the topology tree" - ), - }) - } -} - -/// `GetLooseEnds` — read the target's loose ends. `None` / own / a subtree -/// descendant resolve freely (a parent sees its subtree, the root sees all); -/// any other named agent needs `QueryAgentState`; `"*"` is a hive-wide sweep -/// gated on `QueryAgentState`. -fn handle_get_loose_ends( - coord: &Arc, - agent: &str, - target: Option<&str>, -) -> AgentResponse { - let result = if target == Some("*") { - if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::QueryAgentState) { - return AgentResponse::Err { - message: "query_agent_state capability required for hive-wide loose ends" - .to_owned(), - }; - } - crate::loose_ends::hive_wide(coord) - } else { - match resolve_agent_state_target(agent, target) { - Ok(name) => crate::loose_ends::for_agent(coord, name), - Err(message) => return AgentResponse::Err { message }, - } - }; - match result { - Ok(loose_ends) => AgentResponse::LooseEnds { loose_ends }, - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - } -} - -/// `CountPendingReminders` — resolve the target (own / subtree free, else -/// `QueryAgentState`) then count its pending reminders. -fn handle_count_pending_reminders( - coord: &Arc, - agent: &str, - target: Option<&str>, -) -> AgentResponse { - match resolve_agent_state_target(agent, target) { - Ok(name) => match coord.broker.count_pending_reminders_for(name) { - Ok(count) => AgentResponse::PendingRemindersCount { count }, - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - }, - Err(message) => AgentResponse::Err { message }, - } -} - -/// `ReminderRollup` — resolve the target (own / subtree free, else -/// `QueryAgentState`) then roll up its reminders fired in the last -/// `since_secs`. -fn handle_reminder_rollup( - coord: &Arc, - agent: &str, - target: Option<&str>, - since_secs: u64, -) -> AgentResponse { - match resolve_agent_state_target(agent, target) { - Ok(name) => match coord.broker.reminder_rollup_for(name, since_secs) { - Ok(stats) => AgentResponse::ReminderRollup(stats), - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - }, - Err(message) => AgentResponse::Err { message }, - } -} - -/// `Start` — start a container, kicking its next turn. The caller must be an -/// ancestor of `name` in the topology (the root covers every agent). -async fn handle_start(coord: &Arc, agent: &str, name: &str) -> AgentResponse { - if let Some(err) = require_descendant(agent, name, "start") { - return err; - } - tracing::info!(%agent, %name, "start container"); - // If the hyperhive rev is stale, route through the rebuild queue so the - // container runs current nix derivations before it starts. Same logic as - // `run_start`; this covers the MCP `start` tool path. - let current_rev = crate::auto_update::current_flake_rev(&coord.hyperhive_flake); - if let Some(ref rev) = current_rev { - let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(name)).ok(); - if stored.as_deref() != Some(rev.as_str()) { - tracing::info!(%agent, %name, "start: rev stale — enqueuing rebuild"); - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name.to_owned(), - crate::rebuild_queue::QueueSource::Manual, - format!("start {name}: rev stale — rebuilding first"), - None, - ); - coord.emit_rebuild_queue_snapshot(); - return AgentResponse::Ok; - } - } - match crate::lifecycle::start(name).await { - Ok(()) => { - coord.kick_agent(name, "container started"); - AgentResponse::Ok - } - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - } -} - -/// `Restart` — enqueue a restart for a container. The caller must be an -/// ancestor of `name` in the topology. The infra-container branch is -/// orthogonal: it is gated on the `infra_admin` capability and audited, so it -/// stays ahead of the topology guard. -async fn handle_restart(coord: &Arc, agent: &str, name: &str) -> AgentResponse { - // Infra-container restart: an agent holding the `infra_admin` - // capability can restart a hive infrastructure container (hive-ci / - // hive-gateway / hive-forge / hive-matrix) by passing its name to the - // same restart tool. The `InfraContainer` enum parse both recognises - // these (never agent children, so disjoint from the child path below) - // and yields the typed value the restart path needs. - if let Ok(container) = name.parse::() { - return handle_restart_infra(coord, agent, container).await; - } - if let Some(err) = require_descendant(agent, name, "restart") { - return err; - } - tracing::info!(%agent, %name, "enqueue restart"); - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Restart, - name.to_owned(), - crate::rebuild_queue::QueueSource::Manual, - format!("agent `{agent}` restart tool"), - None, - ); - coord.emit_rebuild_queue_snapshot(); - AgentResponse::Ok -} - -/// Restart a hive infrastructure container on behalf of an agent that -/// holds the `infra_admin` capability. The `container` is already a valid -/// [`InfraContainer`] (the caller parsed it); this gates on the capability -/// and routes the systemctl restart through hive-priv. Direct, not -/// approval-gated. -async fn handle_restart_infra( - coord: &Arc, - agent: &str, - container: hive_sh4re::priv_proto::InfraContainer, -) -> AgentResponse { - let name = container.unit_name(); - // Record the attempt in the operator-visible privileged-action audit - // trail, then emit a live `AuditEntryAdded` so the dashboard audit view - // appends it off `/dashboard/stream`. Best-effort: `record` returns the - // canonical row (or `None` on a sqlite blip), and we stream exactly that - // row so the stored + streamed views can't drift. `action` is stable so - // the dashboard can group/filter. - let audit = |outcome: crate::audit_log::AuditOutcome, detail: Option<&str>| { - if let Some(entry) = coord - .audit_log - .record(agent, "restart_infra", name, outcome, detail) - { - coord.emit_audit_entry(entry); - } - }; - if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::InfraAdmin) { - tracing::warn!(%agent, %name, "agent: infra restart denied (no infra_admin capability)"); - audit( - crate::audit_log::AuditOutcome::Err, - Some("denied: missing infra_admin capability"), - ); - return AgentResponse::Err { - message: format!( - "restarting infra container `{name}` requires the `infra_admin` capability" - ), - }; - } - tracing::info!(%agent, %name, "agent: restart infra container"); - match crate::priv_client::restart_infra_container(container).await { - Ok(()) => { - audit(crate::audit_log::AuditOutcome::Ok, None); - AgentResponse::Ok - } - Err(e) => { - let msg = format!("{e:#}"); - audit(crate::audit_log::AuditOutcome::Err, Some(&msg)); - AgentResponse::Err { message: msg } - } - } -} - -/// `Kill` — kill a container, unregister it, notify the manager. The caller -/// must be an ancestor of `name` in the topology. -async fn handle_kill(coord: &Arc, agent: &str, name: &str) -> AgentResponse { - if let Some(err) = require_descendant(agent, name, "kill") { - return err; - } - tracing::info!(%agent, %name, "kill container"); - let result: anyhow::Result<()> = async { - crate::lifecycle::kill(name).await?; - coord.unregister_agent(name); - Ok(()) - } - .await; - match result { - Ok(()) => { - coord.notify_manager(&hive_sh4re::HelperEvent::Killed { - agent: name.to_owned(), - }); - AgentResponse::Ok - } - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - } -} - -/// `Update` — enqueue a rebuild for a container. The caller must be an -/// ancestor of `name` in the topology. -fn handle_update(coord: &Arc, agent: &str, name: &str) -> AgentResponse { - if let Some(err) = require_descendant(agent, name, "rebuild") { - return err; - } - tracing::info!(%agent, %name, "enqueue rebuild"); - coord.rebuild_queue.enqueue( - crate::rebuild_queue::QueueKind::Rebuild, - name.to_owned(), - crate::rebuild_queue::QueueSource::Manual, - format!("agent `{agent}` update tool"), - None, - ); - coord.emit_rebuild_queue_snapshot(); - AgentResponse::Ok -} - -/// `ListDescendants` — every topological descendant of `agent` with -/// its running/stopped state, parents before children. -async fn handle_list_descendants(agent: &str) -> AgentResponse { - tracing::debug!(%agent, "agent: list descendants"); - // All containers known to nixos-container (running only). - let running_set: std::collections::HashSet = match crate::lifecycle::list().await { - Ok(names) => names - .into_iter() - .filter_map(|c| { - c.strip_prefix(crate::lifecycle::AGENT_PREFIX) - .map(str::to_owned) - }) - .collect(), - Err(e) => { - return AgentResponse::Err { - message: format!("list containers failed: {e:#}"), - }; - } - }; - // Walk the full topology and collect every descendant. - let topo = crate::topology::read(); - let mut names: Vec = topo - .keys() - .filter(|name| crate::topology::is_descendant_of(name, agent)) - .cloned() - .collect(); - // Parents before children, then alpha within each tier. - crate::auto_update::topology_sort(&mut names, &topo); - let containers = names - .into_iter() - .map(|name| { - let running = running_set.contains(&name); - hive_sh4re::ContainerInfo { name, running } - }) - .collect(); - AgentResponse::Containers { containers } -} - -/// `RequestInitConfig` — queue an `InitConfig` approval for an agent. The -/// `name` must be brand-new (absent from the topology) or already in the -/// caller's subtree; the requester is recorded as the new agent's parent (the -/// root requesting a new agent → a top-level agent, matching reconcile's -/// default). -fn handle_request_init_config( - coord: &Arc, - agent: &str, - name: &str, - description: Option, -) -> AgentResponse { - if let Some(err) = require_new_child(agent, name, "request_init_config for") { - return err; - } - tracing::info!(%agent, %name, "request_init_config"); - match submit_init_config(coord, name, Some(agent), description) { - Ok(_id) => AgentResponse::Ok, - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - } -} - -/// `RequestApplyCommit` — queue an apply-commit approval for an agent. The -/// target must be in the caller's subtree (the root covers every agent). -async fn handle_request_apply_commit( - coord: &Arc, - agent: &str, - target_agent: &str, - commit_ref: &str, - description: Option<&str>, -) -> AgentResponse { - if let Some(err) = require_new_child(agent, target_agent, "request_apply_commit for") { - return err; - } - tracing::info!(%agent, %target_agent, %commit_ref, "request_apply_commit"); - match submit_apply_commit(coord, target_agent, commit_ref, description, agent).await { - Ok((id, sha)) => { - tracing::info!(%id, %target_agent, %sha, "apply_commit approval queued"); - AgentResponse::Ok - } - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - } -} - -/// Field-named journal-query knobs for [`dispatch_host_journal`]. -/// Borrows straight from the matched `GetHostJournal` request variant. -pub struct HostJournalArgs<'a> { - pub unit: &'a Option, - pub container: &'a Option, - pub lines: &'a Option, - pub priority: &'a Option, - pub grep: &'a Option, - pub since: &'a Option, - pub until: &'a Option, -} - -/// Handle `GetHostJournal` from both the agent and manager sockets. -/// Capability-gated: the calling agent must hold `read_host_journal` in -/// `meta/capabilities.json`. Runs `journalctl` host-side and returns -/// the output as a `HostJournal` response. -/// -/// The manager is not exempt - grant `read_host_journal` in -/// `meta/capabilities.json` to enable it for any agent including the manager. -pub async fn dispatch_host_journal(agent: &str, args: HostJournalArgs<'_>) -> AgentResponse { - let HostJournalArgs { - unit, - container, - lines, - priority, - grep, - since, - until, - } = args; - if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::ReadHostJournal) { - return AgentResponse::Err { - message: "agent does not have the read_host_journal capability".to_owned(), - }; - } - let n = lines.unwrap_or(30).min(100); - - // A container (`-M`) read enters the container namespace and needs - // root, so it's delegated to hive-priv. A host read (no container) - // the unprivileged hive-core user can do directly via its - // systemd-journal group membership. - if let Some(c) = container { - tracing::info!(%agent, machine = %c, %n, "get_host_journal (container)"); - return match crate::priv_client::read_container_journal( - c, - hive_sh4re::priv_proto::JournalQuery { - lines: n, - unit: unit.clone(), - priority: priority.as_ref().map(|p| p.as_str().to_owned()), - grep: grep.clone(), - since: since.clone(), - until: until.clone(), - ..Default::default() - }, - ) - .await - { - Ok((stdout, stderr)) => { - let content = if stdout.is_empty() { stderr } else { stdout }; - AgentResponse::HostJournal { content } - } - Err(e) => AgentResponse::Err { - message: format!("journal read: {e:#}"), - }, - }; - } - - let mut args: Vec = vec![ - "--no-pager".to_owned(), - "--output=short".to_owned(), - "-n".to_owned(), - n.to_string(), - ]; - if let Some(u) = unit { - args.push("-u".to_owned()); - args.push(u.clone()); - } - if let Some(p) = priority { - args.push("-p".to_owned()); - args.push(p.as_str().to_owned()); - } - if let Some(g) = grep { - args.push(format!("--grep={g}")); - } - if let Some(s) = since { - args.push(format!("--since={s}")); - } - if let Some(u) = until { - args.push(format!("--until={u}")); - } - tracing::info!(%agent, ?args, "get_host_journal"); - match tokio::process::Command::new("journalctl") - .args(&args) - .output() - .await - { - Ok(out) => { - let content = if out.status.success() || !out.stdout.is_empty() { - String::from_utf8_lossy(&out.stdout).into_owned() - } else { - let stderr = String::from_utf8_lossy(&out.stderr); - format!("journalctl exited {}: {stderr}", out.status) - }; - AgentResponse::HostJournal { content } - } - Err(e) => AgentResponse::Err { - message: format!("journalctl spawn failed: {e:#}"), - }, - } -} - -/// Fan out one message to each recipient in `targets`. Skips the sender -/// itself. Returns a list of `": "` strings for any delivery -/// failures (empty = all good). -pub(crate) fn fan_out_send( - coord: &Arc, - from: &str, - body: &str, - in_reply_to: Option, - targets: &[String], -) -> Vec { - let mut errors = Vec::new(); - for target in targets { - if target == from { - continue; - } - if let Err(e) = coord.broker.send(&Message { - from: from.to_owned(), - to: target.clone(), - body: body.to_owned(), - in_reply_to, - }) { - errors.push(format!("{target}: {e}")); - } - } - errors -} - -/// Common Send handler shared between dispatch arms. Applies the -/// 4 KiB body cap, then routes broadcast (`to == "*"`) / children fan-out -/// (`to == ""`) / unicast through their respective broker calls. -/// `pub(crate)` so `dispatch_shared` can use it across both socket paths. -pub(crate) fn handle_send( - coord: &Arc, - agent: &str, - to: &str, - body: &str, - in_reply_to: Option, -) -> AgentResponse { - if let Err(message) = crate::limits::check_size("send", body) { - return AgentResponse::Err { message }; - } - if to == "*" { - let errors = coord.broadcast_send(agent, body); - return if errors.is_empty() { - AgentResponse::Ok - } else { - AgentResponse::Err { - message: format!("broadcast failed for agents: {}", errors.join(", ")), - } - }; - } - // ``: fan out to every direct descendant of the sender per - // topology.json. Bypasses the allow-list check — structural fan-out - // targets are never user-listed peers. No-op (returns Ok) for leaf - // agents that have no children. - if to == hive_sh4re::CHILDREN_RECIPIENT { - let children = crate::topology::children_of(agent); - let errors = fan_out_send(coord, agent, body, in_reply_to, &children); - return if errors.is_empty() { - AgentResponse::Ok - } else { - AgentResponse::Err { - message: format!("children fan-out failed for agents: {}", errors.join(", ")), - } - }; - } - // Resolve magic-recipient sentinels (``) against topology.json; - // no-op for ordinary names. Lets agents address structural roles without - // learning the label — runtime reparenting propagates for free. See - // `docs/conventions.md::Recipient sentinels`. - let resolved = crate::topology::resolve_recipient(agent, to); - // Validate that the resolved recipient is a known local agent or the - // special "operator" recipient. Without this check a typo in `to` - // silently queues a message nobody will ever read. - // - // Cross-hive messaging (`name@hive` qualified names) is not routed - // through the broker — use the Matrix MCP tools for that instead. - if resolved.contains('@') { - return AgentResponse::Err { - message: format!( - "send failed: cross-hive recipient `{resolved}` is not supported \ - via the broker — use Matrix MCP tools for cross-hive messaging" - ), - }; - } - if resolved != hive_sh4re::OPERATOR_RECIPIENT { - let state_root = crate::coordinator::Coordinator::agent_state_root(&resolved); - if !state_root.exists() { - return AgentResponse::Err { - message: format!( - "send failed: unknown recipient `{resolved}` \ - (no agent with that name exists on this hive)" - ), - }; - } - } - match coord.broker.send(&Message { - from: agent.to_owned(), - to: resolved, - body: body.to_owned(), - in_reply_to, - }) { - Ok(()) => AgentResponse::Ok, - Err(e) => AgentResponse::Err { - message: format!("{e:#}"), - }, - } -} - -fn handle_remind( - coord: &Arc, - agent: &str, - message: &str, - timing: &hive_sh4re::ReminderTiming, - file_path: Option<&str>, -) -> AgentResponse { - match store_remind(coord, agent, message, timing, file_path) { - Ok(()) => AgentResponse::Ok, - Err(message) => AgentResponse::Err { message }, - } -} - -/// Shared remind-storage path used by both the agent and the manager -/// dispatchers. Validates timing, applies the auto-file overflow -/// dance (see [`prepare_remind_storage`]), and writes the reminder -/// row. Returns `Ok(())` on success, or a caller-ready error string -/// the dispatcher wraps in `*Response::Err`. -/// Maximum pending (un-delivered) reminders per agent. Exceeding this -/// causes `store_remind` to return an error so the agent knows to back -/// off instead of silently dropping. Override via -/// `HIVE_REMIND_MAX_PENDING_PER_AGENT`; set to `0` to disable the cap -/// (not recommended — a runaway agent can still flood the scheduler). -const DEFAULT_REMIND_MAX_PENDING: u64 = 50; - -fn remind_max_pending() -> u64 { - std::env::var("HIVE_REMIND_MAX_PENDING_PER_AGENT") - .ok() - .and_then(|s| s.trim().parse::().ok()) - .unwrap_or(DEFAULT_REMIND_MAX_PENDING) -} - -pub(crate) fn store_remind( - coord: &Arc, - agent: &str, - message: &str, - timing: &hive_sh4re::ReminderTiming, - file_path: Option<&str>, -) -> Result<(), String> { - let max = remind_max_pending(); - if max > 0 { - let pending = coord.broker.count_pending_reminders_for(agent).unwrap_or(0); - if pending >= max { - return Err(format!( - "reminder rejected: agent `{agent}` already has {pending} pending \ - reminders (cap {max}). Cancel some via `cancel_loose_end` or wait \ - for them to fire before scheduling more. Override the cap with \ - `HIVE_REMIND_MAX_PENDING_PER_AGENT`." - )); - } - } - let due_at = resolve_due_at(timing).map_err(|e| format!("invalid reminder timing: {e:#}"))?; - let (stored_message, stored_path) = prepare_remind_storage(agent, message, file_path)?; - let id = coord - .broker - .store_reminder(agent, &stored_message, stored_path.as_deref(), due_at) - .map_err(|e| format!("failed to store reminder: {e:#}"))?; - tracing::info!(%id, %agent, %due_at, "reminder scheduled"); - coord.emit_reminders_snapshot(); - Ok(()) -} - -/// Decide what we actually store in the reminders row, applying the -/// same byte cap as the rest of the wire protocol -/// ([`crate::limits::MESSAGE_MAX_BYTES`]). Three outcomes: -/// -/// 1. Body within the cap → stored verbatim, with whatever `file_path` -/// the caller passed (None or Some). The scheduler honours -/// `file_path` at delivery time as before. -/// 2. Body over the cap, no caller `file_path` → auto-generate a path -/// under `/agents//state/reminders/auto-.md`, write the -/// body to disk now, store a short pointer hint as the message and -/// clear `file_path` (so the scheduler doesn't re-write at -/// delivery and overwrite the body with the hint). -/// 3. Body over the cap, caller provided `file_path` → honour the -/// caller's path: write the body to it now, store the same hint -/// and clear `file_path` for the same reason as (2). -/// -/// Returns `(stored_message, stored_file_path)` on success, or a -/// caller-ready error string on auto-save failure (which is the only -/// way a Remind request can be refused for size — the agent never has -/// to think about the cap). -fn prepare_remind_storage( - agent: &str, - message: &str, - file_path: Option<&str>, -) -> Result<(String, Option), String> { - if message.len() <= crate::limits::MESSAGE_MAX_BYTES { - return Ok((message.to_owned(), file_path.map(str::to_owned))); - } - let req_path = match file_path { - Some(p) => p.to_owned(), - None => auto_reminder_path(agent), - }; - let host_path = crate::reminder_scheduler::resolve_host_path(agent, &req_path) - .map_err(|reason| format!("auto-save path `{req_path}` rejected: {reason}"))?; - crate::reminder_scheduler::write_payload(agent, &host_path, message).map_err(|reason| { - format!("auto-save of large reminder body to `{req_path}` failed: {reason}") - })?; - let hint = format!( - "[reminder body of {} bytes auto-saved to `{req_path}`; read with your filesystem tools]", - message.len() - ); - Ok((hint, None)) -} - -/// Generate a per-agent path for an auto-saved reminder body. Uses -/// `unix_nanos` plus the agent name to keep collisions infinitesimal -/// across the agent's own state subtree (we're not stamping a hostname -/// since hive-c0re is single-host). -fn auto_reminder_path(agent: &str) -> String { - let ts_ns = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_nanos()); - format!("/agents/{agent}/state/reminders/auto-{ts_ns}.md") -} - -/// Resolve the target agent name for a *named* `GetLooseEnds` / -/// `CountPendingReminders` / `ReminderRollup` query. Rules: -/// -/// - `None` (or `Some(caller)`) → the caller's own threads (always allowed). -/// - `Some(descendant)` in the caller's subtree (the root's subtree is the whole hive) → allowed, no extra capability. -/// - `Some(other)` outside the subtree → requires the `query_agent_state` capability; error otherwise. -/// - `Some("*")` → rejected here; the hive-wide sweep is handled by `handle_get_loose_ends` under the same `query_agent_state` gate. -fn resolve_agent_state_target<'a>( - caller: &'a str, - target: Option<&'a str>, -) -> Result<&'a str, String> { - match target { - None => Ok(caller), - Some("*") => Err( - "hive-wide query (agent=\"*\") is only valid for loose-ends; \ - not available for this query" - .to_owned(), - ), - Some(name) => { - // Own subtree (the root covers all) is visible without extra - // capability; `is_descendant_of` returns true for `name == caller`. - if crate::topology::is_descendant_of(name, caller) { - return Ok(name); - } - if crate::capabilities::has_cap(caller, hive_sh4re::Capability::QueryAgentState) { - Ok(name) - } else { - Err(format!( - "agent `{caller}` cannot query `{name}`: not in its subtree and \ - `query_agent_state` capability is not granted" - )) - } - } - } -} - -/// Resolve the `due_at` unix timestamp for a Remind request. Returns -/// distinct error messages for each failure mode (overflow on -/// `InSeconds`, pre-epoch clock, `i64` cast wrap) so the caller can tell -/// what went wrong without inspecting the chain. -fn resolve_due_at(timing: &hive_sh4re::ReminderTiming) -> anyhow::Result { - use hive_sh4re::ReminderTiming; - match timing { - ReminderTiming::InSeconds { seconds } => { - let now = std::time::SystemTime::now(); - let future = now - .checked_add(std::time::Duration::from_secs(*seconds)) - .ok_or_else(|| { - anyhow::anyhow!("InSeconds overflow: {seconds}s exceeds system time range") - })?; - let duration = future - .duration_since(std::time::UNIX_EPOCH) - .map_err(|e| anyhow::anyhow!("system time before UNIX_EPOCH: {e}"))?; - i64::try_from(duration.as_secs()) - .map_err(|e| anyhow::anyhow!("unix timestamp exceeds i64 range: {e}")) - } - ReminderTiming::At { unix_timestamp } => Ok(*unix_timestamp), - } -} - -// --------------------------------------------------------------------------- -// Orchestration handlers + submit/schedule helpers. -// The schedule / meta-input handlers are reached via the tool-group-gated arms -// in `dispatch_orchestration`; `submit_init_config` / `submit_apply_commit` -// are re-used by the lifecycle handlers; `schedule_to_wire_public` / -// `filter_ghost_schedule_targets` are re-used by the dashboard. -// --------------------------------------------------------------------------- - -/// `RequestUpdateMetaInputs` — queue an `UpdateMetaInputs` approval -/// carrying the JSON-encoded input list in `commit_ref` (no git commit -/// is involved; the field is the payload the approval handler decodes). -fn handle_request_update_meta_inputs( - coord: &Arc, - requester: &str, - inputs: &[String], - description: Option<&str>, -) -> AgentResponse { - let label = if inputs.is_empty() { - "all inputs".to_string() - } else { - inputs.join(", ") - }; - tracing::info!(%requester, %label, "request_update_meta_inputs"); - let commit_ref = serde_json::to_string(inputs).unwrap_or_default(); - let id = match coord - .approvals - .submit_kind( - requester, - hive_sh4re::ApprovalKind::UpdateMetaInputs, - &commit_ref, - description, - requester, - ) - .map_err(|e| anyhow::anyhow!("{e:#}")) - { - Ok(id) => id, - Err(e) => { - return AgentResponse::Err { - message: format!("queue update_meta_inputs approval: {e:#}"), - }; - } - }; - tracing::info!(%id, %label, "update_meta_inputs approval queued"); - coord.emit_approval_added(crate::coordinator::ApprovalAdded { - id, - agent: requester, - approval_kind: "update_meta_inputs", - sha_short: None, - diff: None, - description: description.map(str::to_owned), - pr_number: None, - }); - AgentResponse::Ok -} - -/// `ListSchedules` — snapshot every scheduled prompt onto the wire. -fn handle_list_schedules(coord: &Arc) -> AgentResponse { - match coord.scheduled_prompts.list() { - Ok(schedules) => AgentResponse::Schedules { - schedules: schedules.into_iter().map(schedule_to_wire).collect(), - }, - Err(e) => AgentResponse::Err { - message: format!("list scheduled prompts: {e:#}"), - }, - } -} - -/// `GetLogs` — read a child container's journal via hive-priv (the -/// `-M` read needs root). `journalctl -M` wants the `h-` machine -/// name, which `container_name` derives. -async fn handle_get_logs(agent: &str, lines: Option) -> AgentResponse { - let n = lines.unwrap_or(50); - let machine = crate::lifecycle::container_name(agent); - tracing::info!(%agent, %machine, %n, "manager: get_logs"); - match crate::priv_client::read_container_journal( - &machine, - hive_sh4re::priv_proto::JournalQuery { - lines: n, - ..Default::default() - }, - ) - .await - { - Ok((stdout, stderr)) => { - let content = if stdout.is_empty() { stderr } else { stdout }; - AgentResponse::Logs { content } - } - Err(e) => AgentResponse::Err { - message: format!("get_logs: {e:#}"), - }, - } -} - -/// Submit a `RequestSchedulePrompt` payload as an `ApprovalKind::SchedulePrompt` -/// row. Encodes the payload into the approval's `commit_ref` so the -/// approve handler can re-parse it without a side table. Validates -/// inputs (non-empty targets, non-empty body, sane interval) at -/// submit time — the operator should never see a malformed schedule -/// pending approval. -fn handle_request_schedule_prompt( - coord: &Arc, - requester: &str, - payload: &hive_sh4re::SchedulePromptPayload, -) -> AgentResponse { - if payload.targets.is_empty() { - return AgentResponse::Err { - message: "schedule must have at least one target".into(), - }; - } - if payload.body.trim().is_empty() { - return AgentResponse::Err { - message: "schedule body must be non-empty".into(), - }; - } - if let Some(0) = payload.interval_seconds { - return AgentResponse::Err { - message: "interval_seconds must be > 0 (use None for one-shot)".into(), - }; - } - let commit_ref = match serde_json::to_string(payload) { - Ok(s) => s, - Err(e) => { - return AgentResponse::Err { - message: format!("encode SchedulePromptPayload: {e:#}"), - }; - } - }; - let id = match coord.approvals.submit_kind( - requester, - hive_sh4re::ApprovalKind::SchedulePrompt, - &commit_ref, - payload.description.as_deref(), - requester, - ) { - Ok(id) => id, - Err(e) => { - return AgentResponse::Err { - message: format!("queue schedule_prompt approval: {e:#}"), - }; - } - }; - tracing::info!( - %id, - requester, - targets = ?payload.targets, - first_fire_at = payload.first_fire_at_unix, - interval = ?payload.interval_seconds, - "schedule_prompt approval queued" - ); - coord.emit_approval_added(crate::coordinator::ApprovalAdded { - id, - agent: requester, - approval_kind: "schedule_prompt", - sha_short: None, - diff: None, - description: payload.description.clone(), - pr_number: None, - }); - AgentResponse::Ok -} - -/// Cancel a schedule (whole or per-target). Manager-surface -/// authorization: a manager can cancel its own schedules + any -/// schedule whose owner is one of its sub-agents (topology-walked). -/// The operator surface bypasses this and can cancel anything; -/// agents reaching this path through the manager get the -/// topology-scoped check. -fn handle_cancel_schedule( - coord: &Arc, - requester: &str, - schedule_id: i64, - targets: Option<&[String]>, -) -> AgentResponse { - let schedule = match coord.scheduled_prompts.get(schedule_id) { - Ok(Some(s)) => s, - Ok(None) => { - return AgentResponse::Err { - message: format!("schedule {schedule_id} not found"), - }; - } - Err(e) => { - return AgentResponse::Err { - message: format!("read schedule {schedule_id}: {e:#}"), - }; - } - }; - if !cancel_authorized(requester, &schedule.owner) { - return AgentResponse::Err { - message: format!( - "not authorized: {requester} cannot cancel schedule owned by {owner}", - owner = schedule.owner - ), - }; - } - let result = match targets { - Some(list) if !list.is_empty() => coord - .scheduled_prompts - .cancel_targets(schedule_id, list) - .map_err(|e| format!("cancel targets: {e:#}")), - _ => coord - .scheduled_prompts - .cancel_all(schedule_id) - .map_err(|e| format!("cancel all: {e:#}")), - }; - match result { - Ok(()) => { - coord.emit_schedules_snapshot(); - AgentResponse::Ok - } - Err(message) => AgentResponse::Err { message }, - } -} - -/// Authorize + dispatch a `FireScheduleNow` request from the -/// manager surface. Same ownership rules as `CancelSchedule`: -/// requester can fire its own schedules + any owned by an agent -/// in its subtree. The actual fan-out lives in -/// `scheduled_prompts_worker::fire_now`. -async fn handle_fire_schedule_now( - coord: &Arc, - requester: &str, - schedule_id: i64, -) -> AgentResponse { - let schedule = match coord.scheduled_prompts.get(schedule_id) { - Ok(Some(s)) => s, - Ok(None) => { - return AgentResponse::Err { - message: format!("schedule {schedule_id} not found"), - }; - } - Err(e) => { - return AgentResponse::Err { - message: format!("read schedule {schedule_id}: {e:#}"), - }; - } - }; - if !cancel_authorized(requester, &schedule.owner) { - return AgentResponse::Err { - message: format!( - "not authorized: {requester} cannot fire schedule owned by {owner}", - owner = schedule.owner - ), - }; - } - // MCP fire_schedule_now stays no-reset (cadence intact); the - // reset-timer option is a dashboard-dialog affordance. - match crate::scheduled_prompts_worker::fire_now(coord, schedule_id, false).await { - Ok(_report) => { - coord.emit_schedules_snapshot(); - AgentResponse::Ok - } - Err(e) => AgentResponse::Err { - message: format!("fire schedule {schedule_id} now: {e:#}"), - }, - } -} - -/// Field-named PATCH payload for [`handle_edit_schedule`]. Every -/// field is "leave alone" when `None`; the double-`Option` fields -/// additionally distinguish clear (`Some(None)`) from set -/// (`Some(Some(v))`). -#[allow( - clippy::option_option, - reason = "double-Option carries three-state PATCH semantics: outer None = \ - leave alone, Some(None) = clear, Some(Some(v)) = set" -)] -struct EditSchedulePatch { - body: Option, - description: Option>, - interval_seconds: Option>, - next_fire_at_unix: Option, - targets_add: Option>, - targets_remove: Option>, -} - -/// Authorize + dispatch a `EditSchedule` patch. Same ownership -/// rules as `CancelSchedule` — the manager can edit -/// schedules it owns + any owned by an agent in its subtree. -/// Forwards the partial payload to -/// `ScheduledPrompts::update` which enforces the cancelled-row / -/// zero-interval validation. Returns `Ok` on a clean update; -/// `Err` with the underlying message on any auth / validation -/// failure so the dashboard can surface it verbatim. -fn handle_edit_schedule( - coord: &Arc, - requester: &str, - schedule_id: i64, - patch: EditSchedulePatch, -) -> AgentResponse { - let EditSchedulePatch { - body, - description, - interval_seconds, - next_fire_at_unix, - targets_add, - targets_remove, - } = patch; - let schedule = match coord.scheduled_prompts.get(schedule_id) { - Ok(Some(s)) => s, - Ok(None) => { - return AgentResponse::Err { - message: format!("schedule {schedule_id} not found"), - }; - } - Err(e) => { - return AgentResponse::Err { - message: format!("read schedule {schedule_id}: {e:#}"), - }; - } - }; - if !cancel_authorized(requester, &schedule.owner) { - return AgentResponse::Err { - message: format!( - "not authorized: {requester} cannot edit schedule owned by {owner}", - owner = schedule.owner - ), - }; - } - let patch = crate::scheduled_prompts::UpdateSchedule { - body, - description, - interval_seconds, - next_fire_at_unix, - targets_add, - targets_remove, - }; - match coord.scheduled_prompts.update(schedule_id, patch) { - Ok(()) => { - coord.emit_schedules_snapshot(); - AgentResponse::Ok - } - Err(e) => AgentResponse::Err { - message: format!("edit schedule {schedule_id}: {e:#}"), - }, - } -} - -/// Permission check for `CancelSchedule` on the manager surface. -/// `requester` (always `ruth` here) can cancel its own schedules. -/// Sub-agent ownership is delegated to topology — see -/// `crate::topology::is_descendant_of`. Also reused by -/// `handle_fire_schedule_now` — fire-auth follows the same shape. -fn cancel_authorized(requester: &str, owner: &str) -> bool { - if requester == owner { - return true; - } - if requester == hive_sh4re::OPERATOR_RECIPIENT { - return true; - } - // Manager can cancel anything owned by an agent in its subtree. - // For the current single-manager topology that covers everything, - // but the check stays correct as the tree grows. - crate::topology::is_descendant_of(owner, requester) -} - -/// `request_apply_commit` takes a commit SHA only — not a branch or -/// tag name. A branch is mutable; pinning the proposal to a concrete -/// sha keeps "what the manager asked to deploy" unambiguous and means -/// the `proposal/` tag is a faithful record of the request. -/// Accepts a 7..=40 char hex string (short or full sha); the exact -/// commit is resolved + existence-checked against the proposed repo -/// later in `lifecycle::git_fetch_to_tag`. -pub(crate) fn validate_commit_ref(commit_ref: &str) -> Result<()> { - let n = commit_ref.len(); - let hex = commit_ref.chars().all(|c| c.is_ascii_hexdigit()); - if !(7..=40).contains(&n) || !hex { - anyhow::bail!( - "commit_ref '{commit_ref}' is not a commit sha — request_apply_commit \ - takes a 7-40 char hex sha, not a branch or tag name" - ); - } - Ok(()) -} - -/// Queue an `InitConfig` approval for a brand-new agent whose config repo -/// does not yet exist. Shared between the manager and agent sockets. -/// -/// `parent`, when `Some`, is the agent that will own the new child once -/// the operator approves: it is stashed in the approval's `commit_ref` -/// field (unused for `InitConfig` otherwise — same pattern -/// `UpdateMetaInputs` uses to carry its inputs JSON) and consumed in -/// `run_approval_init_config` to write the `child -> parent` topology -/// edge. Callers pass the requesting agent, so the requester becomes the -/// new agent's parent (the root requesting a new agent → a top-level agent, -/// matching `topology::reconcile`'s default). `None` writes no explicit -/// edge (reconcile-default placement) — retained for that fallback. -pub(crate) fn submit_init_config( - coord: &Arc, - name: &str, - parent: Option<&str>, - description: Option, -) -> anyhow::Result { - let proposed_dir = crate::coordinator::Coordinator::agent_proposed_dir(name); - if proposed_dir.join(".git").exists() { - anyhow::bail!( - "proposed config repo for '{name}' already exists at {} - \ - use request_apply_commit to update an existing agent's config", - proposed_dir.display() - ); - } - let id = coord - .approvals - .submit_kind( - name, - hive_sh4re::ApprovalKind::InitConfig, - parent.unwrap_or(""), - description.as_deref(), - // `parent` is the requesting agent (becomes the new child's - // parent); it's also the submitter the approval events route - // back to. No declared parent = operator-initiated path. - parent.unwrap_or("operator"), - ) - .map_err(|e| anyhow::anyhow!("queue approval row: {e:#}"))?; - tracing::info!(%id, %name, "init_config approval queued"); - coord.emit_approval_added(crate::coordinator::ApprovalAdded { - id, - agent: name, - approval_kind: "init_config", - sha_short: None, - diff: None, - description, - pr_number: None, - }); - Ok(id) -} - -/// Submit-time half of the apply flow: queue the approval row, then -/// fetch the manager's commit from the proposed repo into applied and -/// pin it as `refs/tags/proposal/`. From this point on the manager -/// repo is irrelevant for this approval — even if the manager amends -/// or force-pushes, the canonical sha hive-c0re will eventually -/// approve/deny lives in applied's object DB. -/// -/// If anything fails after the row is inserted (sha missing in -/// proposed, fs error, git plumbing crash) we mark the row failed and -/// surface the error to the manager. We don't try to roll the row -/// back — the failure is part of the audit trail. -pub(crate) async fn submit_apply_commit( - coord: &Arc, - agent: &str, - commit_ref: &str, - description: Option<&str>, - submitter: &str, -) -> anyhow::Result<(i64, String)> { - validate_commit_ref(commit_ref)?; - let proposed_dir = crate::coordinator::Coordinator::agent_proposed_dir(agent); - let applied_dir = crate::coordinator::Coordinator::agent_applied_dir(agent); - if !proposed_dir.exists() { - anyhow::bail!( - "proposed repo missing for agent '{agent}' (expected at {})", - proposed_dir.display() - ); - } - if !applied_dir.join(".git").exists() { - // First deploy: seed the applied repo from proposed so we can plant - // the proposal/ tag below. setup_applied seeds at the root - // (template) commit of proposed, not at main, so deployed/0 is the - // template baseline. This makes the diff mara sees on approval - // show the manager's actual changes rather than an empty diff. - crate::lifecycle::setup_applied(&applied_dir, Some(&proposed_dir), agent) - .await - .context("seed applied repo for first spawn")?; - } - let id = coord - .approvals - .submit_kind( - agent, - hive_sh4re::ApprovalKind::ApplyCommit, - commit_ref, - description, - submitter, - ) - .map_err(|e| anyhow::anyhow!("queue approval row: {e:#}"))?; - let tag = format!("proposal/{id}"); - let sha = - match crate::lifecycle::git_fetch_to_tag(&applied_dir, &proposed_dir, commit_ref, &tag) - .await - { - Ok(s) => s, - Err(e) => { - // Surface the failure on the approval row so the - // dashboard reflects it instead of leaving a phantom - // pending entry. The note doubles as the operator-visible - // explanation of why the approval can't be approved. - let note = format!("{e:#}"); - let _ = coord.approvals.mark_failed(id, ¬e); - coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { - id, - agent, - approval_kind: "apply_commit", - sha_short: None, - status: "failed", - note: Some(note), - description: description.map(str::to_owned), - }); - return Err(anyhow::anyhow!("git_fetch_to_tag: {e:#}")); - } - }; - coord - .approvals - .set_fetched_sha(id, &sha) - .map_err(|e| anyhow::anyhow!("persist fetched_sha: {e:#}"))?; - // Pre-flight gates: both reject the apply before approval if - // the agent's flake state would inflate meta's lock with duplicates - // or lie about what nix will fetch. Both checks independently read - // `:flake.lock` via git — they don't share state. Order matters - // only for early-exit + messaging: sync first means a stale lock - // bails with the actionable "run `nix flake lock`" hint rather than - // a dedup pass on a lock nix would never produce. - // - // Runs after `set_fetched_sha` so the failed row carries the sha - // that broke. Both failure paths mark + emit, then bail. - let sha_short = sha[..sha.len().min(12)].to_owned(); - if let Err(e) = crate::flake_check::check_lock_in_sync(&applied_dir, &tag, id).await { - let note = format!("{e:#}"); - let _ = coord.approvals.mark_failed(id, ¬e); - coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { - id, - agent, - approval_kind: "apply_commit", - sha_short: Some(sha_short.clone()), - status: "failed", - note: Some(note), - description: description.map(str::to_owned), - }); - return Err(anyhow::anyhow!("flake lock-sync check: {e:#}")); - } - if let Err(e) = crate::flake_check::check_no_duplicate_inputs(&applied_dir, &tag).await { - let note = format!("{e:#}"); - let _ = coord.approvals.mark_failed(id, ¬e); - coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { - id, - agent, - approval_kind: "apply_commit", - sha_short: Some(sha_short.clone()), - status: "failed", - note: Some(note), - description: description.map(str::to_owned), - }); - return Err(anyhow::anyhow!("flake dedup check: {e:#}")); - } - // Mirror the freshly-planted proposal/ tag to the forge. - if let Err(e) = crate::forge::push_config(agent).await { - tracing::warn!(%agent, %id, error = ?e, "forge: push_config after submit failed"); - } - // Phase 5b: surface the new pending approval on the dashboard - // event channel. Compute the diff once here so live subscribers - // get a fully-formed row without a snapshot refetch. `sha_short` - // is reused from the dedup gate above. - let diff = crate::dashboard::approval_diff(agent, id).await; - coord.emit_approval_added(crate::coordinator::ApprovalAdded { - id, - agent, - approval_kind: "apply_commit", - sha_short: Some(sha_short), - diff: Some(diff), - description: description.map(str::to_owned), - pr_number: None, - }); - Ok((id, sha)) -} - -/// Map a `scheduled_prompts::Schedule` to its public wire shape. -/// Field-by-field copy — the two types are intentionally identical; -/// the separation keeps hive-sh4re free of hive-c0re-internal types. -/// Public alias `schedule_to_wire_public` re-exports for -/// `dashboard.rs::api_schedules` without crossing the module -/// boundary into the socket-server file. -pub fn schedule_to_wire_public(s: crate::scheduled_prompts::Schedule) -> hive_sh4re::WireSchedule { - schedule_to_wire(s) -} - -/// Drop schedule targets that point at agents which no longer exist, so -/// the dashboard's schedule table doesn't render ghost columns for -/// destroyed agents. `live` is the set of logical agent names from the -/// last `nixos-container list` scan (stopped agents included, destroyed -/// ones absent); the `operator` pseudo-target is always retained since -/// it isn't a container. Applied only to the dashboard wire paths -/// (`api_schedules` + the `SchedulesChanged` SSE emit) — the -/// manager-facing `list_schedules` stays unfiltered so agents can still -/// see and cancel stale targets. This is a view filter: the underlying -/// schedule rows keep every target, so a re-spawned agent's targets -/// reappear on their own. -pub(crate) fn filter_ghost_schedule_targets( - schedules: &mut [hive_sh4re::WireSchedule], - live: &std::collections::HashSet, -) { - for s in schedules.iter_mut() { - s.targets - .retain(|t| t.target == hive_sh4re::OPERATOR_RECIPIENT || live.contains(&t.target)); - } -} - -fn schedule_to_wire(s: crate::scheduled_prompts::Schedule) -> hive_sh4re::WireSchedule { - hive_sh4re::WireSchedule { - id: s.id, - owner: s.owner, - body: s.body, - interval_seconds: s.interval_seconds, - next_fire_at_unix: hive_sh4re::wire_time::from_secs(s.next_fire_at_unix), - created_at_unix: hive_sh4re::wire_time::from_secs(s.created_at_unix), - source: match s.source { - crate::scheduled_prompts::ScheduleSource::Operator => { - hive_sh4re::WireScheduleSource::Operator - } - crate::scheduled_prompts::ScheduleSource::Approval { id } => { - hive_sh4re::WireScheduleSource::Approval { id } - } - }, - cancelled_at_unix: s.cancelled_at_unix.map(hive_sh4re::wire_time::from_secs), - paused_at_unix: s.paused_at_unix.map(hive_sh4re::wire_time::from_secs), - description: s.description, - targets: s - .targets - .into_iter() - .map(|t| hive_sh4re::WireScheduleTarget { - target: t.target, - cancelled_at_unix: t.cancelled_at_unix.map(hive_sh4re::wire_time::from_secs), - last_fired_at_unix: t.last_fired_at_unix.map(hive_sh4re::wire_time::from_secs), - last_result: t.last_result, - }) - .collect(), - } -} - -/// On `Ask { ttl_seconds: Some(n) }`, sleep n seconds and then try to -/// resolve the question with `[expired]`. If the operator (or any -/// other path) already answered it, `answer()` returns Err and we -/// no-op silently. Otherwise fire a `QuestionAnswered` helper event -/// with `answerer = "ttl-watchdog"` so the asker can distinguish a -/// real answer from a deadline trip without parsing the answer text. -const TTL_SENTINEL: &str = "[expired]"; -/// Synthetic `answerer` label used when the ttl watchdog resolves a -/// question instead of a real human / agent. Lives in a distinct -/// namespace from agent names + the operator so the asker can pattern -/// match `event.answerer == "ttl-watchdog"`. -const TTL_ANSWERER: &str = "ttl-watchdog"; - -pub fn spawn_question_watchdog(coord: &Arc, id: i64, ttl_secs: u64) { - let coord = coord.clone(); - tokio::spawn(async move { - tokio::time::sleep(std::time::Duration::from_secs(ttl_secs)).await; - // Watchdog has its own answerer label so the authorisation - // check in `answer()` permits it for any target. We bypass - // the public `answer()` path by calling it with the operator - // identity, since the operator is always permitted; the - // event we fire carries the real watchdog label for observers. - if let Ok((question, asker, target)) = - coord - .questions - .answer(id, TTL_SENTINEL, hive_sh4re::OPERATOR_RECIPIENT) - { - tracing::info!(%id, %asker, "question expired (ttl)"); - coord.notify_agent( - &asker, - &hive_sh4re::HelperEvent::QuestionAnswered { - id, - question, - answer: TTL_SENTINEL.to_owned(), - answerer: TTL_ANSWERER.to_owned(), - }, - ); - coord.emit_question_resolved(id, TTL_SENTINEL, TTL_ANSWERER, false, target.as_deref()); - } - }); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn auto_reminder_path_format() { - let p = auto_reminder_path("damocles"); - assert!(p.starts_with("/agents/damocles/state/reminders/auto-")); - assert!( - std::path::Path::new(&p) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("md")) - ); - } - - #[test] - fn prepare_remind_storage_passthrough_under_cap() { - let (msg, fp) = prepare_remind_storage("foo", "small body", None).unwrap(); - assert_eq!(msg, "small body"); - assert_eq!(fp, None); - } - - #[test] - fn prepare_remind_storage_passthrough_with_caller_file_path() { - let (msg, fp) = - prepare_remind_storage("foo", "small", Some("/agents/foo/state/x.md")).unwrap(); - assert_eq!(msg, "small"); - assert_eq!(fp.as_deref(), Some("/agents/foo/state/x.md")); - } - - fn target(name: &str) -> hive_sh4re::WireScheduleTarget { - hive_sh4re::WireScheduleTarget { - target: name.to_owned(), - cancelled_at_unix: None, - last_fired_at_unix: None, - last_result: None, - } - } - - fn schedule(targets: &[&str]) -> hive_sh4re::WireSchedule { - hive_sh4re::WireSchedule { - id: 1, - owner: "operator".to_owned(), - body: "ping".to_owned(), - interval_seconds: None, - next_fire_at_unix: hive_sh4re::wire_time::from_secs(0), - created_at_unix: hive_sh4re::wire_time::from_secs(0), - source: hive_sh4re::WireScheduleSource::Operator, - cancelled_at_unix: None, - paused_at_unix: None, - description: None, - targets: targets.iter().map(|t| target(t)).collect(), - } - } - - #[test] - fn ghost_filter_drops_dead_agents_keeps_live_and_operator() { - let live: std::collections::HashSet = ["iris".to_owned(), "damocles".to_owned()] - .into_iter() - .collect(); - let mut schedules = vec![schedule(&["iris", "ghost", "operator", "damocles"])]; - filter_ghost_schedule_targets(&mut schedules, &live); - let kept: Vec<&str> = schedules[0] - .targets - .iter() - .map(|t| t.target.as_str()) - .collect(); - // `ghost` (destroyed) dropped; live agents + operator pseudo-target kept. - assert_eq!(kept, vec!["iris", "operator", "damocles"]); - } - - #[test] - fn ghost_filter_can_empty_targets_when_all_dead() { - let live: std::collections::HashSet = std::collections::HashSet::new(); - let mut schedules = vec![schedule(&["gone1", "gone2"])]; - filter_ghost_schedule_targets(&mut schedules, &live); - // operator is never in the live set but is always retained; here - // there's no operator target, so everything drops. - assert!(schedules[0].targets.is_empty()); - } - - #[test] - fn accepts_short_and_full_sha() { - assert!(validate_commit_ref("e194f78").is_ok()); - assert!(validate_commit_ref("e194f7812ab").is_ok()); - assert!(validate_commit_ref(&"a".repeat(40)).is_ok()); - // Uppercase hex resolves fine through `git rev-parse`. - assert!(validate_commit_ref("E194F78").is_ok()); - } - - #[test] - fn rejects_branch_and_tag_names() { - // The exact bug class this guard exists for. - assert!(validate_commit_ref("main").is_err()); - assert!(validate_commit_ref("HEAD").is_err()); - assert!(validate_commit_ref("deployed/0").is_err()); - assert!(validate_commit_ref("feature-branch").is_err()); - } - - #[test] - fn rejects_too_short_too_long_and_empty() { - assert!(validate_commit_ref("").is_err()); - assert!(validate_commit_ref("abc123").is_err()); // 6 chars - assert!(validate_commit_ref(&"a".repeat(41)).is_err()); - } - - #[test] - fn resolve_agent_state_target_self_and_default_are_free() { - // No topology/capability state needed for these: `None` and the - // caller's own name resolve to the caller (`is_descendant_of` short- - // circuits to true when candidate == ancestor); `"*"` is rejected - // (the hive-wide sweep is handled by the loose-ends caller instead). - assert_eq!(resolve_agent_state_target("iris", None), Ok("iris")); - assert_eq!(resolve_agent_state_target("iris", Some("iris")), Ok("iris")); - assert!(resolve_agent_state_target("iris", Some("*")).is_err()); - } -} diff --git a/hive-c0re/src/socket_server/config_approvals.rs b/hive-c0re/src/socket_server/config_approvals.rs new file mode 100644 index 00000000..cabfc994 --- /dev/null +++ b/hive-c0re/src/socket_server/config_approvals.rs @@ -0,0 +1,343 @@ +//! Config-approval request handlers: `RequestInitConfig` / +//! `RequestApplyCommit` / `RequestUpdateMetaInputs`, plus the shared +//! submit helpers (`submit_init_config` / `submit_apply_commit`) and the +//! commit-sha shape check (`validate_commit_ref`). + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use hive_sh4re::AgentResponse; + +use super::require_new_child; +use crate::coordinator::Coordinator; + +/// `RequestInitConfig` — queue an `InitConfig` approval for an agent. The +/// `name` must be brand-new (absent from the topology) or already in the +/// caller's subtree; the requester is recorded as the new agent's parent (the +/// root requesting a new agent → a top-level agent, matching reconcile's +/// default). +pub(super) fn handle_request_init_config( + coord: &Arc, + agent: &str, + name: &str, + description: Option, +) -> AgentResponse { + if let Some(err) = require_new_child(agent, name, "request_init_config for") { + return err; + } + tracing::info!(%agent, %name, "request_init_config"); + match submit_init_config(coord, name, Some(agent), description) { + Ok(_id) => AgentResponse::Ok, + Err(e) => AgentResponse::Err { + message: format!("{e:#}"), + }, + } +} + +/// `RequestApplyCommit` — queue an apply-commit approval for an agent. The +/// target must be in the caller's subtree (the root covers every agent). +pub(super) async fn handle_request_apply_commit( + coord: &Arc, + agent: &str, + target_agent: &str, + commit_ref: &str, + description: Option<&str>, +) -> AgentResponse { + if let Some(err) = require_new_child(agent, target_agent, "request_apply_commit for") { + return err; + } + tracing::info!(%agent, %target_agent, %commit_ref, "request_apply_commit"); + match submit_apply_commit(coord, target_agent, commit_ref, description, agent).await { + Ok((id, sha)) => { + tracing::info!(%id, %target_agent, %sha, "apply_commit approval queued"); + AgentResponse::Ok + } + Err(e) => AgentResponse::Err { + message: format!("{e:#}"), + }, + } +} + +/// `RequestUpdateMetaInputs` — queue an `UpdateMetaInputs` approval +/// carrying the JSON-encoded input list in `commit_ref` (no git commit +/// is involved; the field is the payload the approval handler decodes). +pub(super) fn handle_request_update_meta_inputs( + coord: &Arc, + requester: &str, + inputs: &[String], + description: Option<&str>, +) -> AgentResponse { + let label = if inputs.is_empty() { + "all inputs".to_string() + } else { + inputs.join(", ") + }; + tracing::info!(%requester, %label, "request_update_meta_inputs"); + let commit_ref = serde_json::to_string(inputs).unwrap_or_default(); + let id = match coord + .approvals + .submit_kind( + requester, + hive_sh4re::ApprovalKind::UpdateMetaInputs, + &commit_ref, + description, + requester, + ) + .map_err(|e| anyhow::anyhow!("{e:#}")) + { + Ok(id) => id, + Err(e) => { + return AgentResponse::Err { + message: format!("queue update_meta_inputs approval: {e:#}"), + }; + } + }; + tracing::info!(%id, %label, "update_meta_inputs approval queued"); + coord.emit_approval_added(crate::coordinator::ApprovalAdded { + id, + agent: requester, + approval_kind: "update_meta_inputs", + sha_short: None, + diff: None, + description: description.map(str::to_owned), + pr_number: None, + }); + AgentResponse::Ok +} + +/// `request_apply_commit` takes a commit SHA only — not a branch or +/// tag name. A branch is mutable; pinning the proposal to a concrete +/// sha keeps "what the manager asked to deploy" unambiguous and means +/// the `proposal/` tag is a faithful record of the request. +/// Accepts a 7..=40 char hex string (short or full sha); the exact +/// commit is resolved + existence-checked against the proposed repo +/// later in `lifecycle::git_fetch_to_tag`. +pub(crate) fn validate_commit_ref(commit_ref: &str) -> Result<()> { + let n = commit_ref.len(); + let hex = commit_ref.chars().all(|c| c.is_ascii_hexdigit()); + if !(7..=40).contains(&n) || !hex { + anyhow::bail!( + "commit_ref '{commit_ref}' is not a commit sha — request_apply_commit \ + takes a 7-40 char hex sha, not a branch or tag name" + ); + } + Ok(()) +} + +/// Queue an `InitConfig` approval for a brand-new agent whose config repo +/// does not yet exist. Shared between the manager and agent sockets. +/// +/// `parent`, when `Some`, is the agent that will own the new child once +/// the operator approves: it is stashed in the approval's `commit_ref` +/// field (unused for `InitConfig` otherwise — same pattern +/// `UpdateMetaInputs` uses to carry its inputs JSON) and consumed in +/// `run_approval_init_config` to write the `child -> parent` topology +/// edge. Callers pass the requesting agent, so the requester becomes the +/// new agent's parent (the root requesting a new agent → a top-level agent, +/// matching `topology::reconcile`'s default). `None` writes no explicit +/// edge (reconcile-default placement) — retained for that fallback. +pub(crate) fn submit_init_config( + coord: &Arc, + name: &str, + parent: Option<&str>, + description: Option, +) -> anyhow::Result { + let proposed_dir = crate::coordinator::Coordinator::agent_proposed_dir(name); + if proposed_dir.join(".git").exists() { + anyhow::bail!( + "proposed config repo for '{name}' already exists at {} - \ + use request_apply_commit to update an existing agent's config", + proposed_dir.display() + ); + } + let id = coord + .approvals + .submit_kind( + name, + hive_sh4re::ApprovalKind::InitConfig, + parent.unwrap_or(""), + description.as_deref(), + // `parent` is the requesting agent (becomes the new child's + // parent); it's also the submitter the approval events route + // back to. No declared parent = operator-initiated path. + parent.unwrap_or("operator"), + ) + .map_err(|e| anyhow::anyhow!("queue approval row: {e:#}"))?; + tracing::info!(%id, %name, "init_config approval queued"); + coord.emit_approval_added(crate::coordinator::ApprovalAdded { + id, + agent: name, + approval_kind: "init_config", + sha_short: None, + diff: None, + description, + pr_number: None, + }); + Ok(id) +} + +/// Submit-time half of the apply flow: queue the approval row, then +/// fetch the manager's commit from the proposed repo into applied and +/// pin it as `refs/tags/proposal/`. From this point on the manager +/// repo is irrelevant for this approval — even if the manager amends +/// or force-pushes, the canonical sha hive-c0re will eventually +/// approve/deny lives in applied's object DB. +/// +/// If anything fails after the row is inserted (sha missing in +/// proposed, fs error, git plumbing crash) we mark the row failed and +/// surface the error to the manager. We don't try to roll the row +/// back — the failure is part of the audit trail. +pub(crate) async fn submit_apply_commit( + coord: &Arc, + agent: &str, + commit_ref: &str, + description: Option<&str>, + submitter: &str, +) -> anyhow::Result<(i64, String)> { + validate_commit_ref(commit_ref)?; + let proposed_dir = crate::coordinator::Coordinator::agent_proposed_dir(agent); + let applied_dir = crate::coordinator::Coordinator::agent_applied_dir(agent); + if !proposed_dir.exists() { + anyhow::bail!( + "proposed repo missing for agent '{agent}' (expected at {})", + proposed_dir.display() + ); + } + if !applied_dir.join(".git").exists() { + // First deploy: seed the applied repo from proposed so we can plant + // the proposal/ tag below. setup_applied seeds at the root + // (template) commit of proposed, not at main, so deployed/0 is the + // template baseline. This makes the diff mara sees on approval + // show the manager's actual changes rather than an empty diff. + crate::lifecycle::setup_applied(&applied_dir, Some(&proposed_dir), agent) + .await + .context("seed applied repo for first spawn")?; + } + let id = coord + .approvals + .submit_kind( + agent, + hive_sh4re::ApprovalKind::ApplyCommit, + commit_ref, + description, + submitter, + ) + .map_err(|e| anyhow::anyhow!("queue approval row: {e:#}"))?; + let tag = format!("proposal/{id}"); + let sha = + match crate::lifecycle::git_fetch_to_tag(&applied_dir, &proposed_dir, commit_ref, &tag) + .await + { + Ok(s) => s, + Err(e) => { + // Surface the failure on the approval row so the + // dashboard reflects it instead of leaving a phantom + // pending entry. The note doubles as the operator-visible + // explanation of why the approval can't be approved. + let note = format!("{e:#}"); + let _ = coord.approvals.mark_failed(id, ¬e); + coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { + id, + agent, + approval_kind: "apply_commit", + sha_short: None, + status: "failed", + note: Some(note), + description: description.map(str::to_owned), + }); + return Err(anyhow::anyhow!("git_fetch_to_tag: {e:#}")); + } + }; + coord + .approvals + .set_fetched_sha(id, &sha) + .map_err(|e| anyhow::anyhow!("persist fetched_sha: {e:#}"))?; + // Pre-flight gates: both reject the apply before approval if + // the agent's flake state would inflate meta's lock with duplicates + // or lie about what nix will fetch. Both checks independently read + // `:flake.lock` via git — they don't share state. Order matters + // only for early-exit + messaging: sync first means a stale lock + // bails with the actionable "run `nix flake lock`" hint rather than + // a dedup pass on a lock nix would never produce. + // + // Runs after `set_fetched_sha` so the failed row carries the sha + // that broke. Both failure paths mark + emit, then bail. + let sha_short = sha[..sha.len().min(12)].to_owned(); + if let Err(e) = crate::flake_check::check_lock_in_sync(&applied_dir, &tag, id).await { + let note = format!("{e:#}"); + let _ = coord.approvals.mark_failed(id, ¬e); + coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { + id, + agent, + approval_kind: "apply_commit", + sha_short: Some(sha_short.clone()), + status: "failed", + note: Some(note), + description: description.map(str::to_owned), + }); + return Err(anyhow::anyhow!("flake lock-sync check: {e:#}")); + } + if let Err(e) = crate::flake_check::check_no_duplicate_inputs(&applied_dir, &tag).await { + let note = format!("{e:#}"); + let _ = coord.approvals.mark_failed(id, ¬e); + coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { + id, + agent, + approval_kind: "apply_commit", + sha_short: Some(sha_short.clone()), + status: "failed", + note: Some(note), + description: description.map(str::to_owned), + }); + return Err(anyhow::anyhow!("flake dedup check: {e:#}")); + } + // Mirror the freshly-planted proposal/ tag to the forge. + if let Err(e) = crate::forge::push_config(agent).await { + tracing::warn!(%agent, %id, error = ?e, "forge: push_config after submit failed"); + } + // Phase 5b: surface the new pending approval on the dashboard + // event channel. Compute the diff once here so live subscribers + // get a fully-formed row without a snapshot refetch. `sha_short` + // is reused from the dedup gate above. + let diff = crate::dashboard::approval_diff(agent, id).await; + coord.emit_approval_added(crate::coordinator::ApprovalAdded { + id, + agent, + approval_kind: "apply_commit", + sha_short: Some(sha_short), + diff: Some(diff), + description: description.map(str::to_owned), + pr_number: None, + }); + Ok((id, sha)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_short_and_full_sha() { + assert!(validate_commit_ref("e194f78").is_ok()); + assert!(validate_commit_ref("e194f7812ab").is_ok()); + assert!(validate_commit_ref(&"a".repeat(40)).is_ok()); + // Uppercase hex resolves fine through `git rev-parse`. + assert!(validate_commit_ref("E194F78").is_ok()); + } + + #[test] + fn rejects_branch_and_tag_names() { + // The exact bug class this guard exists for. + assert!(validate_commit_ref("main").is_err()); + assert!(validate_commit_ref("HEAD").is_err()); + assert!(validate_commit_ref("deployed/0").is_err()); + assert!(validate_commit_ref("feature-branch").is_err()); + } + + #[test] + fn rejects_too_short_too_long_and_empty() { + assert!(validate_commit_ref("").is_err()); + assert!(validate_commit_ref("abc123").is_err()); // 6 chars + assert!(validate_commit_ref(&"a".repeat(41)).is_err()); + } +} diff --git a/hive-c0re/src/socket_server/lifecycle_handlers.rs b/hive-c0re/src/socket_server/lifecycle_handlers.rs new file mode 100644 index 00000000..f6640c72 --- /dev/null +++ b/hive-c0re/src/socket_server/lifecycle_handlers.rs @@ -0,0 +1,201 @@ +//! Container-lifecycle request handlers (`Start` / `Restart` / `Kill` / +//! `Update` / `ListDescendants`), including the capability-gated +//! infra-container restart path. All are topology-guarded via +//! `super::require_descendant`. + +use std::sync::Arc; + +use hive_sh4re::AgentResponse; + +use super::require_descendant; +use crate::coordinator::Coordinator; + +/// `Start` — start a container, kicking its next turn. The caller must be an +/// ancestor of `name` in the topology (the root covers every agent). +pub(super) fn handle_start(coord: &Arc, agent: &str, name: &str) -> AgentResponse { + if let Some(err) = require_descendant(agent, name, "start") { + return err; + } + tracing::info!(%agent, %name, "start container"); + // Persist `wanted = Up` and submit the Start DAG; the submit layer + // upgrades a stale-rev start to a full rebuild so the container + // runs current nix derivations before it starts. + crate::job_queue::submit::start( + coord, + name, + crate::job_queue::Source::Manual, + format!("agent `{agent}` start tool"), + ); + AgentResponse::Ok +} + +/// `Restart` — enqueue a restart for a container. The caller must be an +/// ancestor of `name` in the topology. The infra-container branch is +/// orthogonal: it is gated on the `infra_admin` capability and audited, so it +/// stays ahead of the topology guard. +pub(super) async fn handle_restart( + coord: &Arc, + agent: &str, + name: &str, +) -> AgentResponse { + // Infra-container restart: an agent holding the `infra_admin` + // capability can restart a hive infrastructure container (hive-ci / + // hive-gateway / hive-forge / hive-matrix) by passing its name to the + // same restart tool. The `InfraContainer` enum parse both recognises + // these (never agent children, so disjoint from the child path below) + // and yields the typed value the restart path needs. + if let Ok(container) = name.parse::() { + return handle_restart_infra(coord, agent, container).await; + } + if let Some(err) = require_descendant(agent, name, "restart") { + return err; + } + tracing::info!(%agent, %name, "submit restart"); + crate::job_queue::submit::restart( + coord, + name, + crate::job_queue::Source::Manual, + format!("agent `{agent}` restart tool"), + ); + AgentResponse::Ok +} + +/// Restart a hive infrastructure container on behalf of an agent that +/// holds the `infra_admin` capability. The `container` is already a valid +/// [`InfraContainer`] (the caller parsed it); this gates on the capability +/// and routes the systemctl restart through hive-priv. Direct, not +/// approval-gated. +async fn handle_restart_infra( + coord: &Arc, + agent: &str, + container: hive_sh4re::priv_proto::InfraContainer, +) -> AgentResponse { + let name = container.unit_name(); + // Record the attempt in the operator-visible privileged-action audit + // trail, then emit a live `AuditEntryAdded` so the dashboard audit view + // appends it off `/dashboard/stream`. Best-effort: `record` returns the + // canonical row (or `None` on a sqlite blip), and we stream exactly that + // row so the stored + streamed views can't drift. `action` is stable so + // the dashboard can group/filter. + let audit = |outcome: crate::audit_log::AuditOutcome, detail: Option<&str>| { + if let Some(entry) = coord + .audit_log + .record(agent, "restart_infra", name, outcome, detail) + { + coord.emit_audit_entry(entry); + } + }; + if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::InfraAdmin) { + tracing::warn!(%agent, %name, "agent: infra restart denied (no infra_admin capability)"); + audit( + crate::audit_log::AuditOutcome::Err, + Some("denied: missing infra_admin capability"), + ); + return AgentResponse::Err { + message: format!( + "restarting infra container `{name}` requires the `infra_admin` capability" + ), + }; + } + tracing::info!(%agent, %name, "agent: restart infra container"); + match crate::priv_client::restart_infra_container(container).await { + Ok(()) => { + audit(crate::audit_log::AuditOutcome::Ok, None); + AgentResponse::Ok + } + Err(e) => { + let msg = format!("{e:#}"); + audit(crate::audit_log::AuditOutcome::Err, Some(&msg)); + AgentResponse::Err { message: msg } + } + } +} + +/// `Kill` — kill a container, unregister it, notify the manager. The caller +/// must be an ancestor of `name` in the topology. +pub(super) async fn handle_kill( + coord: &Arc, + agent: &str, + name: &str, +) -> AgentResponse { + if let Some(err) = require_descendant(agent, name, "kill") { + return err; + } + tracing::info!(%agent, %name, "kill container"); + // Persist the intent even if the kill fails — otherwise the next + // reconcile would restart the container. + if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) { + tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed"); + } + let result: anyhow::Result<()> = async { + crate::lifecycle::kill(name).await?; + coord.unregister_agent(name); + Ok(()) + } + .await; + match result { + Ok(()) => { + coord.notify_manager(&hive_sh4re::HelperEvent::Killed { + agent: name.to_owned(), + }); + AgentResponse::Ok + } + Err(e) => AgentResponse::Err { + message: format!("{e:#}"), + }, + } +} + +/// `Update` — enqueue a rebuild for a container. The caller must be an +/// ancestor of `name` in the topology. +pub(super) fn handle_update(coord: &Arc, agent: &str, name: &str) -> AgentResponse { + if let Some(err) = require_descendant(agent, name, "rebuild") { + return err; + } + tracing::info!(%agent, %name, "submit rebuild"); + crate::job_queue::submit::rebuild( + coord, + name, + crate::job_queue::Source::Manual, + format!("agent `{agent}` update tool"), + ); + AgentResponse::Ok +} + +/// `ListDescendants` — every topological descendant of `agent` with +/// its running/stopped state, parents before children. +pub(super) async fn handle_list_descendants(agent: &str) -> AgentResponse { + tracing::debug!(%agent, "agent: list descendants"); + // All containers known to nixos-container (running only). + let running_set: std::collections::HashSet = match crate::lifecycle::list().await { + Ok(names) => names + .into_iter() + .filter_map(|c| { + c.strip_prefix(crate::lifecycle::AGENT_PREFIX) + .map(str::to_owned) + }) + .collect(), + Err(e) => { + return AgentResponse::Err { + message: format!("list containers failed: {e:#}"), + }; + } + }; + // Walk the full topology and collect every descendant. + let topo = crate::topology::read(); + let mut names: Vec = topo + .keys() + .filter(|name| crate::topology::is_descendant_of(name, agent)) + .cloned() + .collect(); + // Parents before children, then alpha within each tier. + crate::auto_update::topology_sort(&mut names, &topo); + let containers = names + .into_iter() + .map(|name| { + let running = running_set.contains(&name); + hive_sh4re::ContainerInfo { name, running } + }) + .collect(); + AgentResponse::Containers { containers } +} diff --git a/hive-c0re/src/socket_server/mod.rs b/hive-c0re/src/socket_server/mod.rs new file mode 100644 index 00000000..641b3729 --- /dev/null +++ b/hive-c0re/src/socket_server/mod.rs @@ -0,0 +1,1092 @@ +//! Unix-socket request server, shared by the per-agent sockets and the +//! (pure-transport) manager socket. The socket file's existence on disk +//! authenticates the caller: connecting to `<.../agents/foo/mcp.sock>` means +//! you are `foo`; the manager socket simply serves as `ruth`. There is no +//! privilege flag — both transports run the same [`serve`] / [`dispatch`] +//! code, and authority derives uniformly from the caller's identity: +//! topology (`is_descendant_of`) for subtree-relational verbs, capabilities +//! for hive-wide queries, and tool-group membership for the orchestration +//! verbs. `ruth` reaches every agent only as a consequence of being the +//! topology root, not via any hardcoded name match. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use hive_sh4re::{AgentRequest, AgentResponse, MANAGER_AGENT, Message}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{UnixListener, UnixStream}; +use tokio::task::JoinHandle; + +use crate::coordinator::Coordinator; + +mod config_approvals; +mod lifecycle_handlers; +mod reminders; +mod schedules; + +pub(crate) use schedules::filter_ghost_schedule_targets; +pub use schedules::schedule_to_wire_public; + +use config_approvals::{ + handle_request_apply_commit, handle_request_init_config, handle_request_update_meta_inputs, +}; +use lifecycle_handlers::{ + handle_kill, handle_list_descendants, handle_restart, handle_start, handle_update, +}; +use reminders::{handle_remind, resolve_agent_state_target}; +use schedules::{ + EditSchedulePatch, handle_cancel_schedule, handle_edit_schedule, handle_fire_schedule_now, + handle_list_schedules, handle_request_schedule_prompt, +}; + +pub struct AgentSocket { + pub path: PathBuf, + pub handle: JoinHandle<()>, +} + +pub fn start(agent: &str, socket_path: &Path, coord: Arc) -> Result { + use std::os::unix::fs::PermissionsExt as _; + let agent = agent.to_owned(); + if let Some(parent) = socket_path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create agent socket dir {}", parent.display()))?; + } + if socket_path.exists() { + std::fs::remove_file(socket_path).context("remove stale agent socket")?; + } + let listener = UnixListener::bind(socket_path) + .with_context(|| format!("bind agent socket {}", socket_path.display()))?; + // The socket is bind-mounted into exactly one container as + // `/run/hive/mcp.sock` (`lifecycle::set_nspawn_flags`); the + // in-container harness connects as the per-agent unix user, + // not root, so the default `tokio::net::UnixListener::bind` + // perms (0755) lock it out. 0666 lets the agent user connect; + // the bind source dir is per-agent on host so blast radius is + // unchanged. + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o666)) + .with_context(|| format!("chmod agent socket {}", socket_path.display()))?; + tracing::info!(%agent, socket = %socket_path.display(), "agent socket listening"); + + let path = socket_path.to_path_buf(); + let handle = tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => { + let agent = agent.clone(); + let coord = coord.clone(); + tokio::spawn(async move { + if let Err(e) = serve(stream, agent, coord).await { + tracing::warn!(error = ?e, "agent connection failed"); + } + }); + } + Err(e) => { + tracing::warn!(error = ?e, "agent listener accept failed; exiting"); + return; + } + } + } + }); + Ok(AgentSocket { path, handle }) +} + +/// Bind + serve the manager socket. This is now **pure transport**: it grants +/// no authority of its own — it just serves requests as `agent = MANAGER_AGENT` +/// ("ruth"), and ruth's reach comes entirely from being the topology root +/// (`is_descendant_of` covers every agent) plus the capabilities / tool-groups +/// it holds, identical to connecting on a per-agent socket — ruth uses the +/// standard per-agent runtime dir + socket, with no dedicated helpers. +pub fn start_manager(coord: Arc) -> Result<()> { + use std::os::unix::fs::PermissionsExt as _; + let dir = Coordinator::agent_dir(crate::lifecycle::MANAGER_NAME); + std::fs::create_dir_all(&dir) + .with_context(|| format!("create manager dir {}", dir.display()))?; + let socket = Coordinator::socket_path(crate::lifecycle::MANAGER_NAME); + if socket.exists() { + std::fs::remove_file(&socket).context("remove stale manager socket")?; + } + let listener = UnixListener::bind(&socket) + .with_context(|| format!("bind manager socket {}", socket.display()))?; + // 0666 so the in-container root user (non-root) can connect; the bind + // source dir is manager-only on host (see the per-agent socket above). + std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666)) + .with_context(|| format!("chmod manager socket {}", socket.display()))?; + tracing::info!(socket = %socket.display(), "manager socket listening"); + + tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => { + let coord = coord.clone(); + tokio::spawn(async move { + // Pure transport: serve as `ruth`, no privilege grant. + if let Err(e) = serve(stream, MANAGER_AGENT.to_owned(), coord).await { + tracing::warn!(error = ?e, "manager connection failed"); + } + }); + } + Err(e) => { + tracing::warn!(error = ?e, "manager listener accept failed"); + return; + } + } + } + }); + Ok(()) +} + +async fn serve(stream: UnixStream, agent: String, coord: Arc) -> Result<()> { + let (read, mut write) = stream.into_split(); + let mut reader = BufReader::new(read); + let mut line = String::new(); + loop { + line.clear(); + let n = reader.read_line(&mut line).await?; + if n == 0 { + return Ok(()); + } + let resp = match serde_json::from_str::(line.trim()) { + Ok(req) => dispatch(&req, &agent, &coord).await, + Err(e) => AgentResponse::Err { + message: format!("parse error: {e}"), + }, + }; + let mut payload = serde_json::to_string(&resp)?; + payload.push('\n'); + write.write_all(payload.as_bytes()).await?; + write.flush().await?; + } +} + +/// Max long-poll window the caller can ask for; values above the +/// cap are clamped. 180s keeps us under typical TCP/proxy idle +/// limits while still letting agents park their turn until a +/// message arrives. Omitting `wait_seconds` (or passing `0`) means +/// "peek, don't wait" — claude can call recv whenever it wants a +/// cheap "is there anything pending?" check without blocking the +/// turn for 30 seconds. To actually park, the caller passes a +/// positive `wait_seconds`. +pub(crate) const RECV_LONG_POLL_MAX: std::time::Duration = std::time::Duration::from_mins(3); + +/// Server-side hard cap on `Recv.max` — canonical value lives in +/// `hive_sh4re::RECV_BATCH_MAX` so the harness's wake-prompt hint and +/// this enforcement site can't drift apart. +pub(crate) const RECV_BATCH_MAX: u32 = hive_sh4re::RECV_BATCH_MAX; + +pub(crate) fn recv_timeout(wait_seconds: Option) -> std::time::Duration { + match wait_seconds { + Some(s) => std::time::Duration::from_secs(s).min(RECV_LONG_POLL_MAX), + None => std::time::Duration::ZERO, + } +} + +/// Handle the subset of `Request` variants that are identical on both +/// the agent socket and the manager socket. Returns `Some(response)` for +/// every variant it handles; returns `None` for variants with socket-specific +/// semantics (e.g. `GetLooseEnds` / `CountPendingReminders` / `ReminderRollup` +/// where the manager can target other agents) or for manager-only variants. +/// +/// The unified `dispatch` calls this first; the remaining arms (which gate +/// on topology / capabilities / tool-groups) are handled there. +pub(crate) async fn dispatch_shared( + req: &hive_sh4re::Request, + agent: &str, + coord: &Arc, +) -> Option { + Some(match req { + hive_sh4re::Request::Send { + to, + body, + in_reply_to, + } => handle_send(coord, agent, to, body, *in_reply_to), + hive_sh4re::Request::Recv { wait_seconds, max } => { + handle_recv(coord, agent, *wait_seconds, *max).await + } + hive_sh4re::Request::Status => handle_status(coord, agent), + hive_sh4re::Request::OperatorMsg { body } => handle_operator_msg(coord, agent, body), + hive_sh4re::Request::Wake { from, body } => handle_wake(coord, agent, from, body), + hive_sh4re::Request::Recent { limit } => handle_recent(coord, agent, *limit), + hive_sh4re::Request::Ask { + question, + options, + multi, + ttl_seconds, + to, + } => crate::questions::handle_ask( + coord, + agent, + question, + options, + *multi, + *ttl_seconds, + to.as_deref(), + ) + .map_or_else( + |message| hive_sh4re::Response::Err { message }, + |id| hive_sh4re::Response::QuestionQueued { id }, + ), + hive_sh4re::Request::Answer { id, answer } => { + crate::questions::handle_answer(coord, agent, *id, answer).map_or_else( + |message| hive_sh4re::Response::Err { message }, + |()| hive_sh4re::Response::Ok, + ) + } + hive_sh4re::Request::Remind { + message, + timing, + file_path, + } => handle_remind(coord, agent, message, timing, file_path.as_deref()), + hive_sh4re::Request::SetStatus { text } => handle_set_status(coord, text), + hive_sh4re::Request::GetAgentMeta { name } => { + handle_get_agent_meta(coord, agent, name.as_deref()).await + } + hive_sh4re::Request::CancelLooseEnd { kind, id } => { + crate::questions::handle_cancel_loose_end(coord, agent, *kind, *id).map_or_else( + |message| hive_sh4re::Response::Err { message }, + |()| hive_sh4re::Response::Ok, + ) + } + hive_sh4re::Request::CreateRepo { repo } => handle_create_repo(agent, repo).await, + hive_sh4re::Request::AckTurn => handle_ack_turn(coord, agent), + hive_sh4re::Request::AckUntil { up_to } => handle_ack_until(coord, agent, *up_to), + hive_sh4re::Request::RequeueInflight => handle_requeue_inflight(coord, agent), + hive_sh4re::Request::GracefulStopComplete => { + // Harness drained + is exiting: clear the fence so the + // `GracefulStop` orchestration (which polls this flag) proceeds + // to stop the container without waiting out its timeout. + coord.clear_graceful_stop(agent); + hive_sh4re::Response::Ok + } + hive_sh4re::Request::GetHostJournal { + unit, + container, + lines, + priority, + grep, + since, + until, + } => { + dispatch_host_journal( + agent, + HostJournalArgs { + unit, + container, + lines, + priority, + grep, + since, + until, + }, + ) + .await + } + // Not a shared variant. + _ => return None, + }) +} + +/// `Recv` — long-poll the broker for up to `max` messages (capped at +/// `RECV_BATCH_MAX`), mapping deliveries onto the wire response. +async fn handle_recv( + coord: &Arc, + agent: &str, + wait_seconds: Option, + max: Option, +) -> hive_sh4re::Response { + // Graceful-stop fence: while a graceful stop is pending for this agent, + // return `GracefulStop` instead of polling the broker. The harness runs + // one stop-checkpoint turn then exits; new sends keep queueing in the + // broker for the agent's next start. Checked before the (blocking) poll + // so a flag set between polls is seen on the next Recv — the orchestration + // also fires a transient wake to break an in-flight long-poll. + if coord.is_graceful_stop_pending(agent) { + return hive_sh4re::Response::GracefulStop; + } + let cap = max.unwrap_or(1).min(RECV_BATCH_MAX) as usize; + match coord + .broker + .recv_blocking_batch(agent, recv_timeout(wait_seconds), cap) + .await + { + Ok(deliveries) => hive_sh4re::Response::Messages { + messages: deliveries + .into_iter() + .map(|d| hive_sh4re::DeliveredMessage { + from: d.message.from, + body: d.message.body, + id: d.id, + redelivered: d.redelivered, + in_reply_to: d.message.in_reply_to, + }) + .collect(), + }, + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// `Wake` — inject a wake into `agent`'s own inbox. Persisted through +/// the sqlite broker like any other message so the agent can ack it +/// via `AckUntil` and it appears in message history for post-mortem. +fn handle_wake( + coord: &Arc, + agent: &str, + from: &str, + body: &str, +) -> hive_sh4re::Response { + match coord.broker.send(&Message { + from: from.to_owned(), + to: agent.to_owned(), + body: body.to_owned(), + in_reply_to: None, + }) { + Ok(()) => hive_sh4re::Response::Ok, + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// `SetStatus` — validate the status text, then trigger a dashboard +/// rescan. The harness has already written the status file to its own +/// `state/` dir (it runs as the agent user), so this only refreshes the +/// dashboard's view. +fn handle_set_status(coord: &Arc, text: &str) -> hive_sh4re::Response { + if let Err(message) = crate::limits::check_status_text(text) { + return hive_sh4re::Response::Err { message }; + } + let coord2 = Arc::clone(coord); + tokio::spawn(async move { coord2.rescan_containers_and_emit().await }); + hive_sh4re::Response::Ok +} + +/// Validate an agent-supplied repo name: a single safe slug segment, no +/// path traversal. Forgejo validates server-side too, but rejecting early +/// gives a clear message and avoids building odd API paths. +fn valid_repo_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 100 + && !name.starts_with(['-', '.']) + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) +} + +/// `CreateRepo` — create a repo for `agent` *through hive-c0re* in the +/// c0re-owned `agents` org with operator-team branch protection. +/// The sanctioned create path now that agents can't create repos directly. +async fn handle_create_repo(agent: &str, repo: &str) -> hive_sh4re::Response { + if !valid_repo_name(repo) { + return hive_sh4re::Response::Err { + message: format!( + "invalid repo name {repo:?} — single segment of letters, digits, '-', '_', '.' \ + (no leading '-'/'.', max 100 chars)" + ), + }; + } + let Some(core_token) = crate::forge::core_token() else { + return hive_sh4re::Response::Err { + message: "forge unavailable (no core token) — cannot create repo".to_owned(), + }; + }; + match crate::forge::create_agent_repo(agent, repo, &core_token).await { + Ok(full_name) => hive_sh4re::Response::RepoCreated { + clone_url: format!("{}/{full_name}.git", crate::forge::FORGE_HTTP), + full_name, + }, + Err(e) => hive_sh4re::Response::Err { + message: format!("create repo {repo:?} failed: {e:#}"), + }, + } +} + +/// `GetAgentMeta` — identity + live status for `name` (defaults to the +/// caller). Reads the live container-view status and the hive/swarm +/// display names. +async fn handle_get_agent_meta( + coord: &Arc, + agent: &str, + name: Option<&str>, +) -> hive_sh4re::Response { + let target = name.unwrap_or(agent); + let (status_text, status_set_at, running) = + crate::container_view::read_agent_status_live(target).await; + let (hive_name, swarm_name) = crate::container_view::hive_swarm_names(); + hive_sh4re::Response::AgentMeta { + name: target.to_owned(), + running, + hyperhive_rev: crate::auto_update::current_flake_rev(&coord.hyperhive_flake), + status_text, + status_set_at, + hive_name, + swarm_name, + matrix_accounts: read_agent_matrix_identities(target), + } +} + +/// Read the target agent's matrix identities from the daemon's +/// `matrix-accounts.json` snapshot (under the agent's state dir). +/// Best-effort: an absent / unparseable snapshot (no matrix provisioning, +/// or the daemon not up yet) yields an empty list. The `MatrixIdentity` +/// serde shape matches the snapshot entries; the snapshot's `live` field is +/// ignored (only live accounts are written). +fn read_agent_matrix_identities(agent: &str) -> Vec { + let path = Coordinator::agent_notes_dir(agent).join("matrix-accounts.json"); + std::fs::read_to_string(&path) + .ok() + .and_then(|s| serde_json::from_str::>(&s).ok()) + .unwrap_or_default() +} + +/// `Status` — count of pending (unread) inbox messages for `agent`. +fn handle_status(coord: &Arc, agent: &str) -> hive_sh4re::Response { + match coord.broker.count_pending(agent) { + Ok(unread) => hive_sh4re::Response::Status { unread }, + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// `OperatorMsg` — deliver an operator-authored message into `agent`'s +/// inbox (from the `operator` recipient). +fn handle_operator_msg(coord: &Arc, agent: &str, body: &str) -> hive_sh4re::Response { + match coord.broker.send(&Message { + from: hive_sh4re::OPERATOR_RECIPIENT.to_owned(), + to: agent.to_owned(), + body: body.to_owned(), + in_reply_to: None, + }) { + Ok(()) => hive_sh4re::Response::Ok, + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// `Recent` — the last `limit` inbox rows for `agent` (read-only, +/// doesn't consume). +fn handle_recent(coord: &Arc, agent: &str, limit: u64) -> hive_sh4re::Response { + match coord.broker.recent_for(agent, limit) { + Ok(rows) => hive_sh4re::Response::Recent { rows }, + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// `AckTurn` — mark `agent`'s in-flight delivered messages acked so +/// they don't redeliver on the next turn. +fn handle_ack_turn(coord: &Arc, agent: &str) -> hive_sh4re::Response { + match coord.broker.ack_turn(agent) { + Ok(_n) => hive_sh4re::Response::Ok, + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// `AckUntil` — bulk-ack every message addressed to `agent` with row +/// id `<= up_to` (the agent-side backlog-triage escape hatch). +fn handle_ack_until(coord: &Arc, agent: &str, up_to: i64) -> hive_sh4re::Response { + match coord.broker.ack_until(agent, up_to) { + Ok(count) => hive_sh4re::Response::Acked { count }, + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// `RequeueInflight` — resurface `agent`'s unacked in-flight messages +/// (crash recovery on harness boot). +fn handle_requeue_inflight(coord: &Arc, agent: &str) -> hive_sh4re::Response { + match coord.broker.requeue_inflight(agent) { + Ok(n) => { + if n > 0 { + tracing::info!(%agent, requeued = %n, "requeued in-flight messages"); + } + hive_sh4re::Response::Ok + } + Err(e) => hive_sh4re::Response::Err { + message: format!("{e:#}"), + }, + } +} + +/// Unified dispatch for every socket connection — per-agent sockets and the +/// (now pure-transport) manager socket alike. There is no privilege bit; +/// authority derives uniformly from the caller's identity: subtree-relational +/// verbs (lifecycle/config/logs) require the caller to be an ancestor of the +/// target (`is_descendant_of`, so the root covers all); hive-wide agent-state +/// queries require the `QueryAgentState` capability; hive-wide orchestration +/// verbs (schedules / meta-inputs) require the matching tool-group (the +/// grantable capability). +async fn dispatch(req: &AgentRequest, agent: &str, coord: &Arc) -> AgentResponse { + if let Some(resp) = dispatch_shared(req, agent, coord).await { + return resp; + } + match req { + // Lifecycle + config: caller must be an ancestor of the target + // (a parent owns its whole subtree; the root covers every agent). + AgentRequest::Start { name } => handle_start(coord, agent, name), + AgentRequest::Restart { name } => handle_restart(coord, agent, name).await, + AgentRequest::Kill { name } => handle_kill(coord, agent, name).await, + AgentRequest::Update { name } => handle_update(coord, agent, name), + AgentRequest::ListDescendants => handle_list_descendants(agent).await, + AgentRequest::RequestInitConfig { name, description } => { + handle_request_init_config(coord, agent, name, description.clone()) + } + AgentRequest::RequestApplyCommit { + agent: target_agent, + commit_ref, + description, + } => { + handle_request_apply_commit( + coord, + agent, + target_agent, + commit_ref, + description.as_deref(), + ) + .await + } + // Agent-state queries: own subtree is free; other agents + the + // hive-wide `"*"` sweep require `QueryAgentState`. + AgentRequest::GetLooseEnds { agent: target } => { + handle_get_loose_ends(coord, agent, target.as_deref()) + } + AgentRequest::CountPendingReminders { agent: target } => { + handle_count_pending_reminders(coord, agent, target.as_deref()) + } + AgentRequest::ReminderRollup { + since_secs, + agent: target, + } => handle_reminder_rollup(coord, agent, target.as_deref(), *since_secs), + // Orchestration / diagnostics verbs — gated per-verb on tool-group + // membership or topology (see `dispatch_orchestration`). + _ => dispatch_orchestration(req, agent, coord).await, + } +} + +/// Handle the hive-wide orchestration verbs (scheduling, meta-input updates) +/// plus container-log reads. No blanket socket gate: each verb gates on the +/// grantable capability that authorises it — the matching tool-group +/// (`scheduling` / `approvals`), or `is_descendant_of` for `get_logs` +/// (a parent reads its subtree's logs). Any other variant is a host-admin / +/// unknown request invalid on either socket. +async fn dispatch_orchestration( + req: &AgentRequest, + agent: &str, + coord: &Arc, +) -> AgentResponse { + match req { + AgentRequest::RequestUpdateMetaInputs { + inputs, + description, + } => { + if let Some(err) = require_group(agent, "approvals", "request update_meta_inputs") { + return err; + } + handle_request_update_meta_inputs(coord, agent, inputs, description.as_deref()) + } + AgentRequest::RequestSchedulePrompt(payload) => { + if let Some(err) = require_group(agent, "scheduling", "schedule a prompt") { + return err; + } + handle_request_schedule_prompt(coord, agent, payload) + } + AgentRequest::CancelSchedule { id, targets } => { + if let Some(err) = require_group(agent, "scheduling", "cancel a schedule") { + return err; + } + handle_cancel_schedule(coord, agent, *id, targets.as_deref()) + } + AgentRequest::EditSchedule { + id, + body, + description, + interval_seconds, + next_fire_at_unix, + targets_add, + targets_remove, + } => { + if let Some(err) = require_group(agent, "scheduling", "edit a schedule") { + return err; + } + handle_edit_schedule( + coord, + agent, + *id, + EditSchedulePatch { + body: body.clone(), + description: description.clone(), + interval_seconds: *interval_seconds, + next_fire_at_unix: *next_fire_at_unix, + targets_add: targets_add.clone(), + targets_remove: targets_remove.clone(), + }, + ) + } + AgentRequest::ListSchedules => { + if let Some(err) = require_group(agent, "scheduling", "list schedules") { + return err; + } + handle_list_schedules(coord) + } + AgentRequest::FireScheduleNow { id } => { + if let Some(err) = require_group(agent, "scheduling", "fire a schedule") { + return err; + } + handle_fire_schedule_now(coord, agent, *id).await + } + AgentRequest::GetLogs { + agent: target, + lines, + } => { + if let Some(err) = require_descendant(agent, target, "read logs of") { + return err; + } + handle_get_logs(target, *lines).await + } + // Host-admin-only / unknown variants: never valid on either socket. + _ => AgentResponse::Err { + message: "request not handled on this socket".to_owned(), + }, + } +} + +/// Topology guard for the subtree-relational lifecycle/config/log tools: the +/// `target` must be the caller itself or one of its topology descendants — a +/// parent owns its whole subtree, and the root (`ruth`) covers every agent as +/// a consequence, with no positional/hardcoded privilege. Returns `Some(Err)` +/// to short-circuit the dispatch arm when it isn't, `None` when authorised. +/// `action` is the verb phrase for the message (e.g. `"start"`). +fn require_descendant(agent: &str, target: &str, action: &str) -> Option { + if crate::topology::is_descendant_of(target, agent) { + None + } else { + Some(AgentResponse::Err { + message: format!( + "agent `{agent}` cannot {action} `{target}`: \ + not in its subtree (topology)" + ), + }) + } +} + +/// Capability guard for the hive-wide orchestration verbs: the caller must +/// hold the given tool-group. The tool-group (c0re-owned `tool_groups.json`, +/// read server-side via [`crate::tool_groups::groups_for`]) is the grantable +/// capability — granting it to an orchestrator (e.g. the root) authorises +/// these verbs without any positional/hardcoded privilege. `action` is the +/// verb phrase for the message. +fn require_group(agent: &str, group: &str, action: &str) -> Option { + if crate::tool_groups::groups_for(agent) + .iter() + .any(|g| g == group) + { + None + } else { + Some(AgentResponse::Err { + message: format!("agent `{agent}` cannot {action}: requires the `{group}` tool group"), + }) + } +} + +/// Topology guard for `request_init_config` / `request_apply_commit`, +/// which may legitimately target a child that does not exist *yet* +/// (spawning a brand-new sub-agent). The caller may act on a +/// `target` that is EITHER already its direct child (re-init / config +/// update of an existing child) OR brand-new (absent from the topology +/// tree — the requester becomes its parent). A name that already +/// belongs to a *different* parent (or is a root agent) is refused so +/// one agent can't hijack another's sub-tree. +/// +/// Also re-runs the agent-name format check (a traversal / malformed name +/// could never be a descendant): a brand-new name now flows straight to +/// `submit_init_config`, which builds filesystem paths from it, so validate +/// before that. +fn require_new_child(agent: &str, target: &str, action: &str) -> Option { + if let Some(reason) = crate::dashboard::validate_agent_name(target) { + return Some(AgentResponse::Err { + message: format!("agent `{agent}` cannot {action} `{target}`: {reason}"), + }); + } + // brand-new name (absent from topology) — requester becomes the parent on + // approval; allowed for any caller. + if !crate::topology::read().contains_key(target) { + return None; + } + // existing agent — allowed only if it's in the caller's subtree + // (re-init / config update of an agent the caller owns; the root owns + // every existing agent). Refuses an agent outside the caller's subtree + // so one agent can't hijack another's config. + if crate::topology::is_descendant_of(target, agent) { + None + } else { + Some(AgentResponse::Err { + message: format!( + "agent `{agent}` cannot {action} `{target}`: it already exists \ + outside its subtree in the topology tree" + ), + }) + } +} + +/// `GetLooseEnds` — read the target's loose ends. `None` / own / a subtree +/// descendant resolve freely (a parent sees its subtree, the root sees all); +/// any other named agent needs `QueryAgentState`; `"*"` is a hive-wide sweep +/// gated on `QueryAgentState`. +fn handle_get_loose_ends( + coord: &Arc, + agent: &str, + target: Option<&str>, +) -> AgentResponse { + let result = if target == Some("*") { + if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::QueryAgentState) { + return AgentResponse::Err { + message: "query_agent_state capability required for hive-wide loose ends" + .to_owned(), + }; + } + crate::loose_ends::hive_wide(coord) + } else { + match resolve_agent_state_target(agent, target) { + Ok(name) => crate::loose_ends::for_agent(coord, name), + Err(message) => return AgentResponse::Err { message }, + } + }; + match result { + Ok(loose_ends) => AgentResponse::LooseEnds { loose_ends }, + Err(e) => AgentResponse::Err { + message: format!("{e:#}"), + }, + } +} + +/// `CountPendingReminders` — resolve the target (own / subtree free, else +/// `QueryAgentState`) then count its pending reminders. +fn handle_count_pending_reminders( + coord: &Arc, + agent: &str, + target: Option<&str>, +) -> AgentResponse { + match resolve_agent_state_target(agent, target) { + Ok(name) => match coord.broker.count_pending_reminders_for(name) { + Ok(count) => AgentResponse::PendingRemindersCount { count }, + Err(e) => AgentResponse::Err { + message: format!("{e:#}"), + }, + }, + Err(message) => AgentResponse::Err { message }, + } +} + +/// `ReminderRollup` — resolve the target (own / subtree free, else +/// `QueryAgentState`) then roll up its reminders fired in the last +/// `since_secs`. +fn handle_reminder_rollup( + coord: &Arc, + agent: &str, + target: Option<&str>, + since_secs: u64, +) -> AgentResponse { + match resolve_agent_state_target(agent, target) { + Ok(name) => match coord.broker.reminder_rollup_for(name, since_secs) { + Ok(stats) => AgentResponse::ReminderRollup(stats), + Err(e) => AgentResponse::Err { + message: format!("{e:#}"), + }, + }, + Err(message) => AgentResponse::Err { message }, + } +} + +/// Field-named journal-query knobs for [`dispatch_host_journal`]. +/// Borrows straight from the matched `GetHostJournal` request variant. +pub struct HostJournalArgs<'a> { + pub unit: &'a Option, + pub container: &'a Option, + pub lines: &'a Option, + pub priority: &'a Option, + pub grep: &'a Option, + pub since: &'a Option, + pub until: &'a Option, +} + +/// Handle `GetHostJournal` from both the agent and manager sockets. +/// Capability-gated: the calling agent must hold `read_host_journal` in +/// `meta/capabilities.json`. Runs `journalctl` host-side and returns +/// the output as a `HostJournal` response. +/// +/// The manager is not exempt - grant `read_host_journal` in +/// `meta/capabilities.json` to enable it for any agent including the manager. +pub async fn dispatch_host_journal(agent: &str, args: HostJournalArgs<'_>) -> AgentResponse { + let HostJournalArgs { + unit, + container, + lines, + priority, + grep, + since, + until, + } = args; + if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::ReadHostJournal) { + return AgentResponse::Err { + message: "agent does not have the read_host_journal capability".to_owned(), + }; + } + let n = lines.unwrap_or(30).min(100); + + // A container (`-M`) read enters the container namespace and needs + // root, so it's delegated to hive-priv. A host read (no container) + // the unprivileged hive-core user can do directly via its + // systemd-journal group membership. + if let Some(c) = container { + tracing::info!(%agent, machine = %c, %n, "get_host_journal (container)"); + return match crate::priv_client::read_container_journal( + c, + hive_sh4re::priv_proto::JournalQuery { + lines: n, + unit: unit.clone(), + priority: priority.as_ref().map(|p| p.as_str().to_owned()), + grep: grep.clone(), + since: since.clone(), + until: until.clone(), + ..Default::default() + }, + ) + .await + { + Ok((stdout, stderr)) => { + let content = if stdout.is_empty() { stderr } else { stdout }; + AgentResponse::HostJournal { content } + } + Err(e) => AgentResponse::Err { + message: format!("journal read: {e:#}"), + }, + }; + } + + let mut args: Vec = vec![ + "--no-pager".to_owned(), + "--output=short".to_owned(), + "-n".to_owned(), + n.to_string(), + ]; + if let Some(u) = unit { + args.push("-u".to_owned()); + args.push(u.clone()); + } + if let Some(p) = priority { + args.push("-p".to_owned()); + args.push(p.as_str().to_owned()); + } + if let Some(g) = grep { + args.push(format!("--grep={g}")); + } + if let Some(s) = since { + args.push(format!("--since={s}")); + } + if let Some(u) = until { + args.push(format!("--until={u}")); + } + tracing::info!(%agent, ?args, "get_host_journal"); + match tokio::process::Command::new("journalctl") + .args(&args) + .output() + .await + { + Ok(out) => { + let content = if out.status.success() || !out.stdout.is_empty() { + String::from_utf8_lossy(&out.stdout).into_owned() + } else { + let stderr = String::from_utf8_lossy(&out.stderr); + format!("journalctl exited {}: {stderr}", out.status) + }; + AgentResponse::HostJournal { content } + } + Err(e) => AgentResponse::Err { + message: format!("journalctl spawn failed: {e:#}"), + }, + } +} + +/// Fan out one message to each recipient in `targets`. Skips the sender +/// itself. Returns a list of `": "` strings for any delivery +/// failures (empty = all good). +pub(crate) fn fan_out_send( + coord: &Arc, + from: &str, + body: &str, + in_reply_to: Option, + targets: &[String], +) -> Vec { + let mut errors = Vec::new(); + for target in targets { + if target == from { + continue; + } + if let Err(e) = coord.broker.send(&Message { + from: from.to_owned(), + to: target.clone(), + body: body.to_owned(), + in_reply_to, + }) { + errors.push(format!("{target}: {e}")); + } + } + errors +} + +/// Common Send handler shared between dispatch arms. Applies the +/// 4 KiB body cap, then routes broadcast (`to == "*"`) / children fan-out +/// (`to == ""`) / unicast through their respective broker calls. +/// `pub(crate)` so `dispatch_shared` can use it across both socket paths. +pub(crate) fn handle_send( + coord: &Arc, + agent: &str, + to: &str, + body: &str, + in_reply_to: Option, +) -> AgentResponse { + if let Err(message) = crate::limits::check_size("send", body) { + return AgentResponse::Err { message }; + } + if to == "*" { + let errors = coord.broadcast_send(agent, body); + return if errors.is_empty() { + AgentResponse::Ok + } else { + AgentResponse::Err { + message: format!("broadcast failed for agents: {}", errors.join(", ")), + } + }; + } + // ``: fan out to every direct descendant of the sender per + // topology.json. Bypasses the allow-list check — structural fan-out + // targets are never user-listed peers. No-op (returns Ok) for leaf + // agents that have no children. + if to == hive_sh4re::CHILDREN_RECIPIENT { + let children = crate::topology::children_of(agent); + let errors = fan_out_send(coord, agent, body, in_reply_to, &children); + return if errors.is_empty() { + AgentResponse::Ok + } else { + AgentResponse::Err { + message: format!("children fan-out failed for agents: {}", errors.join(", ")), + } + }; + } + // Resolve magic-recipient sentinels (``) against topology.json; + // no-op for ordinary names. Lets agents address structural roles without + // learning the label — runtime reparenting propagates for free. See + // `docs/conventions.md::Recipient sentinels`. + let resolved = crate::topology::resolve_recipient(agent, to); + // Validate that the resolved recipient is a known local agent or the + // special "operator" recipient. Without this check a typo in `to` + // silently queues a message nobody will ever read. + // + // Cross-hive messaging (`name@hive` qualified names) is not routed + // through the broker — use the Matrix MCP tools for that instead. + if resolved.contains('@') { + return AgentResponse::Err { + message: format!( + "send failed: cross-hive recipient `{resolved}` is not supported \ + via the broker — use Matrix MCP tools for cross-hive messaging" + ), + }; + } + if resolved != hive_sh4re::OPERATOR_RECIPIENT { + let state_root = crate::coordinator::Coordinator::agent_state_root(&resolved); + if !state_root.exists() { + return AgentResponse::Err { + message: format!( + "send failed: unknown recipient `{resolved}` \ + (no agent with that name exists on this hive)" + ), + }; + } + } + match coord.broker.send(&Message { + from: agent.to_owned(), + to: resolved, + body: body.to_owned(), + in_reply_to, + }) { + Ok(()) => AgentResponse::Ok, + Err(e) => AgentResponse::Err { + message: format!("{e:#}"), + }, + } +} + +/// `GetLogs` — read a child container's journal via hive-priv (the +/// `-M` read needs root). `journalctl -M` wants the `h-` machine +/// name, which `container_name` derives. +async fn handle_get_logs(agent: &str, lines: Option) -> AgentResponse { + let n = lines.unwrap_or(50); + let machine = crate::lifecycle::container_name(agent); + tracing::info!(%agent, %machine, %n, "manager: get_logs"); + match crate::priv_client::read_container_journal( + &machine, + hive_sh4re::priv_proto::JournalQuery { + lines: n, + ..Default::default() + }, + ) + .await + { + Ok((stdout, stderr)) => { + let content = if stdout.is_empty() { stderr } else { stdout }; + AgentResponse::Logs { content } + } + Err(e) => AgentResponse::Err { + message: format!("get_logs: {e:#}"), + }, + } +} + +/// On `Ask { ttl_seconds: Some(n) }`, sleep n seconds and then try to +/// resolve the question with `[expired]`. If the operator (or any +/// other path) already answered it, `answer()` returns Err and we +/// no-op silently. Otherwise fire a `QuestionAnswered` helper event +/// with `answerer = "ttl-watchdog"` so the asker can distinguish a +/// real answer from a deadline trip without parsing the answer text. +const TTL_SENTINEL: &str = "[expired]"; +/// Synthetic `answerer` label used when the ttl watchdog resolves a +/// question instead of a real human / agent. Lives in a distinct +/// namespace from agent names + the operator so the asker can pattern +/// match `event.answerer == "ttl-watchdog"`. +const TTL_ANSWERER: &str = "ttl-watchdog"; + +pub fn spawn_question_watchdog(coord: &Arc, id: i64, ttl_secs: u64) { + let coord = coord.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(ttl_secs)).await; + // Watchdog has its own answerer label so the authorisation + // check in `answer()` permits it for any target. We bypass + // the public `answer()` path by calling it with the operator + // identity, since the operator is always permitted; the + // event we fire carries the real watchdog label for observers. + if let Ok((question, asker, target)) = + coord + .questions + .answer(id, TTL_SENTINEL, hive_sh4re::OPERATOR_RECIPIENT) + { + tracing::info!(%id, %asker, "question expired (ttl)"); + coord.notify_agent( + &asker, + &hive_sh4re::HelperEvent::QuestionAnswered { + id, + question, + answer: TTL_SENTINEL.to_owned(), + answerer: TTL_ANSWERER.to_owned(), + }, + ); + coord.emit_question_resolved(id, TTL_SENTINEL, TTL_ANSWERER, false, target.as_deref()); + } + }); +} diff --git a/hive-c0re/src/socket_server/reminders.rs b/hive-c0re/src/socket_server/reminders.rs new file mode 100644 index 00000000..674ebc1a --- /dev/null +++ b/hive-c0re/src/socket_server/reminders.rs @@ -0,0 +1,229 @@ +//! Reminder request handling: the `Remind` handler, the shared +//! `store_remind` storage path with its pending-cap and large-body +//! auto-save dance, timing resolution, and the agent-state target +//! resolution shared by the loose-ends / reminder query handlers. + +use std::sync::Arc; + +use hive_sh4re::AgentResponse; + +use crate::coordinator::Coordinator; + +pub(super) fn handle_remind( + coord: &Arc, + agent: &str, + message: &str, + timing: &hive_sh4re::ReminderTiming, + file_path: Option<&str>, +) -> AgentResponse { + match store_remind(coord, agent, message, timing, file_path) { + Ok(()) => AgentResponse::Ok, + Err(message) => AgentResponse::Err { message }, + } +} + +/// Shared remind-storage path used by both the agent and the manager +/// dispatchers. Validates timing, applies the auto-file overflow +/// dance (see [`prepare_remind_storage`]), and writes the reminder +/// row. Returns `Ok(())` on success, or a caller-ready error string +/// the dispatcher wraps in `*Response::Err`. +/// Maximum pending (un-delivered) reminders per agent. Exceeding this +/// causes `store_remind` to return an error so the agent knows to back +/// off instead of silently dropping. Override via +/// `HIVE_REMIND_MAX_PENDING_PER_AGENT`; set to `0` to disable the cap +/// (not recommended — a runaway agent can still flood the scheduler). +const DEFAULT_REMIND_MAX_PENDING: u64 = 50; + +fn remind_max_pending() -> u64 { + std::env::var("HIVE_REMIND_MAX_PENDING_PER_AGENT") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(DEFAULT_REMIND_MAX_PENDING) +} + +pub(crate) fn store_remind( + coord: &Arc, + agent: &str, + message: &str, + timing: &hive_sh4re::ReminderTiming, + file_path: Option<&str>, +) -> Result<(), String> { + let max = remind_max_pending(); + if max > 0 { + let pending = coord.broker.count_pending_reminders_for(agent).unwrap_or(0); + if pending >= max { + return Err(format!( + "reminder rejected: agent `{agent}` already has {pending} pending \ + reminders (cap {max}). Cancel some via `cancel_loose_end` or wait \ + for them to fire before scheduling more. Override the cap with \ + `HIVE_REMIND_MAX_PENDING_PER_AGENT`." + )); + } + } + let due_at = resolve_due_at(timing).map_err(|e| format!("invalid reminder timing: {e:#}"))?; + let (stored_message, stored_path) = prepare_remind_storage(agent, message, file_path)?; + let id = coord + .broker + .store_reminder(agent, &stored_message, stored_path.as_deref(), due_at) + .map_err(|e| format!("failed to store reminder: {e:#}"))?; + tracing::info!(%id, %agent, %due_at, "reminder scheduled"); + coord.emit_reminders_snapshot(); + Ok(()) +} + +/// Decide what we actually store in the reminders row, applying the +/// same byte cap as the rest of the wire protocol +/// ([`crate::limits::MESSAGE_MAX_BYTES`]). Three outcomes: +/// +/// 1. Body within the cap → stored verbatim, with whatever `file_path` +/// the caller passed (None or Some). The scheduler honours +/// `file_path` at delivery time as before. +/// 2. Body over the cap, no caller `file_path` → auto-generate a path +/// under `/agents//state/reminders/auto-.md`, write the +/// body to disk now, store a short pointer hint as the message and +/// clear `file_path` (so the scheduler doesn't re-write at +/// delivery and overwrite the body with the hint). +/// 3. Body over the cap, caller provided `file_path` → honour the +/// caller's path: write the body to it now, store the same hint +/// and clear `file_path` for the same reason as (2). +/// +/// Returns `(stored_message, stored_file_path)` on success, or a +/// caller-ready error string on auto-save failure (which is the only +/// way a Remind request can be refused for size — the agent never has +/// to think about the cap). +fn prepare_remind_storage( + agent: &str, + message: &str, + file_path: Option<&str>, +) -> Result<(String, Option), String> { + if message.len() <= crate::limits::MESSAGE_MAX_BYTES { + return Ok((message.to_owned(), file_path.map(str::to_owned))); + } + let req_path = match file_path { + Some(p) => p.to_owned(), + None => auto_reminder_path(agent), + }; + let host_path = crate::reminder_scheduler::resolve_host_path(agent, &req_path) + .map_err(|reason| format!("auto-save path `{req_path}` rejected: {reason}"))?; + crate::reminder_scheduler::write_payload(agent, &host_path, message).map_err(|reason| { + format!("auto-save of large reminder body to `{req_path}` failed: {reason}") + })?; + let hint = format!( + "[reminder body of {} bytes auto-saved to `{req_path}`; read with your filesystem tools]", + message.len() + ); + Ok((hint, None)) +} + +/// Generate a per-agent path for an auto-saved reminder body. Uses +/// `unix_nanos` plus the agent name to keep collisions infinitesimal +/// across the agent's own state subtree (we're not stamping a hostname +/// since hive-c0re is single-host). +fn auto_reminder_path(agent: &str) -> String { + let ts_ns = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + format!("/agents/{agent}/state/reminders/auto-{ts_ns}.md") +} + +/// Resolve the target agent name for a *named* `GetLooseEnds` / +/// `CountPendingReminders` / `ReminderRollup` query. Rules: +/// +/// - `None` (or `Some(caller)`) → the caller's own threads (always allowed). +/// - `Some(descendant)` in the caller's subtree (the root's subtree is the whole hive) → allowed, no extra capability. +/// - `Some(other)` outside the subtree → requires the `query_agent_state` capability; error otherwise. +/// - `Some("*")` → rejected here; the hive-wide sweep is handled by `handle_get_loose_ends` under the same `query_agent_state` gate. +pub(super) fn resolve_agent_state_target<'a>( + caller: &'a str, + target: Option<&'a str>, +) -> Result<&'a str, String> { + match target { + None => Ok(caller), + Some("*") => Err( + "hive-wide query (agent=\"*\") is only valid for loose-ends; \ + not available for this query" + .to_owned(), + ), + Some(name) => { + // Own subtree (the root covers all) is visible without extra + // capability; `is_descendant_of` returns true for `name == caller`. + if crate::topology::is_descendant_of(name, caller) { + return Ok(name); + } + if crate::capabilities::has_cap(caller, hive_sh4re::Capability::QueryAgentState) { + Ok(name) + } else { + Err(format!( + "agent `{caller}` cannot query `{name}`: not in its subtree and \ + `query_agent_state` capability is not granted" + )) + } + } + } +} + +/// Resolve the `due_at` unix timestamp for a Remind request. Returns +/// distinct error messages for each failure mode (overflow on +/// `InSeconds`, pre-epoch clock, `i64` cast wrap) so the caller can tell +/// what went wrong without inspecting the chain. +fn resolve_due_at(timing: &hive_sh4re::ReminderTiming) -> anyhow::Result { + use hive_sh4re::ReminderTiming; + match timing { + ReminderTiming::InSeconds { seconds } => { + let now = std::time::SystemTime::now(); + let future = now + .checked_add(std::time::Duration::from_secs(*seconds)) + .ok_or_else(|| { + anyhow::anyhow!("InSeconds overflow: {seconds}s exceeds system time range") + })?; + let duration = future + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| anyhow::anyhow!("system time before UNIX_EPOCH: {e}"))?; + i64::try_from(duration.as_secs()) + .map_err(|e| anyhow::anyhow!("unix timestamp exceeds i64 range: {e}")) + } + ReminderTiming::At { unix_timestamp } => Ok(*unix_timestamp), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn auto_reminder_path_format() { + let p = auto_reminder_path("damocles"); + assert!(p.starts_with("/agents/damocles/state/reminders/auto-")); + assert!( + std::path::Path::new(&p) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("md")) + ); + } + + #[test] + fn prepare_remind_storage_passthrough_under_cap() { + let (msg, fp) = prepare_remind_storage("foo", "small body", None).unwrap(); + assert_eq!(msg, "small body"); + assert_eq!(fp, None); + } + + #[test] + fn prepare_remind_storage_passthrough_with_caller_file_path() { + let (msg, fp) = + prepare_remind_storage("foo", "small", Some("/agents/foo/state/x.md")).unwrap(); + assert_eq!(msg, "small"); + assert_eq!(fp.as_deref(), Some("/agents/foo/state/x.md")); + } + + #[test] + fn resolve_agent_state_target_self_and_default_are_free() { + // No topology/capability state needed for these: `None` and the + // caller's own name resolve to the caller (`is_descendant_of` short- + // circuits to true when candidate == ancestor); `"*"` is rejected + // (the hive-wide sweep is handled by the loose-ends caller instead). + assert_eq!(resolve_agent_state_target("iris", None), Ok("iris")); + assert_eq!(resolve_agent_state_target("iris", Some("iris")), Ok("iris")); + assert!(resolve_agent_state_target("iris", Some("*")).is_err()); + } +} diff --git a/hive-c0re/src/socket_server/schedules.rs b/hive-c0re/src/socket_server/schedules.rs new file mode 100644 index 00000000..a8aa0783 --- /dev/null +++ b/hive-c0re/src/socket_server/schedules.rs @@ -0,0 +1,404 @@ +//! Scheduled-prompt request handlers (`ListSchedules` / +//! `RequestSchedulePrompt` / `CancelSchedule` / `EditSchedule` / +//! `FireScheduleNow`), their shared ownership check, and the +//! schedule-to-wire mapping reused by the dashboard +//! (`schedule_to_wire_public` / `filter_ghost_schedule_targets`). + +use std::sync::Arc; + +use hive_sh4re::AgentResponse; + +use crate::coordinator::Coordinator; + +/// `ListSchedules` — snapshot every scheduled prompt onto the wire. +pub(super) fn handle_list_schedules(coord: &Arc) -> AgentResponse { + match coord.scheduled_prompts.list() { + Ok(schedules) => AgentResponse::Schedules { + schedules: schedules.into_iter().map(schedule_to_wire).collect(), + }, + Err(e) => AgentResponse::Err { + message: format!("list scheduled prompts: {e:#}"), + }, + } +} + +/// Submit a `RequestSchedulePrompt` payload as an `ApprovalKind::SchedulePrompt` +/// row. Encodes the payload into the approval's `commit_ref` so the +/// approve handler can re-parse it without a side table. Validates +/// inputs (non-empty targets, non-empty body, sane interval) at +/// submit time — the operator should never see a malformed schedule +/// pending approval. +pub(super) fn handle_request_schedule_prompt( + coord: &Arc, + requester: &str, + payload: &hive_sh4re::SchedulePromptPayload, +) -> AgentResponse { + if payload.targets.is_empty() { + return AgentResponse::Err { + message: "schedule must have at least one target".into(), + }; + } + if payload.body.trim().is_empty() { + return AgentResponse::Err { + message: "schedule body must be non-empty".into(), + }; + } + if let Some(0) = payload.interval_seconds { + return AgentResponse::Err { + message: "interval_seconds must be > 0 (use None for one-shot)".into(), + }; + } + let commit_ref = match serde_json::to_string(payload) { + Ok(s) => s, + Err(e) => { + return AgentResponse::Err { + message: format!("encode SchedulePromptPayload: {e:#}"), + }; + } + }; + let id = match coord.approvals.submit_kind( + requester, + hive_sh4re::ApprovalKind::SchedulePrompt, + &commit_ref, + payload.description.as_deref(), + requester, + ) { + Ok(id) => id, + Err(e) => { + return AgentResponse::Err { + message: format!("queue schedule_prompt approval: {e:#}"), + }; + } + }; + tracing::info!( + %id, + requester, + targets = ?payload.targets, + first_fire_at = payload.first_fire_at_unix, + interval = ?payload.interval_seconds, + "schedule_prompt approval queued" + ); + coord.emit_approval_added(crate::coordinator::ApprovalAdded { + id, + agent: requester, + approval_kind: "schedule_prompt", + sha_short: None, + diff: None, + description: payload.description.clone(), + pr_number: None, + }); + AgentResponse::Ok +} + +/// Cancel a schedule (whole or per-target). Manager-surface +/// authorization: a manager can cancel its own schedules + any +/// schedule whose owner is one of its sub-agents (topology-walked). +/// The operator surface bypasses this and can cancel anything; +/// agents reaching this path through the manager get the +/// topology-scoped check. +pub(super) fn handle_cancel_schedule( + coord: &Arc, + requester: &str, + schedule_id: i64, + targets: Option<&[String]>, +) -> AgentResponse { + let schedule = match coord.scheduled_prompts.get(schedule_id) { + Ok(Some(s)) => s, + Ok(None) => { + return AgentResponse::Err { + message: format!("schedule {schedule_id} not found"), + }; + } + Err(e) => { + return AgentResponse::Err { + message: format!("read schedule {schedule_id}: {e:#}"), + }; + } + }; + if !cancel_authorized(requester, &schedule.owner) { + return AgentResponse::Err { + message: format!( + "not authorized: {requester} cannot cancel schedule owned by {owner}", + owner = schedule.owner + ), + }; + } + let result = match targets { + Some(list) if !list.is_empty() => coord + .scheduled_prompts + .cancel_targets(schedule_id, list) + .map_err(|e| format!("cancel targets: {e:#}")), + _ => coord + .scheduled_prompts + .cancel_all(schedule_id) + .map_err(|e| format!("cancel all: {e:#}")), + }; + match result { + Ok(()) => { + coord.emit_schedules_snapshot(); + AgentResponse::Ok + } + Err(message) => AgentResponse::Err { message }, + } +} + +/// Authorize + dispatch a `FireScheduleNow` request from the +/// manager surface. Same ownership rules as `CancelSchedule`: +/// requester can fire its own schedules + any owned by an agent +/// in its subtree. The actual fan-out lives in +/// `scheduled_prompts_worker::fire_now`. +pub(super) async fn handle_fire_schedule_now( + coord: &Arc, + requester: &str, + schedule_id: i64, +) -> AgentResponse { + let schedule = match coord.scheduled_prompts.get(schedule_id) { + Ok(Some(s)) => s, + Ok(None) => { + return AgentResponse::Err { + message: format!("schedule {schedule_id} not found"), + }; + } + Err(e) => { + return AgentResponse::Err { + message: format!("read schedule {schedule_id}: {e:#}"), + }; + } + }; + if !cancel_authorized(requester, &schedule.owner) { + return AgentResponse::Err { + message: format!( + "not authorized: {requester} cannot fire schedule owned by {owner}", + owner = schedule.owner + ), + }; + } + // MCP fire_schedule_now stays no-reset (cadence intact); the + // reset-timer option is a dashboard-dialog affordance. + match crate::scheduled_prompts_worker::fire_now(coord, schedule_id, false).await { + Ok(_report) => { + coord.emit_schedules_snapshot(); + AgentResponse::Ok + } + Err(e) => AgentResponse::Err { + message: format!("fire schedule {schedule_id} now: {e:#}"), + }, + } +} + +/// Field-named PATCH payload for [`handle_edit_schedule`]. Every +/// field is "leave alone" when `None`; the double-`Option` fields +/// additionally distinguish clear (`Some(None)`) from set +/// (`Some(Some(v))`). +#[allow( + clippy::option_option, + reason = "double-Option carries three-state PATCH semantics: outer None = \ + leave alone, Some(None) = clear, Some(Some(v)) = set" +)] +pub(super) struct EditSchedulePatch { + pub(super) body: Option, + pub(super) description: Option>, + pub(super) interval_seconds: Option>, + pub(super) next_fire_at_unix: Option, + pub(super) targets_add: Option>, + pub(super) targets_remove: Option>, +} + +/// Authorize + dispatch a `EditSchedule` patch. Same ownership +/// rules as `CancelSchedule` — the manager can edit +/// schedules it owns + any owned by an agent in its subtree. +/// Forwards the partial payload to +/// `ScheduledPrompts::update` which enforces the cancelled-row / +/// zero-interval validation. Returns `Ok` on a clean update; +/// `Err` with the underlying message on any auth / validation +/// failure so the dashboard can surface it verbatim. +pub(super) fn handle_edit_schedule( + coord: &Arc, + requester: &str, + schedule_id: i64, + patch: EditSchedulePatch, +) -> AgentResponse { + let EditSchedulePatch { + body, + description, + interval_seconds, + next_fire_at_unix, + targets_add, + targets_remove, + } = patch; + let schedule = match coord.scheduled_prompts.get(schedule_id) { + Ok(Some(s)) => s, + Ok(None) => { + return AgentResponse::Err { + message: format!("schedule {schedule_id} not found"), + }; + } + Err(e) => { + return AgentResponse::Err { + message: format!("read schedule {schedule_id}: {e:#}"), + }; + } + }; + if !cancel_authorized(requester, &schedule.owner) { + return AgentResponse::Err { + message: format!( + "not authorized: {requester} cannot edit schedule owned by {owner}", + owner = schedule.owner + ), + }; + } + let patch = crate::scheduled_prompts::UpdateSchedule { + body, + description, + interval_seconds, + next_fire_at_unix, + targets_add, + targets_remove, + }; + match coord.scheduled_prompts.update(schedule_id, patch) { + Ok(()) => { + coord.emit_schedules_snapshot(); + AgentResponse::Ok + } + Err(e) => AgentResponse::Err { + message: format!("edit schedule {schedule_id}: {e:#}"), + }, + } +} + +/// Permission check for `CancelSchedule` on the manager surface. +/// `requester` (always `ruth` here) can cancel its own schedules. +/// Sub-agent ownership is delegated to topology — see +/// `crate::topology::is_descendant_of`. Also reused by +/// `handle_fire_schedule_now` — fire-auth follows the same shape. +fn cancel_authorized(requester: &str, owner: &str) -> bool { + if requester == owner { + return true; + } + if requester == hive_sh4re::OPERATOR_RECIPIENT { + return true; + } + // Manager can cancel anything owned by an agent in its subtree. + // For the current single-manager topology that covers everything, + // but the check stays correct as the tree grows. + crate::topology::is_descendant_of(owner, requester) +} + +/// Map a `scheduled_prompts::Schedule` to its public wire shape. +/// Field-by-field copy — the two types are intentionally identical; +/// the separation keeps hive-sh4re free of hive-c0re-internal types. +/// Public alias `schedule_to_wire_public` re-exports for +/// `dashboard.rs::api_schedules` without crossing the module +/// boundary into the socket-server file. +pub fn schedule_to_wire_public(s: crate::scheduled_prompts::Schedule) -> hive_sh4re::WireSchedule { + schedule_to_wire(s) +} + +/// Drop schedule targets that point at agents which no longer exist, so +/// the dashboard's schedule table doesn't render ghost columns for +/// destroyed agents. `live` is the set of logical agent names from the +/// last `nixos-container list` scan (stopped agents included, destroyed +/// ones absent); the `operator` pseudo-target is always retained since +/// it isn't a container. Applied only to the dashboard wire paths +/// (`api_schedules` + the `SchedulesChanged` SSE emit) — the +/// manager-facing `list_schedules` stays unfiltered so agents can still +/// see and cancel stale targets. This is a view filter: the underlying +/// schedule rows keep every target, so a re-spawned agent's targets +/// reappear on their own. +pub(crate) fn filter_ghost_schedule_targets( + schedules: &mut [hive_sh4re::WireSchedule], + live: &std::collections::HashSet, +) { + for s in schedules.iter_mut() { + s.targets + .retain(|t| t.target == hive_sh4re::OPERATOR_RECIPIENT || live.contains(&t.target)); + } +} + +fn schedule_to_wire(s: crate::scheduled_prompts::Schedule) -> hive_sh4re::WireSchedule { + hive_sh4re::WireSchedule { + id: s.id, + owner: s.owner, + body: s.body, + interval_seconds: s.interval_seconds, + next_fire_at_unix: hive_sh4re::wire_time::from_secs(s.next_fire_at_unix), + created_at_unix: hive_sh4re::wire_time::from_secs(s.created_at_unix), + source: match s.source { + crate::scheduled_prompts::ScheduleSource::Operator => { + hive_sh4re::WireScheduleSource::Operator + } + crate::scheduled_prompts::ScheduleSource::Approval { id } => { + hive_sh4re::WireScheduleSource::Approval { id } + } + }, + cancelled_at_unix: s.cancelled_at_unix.map(hive_sh4re::wire_time::from_secs), + paused_at_unix: s.paused_at_unix.map(hive_sh4re::wire_time::from_secs), + description: s.description, + targets: s + .targets + .into_iter() + .map(|t| hive_sh4re::WireScheduleTarget { + target: t.target, + cancelled_at_unix: t.cancelled_at_unix.map(hive_sh4re::wire_time::from_secs), + last_fired_at_unix: t.last_fired_at_unix.map(hive_sh4re::wire_time::from_secs), + last_result: t.last_result, + }) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn target(name: &str) -> hive_sh4re::WireScheduleTarget { + hive_sh4re::WireScheduleTarget { + target: name.to_owned(), + cancelled_at_unix: None, + last_fired_at_unix: None, + last_result: None, + } + } + + fn schedule(targets: &[&str]) -> hive_sh4re::WireSchedule { + hive_sh4re::WireSchedule { + id: 1, + owner: "operator".to_owned(), + body: "ping".to_owned(), + interval_seconds: None, + next_fire_at_unix: hive_sh4re::wire_time::from_secs(0), + created_at_unix: hive_sh4re::wire_time::from_secs(0), + source: hive_sh4re::WireScheduleSource::Operator, + cancelled_at_unix: None, + paused_at_unix: None, + description: None, + targets: targets.iter().map(|t| target(t)).collect(), + } + } + + #[test] + fn ghost_filter_drops_dead_agents_keeps_live_and_operator() { + let live: std::collections::HashSet = ["iris".to_owned(), "damocles".to_owned()] + .into_iter() + .collect(); + let mut schedules = vec![schedule(&["iris", "ghost", "operator", "damocles"])]; + filter_ghost_schedule_targets(&mut schedules, &live); + let kept: Vec<&str> = schedules[0] + .targets + .iter() + .map(|t| t.target.as_str()) + .collect(); + // `ghost` (destroyed) dropped; live agents + operator pseudo-target kept. + assert_eq!(kept, vec!["iris", "operator", "damocles"]); + } + + #[test] + fn ghost_filter_can_empty_targets_when_all_dead() { + let live: std::collections::HashSet = std::collections::HashSet::new(); + let mut schedules = vec![schedule(&["gone1", "gone2"])]; + filter_ghost_schedule_targets(&mut schedules, &live); + // operator is never in the live set but is always retained; here + // there's no operator target, so everything drops. + assert!(schedules[0].targets.is_empty()); + } +} diff --git a/hive-c0re/src/container_stats.rs b/hive-c0re/src/stats/container_stats.rs similarity index 100% rename from hive-c0re/src/container_stats.rs rename to hive-c0re/src/stats/container_stats.rs diff --git a/hive-c0re/src/hive_stats.rs b/hive-c0re/src/stats/hive_stats.rs similarity index 98% rename from hive-c0re/src/hive_stats.rs rename to hive-c0re/src/stats/hive_stats.rs index f3bd6e2e..da1f2d49 100644 --- a/hive-c0re/src/hive_stats.rs +++ b/hive-c0re/src/stats/hive_stats.rs @@ -18,12 +18,13 @@ use std::collections::HashMap; use std::path::Path; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use rusqlite::{Connection, OpenFlags}; use serde::{Deserialize, Serialize}; use crate::coordinator::Coordinator; +use hive_sh4re::wire_time::now_unix; /// Window accepted by `/api/stats-hive?window=`. Maps to a lookback /// span; the hive view is a flat rollup (no per-bucket trend — the @@ -225,12 +226,6 @@ struct AgentAgg { bash: HashMap, } -fn now_secs() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX)) -} - #[allow( clippy::cast_sign_loss, clippy::cast_possible_truncation, @@ -322,7 +317,7 @@ fn read_bash_heads(conn: &Connection, from: i64) -> HashMap { /// per-agent db is skipped (logged), never fatal. #[must_use] pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats { - let now = now_secs(); + let now = now_unix(); // Fixed windows look back a constant span; `all` aggregates every // recorded turn (`from == 0`). The hive rollup isn't time-bucketed, so // unlike the per-agent snapshot it needs no adaptive bucket sizing. @@ -426,7 +421,7 @@ mod tests { let conn = Connection::open_in_memory().unwrap(); conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);") .unwrap(); - let now = now_secs(); + let now = now_unix(); for (ts, head) in [ (now - 100, "cargo"), (now - 200, "cargo"), diff --git a/hive-c0re/src/host_stats.rs b/hive-c0re/src/stats/host_stats.rs similarity index 100% rename from hive-c0re/src/host_stats.rs rename to hive-c0re/src/stats/host_stats.rs diff --git a/hive-c0re/src/stats/mod.rs b/hive-c0re/src/stats/mod.rs new file mode 100644 index 00000000..ee528cdc --- /dev/null +++ b/hive-c0re/src/stats/mod.rs @@ -0,0 +1,8 @@ +//! Metrics aggregation for the dashboard: hive-wide turn-stats +//! rollups, host-system probes / server warnings, and live +//! per-container cgroup load. Each submodule is re-exported at the +//! crate root, so `crate::hive_stats::…` etc. keep working unchanged. + +pub mod container_stats; +pub mod hive_stats; +pub mod host_stats; diff --git a/hive-c0re/src/approvals.rs b/hive-c0re/src/stores/approvals.rs similarity index 81% rename from hive-c0re/src/approvals.rs rename to hive-c0re/src/stores/approvals.rs index 6fb8dc6a..f9235a2d 100644 --- a/hive-c0re/src/approvals.rs +++ b/hive-c0re/src/stores/approvals.rs @@ -4,9 +4,9 @@ use std::path::Path; use std::sync::Mutex; -use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; +use hive_sh4re::wire_time::now_unix; use hive_sh4re::{Approval, ApprovalKind, ApprovalStatus}; use rusqlite::{Connection, OptionalExtension, params}; @@ -24,66 +24,26 @@ CREATE INDEX IF NOT EXISTS idx_approvals_pending ON approvals (id) WHERE status = 'pending'; "; -/// Add the `description` column to pre-description databases. Manager-supplied -/// note shown on the dashboard approval card at submission time (distinct from -/// `note` which is set on denial/failure). -fn ensure_description_column(conn: &Connection) -> Result<()> { - let has: bool = conn - .prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'description'")? - .exists([])?; - if !has { - conn.execute_batch("ALTER TABLE approvals ADD COLUMN description TEXT;") - .context("add approvals.description column")?; - } - Ok(()) -} - -/// Add the `kind` column to pre-Phase-8 databases. ALTER TABLE ADD COLUMN is -/// idempotent here only via a column-existence check (sqlite doesn't support -/// IF NOT EXISTS on ADD COLUMN). Defaults legacy rows to `apply_commit`, -/// which matches their actual semantics. -fn ensure_kind_column(conn: &Connection) -> Result<()> { - let has_kind: bool = conn - .prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'kind'")? - .exists([])?; - if !has_kind { - conn.execute_batch( - "ALTER TABLE approvals ADD COLUMN kind TEXT NOT NULL DEFAULT 'apply_commit';", - ) - .context("add approvals.kind column")?; - } - Ok(()) -} - -/// Same shape as `ensure_kind_column` but for `fetched_sha` — the -/// canonical sha hive-c0re vouched for at `request_apply_commit` time. -/// Distinct from `commit_ref` (manager-supplied, may not even resolve -/// in proposed by the time we approve). -fn ensure_fetched_sha_column(conn: &Connection) -> Result<()> { - let has: bool = conn - .prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'fetched_sha'")? - .exists([])?; - if !has { - conn.execute_batch("ALTER TABLE approvals ADD COLUMN fetched_sha TEXT;") - .context("add approvals.fetched_sha column")?; - } - Ok(()) -} - -/// Same shape as `ensure_fetched_sha_column` but for `submitter` — the -/// agent that submitted the approval (the authenticated socket caller). -/// Approval-scoped helper events route to this agent. Legacy rows have -/// NULL; callers fall back to the root agent for those. -fn ensure_submitter_column(conn: &Connection) -> Result<()> { - let has: bool = conn - .prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'submitter'")? - .exists([])?; - if !has { - conn.execute_batch("ALTER TABLE approvals ADD COLUMN submitter TEXT;") - .context("add approvals.submitter column")?; - } - Ok(()) -} +/// Additive column migrations for pre-existing databases, applied via +/// `db::apply_migrations` (try-and-ignore-duplicate-column). +const MIGRATIONS: &[&str] = &[ + // `kind` (pre-Phase-8 dbs): legacy rows default to `apply_commit`, + // which matches their actual semantics. + "ALTER TABLE approvals ADD COLUMN kind TEXT NOT NULL DEFAULT 'apply_commit'", + // `description`: manager-supplied note shown on the dashboard + // approval card at submission time (distinct from `note`, set on + // denial/failure). + "ALTER TABLE approvals ADD COLUMN description TEXT", + // `fetched_sha`: the canonical sha hive-c0re vouched for at + // `request_apply_commit` time. Distinct from `commit_ref` + // (manager-supplied, may not even resolve by approve time). + "ALTER TABLE approvals ADD COLUMN fetched_sha TEXT", + // `submitter`: the agent that submitted the approval (the + // authenticated socket caller); approval-scoped helper events + // route to it. Legacy rows are NULL → callers fall back to the + // root agent. + "ALTER TABLE approvals ADD COLUMN submitter TEXT", +]; pub struct Approvals { conn: Mutex, @@ -91,18 +51,10 @@ pub struct Approvals { impl Approvals { pub fn open(path: &Path) -> Result { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("create approvals db parent {}", parent.display()))?; - } - let conn = Connection::open(path) - .with_context(|| format!("open approvals db {}", path.display()))?; + let conn = crate::db::open(path, "approvals")?; conn.execute_batch(SCHEMA) .context("apply approvals schema")?; - ensure_kind_column(&conn).context("migrate approvals.kind")?; - ensure_fetched_sha_column(&conn).context("migrate approvals.fetched_sha")?; - ensure_description_column(&conn).context("migrate approvals.description")?; - ensure_submitter_column(&conn).context("migrate approvals.submitter")?; + crate::db::apply_migrations(&conn, "approvals", MIGRATIONS)?; Ok(Self { conn: Mutex::new(conn), }) @@ -123,7 +75,7 @@ impl Approvals { VALUES (?1, ?2, ?3, ?4, 'pending', ?5, ?6)", params![ agent, - kind_to_str(kind), + kind.as_str(), commit_ref, now_unix(), description, @@ -415,21 +367,6 @@ fn row_to_approval(row: &rusqlite::Row<'_>) -> rusqlite::Result { }) } -/// Stable kind→str mapping used wherever we emit `ApprovalResolved` -/// or persist a kind to sqlite. `pub(crate)` so callers like -/// `questions::handle_cancel_loose_end` don't have to duplicate the -/// match; bumping a kind here is the single source of truth. -pub(crate) fn kind_to_str(kind: ApprovalKind) -> &'static str { - match kind { - ApprovalKind::ApplyCommit => "apply_commit", - ApprovalKind::Spawn => "spawn", - ApprovalKind::InitConfig => "init_config", - ApprovalKind::UpdateMetaInputs => "update_meta_inputs", - ApprovalKind::SchedulePrompt => "schedule_prompt", - ApprovalKind::MergeConfigPr => "merge_config_pr", - } -} - fn kind_from_str(s: &str) -> Result { Ok(match s { "apply_commit" => ApprovalKind::ApplyCommit, @@ -442,14 +379,6 @@ fn kind_from_str(s: &str) -> Result { }) } -fn now_unix() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - #[cfg(test)] mod tests { use super::*; diff --git a/hive-c0re/src/audit_log.rs b/hive-c0re/src/stores/audit_log.rs similarity index 95% rename from hive-c0re/src/audit_log.rs rename to hive-c0re/src/stores/audit_log.rs index c145dde9..6dee6ea0 100644 --- a/hive-c0re/src/audit_log.rs +++ b/hive-c0re/src/stores/audit_log.rs @@ -21,11 +21,11 @@ use std::path::Path; use std::sync::{Arc, Mutex, OnceLock}; -use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; +use hive_sh4re::wire_time::now_unix; use rusqlite::{Connection, params}; use serde::Serialize; @@ -119,11 +119,8 @@ impl AuditLog { /// Returns an error if the directory can't be created, the sqlite /// file can't be opened, or applying the schema fails. pub fn open(db_dir: &Path) -> Result { - std::fs::create_dir_all(db_dir) - .with_context(|| format!("create audit_log db parent {}", db_dir.display()))?; let path = db_dir.join("audit_log.sqlite"); - let conn = Connection::open(&path) - .with_context(|| format!("open audit_log db {}", path.display()))?; + let conn = crate::db::open(&path, "audit_log")?; conn.execute_batch(SCHEMA) .context("apply audit_log schema")?; Ok(Self { @@ -151,7 +148,7 @@ impl AuditLog { outcome: AuditOutcome, detail: Option<&str>, ) -> Option { - let now = now_secs(); + let now = now_unix(); let conn = self.conn.lock().unwrap(); match conn.execute( "INSERT INTO audit_log (ts_unix, agent, action, target, outcome, detail) @@ -218,7 +215,7 @@ impl AuditLog { /// # Errors /// Returns an error if the `DELETE` query fails. pub fn vacuum(&self) -> Result { - let cutoff = now_secs() - KEEP_SECS; + let cutoff = now_unix() - KEEP_SECS; let conn = self.conn.lock().unwrap(); let removed = conn.execute("DELETE FROM audit_log WHERE ts_unix < ?1", params![cutoff])?; Ok(u64::try_from(removed).unwrap_or(0)) @@ -262,14 +259,6 @@ fn row_to_entry(r: &rusqlite::Row) -> rusqlite::Result { }) } -fn now_secs() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - #[cfg(test)] mod tests { use super::*; @@ -329,7 +318,7 @@ mod tests { let conn = db.conn.lock().unwrap(); conn.execute( "UPDATE audit_log SET ts_unix = ?1", - params![now_secs() - KEEP_SECS - 60], + params![now_unix() - KEEP_SECS - 60], ) .unwrap(); } diff --git a/hive-c0re/src/broker.rs b/hive-c0re/src/stores/broker.rs similarity index 97% rename from hive-c0re/src/broker.rs rename to hive-c0re/src/stores/broker.rs index c80d1ef4..c816029e 100644 --- a/hive-c0re/src/broker.rs +++ b/hive-c0re/src/stores/broker.rs @@ -4,11 +4,11 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; use std::sync::Mutex; -use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; +use hive_sh4re::wire_time::now_unix; use hive_sh4re::{InboxRow, Message}; use rusqlite::{Connection, OptionalExtension, params}; use serde::Serialize; @@ -157,12 +157,7 @@ pub struct Broker { impl Broker { pub fn open(path: &Path) -> Result { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("create db parent {}", parent.display()))?; - } - let conn = - Connection::open(path).with_context(|| format!("open broker db {}", path.display()))?; + let conn = crate::db::open(path, "broker")?; conn.execute_batch(SCHEMA).context("apply broker schema")?; ensure_message_columns(&conn).context("migrate messages columns")?; ensure_reminder_columns(&conn).context("migrate reminders columns")?; @@ -1126,41 +1121,19 @@ fn ensure_message_columns(conn: &Connection) -> Result<()> { Ok(()) } -/// Idempotent reminder-table migrations. `ALTER TABLE ADD COLUMN` -/// has no `IF NOT EXISTS` form in sqlite, so we probe -/// `pragma_table_info` per column. New deploys (table created by -/// SCHEMA in this commit cycle) skip the ALTER; pre-existing -/// broker.sqlite files get the columns added on next boot. +/// Idempotent reminder-table migrations — plain additive columns, via +/// `db::apply_migrations`. (The messages-table migration above stays +/// bespoke: its backfill must run only when `acked_at` was just +/// created, which try-and-ignore can't express.) fn ensure_reminder_columns(conn: &Connection) -> Result<()> { - for (name, sql) in [ - ( - "attempt_count", - "ALTER TABLE reminders ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0;", - ), - ( - "last_error", - "ALTER TABLE reminders ADD COLUMN last_error TEXT;", - ), - ] { - let has: bool = conn - .prepare(&format!( - "SELECT 1 FROM pragma_table_info('reminders') WHERE name = '{name}'" - ))? - .exists([])?; - if !has { - conn.execute_batch(sql) - .with_context(|| format!("add reminders.{name} column"))?; - } - } - Ok(()) -} - -fn now_unix() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) + crate::db::apply_migrations( + conn, + "broker reminders", + &[ + "ALTER TABLE reminders ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE reminders ADD COLUMN last_error TEXT", + ], + ) } #[cfg(test)] diff --git a/hive-c0re/src/build_logs.rs b/hive-c0re/src/stores/build_logs.rs similarity index 97% rename from hive-c0re/src/build_logs.rs rename to hive-c0re/src/stores/build_logs.rs index 76679abc..87f271c6 100644 --- a/hive-c0re/src/build_logs.rs +++ b/hive-c0re/src/stores/build_logs.rs @@ -5,9 +5,9 @@ use std::path::Path; use std::sync::{Arc, Mutex, OnceLock}; -use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; +use hive_sh4re::wire_time::now_unix; use rusqlite::{Connection, OptionalExtension, params}; use serde::Serialize; use tokio::sync::broadcast; @@ -147,11 +147,8 @@ pub struct BuildLogs { impl BuildLogs { pub fn open(db_dir: &Path) -> Result { - std::fs::create_dir_all(db_dir) - .with_context(|| format!("create build_logs db parent {}", db_dir.display()))?; let path = db_dir.join("build_logs.sqlite"); - let conn = Connection::open(&path) - .with_context(|| format!("open build_logs db {}", path.display()))?; + let conn = crate::db::open(&path, "build_logs")?; conn.execute_batch(SCHEMA) .context("apply build_logs schema")?; let (notify_tx, _) = broadcast::channel(NOTIFY_CAP); @@ -172,7 +169,7 @@ impl BuildLogs { /// — the caller threads it through `append_stdout` / `append_stderr` /// while the child runs and into `finish` once it exits. pub fn start(&self, agent: &str, kind: &str, cmdline: &str) -> Result { - let now = now_secs(); + let now = now_unix(); let conn = self.conn.lock().unwrap(); conn.execute( "INSERT INTO build_logs (agent, kind, cmdline, started_at) VALUES (?1, ?2, ?3, ?4)", @@ -222,7 +219,7 @@ impl BuildLogs { /// Finalize a build attempt. Sets `finished_at` to now and /// `status` to the terminal state. Best-effort. pub fn finish(&self, id: i64, status: BuildStatus) { - let now = now_secs(); + let now = now_unix(); let conn = self.conn.lock().unwrap(); if let Err(e) = conn.execute( "UPDATE build_logs SET finished_at = ?1, status = ?2 WHERE id = ?3", @@ -377,7 +374,7 @@ impl BuildLogs { /// a long-running build shouldn't disappear from its own log /// viewer mid-stream. pub fn vacuum(&self) -> Result { - let now = now_secs(); + let now = now_unix(); let conn = self.conn.lock().unwrap(); let fail_cutoff = now - KEEP_FAIL_SECS; let ok_cutoff = now - KEEP_OK_SECS; @@ -441,14 +438,6 @@ fn row_to_header(r: &rusqlite::Row) -> rusqlite::Result { }) } -fn now_secs() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - #[cfg(test)] mod tests { use super::*; @@ -536,7 +525,7 @@ mod tests { // stays within KEEP_FAIL_SECS so it survives; old_fail goes // beyond; old_ok goes past KEEP_OK_SECS but inside // KEEP_FAIL_SECS — proves the per-status rule. - let now = now_secs(); + let now = now_unix(); { let conn = db.conn.lock().unwrap(); conn.execute( diff --git a/hive-c0re/src/stores/db.rs b/hive-c0re/src/stores/db.rs new file mode 100644 index 00000000..f483d3d2 --- /dev/null +++ b/hive-c0re/src/stores/db.rs @@ -0,0 +1,58 @@ +//! Shared sqlite connection setup for hive-c0re's host-side stores. +//! +//! Several modules keep their own tables — and their own +//! `Mutex` — in the coordinator DB +//! (`db/broker.sqlite`: broker, approvals, operator questions, +//! scheduled prompts, agent power) or in a sibling file under the same +//! `db/` dir (`build_logs.sqlite`, `audit_log.sqlite`). The open dance +//! is identical everywhere: ensure the parent dir exists, open the +//! connection, set a busy timeout so concurrent same-process writers +//! wait each other out instead of surfacing `SQLITE_BUSY`. This helper +//! owns that dance; schema creation + column migrations stay with each +//! store (they're per-table concerns). + +use std::path::Path; +use std::time::Duration; + +use anyhow::{Context, Result}; +use rusqlite::Connection; + +/// How long a write waits on another connection's lock before erroring. +/// Generous relative to the stores' tiny transactions — a timeout here +/// means something is genuinely wedged, not ordinary contention. +const BUSY_TIMEOUT: Duration = Duration::from_secs(5); + +/// Apply additive, idempotent migrations: run each statement and +/// ignore `duplicate column name` errors (sqlite has no +/// `ADD COLUMN IF NOT EXISTS`; try-and-ignore is the portable path — +/// same pattern as hive-ag3nt's `turn_stats`). Any other error +/// propagates. Fits plain `ALTER TABLE ADD COLUMN` (new columns must +/// carry a default or tolerate NULL) and `IF NOT EXISTS` index DDL; +/// migrations with creation-conditional backfills (broker's +/// `acked_at`) stay bespoke in their store. +pub fn apply_migrations(conn: &Connection, subsystem: &str, statements: &[&str]) -> Result<()> { + for stmt in statements { + if let Err(e) = conn.execute_batch(stmt) { + if e.to_string().contains("duplicate column name") { + continue; + } + return Err(e).with_context(|| format!("{subsystem} migration failed: {stmt}")); + } + } + Ok(()) +} + +/// Open a connection to the sqlite file at `path`, creating the parent +/// directory if needed. `subsystem` labels error contexts (`"broker"`, +/// `"approvals"`, …). +pub fn open(path: &Path, subsystem: &str) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create {subsystem} db parent {}", parent.display()))?; + } + let conn = Connection::open(path) + .with_context(|| format!("open {subsystem} db {}", path.display()))?; + conn.busy_timeout(BUSY_TIMEOUT) + .with_context(|| format!("set {subsystem} busy_timeout"))?; + Ok(conn) +} diff --git a/hive-c0re/src/stores/mod.rs b/hive-c0re/src/stores/mod.rs new file mode 100644 index 00000000..1c02820c --- /dev/null +++ b/hive-c0re/src/stores/mod.rs @@ -0,0 +1,14 @@ +//! Sqlite-backed host-side stores (broker, approval / question / +//! schedule queues, build logs, audit trail, power intent) plus the +//! shared connection open/migration helper (`db`). Each submodule is +//! re-exported at the crate root, so `crate::broker::…` etc. keep +//! working unchanged. + +pub mod approvals; +pub mod audit_log; +pub mod broker; +pub mod build_logs; +pub mod db; +pub mod operator_questions; +pub mod power; +pub mod scheduled_prompts; diff --git a/hive-c0re/src/operator_questions.rs b/hive-c0re/src/stores/operator_questions.rs similarity index 83% rename from hive-c0re/src/operator_questions.rs rename to hive-c0re/src/stores/operator_questions.rs index 604c43c8..2a99699f 100644 --- a/hive-c0re/src/operator_questions.rs +++ b/hive-c0re/src/stores/operator_questions.rs @@ -11,11 +11,11 @@ use std::path::Path; use std::sync::Mutex; -use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; use chrono::{DateTime, Utc}; +use hive_sh4re::wire_time::now_unix; use rusqlite::{Connection, OptionalExtension, params}; use serde::Serialize; @@ -33,41 +33,18 @@ CREATE INDEX IF NOT EXISTS idx_operator_questions_pending ON operator_questions (id) WHERE answered_at IS NULL; "; -/// Add late-added columns to pre-existing databases. `ALTER TABLE -/// ADD COLUMN` has no `IF NOT EXISTS` form in sqlite, so we check -/// `pragma_table_info` first per column. -fn ensure_columns(conn: &Connection) -> Result<()> { - for (name, sql) in [ - ( - "multi", - "ALTER TABLE operator_questions ADD COLUMN multi INTEGER NOT NULL DEFAULT 0;", - ), - ( - "deadline_at", - "ALTER TABLE operator_questions ADD COLUMN deadline_at INTEGER;", - ), - // `target` = recipient of the question. NULL = operator - // (back-compat default for rows written before agent-to-agent - // questions existed); a non-null agent name = peer-to-peer - // question. Dashboard's `pending()` filters on `target IS NULL` - // so peer questions never leak into the operator's queue. - ( - "target", - "ALTER TABLE operator_questions ADD COLUMN target TEXT;", - ), - ] { - let has: bool = conn - .prepare(&format!( - "SELECT 1 FROM pragma_table_info('operator_questions') WHERE name = '{name}'" - ))? - .exists([])?; - if !has { - conn.execute_batch(sql) - .with_context(|| format!("add operator_questions.{name} column"))?; - } - } - Ok(()) -} +/// Additive column migrations for pre-existing databases, applied via +/// `db::apply_migrations` (try-and-ignore-duplicate-column). +const MIGRATIONS: &[&str] = &[ + "ALTER TABLE operator_questions ADD COLUMN multi INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE operator_questions ADD COLUMN deadline_at INTEGER", + // `target` = recipient of the question. NULL = operator + // (back-compat default for rows written before agent-to-agent + // questions existed); a non-null agent name = peer-to-peer + // question. Dashboard's `pending()` filters on `target IS NULL` + // so peer questions never leak into the operator's queue. + "ALTER TABLE operator_questions ADD COLUMN target TEXT", +]; #[derive(Debug, Clone, Serialize)] pub struct OpQuestion { @@ -97,16 +74,10 @@ pub struct OperatorQuestions { impl OperatorQuestions { pub fn open(path: &Path) -> Result { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).with_context(|| { - format!("create operator_questions db parent {}", parent.display()) - })?; - } - let conn = Connection::open(path) - .with_context(|| format!("open operator_questions db {}", path.display()))?; + let conn = crate::db::open(path, "operator_questions")?; conn.execute_batch(SCHEMA) .context("apply operator_questions schema")?; - ensure_columns(&conn).context("migrate operator_questions columns")?; + crate::db::apply_migrations(&conn, "operator_questions", MIGRATIONS)?; Ok(Self { conn: Mutex::new(conn), }) @@ -300,11 +271,3 @@ fn row_to_question(row: &rusqlite::Row<'_>) -> rusqlite::Result { target: row.get(9)?, }) } - -fn now_unix() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} diff --git a/hive-c0re/src/stores/power.rs b/hive-c0re/src/stores/power.rs new file mode 100644 index 00000000..a5900adb --- /dev/null +++ b/hive-c0re/src/stores/power.rs @@ -0,0 +1,198 @@ +//! Durable per-agent power *intent* (`wanted: Up | Offline`) — the +//! spec half of spec-vs-status desired-state reconciliation. +//! `container_view` remains the observed *status*; the job queue's +//! `Reconcile` nodes are the mechanism that converges the two. +//! +//! Stored as the `agent_power` table in the coordinator DB +//! (`/var/lib/hyperhive/db/broker.sqlite`, one tiny row per agent) — +//! same one-file-many-modules pattern as `approvals` / +//! `operator_questions` / `scheduled_prompts`, each with its own +//! connection. Intent persists across hive-c0re restarts; in-flight +//! queue work deliberately does not. Setting `wanted` is never a +//! queued node: operator/intent actions update the row synchronously +//! at request time, then submit the DAG whose terminal `Reconcile` +//! reads the fresh value — rapid toggles are last-writer-wins and the +//! reconciles converge. Power toggles never commit to the meta repo. + +use std::path::Path; +use std::sync::Mutex; + +use anyhow::{Context, Result}; +use hive_sh4re::wire_time::now_unix; +use rusqlite::{Connection, OptionalExtension, params}; + +const SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS agent_power ( + agent TEXT PRIMARY KEY, + wanted TEXT NOT NULL, + updated_at INTEGER NOT NULL +); +"; + +/// Per-agent power intent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Wanted { + Up, + Offline, +} + +impl Wanted { + pub fn as_str(self) -> &'static str { + match self { + Wanted::Up => "up", + Wanted::Offline => "offline", + } + } + + fn parse(s: &str) -> Option { + match s { + "up" => Some(Wanted::Up), + "offline" => Some(Wanted::Offline), + _ => None, + } + } + + /// Seed value from an observed running state (first boot after + /// this store lands, or an agent spawned outside the normal path). + pub fn from_running(running: bool) -> Self { + if running { Wanted::Up } else { Wanted::Offline } + } +} + +/// What a `Reconcile` should do given intent + observation. Pure so +/// the `{Up,Offline} × {up,down}` matrix is unit-testable. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReconcileAction { + Start, + Stop, + Noop, +} + +#[must_use] +pub fn reconcile_action(wanted: Wanted, running: bool) -> ReconcileAction { + match (wanted, running) { + (Wanted::Up, false) => ReconcileAction::Start, + (Wanted::Offline, true) => ReconcileAction::Stop, + (Wanted::Up, true) | (Wanted::Offline, false) => ReconcileAction::Noop, + } +} + +/// Sqlite-backed store. `Arc`-friendly: all methods take `&self`, the +/// internal `Mutex` serializes access. +pub struct PowerStore { + conn: Mutex, +} + +impl PowerStore { + /// Open (a connection to) the shared coordinator DB and ensure the + /// `agent_power` table exists. `db_path` is the same sqlite file + /// the broker / approvals / questions stores open. + pub fn open(db_path: &Path) -> Result { + let conn = crate::db::open(db_path, "agent_power")?; + conn.execute_batch(SCHEMA) + .context("apply agent_power schema")?; + Ok(Self { + conn: Mutex::new(conn), + }) + } + + /// In-memory store for tests. + #[cfg(test)] + pub fn open_in_memory() -> Result { + let conn = Connection::open_in_memory().context("open in-memory agent_power db")?; + conn.execute_batch(SCHEMA) + .context("apply agent_power schema")?; + Ok(Self { + conn: Mutex::new(conn), + }) + } + + /// Read an agent's intent. `None` when the agent has no row yet + /// (callers seed from observed state via [`Self::get_or_seed`]). + pub fn get(&self, agent: &str) -> Result> { + let conn = self.conn.lock().expect("agent_power mutex poisoned"); + let row: Option = conn + .query_row( + "SELECT wanted FROM agent_power WHERE agent = ?1", + params![agent], + |r| r.get(0), + ) + .optional() + .context("select agent_power")?; + Ok(row.and_then(|s| Wanted::parse(&s))) + } + + /// Write an agent's intent (last-writer-wins, synchronous at + /// request time). + pub fn set(&self, agent: &str, wanted: Wanted) -> Result<()> { + let conn = self.conn.lock().expect("agent_power mutex poisoned"); + conn.execute( + "INSERT INTO agent_power (agent, wanted, updated_at) VALUES (?1, ?2, ?3) + ON CONFLICT(agent) DO UPDATE SET wanted = ?2, updated_at = ?3", + params![agent, wanted.as_str(), now_unix()], + ) + .context("upsert agent_power")?; + Ok(()) + } + + /// Read an agent's intent, seeding the row from the observed + /// running state when absent — the migration rule for agents that + /// predate this store (running ⇒ `Up`, stopped ⇒ `Offline`), after + /// which the DB is authoritative. + pub fn get_or_seed(&self, agent: &str, running: bool) -> Result { + if let Some(w) = self.get(agent)? { + return Ok(w); + } + let seeded = Wanted::from_running(running); + self.set(agent, seeded)?; + tracing::info!(%agent, wanted = seeded.as_str(), "agent_power: seeded from observed state"); + Ok(seeded) + } + + /// Drop an agent's row (container destroyed). + pub fn remove(&self, agent: &str) -> Result<()> { + let conn = self.conn.lock().expect("agent_power mutex poisoned"); + conn.execute("DELETE FROM agent_power WHERE agent = ?1", params![agent]) + .context("delete agent_power")?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The full `{Up,Offline} × {up,down}` reconcile matrix: + /// start / stop / noop / noop. + #[test] + fn reconcile_matrix() { + assert_eq!(reconcile_action(Wanted::Up, false), ReconcileAction::Start); + assert_eq!( + reconcile_action(Wanted::Offline, true), + ReconcileAction::Stop + ); + assert_eq!(reconcile_action(Wanted::Up, true), ReconcileAction::Noop); + assert_eq!( + reconcile_action(Wanted::Offline, false), + ReconcileAction::Noop + ); + } + + #[test] + fn get_set_roundtrip_and_seed() { + let store = PowerStore::open_in_memory().expect("open"); + assert_eq!(store.get("alice").expect("get"), None); + // Seed from observed running state, once. + assert_eq!(store.get_or_seed("alice", true).expect("seed"), Wanted::Up); + // Thereafter the DB is authoritative — observed state no longer + // overrides. + assert_eq!( + store.get_or_seed("alice", false).expect("seeded"), + Wanted::Up + ); + store.set("alice", Wanted::Offline).expect("set"); + assert_eq!(store.get("alice").expect("get"), Some(Wanted::Offline)); + store.remove("alice").expect("remove"); + assert_eq!(store.get("alice").expect("get"), None); + } +} diff --git a/hive-c0re/src/scheduled_prompts.rs b/hive-c0re/src/stores/scheduled_prompts.rs similarity index 98% rename from hive-c0re/src/scheduled_prompts.rs rename to hive-c0re/src/stores/scheduled_prompts.rs index 9095e7b5..7aaaff1f 100644 --- a/hive-c0re/src/scheduled_prompts.rs +++ b/hive-c0re/src/stores/scheduled_prompts.rs @@ -16,6 +16,7 @@ use std::path::Path; use std::sync::Mutex; use anyhow::{Context, Result, bail}; +use hive_sh4re::wire_time::now_unix; use rusqlite::{Connection, OptionalExtension, params}; use serde::{Deserialize, Serialize}; @@ -178,13 +179,7 @@ pub struct ScheduledPrompts { impl ScheduledPrompts { pub fn open(path: &Path) -> Result { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).with_context(|| { - format!("create scheduled_prompts db parent {}", parent.display()) - })?; - } - let conn = Connection::open(path) - .with_context(|| format!("open scheduled_prompts db {}", path.display()))?; + let conn = crate::db::open(path, "scheduled_prompts")?; // Required for ON DELETE CASCADE to actually fire — sqlite // ships with FKs disabled per connection by default. conn.execute_batch("PRAGMA foreign_keys = ON;") @@ -192,12 +187,11 @@ impl ScheduledPrompts { conn.execute_batch(SCHEMA) .context("apply scheduled_prompts schema")?; // Migration: add paused_at_unix to existing databases. - // Silently ignores "duplicate column name" errors so this is - // idempotent across daemon restarts on already-migrated DBs. - let _ = conn.execute( - "ALTER TABLE scheduled_prompts ADD COLUMN paused_at_unix INTEGER", - [], - ); + crate::db::apply_migrations( + &conn, + "scheduled_prompts", + &["ALTER TABLE scheduled_prompts ADD COLUMN paused_at_unix INTEGER"], + )?; // Migration: recreate the due-rows index to also exclude paused // rows. `CREATE INDEX IF NOT EXISTS` won't update an existing // index's WHERE clause, so we drop + recreate on every open. @@ -681,14 +675,6 @@ fn load_targets(conn: &Connection, schedule_id: i64) -> Result i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - #[cfg(test)] mod tests { use super::*; diff --git a/hive-c0re/src/agent_sockets.rs b/hive-c0re/src/workers/agent_sockets.rs similarity index 100% rename from hive-c0re/src/agent_sockets.rs rename to hive-c0re/src/workers/agent_sockets.rs diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs new file mode 100644 index 00000000..590ff450 --- /dev/null +++ b/hive-c0re/src/workers/auto_update.rs @@ -0,0 +1,320 @@ +//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path +//! for agents whose per-agent rev marker is stale — a `StartupSweep` +//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for +//! the stale agents whose `wanted` power intent is `Up` — and (b) +//! converge every other drifted agent to its persisted `wanted` via +//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal: +//! +//! 1. **Stale but wanted-offline agents** get no rebuild — their +//! rebuild happens the first time they're started (the start +//! submit path upgrades a stale start to rebuild+start). The sweep +//! parent still runs whenever *any* marker is stale so the meta +//! hyperhive lock is bumped for those later start-upgrades. +//! 2. **Agents whose rev marker matches** the current hyperhive flake +//! path are skipped — nothing changed, no nix work to do. +//! +//! Booting with no config change performs no meta commit — only +//! reconciles. See `docs/coordinator.md::Boot reconcile`. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::Result; + +use crate::coordinator::Coordinator; +use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME}; + +/// Marker file recording the hyperhive rev a sub-agent's container was last +/// built against. Sibling of `applied//` (rather than inside it) to +/// keep it out of the applied repo's git history. Uses a leading dot so a +/// glob over `applied/*` doesn't include it. +pub fn rev_marker_path(name: &str) -> PathBuf { + PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev")) +} + +/// Resolve the current rev of `hyperhive_flake`. For a path on disk we +/// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/... +/// update yields a different string. For anything else we return None. +#[must_use] +pub fn current_flake_rev(hyperhive_flake: &str) -> Option { + let path = Path::new(hyperhive_flake); + if !path.exists() { + return None; + } + std::fs::canonicalize(path) + .ok() + .map(|p| p.display().to_string()) +} + +/// Returns true when the applied repo has commits that have not yet been +/// deployed (i.e. the applied HEAD differs from the sha currently locked in +/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip +/// conveys: "there is a config change ready to apply via rebuild." +/// +/// Async on purpose: this runs per agent inside `container_view::build_all`, +/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and +/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous +/// `git` fork here blocks a tokio worker for the whole exec — under +/// nix-build disk saturation that's long enough that concurrent sweeps +/// starved the runtime and stalled the per-agent sockets. +pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool { + let applied_head = tokio::process::Command::new("git") + .args([ + "-C", + &format!("/var/lib/hyperhive/applied/{name}"), + "rev-parse", + "HEAD", + ]) + .output() + .await + .ok() + .filter(|o| o.status.success()) + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_owned()); + + match (applied_head.as_deref(), deployed_sha) { + (Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head), + _ => false, + } +} + +/// Whether this hive is "ruthless" — running with no root/manager agent at +/// all (no ruth). When true, hive-c0re skips the root-agent create/start +/// sweep entirely. Controlled by the host option +/// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS` +/// env var. Defaults to `false` when the var is unset (back-compat: the +/// root agent was always auto-managed before this opt-out existed); only +/// an explicit `true` / `1` / `yes` enables ruthless mode. +fn ruthless() -> bool { + match std::env::var("HYPERHIVE_RUTHLESS") { + Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"), + Err(_) => false, + } +} + +/// Auto-create the manager container on startup if it isn't already there. +/// hive-c0re manages the manager end-to-end: operators no longer declare +/// `containers.h-ruth` in their host NixOS config. Bypasses the approval +/// queue — the root/manager is auto-managed by default. Operators who +/// don't want a root agent at all set `services.hyperhive.ruthless = true`, +/// which short-circuits this whole function. Idempotent. +pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { + if ruthless() { + tracing::info!( + "ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start" + ); + return Ok(()); + } + let existing = lifecycle::list().await.unwrap_or_default(); + let current_rev = current_flake_rev(&coord.hyperhive_flake); + if existing + .iter() + .any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME)) + { + // Container exists already. If it predates the unified lifecycle + // (no applied flake on disk) we must rebuild — otherwise it's + // running whatever the host-declarative config was at create + // time, with a wrong systemd unit and port. + let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix"); + if !applied_flake.exists() && current_rev.is_some() { + tracing::warn!( + "manager container exists but no applied flake — forcing rebuild to migrate" + ); + if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild( + MANAGER_NAME, + crate::job_queue::Source::AutoUpdate, + "manager migration: no applied flake".to_owned(), + None, + true, + )) { + tracing::warn!(error = ?e, "manager migration rebuild submit failed"); + } + } else { + tracing::debug!("manager container already present"); + } + // hive-c0re auto-manages the root/manager by default, so a + // present-but-stopped root (e.g. a first-start failure on a fresh + // install) is brought back up here: the startup sweep's rebuild only + // restarts a container that was already running, so without this it + // stays down until a manual `nixos-container start`. The sub-agent + // `was_running` guard is intentionally left untouched. (Operators + // opt out of this whole auto-management with + // `services.hyperhive.ruthless = true`, gated at the top of + // this function.) + if !lifecycle::is_running(MANAGER_NAME).await { + tracing::info!("manager container present but not running — starting"); + if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) { + tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed"); + } + if let Err(e) = lifecycle::start(MANAGER_NAME).await { + tracing::warn!(error = ?e, "manager start failed"); + } + } + return Ok(()); + } + tracing::info!("manager container missing — spawning"); + let runtime = coord.ensure_runtime(MANAGER_NAME)?; + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(MANAGER_NAME, runtime); + lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?; + if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) { + tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed"); + } + if let Some(rev) = current_rev { + let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev); + } + Ok(()) +} + +/// Sort `names` in-place so parents precede their children in the topology. +/// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last, +/// alphabetically within their tier. Stable within each depth tier. +pub fn topology_sort( + names: &mut [String], + topo: &std::collections::BTreeMap>, +) { + use std::collections::{HashMap, VecDeque}; + // Build depth map using owned clones so the borrow on `names` is released + // before the sort_by mutable borrow. + let name_set: Vec = names.to_vec(); + let mut depth: HashMap = HashMap::new(); + let mut queue: VecDeque = VecDeque::new(); + // Seed roots: entries with no parent, or names not present in topo at all. + for name in &name_set { + if topo.get(name).is_none_or(Option::is_none) { + depth.insert(name.clone(), 0); + queue.push_back(name.clone()); + } + } + // BFS to assign depths to children. + while let Some(parent) = queue.pop_front() { + let d = depth[&parent] + 1; + for name in &name_set { + let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str()); + if is_child && !depth.contains_key(name) { + depth.insert(name.clone(), d); + queue.push_back(name.clone()); + } + } + } + names.sort_by(|a, b| { + let da = depth.get(a).copied().unwrap_or(usize::MAX); + let db = depth.get(b).copied().unwrap_or(usize::MAX); + da.cmp(&db).then(a.cmp(b)) + }); +} + +/// Boot reconcile (see the module doc): classify every agent by rev +/// freshness + persisted `wanted` intent, submit one `StartupSweep` +/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up +/// agents) when anything is stale, and `Reconcile` DAGs for agents +/// whose observed power state drifted from `wanted`. Returns Ok even +/// if some submissions failed. +pub async fn run(coord: Arc) -> Result<()> { + let containers = match lifecycle::list().await { + Ok(c) => c, + Err(e) => { + tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed"); + return Ok(()); + } + }; + + let current_rev = current_flake_rev(&coord.hyperhive_flake); + + // Resolve container names to logical agent names, then sort by + // topology depth so parents are always rebuilt before their + // children. Root agents (depth 0) go first; agents absent from + // the topology file sort last (stable, alphabetical within tier). + let mut logical_names: Vec = containers + .iter() + .filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned)) + .collect(); + let topo = crate::topology::read(); + topology_sort(&mut logical_names, &topo); + + // Classify. `get_or_seed` doubles as the one-time migration: an + // agent without an `agent_power` row is seeded from its observed + // state (running ⇒ Up), after which the DB is authoritative. + let mut any_stale = false; + let mut fanout: Vec = Vec::new(); // stale ∧ wanted=Up → sweep rebuild + let mut drifted: Vec = Vec::new(); // fresh ∧ wanted≠observed → reconcile + let mut n_deferred = 0usize; + let mut n_skipped = 0usize; + for name in &logical_names { + let running = lifecycle::is_running(name).await; + let wanted = match coord.power.get_or_seed(name, running) { + Ok(w) => w, + Err(e) => { + tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed"); + crate::power::Wanted::from_running(running) + } + }; + let fresh = current_rev.as_ref().is_some_and(|rev| { + std::fs::read_to_string(rev_marker_path(name)) + .is_ok_and(|stored| stored == rev.as_str()) + }); + if fresh { + n_skipped += 1; + } else { + any_stale = true; + if wanted == crate::power::Wanted::Up { + // Rebuild against the post-bump lock; the DAG's tail + // Reconcile brings the agent (back) up — covering both + // the running-stale and stopped-but-wanted-up cases. + fanout.push(name.clone()); + continue; + } + // Stale but wanted offline: no boot-time nix work — the + // start submit path upgrades a stale start to a rebuild. + n_deferred += 1; + tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start"); + } + if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop { + drifted.push(name.clone()); + } + } + + tracing::info!( + total = containers.len(), + rebuilds = fanout.len(), + reconciles = drifted.len(), + deferred = n_deferred, + up_to_date = n_skipped, + "boot reconcile" + ); + + // Sweep parent whenever ANY marker is stale — even when every + // stale agent is wanted-offline: the hyperhive lock bump must land + // now so their later start-upgrade rebuilds build against it. + // No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot. + if any_stale { + let reason = format!( + "startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date", + fanout.len(), + n_deferred, + n_skipped, + ); + if let Err(e) = coord + .job_queue + .submit(crate::job_queue::templates::startup_sweep(reason, fanout)) + { + tracing::warn!(error = ?e, "boot reconcile: sweep submit failed"); + } + } + for name in drifted { + if let Err(e) = coord + .job_queue + .submit(crate::job_queue::templates::reconcile_only( + crate::job_queue::Template::Reconcile, + &name, + crate::job_queue::Source::AutoUpdate, + "boot reconcile".to_owned(), + None, + )) + { + tracing::warn!(%name, error = ?e, "boot reconcile: submit failed"); + } + } + coord.emit_rebuild_queue_snapshot(); + Ok(()) +} diff --git a/hive-c0re/src/crash_watch.rs b/hive-c0re/src/workers/crash_watch.rs similarity index 100% rename from hive-c0re/src/crash_watch.rs rename to hive-c0re/src/workers/crash_watch.rs diff --git a/hive-c0re/src/knowledge.rs b/hive-c0re/src/workers/knowledge.rs similarity index 100% rename from hive-c0re/src/knowledge.rs rename to hive-c0re/src/workers/knowledge.rs diff --git a/hive-c0re/src/workers/mod.rs b/hive-c0re/src/workers/mod.rs new file mode 100644 index 00000000..98654ea3 --- /dev/null +++ b/hive-c0re/src/workers/mod.rs @@ -0,0 +1,12 @@ +//! Background tasks and periodic sweeps: crash/login watcher, the +//! reminder and scheduled-prompt delivery loops, boot-time auto-update +//! reconcile, the agent-sockets.json writer loop, and knowledge-repo +//! sync. Each submodule is re-exported at the crate root, so +//! `crate::crash_watch::…` etc. keep working unchanged. + +pub mod agent_sockets; +pub mod auto_update; +pub mod crash_watch; +pub mod knowledge; +pub mod reminder_scheduler; +pub mod scheduled_prompts_worker; diff --git a/hive-c0re/src/reminder_scheduler.rs b/hive-c0re/src/workers/reminder_scheduler.rs similarity index 100% rename from hive-c0re/src/reminder_scheduler.rs rename to hive-c0re/src/workers/reminder_scheduler.rs diff --git a/hive-c0re/src/scheduled_prompts_worker.rs b/hive-c0re/src/workers/scheduled_prompts_worker.rs similarity index 98% rename from hive-c0re/src/scheduled_prompts_worker.rs rename to hive-c0re/src/workers/scheduled_prompts_worker.rs index 2bdba46b..b4da2909 100644 --- a/hive-c0re/src/scheduled_prompts_worker.rs +++ b/hive-c0re/src/workers/scheduled_prompts_worker.rs @@ -10,6 +10,7 @@ use hive_sh4re::Message; use crate::coordinator::Coordinator; use crate::scheduled_prompts::Schedule; +use hive_sh4re::wire_time::now_unix; /// Per-tick cap. Each schedule fires once per tick at most; /// 100/tick × 5s tick = sustained throughput cap of ~20/sec, @@ -236,14 +237,6 @@ fn notify_operator_missing_target(coord: &Coordinator, schedule: &Schedule, targ } } -fn now_unix() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok() - .and_then(|d| i64::try_from(d.as_secs()).ok()) - .unwrap_or(0) -} - /// Per-target outcome counts for one `fire_now` invocation. /// Returned to the operator so the dashboard can render /// "fired to N (M failed, K missing)" without a follow-up GET. diff --git a/hive-sh4re/src/jobs.rs b/hive-sh4re/src/jobs.rs new file mode 100644 index 00000000..fe4a8fdd --- /dev/null +++ b/hive-sh4re/src/jobs.rs @@ -0,0 +1,191 @@ +//! Wire shapes of hive-c0re's job-DAG queue: what a queued job looks +//! like on the dashboard SSE channel (`rebuild_queue_changed`), the +//! `/api/state.rebuild_queue` snapshot, and the host admin socket's +//! `QueueDag` polling surface (`hivectl`'s wait/progress loop). The +//! queue *internals* — node kinds, dependency edges, scheduling state — +//! live in `hive-c0re::job_queue`; these are the serialized views it +//! produces. Semantics: `docs/coordinator.md::Job queue`. + +use serde::{Deserialize, Serialize}; + +/// What a DAG *means* — the request-level shape. Wire strings match +/// the pre-DAG queue's `kind` values so dashboards key off the same +/// tags. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Template { + /// Rebuild one agent's container (prebuild → stop → profile-swap → + /// reconcile). + Rebuild, + /// Bump meta flake locks; child `Rebuild` DAGs fan out on + /// completion for every affected agent. + MetaUpdate, + /// First-deploy spawn (approval-driven). + Spawn, + /// Reserved for a future destroy integration. + Destroy, + /// Boot-time config sweep (hyperhive lock bump + stale-agent + /// rebuild fan-out). + StartupSweep, + /// Mechanical stop + converge to `wanted = Up` (a restart). + Restart, + /// Perm-file commit followed by the rebuild subgraph. + PermChange, + /// Quiesce the harness, drain, then stop (`wanted = Offline`). + GracefulStop, + /// Converge to `wanted = Up`. + Start, + /// Converge to `wanted = Offline`. + Stop, + /// Bare converge of observed power state to the persisted intent + /// (boot reconcile). + Reconcile, +} + +impl Template { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Template::Rebuild => "rebuild", + Template::MetaUpdate => "meta_update", + Template::Spawn => "spawn", + Template::Destroy => "destroy", + Template::StartupSweep => "startup_sweep", + Template::Restart => "restart", + Template::PermChange => "perm_change", + Template::GracefulStop => "graceful_stop", + Template::Start => "start", + Template::Stop => "stop", + Template::Reconcile => "reconcile", + } + } +} + +/// Where the submit request originated — drives the "why" chip on the +/// dashboard. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Source { + /// Operator action (dashboard button, CLI, manager tool). + Manual, + /// Cascade child of a `MetaUpdate` DAG's fan-out. + MetaUpdate, + /// Boot-time submission (sweep parent, boot reconciles). + AutoUpdate, + /// Cascade child of a `StartupSweep` DAG's fan-out. + StartupSweep, + /// Crash recovery path (future use). + CrashRecover, + /// Operator approved a pending `Approval` row; `approval_id` on + /// the DAG points back at the source row. + Approval, +} + +impl Source { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Source::Manual => "manual", + Source::MetaUpdate => "meta_update", + Source::AutoUpdate => "auto_update", + Source::StartupSweep => "startup_sweep", + Source::CrashRecover => "crash_recover", + Source::Approval => "approval", + } + } +} + +/// Lifecycle state of a node — and, rolled up, of a DAG. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum State { + Queued, + Running, + Done, + Failed, + Cancelled, +} + +impl State { + #[must_use] + pub fn is_terminal(self) -> bool { + matches!(self, State::Done | State::Failed | State::Cancelled) + } +} + +/// Kind-specific payload for `Template::PermChange` DAGs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum PermPayload { + /// Set the tool groups for one agent (`tool-groups.json`). + ToolGroups { groups: Vec }, + /// Set the capabilities for one agent (`capabilities.json`). + Capabilities { caps: Vec }, + /// Set both perm-types in one entry — the batch + /// `POST /api/permissions` path. `None` leaves that file untouched; + /// present fields commit together and rebuild once. + Combined { + groups: Option>, + caps: Option>, + }, +} + +/// Node id, unique within its DAG. +pub type NodeId = u32; + +/// One node of a queued DAG, as serialized. Step labels, build-log +/// links, errors, and timestamps are per-node; the DAG-level `state` +/// is a roll-up. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeView { + pub id: NodeId, + /// Node primitive tag: `"prebuild"`, `"stop_for_update"`, + /// `"swap"`, `"create"`, `"meta_lock"`, `"reconcile"`, `"signal"`, + /// `"drain"`, `"write_dropin"`, `"write_perm_file"`, + /// `"approval_deploy"`. + pub kind: String, + /// Ids of the nodes this one waits for. + #[serde(default)] + pub deps: Vec, + pub state: State, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub step: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub build_log_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// A queued/running/recent DAG. DAG-level fields mirror the pre-DAG +/// `QueueEntry` names (`kind` = template string, roll-up `state`); +/// everything per-node appears exactly once, inside `nodes`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DagView { + pub id: u64, + pub agent: String, + /// Template wire string — same values the old `kind` field used. + pub kind: Template, + /// Roll-up: `failed` if any node failed, else `running` / + /// `queued` / `cancelled` / `done`. + pub state: State, + pub source: Source, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_id: Option, + pub reason: String, + pub enqueued_at: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inputs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub approval_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub perm_payload: Option, + pub nodes: Vec, +} diff --git a/hive-sh4re/src/lib.rs b/hive-sh4re/src/lib.rs index a9f1d540..0f40a40a 100644 --- a/hive-sh4re/src/lib.rs +++ b/hive-sh4re/src/lib.rs @@ -4,6 +4,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; pub mod assets; +pub mod jobs; pub mod paths; pub mod priv_proto; pub mod wire_time; @@ -69,6 +70,10 @@ pub enum HostRequest { /// matrix GUI disabled). Backs `hivectl open` + the federation /// peer-config block (which reads the bare `domain`). Urls, + /// Fetch one job-queue DAG (plus its live fan-out children, linked + /// via `parent_id`) by id — the polling surface behind `hivectl`'s + /// wait/progress loop. Result: [`HostResponse::dags`]. + QueueDag { id: u64 }, /// List pending approval requests. Pending, /// Approve a pending request by id; the action runs immediately. @@ -170,7 +175,7 @@ pub struct HiveUrls { pub matrix: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct HostResponse { pub ok: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -188,6 +193,16 @@ pub struct HostResponse { /// request kind. #[serde(default, skip_serializing_if = "Option::is_none")] pub agent_statuses: Option>, + /// Ids of the job-queue DAGs this request submitted (rebuild / + /// restart / power ops). Clients poll them via + /// [`HostRequest::QueueDag`]; `None` for non-submitting requests. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queued_dags: Option>, + /// `QueueDag` result — the requested DAG followed by its live + /// fan-out children ([`jobs::DagView`]). Empty when the DAG has + /// been evicted from the queue's history tail. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dags: Option>, } /// One row in the approval queue. `commit_ref` is overloaded per @@ -250,6 +265,24 @@ pub enum ApprovalKind { MergeConfigPr, } +impl ApprovalKind { + /// Wire/UI string — the same value serde's `snake_case` rename + /// produces. The single source of truth for every place that needs + /// the kind as a `&'static str` (sqlite storage, dashboard events), + /// so adding a variant can't silently miss a hand-rolled match. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + ApprovalKind::ApplyCommit => "apply_commit", + ApprovalKind::Spawn => "spawn", + ApprovalKind::InitConfig => "init_config", + ApprovalKind::UpdateMetaInputs => "update_meta_inputs", + ApprovalKind::SchedulePrompt => "schedule_prompt", + ApprovalKind::MergeConfigPr => "merge_config_pr", + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum ApprovalStatus { @@ -280,11 +313,7 @@ impl HostResponse { pub fn success() -> Self { Self { ok: true, - error: None, - agents: None, - approvals: None, - urls: None, - agent_statuses: None, + ..Self::default() } } @@ -293,10 +322,7 @@ impl HostResponse { Self { ok: false, error: Some(message.into()), - agents: None, - approvals: None, - urls: None, - agent_statuses: None, + ..Self::default() } } @@ -304,11 +330,8 @@ impl HostResponse { pub fn list(agents: Vec) -> Self { Self { ok: true, - error: None, agents: Some(agents), - approvals: None, - urls: None, - agent_statuses: None, + ..Self::default() } } @@ -316,11 +339,8 @@ impl HostResponse { pub fn pending(approvals: Vec) -> Self { Self { ok: true, - error: None, - agents: None, approvals: Some(approvals), - urls: None, - agent_statuses: None, + ..Self::default() } } @@ -329,11 +349,8 @@ impl HostResponse { pub fn urls(urls: HiveUrls) -> Self { Self { ok: true, - error: None, - agents: None, - approvals: None, urls: Some(urls), - agent_statuses: None, + ..Self::default() } } @@ -342,11 +359,29 @@ impl HostResponse { pub fn agent_statuses(rows: Vec) -> Self { Self { ok: true, - error: None, - agents: None, - approvals: None, - urls: None, agent_statuses: Some(rows), + ..Self::default() + } + } + + /// A request that submitted job-queue DAGs — carries their ids for + /// the client's wait/progress loop. + #[must_use] + pub fn queued(ids: Vec) -> Self { + Self { + ok: true, + queued_dags: Some(ids), + ..Self::default() + } + } + + /// `QueueDag` result — the polled DAG + its live children. + #[must_use] + pub fn dags(dags: Vec) -> Self { + Self { + ok: true, + dags: Some(dags), + ..Self::default() } } } diff --git a/hive-sh4re/src/wire_time.rs b/hive-sh4re/src/wire_time.rs index b246b1cf..2d952901 100644 --- a/hive-sh4re/src/wire_time.rs +++ b/hive-sh4re/src/wire_time.rs @@ -15,6 +15,19 @@ pub fn from_secs(secs: i64) -> DateTime { DateTime::::from_timestamp(secs, 0).unwrap_or_default() } +/// Current unix timestamp in seconds — the single definition behind +/// every store's `created_at` / `sent_at` / … stamp (this module owns +/// the epoch-seconds convention; a dozen local copies of this fn used +/// to float around both binaries). Clamps to 0 on a pre-epoch clock. +#[must_use] +pub fn now_unix() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|d| i64::try_from(d.as_secs()).ok()) + .unwrap_or(0) +} + #[cfg(test)] mod tests { use chrono::{DateTime, Utc}; diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index 451a5a46..8d748f8f 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -52,6 +52,7 @@ let agent_cpu_quota = cfg.agentCpuQuota; agent_memory_max = cfg.agentMemoryMax; model_prices = cfg.modelPrices; + build_slots = cfg.buildSlots; }; # Stylix theme integration (zero-op auto-detect). When the operator's @@ -771,6 +772,22 @@ in `"2G"`. ''; }; + + buildSlots = lib.mkOption { + type = lib.types.ints.positive; + default = 1; + example = 2; + description = '' + Number of nix-heavy job-queue nodes (container prebuilds, + profile swaps, first-spawn creates, meta lock bumps) hive-c0re + runs concurrently. The default of 1 serializes all heavy nix + work like the pre-DAG rebuild queue did; raise it on hosts with + the cores/RAM to build several agent toplevels at once. + Per-agent correctness is independent of this count — each + agent's container-affecting operations are serialized by its + lifecycle lease regardless. + ''; + }; }; config = lib.mkIf cfg.enable {