Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
2486251b32 | ||
|
|
0e4b5a1120 | ||
|
|
b489454dc2 | ||
|
|
dc6a37b29a | ||
|
|
a17015f01e | ||
|
|
8cdebb1752 | ||
|
|
b0736e6f3e | ||
|
|
c84028ddcf | ||
|
|
d190420946 | ||
|
|
34b21b8038 | ||
|
|
f74c1984d7 | ||
|
|
084e12503c | ||
|
|
58e86a3adf | ||
|
|
3ee87d394c | ||
|
|
9e7af3b6bf | ||
|
|
380c6ad47f | ||
|
|
604e1c2557 | ||
|
|
8349e6f621 | ||
|
|
7946e03fde |
92 changed files with 13734 additions and 11853 deletions
18
Cargo.lock
generated
18
Cargo.lock
generated
|
|
@ -1029,6 +1029,12 @@ version = "0.1.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "fixedbitset"
|
||||
version = "0.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
|
|
@ -1381,6 +1387,7 @@ dependencies = [
|
|||
"hive-sh4re",
|
||||
"libc",
|
||||
"listenfd",
|
||||
"petgraph",
|
||||
"problem_details",
|
||||
"reqwest",
|
||||
"rusqlite",
|
||||
|
|
@ -2554,6 +2561,17 @@ version = "2.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "petgraph"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
|
||||
dependencies = [
|
||||
"fixedbitset",
|
||||
"hashbrown 0.15.5",
|
||||
"indexmap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ reqwest = { version = "0.12", default-features = false, features = [
|
|||
"json",
|
||||
"rustls-tls",
|
||||
] }
|
||||
petgraph = { version = "0.8", default-features = false, features = ["std"] }
|
||||
matrix-sdk = { version = "0.14", default-features = false, features = [
|
||||
"rustls-tls",
|
||||
"sqlite",
|
||||
|
|
|
|||
|
|
@ -336,50 +336,49 @@ approval id to retry. Because tags are first-class git objects,
|
|||
rejected and failed trees stay browsable forever — `git log
|
||||
--tags` in the applied repo is the audit trail.
|
||||
|
||||
### Dispatch via `rebuild_queue`
|
||||
### Dispatch via the job queue
|
||||
|
||||
Long-running approval work — `ApplyCommit`, `UpdateMetaInputs`,
|
||||
`Spawn` — no longer runs inline inside `actions::approve`. Instead
|
||||
the approval handler enqueues a `QueueEntry` into the global
|
||||
`rebuild_queue`:
|
||||
the approval handler submits a DAG to the global job queue
|
||||
(`docs/coordinator.md::Job queue`):
|
||||
|
||||
| `ApprovalKind` | `QueueKind` queued | `QueueSource` |
|
||||
| `ApprovalKind` | DAG submitted | source |
|
||||
|---|---|---|
|
||||
| `ApplyCommit` | `Rebuild` | `Approval` |
|
||||
| `MergeConfigPr` | `Rebuild` | `Approval` |
|
||||
| `UpdateMetaInputs` | `MetaUpdate` | `Approval` |
|
||||
| `Spawn` | `Spawn` | `Approval` |
|
||||
| `ApplyCommit` | `rebuild` (single opaque `ApprovalDeploy` node) | `approval` |
|
||||
| `MergeConfigPr` | `rebuild` (single opaque `ApprovalDeploy` node) | `approval` |
|
||||
| `UpdateMetaInputs` | `meta_update` (`MetaLock` + rebuild fan-out) | `approval` |
|
||||
| `Spawn` | `spawn` (`Create → WriteDropin → Reconcile`) | `approval` |
|
||||
| `InitConfig` | — runs inline (sub-second git seed) | — |
|
||||
| `SchedulePrompt` | — runs inline (single sqlite insert) | — |
|
||||
|
||||
Each queue entry carries the originating `approval_id` so the
|
||||
worker can re-fetch the approval row when it dispatches, run the
|
||||
kind-specific pipeline (`run_approval_apply_commit` /
|
||||
`run_approval_merge_config_pr` / `run_approval_update_meta_inputs` /
|
||||
`run_approval_spawn`), and
|
||||
fire the matching `HelperEvent::*` on completion via
|
||||
`finish_approval`.
|
||||
The DAG carries the originating `approval_id`. The `ApprovalDeploy`
|
||||
node runs the kind-specific pipeline (`run_approval_apply_commit` /
|
||||
`run_approval_merge_config_pr` — the two-phase meta deploy stays
|
||||
inside `actions.rs`) and fires the matching `HelperEvent::*` via
|
||||
`finish_approval` itself; `Spawn` and `UpdateMetaInputs` DAGs resolve
|
||||
through `actions::resolve_approval_dag` when the DAG settles terminal
|
||||
(a spawn additionally runs the post-spawn forge bookkeeping there).
|
||||
|
||||
Two visible consequences:
|
||||
|
||||
- **Operator dashboard**: after clicking APPR0VE the work-in-progress
|
||||
shows up on the *rebuild queue* card (`POST /api/state.rebuild_queue`
|
||||
shows up on the *rebuild queue* card (`/api/state.rebuild_queue`
|
||||
+ live `rebuild_queue_changed` events), not on the approvals panel
|
||||
(which already moved the row to "approved"). A long meta-update
|
||||
cascade renders as a parent entry with one child per per-agent
|
||||
rebuild — see `docs/web-ui.md` for the layout.
|
||||
- **Cancellation**: the dashboard's *× cancel* button on a `Queued`
|
||||
entry calls `POST /api/rebuild-queue/{id}/cancel`, which flips the
|
||||
entry to `Cancelled` before the worker dispatches it. Returns
|
||||
`{"cancelled": true}` on success, `{"cancelled": false}` if the
|
||||
entry already left `Queued` (running / done / failed) — terminal
|
||||
states can't be retroactively rewritten.
|
||||
cascade renders as a parent DAG with one child rebuild per affected
|
||||
agent — see `docs/web-ui.md` for the layout.
|
||||
- **Cancellation**: the dashboard's *× cancel* button on a still-queued
|
||||
DAG calls `POST /api/rebuild-queue/{id}/cancel`, which flips it to
|
||||
`Cancelled` before any node runs (and fails the approval row instead
|
||||
of leaving it dangling). Returns `{"cancelled": true}` on success,
|
||||
`{"cancelled": false}` once any node started — terminal states can't
|
||||
be retroactively rewritten.
|
||||
|
||||
`QueueSource::Approval` carries the `approval_id` so a tail-end
|
||||
build failure surfaces back as a failed approval row, not just a
|
||||
silent queue entry. `QueueSource::Manual` (dashboard ↻ R3BU1LD)
|
||||
and `QueueSource::AutoUpdate` (boot-time sweep) use the same
|
||||
queue but skip the approval row plumbing.
|
||||
The `approval` source + `approval_id` mean a tail-end build failure
|
||||
surfaces back as a failed approval row, not just a silent queue
|
||||
entry. `manual` (dashboard ↻ R3BU1LD) and `auto_update` (boot
|
||||
reconcile) DAGs use the same queue but skip the approval plumbing.
|
||||
|
||||
### Forge mirror
|
||||
|
||||
|
|
@ -414,7 +413,7 @@ tool group can therefore edit, commit, and submit changes for any of
|
|||
its direct children directly inside its container at `/agents/<child>/config/`.
|
||||
|
||||
Agents holding the `can_manage_top_level_agents` topology role
|
||||
(defined as `ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS` in `hive-c0re/src/topology.rs`)
|
||||
(defined as `ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS` in `hive-c0re/src/agent_config/topology.rs`)
|
||||
get additional host-side bind mounts via `set_nspawn_flags`:
|
||||
|
||||
- `/var/lib/hyperhive/agents/` → `/agents/` (RW) — all top-level
|
||||
|
|
|
|||
|
|
@ -6,102 +6,191 @@ and `docs/persistence.md`.
|
|||
|
||||
---
|
||||
|
||||
## Rebuild queue
|
||||
## Job queue
|
||||
|
||||
Every long-running container/meta operation (rebuild, meta-update, first-spawn)
|
||||
goes through the global rebuild queue (`hive-c0re/src/rebuild_queue.rs`). A single
|
||||
background worker drains it in FIFO order so two `nixos-container update` runs on
|
||||
the same agent never overlap, and a fresh agent rebuild never races a meta-update's
|
||||
lock bump.
|
||||
Every container/meta operation (rebuild, meta-update, first-spawn, power
|
||||
changes) is submitted to the global job-DAG queue (`hive-c0re/src/job_queue/`)
|
||||
as a **DAG of primitive nodes**. One scheduler task drives all DAGs;
|
||||
concurrency comes from the resource classes below, not from multiple workers.
|
||||
The old special cases — the graceful-stop watcher thread, the deferred-start
|
||||
fast-lane follow-up, the meta-update cascade pre-enqueue — are all just DAG
|
||||
*shapes* now.
|
||||
|
||||
### Why one queue
|
||||
### Two levels: DAG and node
|
||||
|
||||
Before the rebuild queue landed, four independent call paths could fire
|
||||
`auto_update::rebuild_agent` concurrently:
|
||||
The **DAG** is the unit of dedup / cancel / approval-resolution and the
|
||||
dashboard group; the **node** is the unit of scheduling / execution /
|
||||
build-log / step label. Deps are intra-DAG edges only (`AfterOk` by default:
|
||||
the dep must succeed, a failed/cancelled dep cancels the dependent —
|
||||
cancel-downstream). Cross-DAG ordering comes from the per-agent lease + dedup,
|
||||
never from edges between DAGs. Submit-time validation (petgraph `toposort`)
|
||||
rejects cyclic specs outright, fixing the old queue's "circular dep silently
|
||||
deadlocks" caveat.
|
||||
|
||||
- Dashboard manual rebuild button
|
||||
- `update-all` / `meta-update` cascade
|
||||
- Approval handler (apply-commit / spawn)
|
||||
- Startup auto-update sweep
|
||||
### Node inventory (primitives)
|
||||
|
||||
Nothing serialised them. `nix-daemon` serialises the actual store ops, but the rest
|
||||
of `rebuild_agent` (token sync, kick, rescan, lock-bump emit) interleaved
|
||||
unpredictably. The single-worker queue gives operators a visible, ordered runway and
|
||||
lets the UI render "what's about to happen" instead of "something might be happening
|
||||
somewhere."
|
||||
Nix-heavy — hold one of the `buildSlots` permits for the node's duration:
|
||||
|
||||
### Queue kinds
|
||||
| Node | Wraps |
|
||||
| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `Prebuild` | meta `sync_agents` + optional per-agent relock + `lifecycle::prebuild_toplevel` — build the toplevel out-of-band while the container keeps serving |
|
||||
| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped) + the post-rebuild bookkeeping tail (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan) |
|
||||
| `Create` | first-spawn provisioning + `nixos-container create` (atomic build+create) |
|
||||
| `MetaLock` | meta flake lock bump (`lock_update` / boot-sweep `lock_update_hyperhive`, commit fused — see below); fans out child `Rebuild` DAGs on completion |
|
||||
| `ApprovalDeploy` | the opaque apply-commit / merge-config-PR pipeline (see _Approvals_ below) |
|
||||
|
||||
| Kind | Description |
|
||||
| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `Rebuild` | Single-agent rebuild. Covers manual, approval-driven, auto-update, and meta-update cascade variants — all funnel through the same path. The start-after-rebuild is **deferred to a fast-lane `Start` follow-up** (`parent_id` = this entry) so the build lane is freed as soon as the profile-swap finishes instead of waiting out the container boot — see _Deferred start_ under the rebuild path below. |
|
||||
| `MetaUpdate` | `nix flake update` on the meta flake. The worker runs the lock bump itself, then enqueues a cascade of `Rebuild` entries with `parent_id` set to the meta-update's id. |
|
||||
| `Spawn` | First-deploy of a new agent (approval-driven). Same serialisation as `Rebuild` from the operator's POV. |
|
||||
| `Destroy` | For future use (`destroy --purge` does real I/O). Variant exists so the wire shape doesn't change later; not currently routed through the queue. |
|
||||
| `Restart` | Stop + start a container without touching config (~5-10s). Routed through the queue so it serialises against in-flight rebuilds for the same agent — prevents a restart racing a rebuild mid-flight. Sources: dashboard ↺ button, the `restart` MCP tool. |
|
||||
| `PermChange` | Write a tool-group or capability change to the shared JSON file (`tool-groups.json` / `capabilities.json`), then rebuild the agent so the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes effect. Serialising the file write through the queue prevents concurrent dashboard batch-apply actions from racing on the shared file. After a successful file write, emits `CapabilitiesChanged` or `ToolGroupsChanged` SSE snapshot so the P3RM1SS10NS tab updates live. |
|
||||
| `GracefulStop` | Quiesce then stop a container (the `?graceful=true` path on `/api/kill/<agent>`). Signals the harness (its next `Recv` returns `GracefulStop` — the inbound fence — so it runs a stop-checkpoint turn that flushes durable `/state`, then takes the normal post-turn compaction path if it crossed the watermark, then exits) and **immediately releases the build lane**, spawning a detached watcher that holds the `Stopping` transient across the drain (bounded by a 3-min timeout → hard-stop fallback) and then enqueues a fast-lane `Stop` (`parent_id` = this entry) for the actual `nixos-container stop`. Net: a whole-hive graceful stop signals every agent up front, drains overlap, and only the container teardowns serialise (on the fast lane). Queued so the signal can't race an in-flight rebuild for the same agent. |
|
||||
Cheap — no build slot:
|
||||
|
||||
**Intentionally not queued** (sub-second ops): the _hard_ `start`, `stop`, `kill` via the direct API paths. (A _graceful_ stop is the `GracefulStop` kind above — it takes a checkpoint turn, so it rides the queue.) The queue's fast lane does carry `Start` / `Stop` kinds, but only as **follow-ups** other entries enqueue for themselves — the graceful-stop teardown and the deferred start-after-rebuild — so the container op groups under its parent entry on the dashboard.
|
||||
| Node | Behavior |
|
||||
| --------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop |
|
||||
| `StopForUpdate` | mechanical `nixos-container stop` for the profile swap; never touches `wanted`; noop if already stopped |
|
||||
| `Signal` | set the graceful fence + kick, so the harness runs one stop-checkpoint turn |
|
||||
| `Drain` | await the harness clearing the fence, bounded by the 3-min graceful-stop timeout; resolves ok either way |
|
||||
| `WriteDropin` | `set_nspawn_flags` + `set_resource_limits` + daemon-reload |
|
||||
| `WritePermFile` | commit `tool-groups.json` / `capabilities.json` (single git commit under `META_LOCK`) + emit the P3RM1SS10NS snapshots |
|
||||
|
||||
### Dedup
|
||||
There is deliberately **no `GitCommit` node**: `meta.rs` fuses each mutation
|
||||
with its commit under its internal `META_LOCK` mutex, so a standalone commit
|
||||
node would open a dirty-working-tree window between nodes.
|
||||
|
||||
Enqueueing `(kind, agent)` that already has a `Queued` entry returns the existing
|
||||
entry's id and appends the new request as an "also requested by …" line. Running
|
||||
entries do not dedup — a re-queue during a run is legitimate (something changed
|
||||
since the current run started).
|
||||
Two further layers protect the meta repo across *windows* that span multiple
|
||||
`META_LOCK` acquisitions — above all the approval deploy's prepare→finalize
|
||||
span, which keeps a bumped `flake.lock` **staged uncommitted** for the whole
|
||||
container build:
|
||||
|
||||
### Sources
|
||||
- **The deploy-window gate** (`meta::exclusive()`): every executor that
|
||||
mutates the meta repo (`Prebuild`'s sync+relock, `MetaLock`,
|
||||
`WritePermFile`, `Create`'s agent registration, and `ApprovalDeploy` for
|
||||
its whole span) holds this async mutex for its mutation span, so no commit
|
||||
can land inside another node's staged window. `Prebuild` drops it before
|
||||
the long toplevel build (store reads only), preserving `buildSlots > 1`
|
||||
concurrency.
|
||||
- **Path-limited commits**: the targeted meta committers (perm files,
|
||||
topology, lock bumps, finalize) commit `-- <their paths>` with path-scoped
|
||||
dirty checks, so even a non-queue caller (boot migration, destroy's
|
||||
`sync_agents`) can never sweep someone else's staged content into its
|
||||
commit.
|
||||
|
||||
| Source | Meaning |
|
||||
| -------------- | -------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `Manual` | Operator clicked rebuild / update-all / meta-update on the dashboard, or any other direct human action (CLI, an agent MCP tool). |
|
||||
| `AutoUpdate` | Legacy startup-sweep source (flat, no parent). Replaced by `StartupSweep` for new boots. |
|
||||
| `StartupSweep` | Child of a `StartupSweep` parent entry; boot-time per-agent rebuild with the sweep as the visual group header. |
|
||||
| `Approval` | Triggered by an operator-approved `ApprovalKind::{Spawn, ApplyCommit}`. |
|
||||
### Every operation as a DAG
|
||||
|
||||
### Cascade parent tracking
|
||||
```text
|
||||
rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a)
|
||||
graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
||||
restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
||||
start(a): [wanted=Up] Reconcile(a) (stale rev ⇒ upgraded to rebuild)
|
||||
stop(a): [wanted=Offline] Reconcile(a)
|
||||
spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
|
||||
perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
||||
meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected agent»
|
||||
boot: (if any rev marker stale) MetaLock(hyperhive) → «fan-out rebuild»;
|
||||
plus Reconcile(a) for every drifted agent
|
||||
```
|
||||
|
||||
`MetaUpdate` and `StartupSweep` entries fan out `Rebuild` children, each carrying
|
||||
`parent_id = <parent_id>`. The dashboard groups children under their parent
|
||||
in the queue panel so the operator sees the whole cascade as a tree,
|
||||
not a flat list.
|
||||
Notable collapses:
|
||||
|
||||
### Step labels
|
||||
- **`rebuild` is one uniform shape** — no `was_running` branch.
|
||||
`StopForUpdate` noops when already down; the tail `Reconcile` auto-noops the
|
||||
start when `wanted = Offline` (a rebuild of a deliberately-stopped agent
|
||||
leaves it stopped).
|
||||
- **The swap-failure recovery-start is structural**: `Reconcile` deps on
|
||||
`Swap` with the one `AfterAny` edge in the system — it runs after `Swap`
|
||||
terminal ok *or* fail, bringing a wanted-up agent back on its old config.
|
||||
- **Deferred start is automatic**: `Reconcile` holds no build slot, so the
|
||||
next DAG's `Prebuild` starts as soon as `Swap` frees the slot.
|
||||
- **Graceful stop needs no watcher thread**: `Signal`/`Drain` are cheap, so a
|
||||
whole-hive graceful stop fires every agent's signal immediately and all
|
||||
drains overlap; each DAG's tail `Reconcile` does the actual stop.
|
||||
- **The meta-update cascade fans out on completion**: `MetaLock`'s executor
|
||||
computes the affected agent set after the bump lands and appends child
|
||||
`rebuild` DAGs (`parent_id` set, `relock = false` so the children don't
|
||||
revert the bump). A failed bump fans out nothing — no cancel-children dance.
|
||||
|
||||
Each queue entry has a mutable `step: Option<String>` field that the worker updates
|
||||
as it progresses through lifecycle phases (`"nix build"`, `"nixos-container stop"`,
|
||||
`"nixos-container update"`, `"nixos-container start"`). The dashboard polls
|
||||
`/api/state` and renders the current step beneath the running entry so the operator
|
||||
can see which phase is taking time.
|
||||
### Desired-state (spec vs status)
|
||||
|
||||
### Dependency tracking
|
||||
Per-agent power *intent* — `wanted: Up | Offline` — is durable as the
|
||||
`agent_power` table in the coordinator DB (`hive-c0re/src/stores/power.rs`).
|
||||
`container_view` remains the observed *status*; `Reconcile` nodes converge the
|
||||
two. Setting `wanted` is never a queued node: the submit layer
|
||||
(`job_queue/submit.rs`) writes the row synchronously, then submits the DAG
|
||||
whose `Reconcile` reads the fresh value — rapid toggles are last-writer-wins.
|
||||
Power toggles never commit to the meta repo. Every operator power surface —
|
||||
dashboard buttons, the MCP tools, and `hivectl stop/start/restart/kill` —
|
||||
rides the queue through that submit layer, so intent, lease serialization,
|
||||
and crash-watch suppression can't drift per surface; the only direct starts
|
||||
left are the root-agent bootstrap and infra containers (no lease, no
|
||||
harness). Cancelling a still-queued power DAG reverts `wanted` to the
|
||||
observed state — a cancel means "don't do it", not "do it later". Agents
|
||||
without a row are seeded from observed state on first touch (running ⇒
|
||||
`Up`); destroy removes the row.
|
||||
|
||||
Each entry carries a `depends_on: Vec<u64>` field. The worker skips entries whose
|
||||
dependencies are not yet resolved — a dependency is resolved when its id is either
|
||||
in the queue as a terminal entry (`Done` / `Failed` / `Cancelled`) or no longer in
|
||||
the queue at all (evicted by `trim_history`, which only evicts terminals).
|
||||
The admin-socket responses carry the submitted DAG ids; `hivectl` polls
|
||||
`HostRequest::QueueDag` (~1s) and prints a progress line per DAG — roll-up
|
||||
glyph, template, agent, node chain with the running node's step label — so
|
||||
CLI verbs block until their jobs finish (`--no-wait` opts out; failures exit
|
||||
non-zero). Fan-out children joining a polled parent show up in the same
|
||||
loop.
|
||||
|
||||
Use cases:
|
||||
### Scheduler semantics
|
||||
|
||||
- Chain a `Rebuild` after an explicit prerequisite step without coupling them through
|
||||
the `parent_id` cascade mechanism.
|
||||
- Sequence a `PermChange` + `Rebuild` pair where the rebuild must not start until the
|
||||
perm-file write commits (already handled by the single-worker FIFO today, but
|
||||
`depends_on` allows explicit cross-kind sequencing when parallel workers are added).
|
||||
A node is **ready** when it's `Queued`, every dep is satisfied, and its
|
||||
resources are free. Resources:
|
||||
|
||||
`depends_on` is part of the dedup key: two entries with the same `(kind, agent,
|
||||
parent_id, inputs, approval_id)` but different dep sets are treated as distinct work.
|
||||
1. **Build slots** — `services.hyperhive.c0re.buildSlots` permits (default 1),
|
||||
held by nix-heavy nodes for the node's duration.
|
||||
2. **Per-agent lifecycle lease** — DAG-scoped: acquired at the DAG's first
|
||||
container-affecting node (`StopForUpdate`, `Swap`, `Signal`, `Drain`,
|
||||
`Reconcile`, `WriteDropin`, `Create`, `ApprovalDeploy`), held until the DAG
|
||||
is terminal, so two lifecycle DAGs for one agent never interleave their
|
||||
container ops. **Lease-exempt**: `Prebuild`, `MetaLock`, `WritePermFile` —
|
||||
they touch the store / meta, not the running container, which is exactly
|
||||
why a stop can land while another DAG's prebuild is still building.
|
||||
|
||||
**Worker re-notification**: the worker drains `take_next()` in a tight loop after
|
||||
each entry finishes. When a dep entry transitions to terminal, the loop re-evaluates
|
||||
the queue immediately, so downstream entries are unblocked with no extra wakeup. No
|
||||
additional `notify_one()` call is needed.
|
||||
Among simultaneously-ready nodes competing for a resource, DAG-submit order
|
||||
wins (FIFO) so bulk operations drain predictably. The scheduler also owns the
|
||||
DAG-lifetime transient guard (dashboard pill + crash-watch suppression),
|
||||
created on lease acquisition and dropped when the DAG settles terminal.
|
||||
|
||||
**Circular-dep caveat**: if A depends on B and B depends on A, neither entry ever
|
||||
becomes runnable — the worker skips both indefinitely with no error. Callers must
|
||||
ensure acyclic dep graphs. Cycle detection is deferred to a future iteration (when
|
||||
parallel workers make a stuck queue more visible).
|
||||
The queue is in-memory only and lost on hive-c0re restart — deliberate:
|
||||
desired state is re-derived at boot from the DB + rev markers (see _Boot
|
||||
reconcile_), so there is no durable-recovery machinery to go wrong.
|
||||
|
||||
### Dedup, cancel, history
|
||||
|
||||
Dedup at **DAG granularity**: a repeat submit against a DAG whose roll-up is
|
||||
still `Queued` with the same `(template, agent, parent_id, approval_id)` —
|
||||
plus `inputs` for meta-updates and the perm-type discriminant for perm
|
||||
changes — returns the existing id and appends an "also requested by …" line.
|
||||
`parent_id` in the key keeps a cascade child from collapsing into a
|
||||
standalone or sweep rebuild. Running/terminal DAGs never dedup.
|
||||
|
||||
Cancel only applies to still-fully-queued DAGs (an in-flight nix build isn't
|
||||
interruptible); `cancel_children` cancels a parent's still-queued child DAGs.
|
||||
Roll-up state: `Failed` if any node failed, else `Running` / `Queued` /
|
||||
`Cancelled` / `Done`. The snapshot retains the 5 most recent terminal DAGs
|
||||
per template.
|
||||
|
||||
### Approvals
|
||||
|
||||
`ApplyCommit` / `MergeConfigPr` approvals ride as single-node
|
||||
`ApprovalDeploy` DAGs: the two-phase `prepare_deploy` / `finalize_deploy` /
|
||||
`abort_deploy` meta orchestration stays inside `actions.rs` in v1
|
||||
(deliberately not modeled as scheduler nodes) and resolves the approval
|
||||
itself. `Spawn` and `UpdateMetaInputs` approvals map onto the ordinary
|
||||
`spawn` / `meta-update` shapes; the scheduler fires
|
||||
`actions::resolve_approval_dag` exactly once when such a DAG settles
|
||||
terminal (including cancelled-while-queued, which fails the approval instead
|
||||
of dangling it).
|
||||
|
||||
### Wire shape
|
||||
|
||||
`RebuildQueueChanged { seq, queue: [DagView…] }` (event name kept). Each
|
||||
`DagView` carries the old entry-level fields (`id`, `kind` = template string,
|
||||
roll-up `state`, `agent`, `source`, `parent_id`, `reason`, timestamps,
|
||||
`inputs`, `approval_id`) plus `nodes: [NodeView…]` — per-node `kind`, `deps`,
|
||||
`state`, `step`, `build_log_id`, timestamps, `error`. Step labels and build
|
||||
logs are **per-node**; the dashboard renders the node chain on each queue
|
||||
card and keys the live-log panel off the running node.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -115,31 +204,31 @@ render.
|
|||
|
||||
---
|
||||
|
||||
## Auto-update sweep
|
||||
## Boot reconcile
|
||||
|
||||
On startup, `auto_update.rs` rebuilds containers that actually need it. Two skip
|
||||
rules keep boot-time work minimal:
|
||||
On startup, `auto_update::run` classifies every agent by rev freshness (the
|
||||
per-agent `.{name}.hyperhive-rev` marker under `/var/lib/hyperhive/applied/`
|
||||
vs the current flake path) and persisted `wanted` intent, then:
|
||||
|
||||
1. **Stopped containers** are deferred: the startup sweep enqueues nothing for them.
|
||||
When the operator later starts a stopped container (via the dashboard or the
|
||||
`start` MCP tool), both `run_start` (queue path) and `handle_start` (socket path)
|
||||
check the rev marker first — if it's stale, the start is silently upgraded to a
|
||||
full rebuild+start so the container runs current nix derivations.
|
||||
1. **Config path** — when *any* marker is stale, submit one `StartupSweep`
|
||||
DAG: a `MetaLock` (hyperhive input bump, non-fatal) that fans out `Rebuild`
|
||||
children for the stale agents whose `wanted = Up` (topology-sorted, parents
|
||||
first). Stale but wanted-offline agents get no boot-time nix work — their
|
||||
rebuild happens on their next start (the start submit path upgrades a
|
||||
stale start to rebuild+start), which is also why the lock bump runs even
|
||||
when every stale agent is offline: those later start-upgrades must build
|
||||
against the bumped lock. Each child rebuild's tail `Reconcile` brings the
|
||||
agent (back) up, covering both the running-stale and stopped-but-wanted-up
|
||||
cases.
|
||||
|
||||
2. **Running containers with a matching rev marker** are skipped: if the per-agent
|
||||
`.{name}.hyperhive-rev` file under `/var/lib/hyperhive/applied/` already holds
|
||||
the current flake rev, no nix work is needed and the entry is omitted entirely.
|
||||
2. **Power path** — every agent whose observed state drifted from `wanted`
|
||||
gets a plain `Reconcile` DAG (`kind = reconcile`, source `auto_update`).
|
||||
|
||||
`auto_update::run` enqueues a single `StartupSweep` parent entry (`kind =
|
||||
startup_sweep`, `agent = "hyperhive"`) followed by per-agent `Rebuild` children
|
||||
for the agents that do need rebuilding (`source = startup_sweep`, `parent_id =
|
||||
sweep_id`). The sweep description records the rebuild / deferred / skipped counts
|
||||
so the operator can see at a glance how much work the boot triggered. The child
|
||||
rebuilds drain sequentially through the queue; the dashboard renders them nested
|
||||
under the parent.
|
||||
|
||||
Before the sweep-grouping change, each boot enqueued flat `Rebuild` entries with
|
||||
`source = AutoUpdate` and no parent — visible but ungrouped.
|
||||
Booting with no config change performs **no meta commit** — only reconciles.
|
||||
The sweep reason records the rebuild / deferred / up-to-date counts so the
|
||||
operator sees at a glance how much work the boot triggered. Agents without an
|
||||
`agent_power` row are seeded from observed state during classification (the
|
||||
one-time migration; thereafter the DB is authoritative).
|
||||
|
||||
## Meta flake
|
||||
|
||||
|
|
@ -157,8 +246,13 @@ Key operations:
|
|||
`RequestApplyCommit` path so a failed `nixos-container update` leaves no orphan
|
||||
commit in meta. Prepare writes the new lock without committing; finalize commits
|
||||
with the deploy message; abort restores the lock.
|
||||
- **`lock_update_hyperhive`** — one-shot for the auto-update path: bumps the
|
||||
`hyperhive` input lock, commits, cascades agent rebuilds.
|
||||
- **`lock_update_hyperhive`** — one-shot for the boot-reconcile path (the
|
||||
sweep DAG's `MetaLock` node): bumps the `hyperhive` input lock and commits;
|
||||
the scheduler fans out the agent rebuilds on completion.
|
||||
|
||||
Every public `meta.rs` operation takes the module's internal `META_LOCK`
|
||||
mutex, so concurrent job-queue nodes (and the approval deploy pipeline) never
|
||||
race on the repo's `.git/index.lock`.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -179,34 +273,26 @@ new profile and skips the in-container `switch-to-configuration`. The subsequent
|
|||
`start` then applies both the new profile and any `EXTRA_NSPAWN_FLAGS` changes in
|
||||
one go, rather than the double-bounce a live `update` would trigger.
|
||||
|
||||
Sequence for a running container:
|
||||
Sequence for a rebuild DAG (each step is its own queue node):
|
||||
|
||||
1. `prebuild_toplevel` — build the new `system.build.toplevel` **before** stopping.
|
||||
The container keeps serving the previous generation while eval + fetch + build
|
||||
happen out-of-band. `nixos-container update` then finds the result cached and
|
||||
skips straight to the profile-swap. Build failures surface here, before the
|
||||
running container is touched.
|
||||
2. `nixos-container stop` — bring the container down.
|
||||
3. `nixos-container update --flake meta#<name>` — profile-swap (near-instant after
|
||||
the prebuild).
|
||||
4. `nixos-container start` — boot into the new generation; the in-container
|
||||
activation script transitions old → new.
|
||||
1. `Prebuild` — build the new `system.build.toplevel` **before** stopping.
|
||||
The container keeps serving the previous generation while eval + fetch +
|
||||
build happen out-of-band. `nixos-container update` then finds the result
|
||||
cached and skips straight to the profile-swap. Build failures surface
|
||||
here, before the running container is touched. (Runs even for a stopped
|
||||
container — same total nix work, one uniform DAG shape.)
|
||||
2. `StopForUpdate` — bring the container down (noop when already stopped).
|
||||
3. `Swap` — `nixos-container update --flake meta#<name>` profile-swap
|
||||
(near-instant after the prebuild).
|
||||
4. `Reconcile` — boot into the new generation when `wanted = Up`; the
|
||||
in-container activation script transitions old → new. Holds no build
|
||||
slot, so the next DAG's `Prebuild` overlaps the container boot — the old
|
||||
"deferred start" split, now structural.
|
||||
|
||||
If the container is already stopped, step 1 is skipped (no downtime to shave — no
|
||||
point evaluating the flake twice).
|
||||
|
||||
**Deferred start (queue-dispatched rebuilds):** step 4 can take a while
|
||||
(container boot), and holding the serialized build lane through it delays the
|
||||
next queued rebuild's nix build for no reason. Queue-dispatched rebuilds
|
||||
therefore pass `defer_start` — `rebuild_no_meta` skips the start and returns
|
||||
`true`, and `rebuild_agent` enqueues a fast-lane `Start` entry instead
|
||||
(`parent_id` = the rebuild entry, so the dashboard groups the follow-up under
|
||||
it — the same split the graceful-stop path uses for its container stop). The
|
||||
build-lane entry completes when the profile-swap finishes; a start failure
|
||||
surfaces on the `Start` entry, which runs with the cold-start fallback. Direct
|
||||
callers (admin-socket CLI, root-agent migration nudge, the apply-commit deploy
|
||||
flow which verifies the agent comes back up before finalizing) keep the start
|
||||
inline.
|
||||
The approval apply-commit pipeline still drives `lifecycle::rebuild_no_meta`
|
||||
(the fused stop/update/start path with an inline start) inside its
|
||||
`ApprovalDeploy` node, because it verifies the agent comes back up before
|
||||
finalizing the deploy tag.
|
||||
|
||||
### Cold-start fallback
|
||||
|
||||
|
|
@ -218,9 +304,8 @@ half-started at that point.
|
|||
Fallback: `stop` (graceful SIGTERM drain) → `kill` (SIGKILL any lingering processes)
|
||||
→ `start` (clean cold-start, no generation transition, new activation runs cleanly).
|
||||
Both errors are preserved and surfaced if the cold-start also fails. The fallback
|
||||
lives in `lifecycle::start_with_fallback`, shared by the inline start-after-rebuild
|
||||
path and the queue's fast-lane `Start` handler (which the deferred
|
||||
start-after-rebuild rides).
|
||||
lives in `lifecycle::start_with_fallback`, shared by the apply-commit deploy's
|
||||
inline start and every `Reconcile` node's start action.
|
||||
|
||||
### Spawn path (new container)
|
||||
|
||||
|
|
@ -241,9 +326,18 @@ or the flake root directly — none of which exist in the rendered meta flake.
|
|||
|
||||
## Host-level resource + performance options
|
||||
|
||||
Three `services.hyperhive.c0re.*` options tune container resource
|
||||
limits and first-spawn latency. All three apply uniformly to every
|
||||
agent container.
|
||||
A handful of `services.hyperhive.c0re.*` options tune container resource
|
||||
limits, build parallelism, and first-spawn latency.
|
||||
|
||||
### Build slots
|
||||
|
||||
`buildSlots` (default `1`) sets how many nix-heavy job-queue nodes
|
||||
(prebuilds, profile swaps, first-spawn creates, meta lock bumps) run
|
||||
concurrently. The default serializes all heavy nix work like the pre-DAG
|
||||
rebuild queue did; raise it on hosts with the cores/RAM to build several
|
||||
agent toplevels at once. Per-agent correctness is independent of the count —
|
||||
each agent's container-affecting ops serialize on its lifecycle lease
|
||||
regardless.
|
||||
|
||||
### Container resource limits
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
|
||||
Where state lives, what survives what, and how it's bounded.
|
||||
|
||||
## Three sqlite databases
|
||||
## Sqlite databases
|
||||
|
||||
### `/var/lib/hyperhive/db/broker.sqlite` (host)
|
||||
|
||||
Six tables, all in one file — four queues plus the schedule
|
||||
header/targets split:
|
||||
Seven tables, all in one file — four queues, the schedule
|
||||
header/targets split, and the per-agent power-intent registry:
|
||||
|
||||
- `messages` — every inter-agent / operator-bound message.
|
||||
`sender / recipient / body / sent_at / delivered_at / acked_at /
|
||||
|
|
@ -44,6 +44,17 @@ header/targets split:
|
|||
last_fired_at_unix / last_result`. `ON DELETE CASCADE` from
|
||||
`scheduled_prompts(id)` — requires `PRAGMA foreign_keys = ON`
|
||||
per connection (set at open).
|
||||
- `agent_power` — one tiny row per agent: `agent PK / wanted (up |
|
||||
offline) / updated_at` — the durable power *intent* behind the job
|
||||
queue's desired-state reconciliation
|
||||
(`docs/coordinator.md::Job queue`; owner: `hive-c0re/src/stores/power.rs`).
|
||||
Written synchronously by every operator/agent power action
|
||||
(dashboard start/stop, `hivectl stop`, the MCP kill/start tools,
|
||||
spawn approval); read by `Reconcile` nodes and the boot reconcile.
|
||||
Intent survives hive-c0re restarts — in-flight queue work
|
||||
deliberately does not. Agents without a row are seeded from
|
||||
observed state on first touch (running ⇒ `up`); destroy deletes
|
||||
the row.
|
||||
|
||||
Retention:
|
||||
|
||||
|
|
@ -63,6 +74,8 @@ Retention:
|
|||
(`cancel_schedule` MCP / dashboard ✗) which tombstones via
|
||||
`cancelled_at_unix`, then `reap_cancelled` drops the row on
|
||||
the next worker pass.
|
||||
- `agent_power` rows live until the agent is destroyed (one row per
|
||||
agent — nothing to vacuum).
|
||||
|
||||
### `/harness/hyperhive-events.sqlite` (per agent)
|
||||
|
||||
|
|
|
|||
|
|
@ -271,8 +271,8 @@ Agent container management. Requires the hive-c0re daemon to be running (connect
|
|||
###### **Subcommands:**
|
||||
|
||||
* `list` — Show all managed agents with their status (running / needs-login / needs-update) and technical state (deployed sha, parent, pending reminders). The host roster overview; reuses the dashboard's per-agent aggregation. Requires the daemon running
|
||||
* `restart` — Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config
|
||||
* `restart-all` — Stop and restart ALL managed agent containers in sequence. Iterates the live container list and restarts each one. Any per-agent failure is reported at the end rather than stopping mid-run, so all containers get a restart attempt
|
||||
* `restart` — Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config. Rides the job queue (serialized against in-flight rebuilds for the same agent); waits with live progress unless `--no-wait`
|
||||
* `restart-all` — Restart ALL managed agent containers via one restart DAG each — unrelated agents overlap, each serializes on its own lease. Waits for the whole set with live progress unless `--no-wait`
|
||||
|
||||
|
||||
|
||||
|
|
@ -290,21 +290,29 @@ Show all managed agents with their status (running / needs-login / needs-update)
|
|||
|
||||
## `hivectl agents restart`
|
||||
|
||||
Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config
|
||||
Stop and start a single agent container without rebuilding config. Useful for "kick the container" when the process is stuck or the container needs a clean restart without changing the NixOS config. Rides the job queue (serialized against in-flight rebuilds for the same agent); waits with live progress unless `--no-wait`
|
||||
|
||||
**Usage:** `hivectl agents restart <NAME>`
|
||||
**Usage:** `hivectl agents restart [OPTIONS] <NAME>`
|
||||
|
||||
###### **Arguments:**
|
||||
|
||||
* `<NAME>` — Agent name (e.g. `damocles`, `ruth`)
|
||||
|
||||
###### **Options:**
|
||||
|
||||
* `--no-wait` — Return immediately after the restart DAG is queued
|
||||
|
||||
|
||||
|
||||
## `hivectl agents restart-all`
|
||||
|
||||
Stop and restart ALL managed agent containers in sequence. Iterates the live container list and restarts each one. Any per-agent failure is reported at the end rather than stopping mid-run, so all containers get a restart attempt
|
||||
Restart ALL managed agent containers via one restart DAG each — unrelated agents overlap, each serializes on its own lease. Waits for the whole set with live progress unless `--no-wait`
|
||||
|
||||
**Usage:** `hivectl agents restart-all`
|
||||
**Usage:** `hivectl agents restart-all [OPTIONS]`
|
||||
|
||||
###### **Options:**
|
||||
|
||||
* `--no-wait` — Return immediately after the restart DAGs are queued
|
||||
|
||||
|
||||
|
||||
|
|
@ -407,7 +415,8 @@ Stop containers hive-wide in one operator action. Bare `hivectl stop` stops **ev
|
|||
* `--forge` — The forge container (`hive-forge`)
|
||||
* `--gateway` — The gateway container (`hive-gateway`)
|
||||
* `--matrix` — The matrix container (`hive-matrix`)
|
||||
* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent is enqueued as a `GracefulStop` on the rebuild queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). Applies to agents only
|
||||
* `--graceful` — Gracefully quiesce each agent before stopping, instead of a hard stop. Each agent gets a graceful-stop DAG on the job queue: the harness is signalled, runs one stop-checkpoint turn to flush durable `/state`, drains, then the container is stopped (bounded by a 3-min timeout that falls back to a hard stop). All drains overlap. Applies to agents only
|
||||
* `--no-wait` — Return immediately after the stop DAGs are queued instead of waiting for them with live per-node progress
|
||||
|
||||
|
||||
|
||||
|
|
@ -425,6 +434,7 @@ Start containers hive-wide — the inverse of `hivectl stop`. Bare `hivectl star
|
|||
* `--forge` — The forge container (`hive-forge`)
|
||||
* `--gateway` — The gateway container (`hive-gateway`)
|
||||
* `--matrix` — The matrix container (`hive-matrix`)
|
||||
* `--no-wait` — Return immediately after the start DAGs are queued instead of waiting for them with live per-node progress
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -131,6 +131,7 @@ const QUEUE_KIND_GLYPH = {
|
|||
graceful_stop: '⏹',
|
||||
start: '▶',
|
||||
stop: '■',
|
||||
reconcile: '⇄',
|
||||
};
|
||||
const QUEUE_STATE_GLYPH = {
|
||||
queued: '⏸',
|
||||
|
|
@ -139,11 +140,33 @@ const QUEUE_STATE_GLYPH = {
|
|||
failed: '✖',
|
||||
cancelled: '⊘',
|
||||
};
|
||||
// Short display labels for the per-DAG node kinds (the primitive ops).
|
||||
const NODE_KIND_LABEL = {
|
||||
prebuild: 'prebuild',
|
||||
stop_for_update: 'stop',
|
||||
swap: 'swap',
|
||||
create: 'create',
|
||||
meta_lock: 'meta lock',
|
||||
reconcile: 'reconcile',
|
||||
signal: 'signal',
|
||||
drain: 'drain',
|
||||
write_dropin: 'dropin',
|
||||
write_perm_file: 'perm file',
|
||||
approval_deploy: 'deploy',
|
||||
};
|
||||
|
||||
// The currently-running node of a DAG (per-node `step` labels + build
|
||||
// logs live on nodes now; the DAG's `state` is a roll-up).
|
||||
function runningNode(entry) {
|
||||
return (entry.nodes || []).find((n) => n.state === 'running') || null;
|
||||
}
|
||||
function firstFailedNode(entry) {
|
||||
return (entry.nodes || []).find((n) => n.state === 'failed') || null;
|
||||
}
|
||||
|
||||
function rebuildQueueEntryFingerprint(entry, isChild) {
|
||||
return JSON.stringify({
|
||||
state: entry.state,
|
||||
step: entry.step,
|
||||
kind: entry.kind,
|
||||
agent: entry.agent,
|
||||
source: entry.source,
|
||||
|
|
@ -151,8 +174,7 @@ function rebuildQueueEntryFingerprint(entry, isChild) {
|
|||
enqueued_at: entry.enqueued_at,
|
||||
finished_at: entry.finished_at,
|
||||
reason: entry.reason,
|
||||
error: entry.error,
|
||||
build_log_id: entry.build_log_id,
|
||||
nodes: (entry.nodes || []).map((n) => [n.kind, n.state, n.step, n.build_log_id, n.error]),
|
||||
isChild,
|
||||
});
|
||||
}
|
||||
|
|
@ -257,22 +279,39 @@ function renderQueueEntry(entry, _byId, isChild) {
|
|||
const r = entry.reason.split('\n')[0];
|
||||
li.append(' ', el('span', { class: 'rqe-reason', title: entry.reason }, '— ' + truncate(r, 60)));
|
||||
}
|
||||
if (entry.step) {
|
||||
li.append(el('div', { class: 'rqe-step' }, '↳ ' + entry.step));
|
||||
// Per-node chain: every DAG node in dependency order with its own
|
||||
// state glyph, live step label, and build-log link. This is the
|
||||
// node-aware render that makes queue jumps / partial progress
|
||||
// visible (e.g. reconcile running while swap failed).
|
||||
const nodes = entry.nodes || [];
|
||||
if (nodes.length) {
|
||||
const chain = el('div', { class: 'rqe-nodes' });
|
||||
nodes.forEach((n, i) => {
|
||||
if (i > 0) chain.append(el('span', { class: 'rqe-node-arrow' }, ' → '));
|
||||
const chip = el('span', {
|
||||
class: 'rqe-node rqe-node-' + n.state,
|
||||
title: n.kind + ' · ' + n.state + (n.error ? ' — ' + n.error : ''),
|
||||
},
|
||||
(QUEUE_STATE_GLYPH[n.state] || '?') + ' ' + (NODE_KIND_LABEL[n.kind] || n.kind));
|
||||
chain.append(chip);
|
||||
if (n.build_log_id != null) {
|
||||
chain.append(el('a', {
|
||||
class: 'rqe-log-link rqe-node-log',
|
||||
href: '/builds.html?id=' + n.build_log_id + '#buildlogs',
|
||||
target: '_blank',
|
||||
title: 'view build log #' + n.build_log_id,
|
||||
}, '⎙'));
|
||||
}
|
||||
});
|
||||
li.append(chain);
|
||||
}
|
||||
if (entry.build_log_id != null) {
|
||||
li.append(
|
||||
' ',
|
||||
el('a', {
|
||||
class: 'rqe-log-link',
|
||||
href: '/builds.html?id=' + entry.build_log_id + '#buildlogs',
|
||||
target: '_blank',
|
||||
title: 'view build log #' + entry.build_log_id,
|
||||
}, 'logs →'),
|
||||
);
|
||||
const running = runningNode(entry);
|
||||
if (running && running.step) {
|
||||
li.append(el('div', { class: 'rqe-step' }, '↳ ' + running.step));
|
||||
}
|
||||
if (entry.error) {
|
||||
li.append(el('pre', { class: 'rqe-error', title: entry.error }, truncate(entry.error, 200)));
|
||||
const failed = firstFailedNode(entry);
|
||||
if (failed && failed.error) {
|
||||
li.append(el('pre', { class: 'rqe-error', title: failed.error }, truncate(failed.error, 200)));
|
||||
}
|
||||
if (entry.state === 'queued') {
|
||||
const cancelForm = el('form', {
|
||||
|
|
@ -296,11 +335,12 @@ function renderQueueEntry(entry, _byId, isChild) {
|
|||
}
|
||||
|
||||
// ─── running-rebuild live log ─────────────────────────────────────────────────
|
||||
// One persistent live-log panel for the currently-running rebuild (the queue
|
||||
// runs one build at a time). Keyed to the running entry's build_log_id and
|
||||
// kept in its own container (#rebuild-live-log) so the queue's row re-render
|
||||
// — which rebuilds rows as the build step advances — never tears down the open
|
||||
// SSE stream.
|
||||
// One persistent live-log panel for the first currently-building node
|
||||
// (build logs are per-node now; with buildSlots = 1 at most one nix
|
||||
// build runs at a time). Keyed to that node's build_log_id and kept in
|
||||
// its own container (#rebuild-live-log) so the queue's row re-render —
|
||||
// which rebuilds rows as the build step advances — never tears down the
|
||||
// open SSE stream.
|
||||
let liveLogEs = null;
|
||||
let liveLogId = null;
|
||||
let liveLogDone = false;
|
||||
|
|
@ -310,23 +350,36 @@ function closeLiveLogStream() {
|
|||
if (liveLogEs) { liveLogEs.close(); liveLogEs = null; }
|
||||
}
|
||||
|
||||
// First (dag, node) pair with a running node that opened a build log.
|
||||
function findLiveBuild(queue) {
|
||||
for (const e of queue || []) {
|
||||
if (e.state !== 'running') continue;
|
||||
for (const n of e.nodes || []) {
|
||||
if (n.state === 'running' && n.build_log_id != null) {
|
||||
return { entry: e, node: n };
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function renderRebuildLiveLog(queue) {
|
||||
const root = $('rebuild-live-log');
|
||||
if (!root) return;
|
||||
const running = (queue || []).find(
|
||||
(e) => e.state === 'running' && e.build_log_id != null);
|
||||
const live = findLiveBuild(queue);
|
||||
|
||||
if (!running) {
|
||||
if (!live) {
|
||||
closeLiveLogStream();
|
||||
liveLogId = null;
|
||||
liveLogDone = false;
|
||||
if (!root.hidden) { root.hidden = true; root.replaceChildren(); }
|
||||
return;
|
||||
}
|
||||
if (running.build_log_id === liveLogId && (liveLogEs || liveLogDone)) return;
|
||||
const { entry: running, node: liveNode } = live;
|
||||
if (liveNode.build_log_id === liveLogId && (liveLogEs || liveLogDone)) return;
|
||||
|
||||
closeLiveLogStream();
|
||||
liveLogId = running.build_log_id;
|
||||
liveLogId = liveNode.build_log_id;
|
||||
liveLogDone = false;
|
||||
root.hidden = false;
|
||||
root.replaceChildren();
|
||||
|
|
@ -351,17 +404,18 @@ function renderRebuildLiveLog(queue) {
|
|||
toggle, ' ',
|
||||
el('span', { class: 'rebuild-live-log-title' }, 'live build log — '),
|
||||
el('code', { class: 'rqe-agent' }, running.agent),
|
||||
' ', el('span', { class: 'rqe-kind' }, running.kind || 'rebuild'),
|
||||
' ', el('span', { class: 'rqe-kind' },
|
||||
(running.kind || 'rebuild') + ' · ' + (NODE_KIND_LABEL[liveNode.kind] || liveNode.kind)),
|
||||
' ', badge, ' ',
|
||||
el('a', {
|
||||
class: 'rebuild-live-log-raw',
|
||||
href: '/api/build-logs/id/' + running.build_log_id + '/raw',
|
||||
download: 'build-log-' + running.build_log_id + '.txt',
|
||||
href: '/api/build-logs/id/' + liveNode.build_log_id + '/raw',
|
||||
download: 'build-log-' + liveNode.build_log_id + '.txt',
|
||||
}, '↓ raw'),
|
||||
);
|
||||
root.append(header, pre);
|
||||
|
||||
liveLogEs = openBuildLogStream(running.build_log_id, pre, {
|
||||
liveLogEs = openBuildLogStream(liveNode.build_log_id, pre, {
|
||||
onDone: (status) => {
|
||||
liveLogDone = true;
|
||||
badge.className = 'rebuild-live-log-badge ' + (status === 'ok' ? 'rll-ok' : 'rll-fail');
|
||||
|
|
|
|||
|
|
@ -811,6 +811,7 @@ export function renderContainers(s) {
|
|||
: op.kind === 'start' ? 'starting'
|
||||
: op.kind === 'stop' ? 'stopping'
|
||||
: op.kind === 'graceful_stop' ? 'stopping'
|
||||
: op.kind === 'reconcile' ? 'reconciling'
|
||||
: 'rebuilding')
|
||||
: (op.kind === 'meta_update' ? 'meta-update queued'
|
||||
: op.kind === 'destroy' ? 'destroy queued'
|
||||
|
|
@ -818,6 +819,7 @@ export function renderContainers(s) {
|
|||
: op.kind === 'start' ? 'start queued'
|
||||
: op.kind === 'stop' ? 'stop queued'
|
||||
: op.kind === 'graceful_stop' ? 'stop queued'
|
||||
: op.kind === 'reconcile' ? 'reconcile queued'
|
||||
: 'rebuild queued')));
|
||||
const opRunning = transientKind != null
|
||||
|| (op != null && op.state === 'running');
|
||||
|
|
|
|||
|
|
@ -134,6 +134,33 @@
|
|||
.rqe-source-approval { color: var(--green); border-color: var(--green); }
|
||||
.rqe-when { color: var(--muted); font-size: 0.85em; }
|
||||
.rqe-reason { color: var(--muted); font-size: 0.85em; flex: 1 1 auto; }
|
||||
/* Per-node DAG chain: one chip per primitive node, dependency order. */
|
||||
.rqe-nodes {
|
||||
flex-basis: 100%;
|
||||
margin: 0.15em 0 0 1.8em;
|
||||
font-size: 0.85em;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: baseline;
|
||||
gap: 0.15em;
|
||||
}
|
||||
.rqe-node {
|
||||
padding: 0.05em 0.45em;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 0.7em;
|
||||
color: var(--muted);
|
||||
white-space: nowrap;
|
||||
}
|
||||
.rqe-node-running {
|
||||
color: var(--purple);
|
||||
border-color: var(--purple);
|
||||
animation: badge-pulse 1.6s ease-in-out infinite;
|
||||
}
|
||||
.rqe-node-done { color: var(--green); border-color: color-mix(in srgb, var(--green) 45%, transparent); }
|
||||
.rqe-node-failed { color: var(--red); border-color: var(--red); }
|
||||
.rqe-node-cancelled { opacity: 0.55; text-decoration: line-through; }
|
||||
.rqe-node-arrow { color: var(--muted); }
|
||||
.rqe-node-log { margin-left: 0.1em; text-decoration: none; }
|
||||
.rqe-step {
|
||||
flex-basis: 100%;
|
||||
margin: 0.1em 0 0 1.8em;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
|
|||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use hive_claude::TokenUsage;
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::broadcast;
|
||||
|
|
@ -213,14 +214,6 @@ pub fn write_forge_cursor<S: std::hash::BuildHasher>(
|
|||
write_harness_json(&v);
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
|
|
|
|||
359
hive-ag3nt/src/mcp/args.rs
Normal file
359
hive-ag3nt/src/mcp/args.rs
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
//! Argument structs for the MCP tools: `serde::Deserialize` +
|
||||
//! `schemars::JsonSchema` derives whose field doc-comments become the
|
||||
//! parameter descriptions claude sees in each tool's input schema.
|
||||
|
||||
use rmcp::schemars;
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct SendArgs {
|
||||
/// Logical agent name to deliver the message to (e.g. `"manager"`,
|
||||
/// `"alice"`, or the literal `"operator"` for the dashboard's T4LK box).
|
||||
pub to: String,
|
||||
/// Message body. Plain text; the broker doesn't parse it.
|
||||
pub body: String,
|
||||
/// Optional broker row-id of the message this is a reply to. Lets
|
||||
/// the dashboard render conversation threads. Pass the `id` from the
|
||||
/// `DeliveredMessage` you're responding to; omit for new threads.
|
||||
/// Silently ignored if the id is unknown or out of retention.
|
||||
#[serde(default)]
|
||||
pub in_reply_to: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RecvArgs {
|
||||
/// How long to long-poll for the FIRST message before returning
|
||||
/// the empty marker. Capped at 60s server-side. Default (None)
|
||||
/// is 30s. Useful when an agent wants to park its turn waiting
|
||||
/// for any new work — pick a longer wait to coalesce bursts.
|
||||
#[serde(default)]
|
||||
pub wait_seconds: Option<u64>,
|
||||
/// Maximum number of messages to pop in this round-trip. Default
|
||||
/// (None) is 1 (single-message behaviour — exactly what you want
|
||||
/// when you're called to drive a turn off the first wake). Pass
|
||||
/// a higher value (capped at 5 server-side) when you've been
|
||||
/// told the inbox has more queued (the wake prompt mentions
|
||||
/// pending count) and want to drain everything in one tool call.
|
||||
/// Once the long-poll wakes up, the call drains up to `max` in
|
||||
/// total before returning — no extra round-trip needed.
|
||||
#[serde(default)]
|
||||
pub max: Option<u32>,
|
||||
}
|
||||
|
||||
/// MCP tool args for `ack_until`.
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AckUntilArgs {
|
||||
/// Highest broker message id to mark handled: every inbox message
|
||||
/// with `id <= up_to` (ids show as `[msg #<id>]` in recv output)
|
||||
/// is acked in one sweep. Pass the highest id you've actually
|
||||
/// seen/triaged — anything above it stays queued for later turns.
|
||||
pub up_to: i64,
|
||||
}
|
||||
|
||||
/// MCP tool args for `remind`. Exactly one of `delay_seconds` or
|
||||
/// `at_unix_timestamp` must be set; both / neither is a tool-side error.
|
||||
/// Hides the tagged `ReminderTiming` enum behind a flatter schema so the
|
||||
/// model picks one field instead of building `{"timing_type": "in_seconds",
|
||||
/// "seconds": 60}` shaped objects.
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RemindArgs {
|
||||
/// Body that lands in your inbox when the reminder fires (sender
|
||||
/// will appear as `reminder`). Soft cap at 4 KiB inline — anything
|
||||
/// larger gets auto-persisted to a file under
|
||||
/// `/agents/<you>/state/reminders/auto-<ts>.md` and the inbox
|
||||
/// message becomes a short pointer. Pass `file_path` if you want
|
||||
/// to control the destination yourself.
|
||||
pub message: String,
|
||||
/// Fire `delay_seconds` from now (relative). Set this OR
|
||||
/// `at_unix_timestamp`, not both.
|
||||
#[serde(default)]
|
||||
pub delay_seconds: Option<u64>,
|
||||
/// Fire at this absolute unix timestamp (seconds since epoch). Set
|
||||
/// this OR `delay_seconds`, not both.
|
||||
#[serde(default)]
|
||||
pub at_unix_timestamp: Option<i64>,
|
||||
/// Optional path to a file the scheduler should reference instead of
|
||||
/// inlining a long `message`. Use this for large payloads (research
|
||||
/// notes, file lists, intermediate state). Path must be reachable from
|
||||
/// the agent's container — typically under `/agents/<you>/state/`.
|
||||
#[serde(default)]
|
||||
pub file_path: Option<String>,
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Privileged tool arg types (lifecycle, approvals, scheduling, diagnostics)
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RequestInitConfigArgs {
|
||||
/// New sub-agent name (≤9 chars). Queues an `InitConfig` approval; on
|
||||
/// approval hive-c0re seeds the proposed config repo at
|
||||
/// `/agents/<name>/config/agent.nix` with the default template. After
|
||||
/// the approval the manager edits + commits the config and calls
|
||||
/// `request_apply_commit` to pin the customised sha for the container's
|
||||
/// first build.
|
||||
pub name: String,
|
||||
/// Optional description shown on the dashboard approval card.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct KillArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct SetStatusArgs {
|
||||
/// Status text to display on the dashboard card. Pass an empty string to clear.
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct CreateRepoArgs {
|
||||
/// Repo name — a single segment of letters, digits, `-`, `_`, `.`
|
||||
/// (no leading `-`/`.`). The repo is created as `agents/<repo>`.
|
||||
pub repo: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct GetAgentMetaArgs {
|
||||
/// Logical name of the agent to query (e.g. `"iris"`, `"manager"`).
|
||||
/// Omit to query your own identity + status — replaces the
|
||||
/// previous `whoami` self-introspection tool.
|
||||
#[serde(default)]
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct StartArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RestartArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct UpdateArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AskArgs {
|
||||
/// The question to surface.
|
||||
pub question: String,
|
||||
/// Optional fixed-choice answers. The dashboard renders these as
|
||||
/// chips alongside a free-text fallback ("Other…") so the operator
|
||||
/// is never trapped by an incomplete list; peer-agent recipients
|
||||
/// see the list in their inbox event and can return any string.
|
||||
#[serde(default)]
|
||||
pub options: Vec<String>,
|
||||
/// When true, options are rendered as checkboxes — the answerer
|
||||
/// can pick any subset. The answer comes back as a single string
|
||||
/// with selections joined by ", ". Ignored when `options` is empty.
|
||||
#[serde(default)]
|
||||
pub multi: bool,
|
||||
/// Optional auto-cancel after `ttl_seconds` (capped server-side at
|
||||
/// 6 hours). On expiry the question resolves with answer
|
||||
/// `[expired]` and the asker receives the usual
|
||||
/// `question_answered` system event (with `answerer:
|
||||
/// "ttl-watchdog"`). `None` (default) = wait indefinitely.
|
||||
#[serde(default)]
|
||||
pub ttl_seconds: Option<u64>,
|
||||
/// Recipient. Omit (or pass `"operator"`) to ask the human
|
||||
/// operator via the dashboard. Pass another agent's logical name
|
||||
/// to ask that peer — they receive a `question_asked` event in
|
||||
/// their inbox and answer via `mcp__hyperhive__answer`.
|
||||
#[serde(default)]
|
||||
pub to: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AnswerArgs {
|
||||
/// Id of the question being answered — comes from the
|
||||
/// `question_asked` event in your inbox.
|
||||
pub id: i64,
|
||||
/// Free-text answer body. Soft-capped at 4 KiB by the same
|
||||
/// `MESSAGE_MAX_BYTES` limit as `send`; keep it short or write the
|
||||
/// detail to a file and pass a path.
|
||||
pub answer: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct CancelLooseEndArgs {
|
||||
/// Which kind of thread to cancel — `"question"` for an open
|
||||
/// `ask` that's still waiting on an answer, `"reminder"` for a
|
||||
/// scheduled `remind` that hasn't fired yet. Use the `kind`
|
||||
/// field straight off the `get_loose_ends` row.
|
||||
pub kind: String,
|
||||
/// Row id from the matching `get_loose_ends` entry (or the
|
||||
/// `question_queued` reply when you submitted it).
|
||||
pub id: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AgentGetLooseEndsArgs {
|
||||
/// Whose loose ends to list. Omit (or `null`) for your own. You may
|
||||
/// also pass a direct child agent's name without any extra capability.
|
||||
/// Pass any other agent name to inspect their threads — requires the
|
||||
/// `query_agent_state` capability; without it the request is rejected
|
||||
/// with an error. The `"*"` hive-wide value is not available on the
|
||||
/// agent socket.
|
||||
#[serde(default)]
|
||||
pub agent: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RequestApplyCommitArgs {
|
||||
/// Logical agent name whose config repo the commit lives in.
|
||||
pub agent: String,
|
||||
/// Commit sha (full or short, 7-40 hex chars) in that agent's
|
||||
/// proposed config repo. Must be a sha — a branch or tag name
|
||||
/// (e.g. `main`) is rejected; the approval pins the exact commit.
|
||||
pub commit_ref: String,
|
||||
/// Optional description shown on the dashboard approval card so the
|
||||
/// operator knows what the change does without opening the diff.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct UpdateMetaInputsArgs {
|
||||
/// Flake input names to update (e.g. `["bitburner-agent", "nixpkgs"]`).
|
||||
/// Pass an empty list to update ALL inputs.
|
||||
#[serde(default)]
|
||||
pub inputs: Vec<String>,
|
||||
/// Optional description shown on the dashboard approval card.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RequestSchedulePromptArgs {
|
||||
/// Recipient agents — one schedule fires to many inboxes at the
|
||||
/// scheduled time. `operator` is a legitimate target (mara: "we
|
||||
/// want to get rid of the manager special case so yes manager
|
||||
/// can be recipient" — the operator slot follows the same rule).
|
||||
pub targets: Vec<String>,
|
||||
/// Message body delivered to each target's inbox at fire time.
|
||||
/// Same size budget as `send` bodies.
|
||||
pub body: String,
|
||||
/// Absolute unix timestamp (seconds) for the FIRST fire. For
|
||||
/// recurring schedules the worker re-arms in
|
||||
/// `interval_seconds` steps from this point on.
|
||||
pub first_fire_at_unix: i64,
|
||||
/// `None` / absent = one-shot. `Some(n > 0)` = recurring every
|
||||
/// `n` seconds. The worker clamps catch-up so a long downtime
|
||||
/// fires ONCE on resume (skipped-cycle count surfaces in the
|
||||
/// per-target `last_result`), not N delayed pulses in a row.
|
||||
#[serde(default)]
|
||||
pub interval_seconds: Option<u64>,
|
||||
/// Optional description shown on the dashboard approval card +
|
||||
/// preserved on the schedule row for later operator reference.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct FireScheduleNowArgs {
|
||||
/// Schedule id to fire out of band. Get this from a prior
|
||||
/// `list_schedules` call or the approval-resolved event for
|
||||
/// the originating `request_schedule_prompt`.
|
||||
pub id: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct CancelScheduleArgs {
|
||||
/// Schedule id from a prior `list_schedules` call or the
|
||||
/// approval-resolved event for a `request_schedule_prompt`.
|
||||
pub id: i64,
|
||||
/// Optional target list. `None` / empty = cancel the entire
|
||||
/// schedule. `Some(["alice", "bob"])` = cancel just those
|
||||
/// recipients (the schedule keeps firing for any remaining
|
||||
/// active targets, and auto-cancels its parent row when every
|
||||
/// target is gone).
|
||||
#[serde(default)]
|
||||
pub targets: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct EditScheduleArgs {
|
||||
/// Schedule id from a prior `list_schedules` call or the
|
||||
/// approval-resolved event for a `request_schedule_prompt`.
|
||||
pub id: i64,
|
||||
/// New body text. Omit to keep the existing one.
|
||||
#[serde(default)]
|
||||
pub body: Option<String>,
|
||||
/// New description. Omit to keep the existing one. (To CLEAR
|
||||
/// the description, use the dashboard PATCH endpoint
|
||||
/// directly — the agent surface intentionally keeps the args
|
||||
/// flat / non-nullable to dodge the doubly-wrapped Option
|
||||
/// schemars quirk; clearing fields is rare and operator-side.)
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
/// Recurring interval in seconds. Omit to keep the existing
|
||||
/// cadence; pass an explicit value to set a new one. Toggling
|
||||
/// recurring↔one-shot (clearing the interval) is operator-only
|
||||
/// for the same reason as `description` above.
|
||||
#[serde(default)]
|
||||
pub interval_seconds: Option<u64>,
|
||||
/// New absolute unix timestamp for the next fire. Omit to
|
||||
/// leave the schedule on its current cadence.
|
||||
#[serde(default)]
|
||||
pub next_fire_at_unix: Option<i64>,
|
||||
/// Names of new targets to add. Replace-on-conflict: re-adding
|
||||
/// a previously cancelled target resets its history (operator
|
||||
/// intent on re-add = "this target is active again").
|
||||
#[serde(default)]
|
||||
pub targets_add: Option<Vec<String>>,
|
||||
/// Names of targets to cancel. Tombstones preserve per-target
|
||||
/// audit; when no active targets remain the schedule
|
||||
/// auto-cancels.
|
||||
#[serde(default)]
|
||||
pub targets_remove: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct GetLogsArgs {
|
||||
/// Logical agent name to fetch logs for (e.g. `gui`, `iris`).
|
||||
/// hive-c0re maps it to the underlying machine name (`h-gui`)
|
||||
/// itself — pass the plain agent name, not the `h-` form.
|
||||
pub agent: String,
|
||||
/// How many journal lines to return (default: 50, max: 500).
|
||||
#[serde(default)]
|
||||
pub lines: Option<u32>,
|
||||
}
|
||||
|
||||
/// Arguments for `get_host_journal` (capability-gated: `read_host_journal`).
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct GetHostJournalArgs {
|
||||
/// Systemd unit to filter (e.g. `hive-c0re.service`). Omit for all units.
|
||||
#[serde(default)]
|
||||
pub unit: Option<String>,
|
||||
/// nspawn machine name verbatim (e.g. `h-iris`). Omit for host journal.
|
||||
/// Agent containers use the `h-` prefix (e.g. `h-iris`); infrastructure
|
||||
/// containers use their full name (e.g. `hive-ci`, `hive-forge`,
|
||||
/// `hive-matrix`, `hive-gateway`).
|
||||
#[serde(default)]
|
||||
pub container: Option<String>,
|
||||
/// Number of lines to return (default 30, max 100).
|
||||
#[serde(default)]
|
||||
pub lines: Option<u32>,
|
||||
/// Minimum syslog priority level.
|
||||
#[serde(default)]
|
||||
pub priority: Option<hive_sh4re::JournalPriority>,
|
||||
/// Regex to match against log message fields (journalctl --grep).
|
||||
#[serde(default)]
|
||||
pub grep: Option<String>,
|
||||
/// Show entries on or newer than this timestamp (e.g. `-1h`).
|
||||
#[serde(default)]
|
||||
pub since: Option<String>,
|
||||
/// Show entries on or older than this timestamp.
|
||||
#[serde(default)]
|
||||
pub until: Option<String>,
|
||||
}
|
||||
|
|
@ -18,12 +18,31 @@ use std::path::PathBuf;
|
|||
|
||||
use anyhow::Result;
|
||||
use rmcp::{
|
||||
ServerHandler, ServiceExt, handler::server::wrapper::Parameters, schemars, tool, tool_handler,
|
||||
ServerHandler, ServiceExt, handler::server::wrapper::Parameters, tool, tool_handler,
|
||||
tool_router, transport::stdio,
|
||||
};
|
||||
|
||||
use crate::client;
|
||||
|
||||
mod args;
|
||||
mod render;
|
||||
|
||||
pub use args::{
|
||||
AckUntilArgs, AgentGetLooseEndsArgs, AnswerArgs, AskArgs, CancelLooseEndArgs,
|
||||
CancelScheduleArgs, CreateRepoArgs, EditScheduleArgs, FireScheduleNowArgs, GetAgentMetaArgs,
|
||||
GetHostJournalArgs, GetLogsArgs, KillArgs, RecvArgs, RemindArgs, RequestApplyCommitArgs,
|
||||
RequestInitConfigArgs, RequestSchedulePromptArgs, RestartArgs, SendArgs, SetStatusArgs,
|
||||
StartArgs, UpdateArgs, UpdateMetaInputsArgs,
|
||||
};
|
||||
pub use render::{
|
||||
IDLE_WAIT_HINT, REDELIVERY_HINT, annotate_retries, format_ack, format_agent_meta, format_recv,
|
||||
};
|
||||
|
||||
use render::{
|
||||
format_matrix_summary, loose_end_kind_label, matrix_unread_summary, parse_loose_end_kind,
|
||||
render_loose_ends, reply_err,
|
||||
};
|
||||
|
||||
/// Write (or remove) the status file in the agent's own `state/` directory.
|
||||
/// Called by `AgentServer::set_status` for both agent and manager flavors
|
||||
/// before dispatching the wire `SetStatus` request (which only triggers a
|
||||
|
|
@ -69,406 +88,6 @@ fn write_status_file(text: &str) -> Result<(), String> {
|
|||
result.map_err(|e| format!("set_status write failed: {e}"))
|
||||
}
|
||||
|
||||
/// Render the three identical failure arms every data-returning tool handler
|
||||
/// repeats: a broker `Err` → `"{tool} failed: {m}"`, an unexpected `Ok` variant
|
||||
/// → `"{tool} unexpected response: …"`, and a transport error → `"{tool}
|
||||
/// transport error: …"`. Handlers match their own happy-path variant and route
|
||||
/// everything else here via a catch-all arm (`other => reply_err(other, tool)`),
|
||||
/// so the triplet lives in exactly one place.
|
||||
fn reply_err(resp: Result<hive_sh4re::Response, anyhow::Error>, tool: &str) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::Err { message }) => format!("{tool} failed: {message}"),
|
||||
Ok(other) => format!("{tool} unexpected response: {other:?}"),
|
||||
Err(e) => format!("{tool} transport error: {e:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format helper for "send-like" tools (anything that expects an `Ok`).
|
||||
/// `tool` and `ok_msg` only appear in the result string; they don't change
|
||||
/// behavior.
|
||||
#[must_use]
|
||||
pub fn format_ack(
|
||||
resp: Result<hive_sh4re::Response, anyhow::Error>,
|
||||
tool: &str,
|
||||
ok_msg: String,
|
||||
) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::Ok) => ok_msg,
|
||||
other => reply_err(other, tool),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format helper for `recv`: renders zero, one, or many popped
|
||||
/// messages. Empty list collapses to "(empty)" so claude doesn't go
|
||||
/// hunting for content; when `waited` is set (the call parked on a
|
||||
/// long-poll that timed out) the empty result also carries
|
||||
/// [`IDLE_WAIT_HINT`] nudging the model toward other work. A single
|
||||
/// message renders as the historical `from: X\n\nbody` block (banner
|
||||
/// first if `redelivered`). A multi-message batch renders with a
|
||||
/// `popped N message(s):` header and `---` separators between bodies
|
||||
/// so the model can tell where one ends and the next begins;
|
||||
/// per-message redelivery banners included.
|
||||
#[must_use]
|
||||
pub fn format_recv(resp: Result<hive_sh4re::Response, anyhow::Error>, waited: bool) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::Messages { messages }) => render_recv_messages(&messages, waited),
|
||||
// A graceful stop is pending — the inbox is fenced. Render a single
|
||||
// explicit directive (not an empty inbox, which claude's "park on recv"
|
||||
// habit would long-poll again, stalling the stop-checkpoint turn until
|
||||
// the drain wait times out into a hard stop) so every recv during the
|
||||
// stop unmissably tells claude to flush + end.
|
||||
Ok(hive_sh4re::Response::GracefulStop) => {
|
||||
render_recv_messages(&[graceful_stop_message()], waited)
|
||||
}
|
||||
other => reply_err(other, "recv"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The synthetic single-message directive rendered for a fenced (graceful-stop)
|
||||
/// inbox — see the `GracefulStop` arm of [`format_recv`].
|
||||
fn graceful_stop_message() -> hive_sh4re::DeliveredMessage {
|
||||
hive_sh4re::DeliveredMessage {
|
||||
from: "graceful-stop".into(),
|
||||
body: "⛔ GRACEFUL STOP IN PROGRESS — the container shuts down as soon as this turn \
|
||||
ends. Flush anything worth keeping to your durable /state files, then END \
|
||||
YOUR TURN now. Do NOT call recv again: the inbox is fenced and recv will \
|
||||
only keep returning this same notice."
|
||||
.into(),
|
||||
id: 0,
|
||||
redelivered: false,
|
||||
in_reply_to: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Render the popped-message payload of a successful `recv` (see `format_recv`
|
||||
/// for the empty/single/batch shapes).
|
||||
fn render_recv_messages(messages: &[hive_sh4re::DeliveredMessage], waited: bool) -> String {
|
||||
use std::fmt::Write as _;
|
||||
if messages.is_empty() {
|
||||
return if waited {
|
||||
format!("(empty){IDLE_WAIT_HINT}")
|
||||
} else {
|
||||
"(empty)".to_owned()
|
||||
};
|
||||
}
|
||||
if messages.len() == 1 {
|
||||
let m = &messages[0];
|
||||
let banner = if m.redelivered { REDELIVERY_HINT } else { "" };
|
||||
return format!("{banner}{}from: {}\n\n{}", msg_id_tag(m.id), m.from, m.body);
|
||||
}
|
||||
let n = messages.len();
|
||||
let mut out = format!("popped {n} message(s):\n\n");
|
||||
for (i, m) in messages.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push_str("\n---\n\n");
|
||||
}
|
||||
let banner = if m.redelivered { REDELIVERY_HINT } else { "" };
|
||||
let _ = write!(
|
||||
out,
|
||||
"{banner}{}from: {}\n\n{}",
|
||||
msg_id_tag(m.id),
|
||||
m.from,
|
||||
m.body
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `[msg #<id>] ` marker prefixed to each recv row so the agent knows
|
||||
/// what to pass to `ack_until` when bulk-triaging a backlog. Transient
|
||||
/// pings carry the sentinel id 0 (in-memory only, nothing in the
|
||||
/// broker to ack) and render without the marker.
|
||||
fn msg_id_tag(id: i64) -> String {
|
||||
if id > 0 {
|
||||
format!("[msg #{id}] ")
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Header prepended to message bodies that were popped by a prior
|
||||
/// harness session, never acked (turn crash / OOM / restart), and
|
||||
/// resurfaced by `RequeueInflight` on this session's boot. Same
|
||||
/// string surfaces in the wake prompt (see the bin loops) and the
|
||||
/// in-turn `recv` tool result so claude sees the warning either way.
|
||||
pub const REDELIVERY_HINT: &str = "[redelivered after harness restart — may already be handled]\n";
|
||||
|
||||
/// Appended to the `recv` empty result when the agent parked on a
|
||||
/// long-poll (`wait_seconds > 0`) that timed out with nothing new.
|
||||
/// Nudges the model to spend the idle time on other useful work
|
||||
/// instead of immediately re-blocking on `recv`.
|
||||
pub const IDLE_WAIT_HINT: &str = " — nothing arrived before the wait timed out. \
|
||||
If you have other useful work (assigned issues, in-flight PRs, a docs sweep, \
|
||||
notes to update), do that now rather than immediately parking on recv again.";
|
||||
|
||||
/// Inner renderer for a `Vec<LooseEnd>` already extracted from the socket
|
||||
/// reply. Called by the `get_loose_ends` handler, which injects the
|
||||
/// `UnreadMatrix` entry before formatting.
|
||||
fn render_loose_ends(loose_ends: &[hive_sh4re::LooseEnd]) -> String {
|
||||
use std::fmt::Write as _;
|
||||
if loose_ends.is_empty() {
|
||||
return "(no loose ends)".to_owned();
|
||||
}
|
||||
let mut out = format!("{} loose end(s):\n", loose_ends.len());
|
||||
for t in loose_ends {
|
||||
match t {
|
||||
hive_sh4re::LooseEnd::Approval {
|
||||
id,
|
||||
agent,
|
||||
commit_ref,
|
||||
description,
|
||||
age_seconds,
|
||||
} => {
|
||||
let desc = description
|
||||
.as_deref()
|
||||
.map(|d| format!(" — {d}"))
|
||||
.unwrap_or_default();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- approval #{id} ({agent} @ {commit_ref}, {age_seconds}s old){desc}"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::Question {
|
||||
id,
|
||||
asker,
|
||||
target,
|
||||
question,
|
||||
age_seconds,
|
||||
} => {
|
||||
let to = target.as_deref().unwrap_or("operator");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- question #{id} ({asker} → {to}, {age_seconds}s old): {question}"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::Reminder {
|
||||
id,
|
||||
owner,
|
||||
message,
|
||||
due_at,
|
||||
age_seconds,
|
||||
} => {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- reminder #{id} ({owner}, scheduled {age_seconds}s ago, due_at={due_at}): {message}"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::PendingMessages { count } => {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- {count} pending inbox message(s) — drain with recv (recv(max: {count}) to batch)"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::UnreadMatrix { rooms, summary } => {
|
||||
let _ = write!(out, "- unread matrix messages in {rooms} room(s)");
|
||||
if summary.is_empty() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" — use list_rooms + read_room to view, mark_read to clear"
|
||||
);
|
||||
} else {
|
||||
let _ = writeln!(out, ":");
|
||||
for line in summary.lines() {
|
||||
let _ = writeln!(out, " {line}");
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" use list_rooms + read_room to view, mark_read to clear"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Per-room unread entry returned by `matrix_unread_summary`. Mirrors
|
||||
/// `hive-matrix-mcp`'s `RoomUnread` but defined locally to avoid a
|
||||
/// cross-crate dep on the matrix-sdk crate tree.
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct MatrixRoomUnread {
|
||||
label: String,
|
||||
count: u32,
|
||||
last_body: Option<String>,
|
||||
last_sender: Option<String>,
|
||||
}
|
||||
|
||||
/// Query the local matrix daemon for per-room unread summaries. Returns
|
||||
/// `None` if the daemon socket is absent or the query fails. Best-effort:
|
||||
/// agents without matrix configured are not penalised.
|
||||
async fn matrix_unread_summary() -> Option<Vec<MatrixRoomUnread>> {
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::UnixStream;
|
||||
let socket = std::env::var_os("HIVE_MATRIX_SOCKET").map_or_else(
|
||||
|| std::path::PathBuf::from("/run/hive-matrix/socket"),
|
||||
std::path::PathBuf::from,
|
||||
);
|
||||
if !socket.exists() {
|
||||
return None;
|
||||
}
|
||||
let mut stream = UnixStream::connect(&socket).await.ok()?;
|
||||
stream
|
||||
.write_all(b"{\"method\":\"unread_summary\"}\n")
|
||||
.await
|
||||
.ok()?;
|
||||
let mut lines = BufReader::new(stream).lines();
|
||||
let line = lines.next_line().await.ok()??;
|
||||
let val: serde_json::Value = serde_json::from_str(&line).ok()?;
|
||||
// Response: {"kind":"ok","payload":[{label, count, last_body?, last_sender?}]}
|
||||
let arr = val.get("payload")?.as_array()?;
|
||||
serde_json::from_value(serde_json::Value::Array(arr.clone())).ok()
|
||||
}
|
||||
|
||||
/// Format a `Vec<MatrixRoomUnread>` into a per-room summary string.
|
||||
/// Single room / single message collapses to one line; multi-room
|
||||
/// expands to a bulleted list. Returns an empty string for empty input.
|
||||
fn format_matrix_summary(rooms: &[MatrixRoomUnread]) -> String {
|
||||
use std::fmt::Write as _;
|
||||
if rooms.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for r in rooms {
|
||||
if r.count == 1
|
||||
&& let (Some(body), Some(sender)) = (&r.last_body, &r.last_sender)
|
||||
{
|
||||
let _ = writeln!(out, "- {}: {sender}: {body}", r.label);
|
||||
continue;
|
||||
}
|
||||
let _ = writeln!(out, "- {}: {} unread", r.label, r.count);
|
||||
}
|
||||
// Remove trailing newline.
|
||||
if out.ends_with('\n') {
|
||||
out.pop();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the user-facing `kind` string for `cancel_loose_end` into the
|
||||
/// wire enum. Accepts a small alias set so claude doesn't have to
|
||||
/// remember the exact spelling (`"q"` / `"r"` shorthand falls out
|
||||
/// for free).
|
||||
fn parse_loose_end_kind(raw: &str) -> Result<hive_sh4re::CancelLooseEndKind, String> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
"question" | "q" => Ok(hive_sh4re::CancelLooseEndKind::Question),
|
||||
"reminder" | "r" => Ok(hive_sh4re::CancelLooseEndKind::Reminder),
|
||||
"approval" | "a" => Ok(hive_sh4re::CancelLooseEndKind::Approval),
|
||||
other => Err(format!(
|
||||
"cancel_loose_end: unknown kind '{other}' \
|
||||
(expected \"question\", \"reminder\", or \"approval\")"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical user-facing label for a `CancelLooseEndKind` — used in
|
||||
/// the success ack so the caller always sees `"question"` /
|
||||
/// `"reminder"` instead of whatever alias they passed in (`"q"` /
|
||||
/// `"r"`).
|
||||
fn loose_end_kind_label(kind: hive_sh4re::CancelLooseEndKind) -> &'static str {
|
||||
match kind {
|
||||
hive_sh4re::CancelLooseEndKind::Question => "question",
|
||||
hive_sh4re::CancelLooseEndKind::Reminder => "reminder",
|
||||
hive_sh4re::CancelLooseEndKind::Approval => "approval",
|
||||
}
|
||||
}
|
||||
|
||||
/// Format helper for `get_agent_meta`: renders an agent's identity +
|
||||
/// current status as a short human-readable block. `name`,
|
||||
/// `hyperhive_rev`, and `running` are always shown; `status` only
|
||||
/// appears when one is set, otherwise the line reads `status: <none>`.
|
||||
/// When `running` is false the host has already cleared `status_text`
|
||||
/// (it would be a stale snapshot from before the stop) so the status
|
||||
/// line is implicitly `<none>` in that case — but the explicit
|
||||
/// `running: no` line tells the caller WHY. See
|
||||
/// `docs/turn-loop/mcp.md::Core tools` (`get_agent_meta`).
|
||||
#[must_use]
|
||||
pub fn format_agent_meta(resp: Result<hive_sh4re::Response, anyhow::Error>) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::AgentMeta {
|
||||
name,
|
||||
running,
|
||||
hyperhive_rev,
|
||||
status_text,
|
||||
status_set_at,
|
||||
hive_name,
|
||||
swarm_name,
|
||||
matrix_accounts,
|
||||
}) => {
|
||||
let rev = hyperhive_rev.as_deref().unwrap_or("<unknown>");
|
||||
let run = if running { "yes" } else { "no" };
|
||||
let mut out = format!("name: {name}\nhyperhive_rev: {rev}\nrunning: {run}");
|
||||
// Surface hive + swarm display names only when set, so
|
||||
// single-hive deployments don't see noisy `<none>` lines.
|
||||
if let Some(hn) = hive_name.as_deref() {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(out, "\nhive_name: {hn}");
|
||||
}
|
||||
if let Some(sn) = swarm_name.as_deref() {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(out, "\nswarm_name: {sn}");
|
||||
}
|
||||
match status_text {
|
||||
None => out.push_str("\nstatus: <none>"),
|
||||
Some(s) => {
|
||||
use std::fmt::Write as _;
|
||||
let age = status_set_at.and_then(|ts| {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?
|
||||
.as_secs();
|
||||
// `ts` is a unix epoch second the agent itself
|
||||
// sourced from `SystemTime` — always positive
|
||||
// in normal operation. Clamp the negative
|
||||
// (clock-skew) edge to 0 before the unsigned
|
||||
// cast so the cast loses no real precision.
|
||||
let ts_secs = u64::try_from(ts).unwrap_or(0);
|
||||
let secs = now.saturating_sub(ts_secs);
|
||||
Some(format_age_secs(secs))
|
||||
});
|
||||
// `write!` into the buffer instead of `push_str(&format!(…))` —
|
||||
// avoids the intermediate allocation clippy::format_push_string
|
||||
// flags. The infallible `String` writer makes this safe to
|
||||
// `let _ =`-ignore.
|
||||
match age {
|
||||
Some(a) => {
|
||||
let _ = write!(out, "\nstatus: {s} (set {a} ago)");
|
||||
}
|
||||
None => {
|
||||
let _ = write!(out, "\nstatus: {s}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Matrix identities the agent can act as (the `account` arg on
|
||||
// the matrix tools). Listed only when matrix is provisioned, so
|
||||
// non-matrix agents don't see an empty line.
|
||||
if !matrix_accounts.is_empty() {
|
||||
use std::fmt::Write as _;
|
||||
out.push_str("\nmatrix_accounts:");
|
||||
for acct in &matrix_accounts {
|
||||
let uid = acct.user_id.as_deref().unwrap_or("?");
|
||||
let _ = write!(out, "\n {} ({uid}) on {}", acct.name, acct.homeserver);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
other => reply_err(other, "get_agent_meta"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a duration in seconds as a human-readable age string.
|
||||
fn format_age_secs(secs: u64) -> String {
|
||||
if secs < 60 {
|
||||
format!("{secs}s")
|
||||
} else if secs < 3600 {
|
||||
format!("{}m", secs / 60)
|
||||
} else if secs < 86400 {
|
||||
format!("{}h", secs / 3600)
|
||||
} else {
|
||||
format!("{}d", secs / 86400)
|
||||
}
|
||||
}
|
||||
|
||||
/// Common envelope around every MCP tool handler: pre-log → run →
|
||||
/// post-log. Tool results stay clean — the inbox-status hint lives in
|
||||
/// the wake prompt + UI header, not appended here.
|
||||
|
|
@ -482,100 +101,6 @@ where
|
|||
result
|
||||
}
|
||||
|
||||
/// Append a short note to a tool result when the underlying socket call
|
||||
/// took retries to land. Lets claude distinguish "my request was wrong"
|
||||
/// from "c0re flickered and the harness rode it out" — without the
|
||||
/// hint, a tool result that took 30s to come back looks identical to a
|
||||
/// content failure and the model would burn a turn retrying it.
|
||||
#[must_use]
|
||||
pub fn annotate_retries(mut s: String, retries: u32) -> String {
|
||||
if retries > 0 {
|
||||
use std::fmt::Write as _;
|
||||
let suffix = if retries == 1 { "retry" } else { "retries" };
|
||||
let _ = write!(
|
||||
s,
|
||||
"\n\n(note: hive socket connect needed {retries} {suffix} — c0re likely \
|
||||
restarted. Your request did succeed on the final attempt; no action needed.)"
|
||||
);
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct SendArgs {
|
||||
/// Logical agent name to deliver the message to (e.g. `"manager"`,
|
||||
/// `"alice"`, or the literal `"operator"` for the dashboard's T4LK box).
|
||||
pub to: String,
|
||||
/// Message body. Plain text; the broker doesn't parse it.
|
||||
pub body: String,
|
||||
/// Optional broker row-id of the message this is a reply to. Lets
|
||||
/// the dashboard render conversation threads. Pass the `id` from the
|
||||
/// `DeliveredMessage` you're responding to; omit for new threads.
|
||||
/// Silently ignored if the id is unknown or out of retention.
|
||||
#[serde(default)]
|
||||
pub in_reply_to: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RecvArgs {
|
||||
/// How long to long-poll for the FIRST message before returning
|
||||
/// the empty marker. Capped at 60s server-side. Default (None)
|
||||
/// is 30s. Useful when an agent wants to park its turn waiting
|
||||
/// for any new work — pick a longer wait to coalesce bursts.
|
||||
#[serde(default)]
|
||||
pub wait_seconds: Option<u64>,
|
||||
/// Maximum number of messages to pop in this round-trip. Default
|
||||
/// (None) is 1 (single-message behaviour — exactly what you want
|
||||
/// when you're called to drive a turn off the first wake). Pass
|
||||
/// a higher value (capped at 5 server-side) when you've been
|
||||
/// told the inbox has more queued (the wake prompt mentions
|
||||
/// pending count) and want to drain everything in one tool call.
|
||||
/// Once the long-poll wakes up, the call drains up to `max` in
|
||||
/// total before returning — no extra round-trip needed.
|
||||
#[serde(default)]
|
||||
pub max: Option<u32>,
|
||||
}
|
||||
|
||||
/// MCP tool args for `ack_until`.
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AckUntilArgs {
|
||||
/// Highest broker message id to mark handled: every inbox message
|
||||
/// with `id <= up_to` (ids show as `[msg #<id>]` in recv output)
|
||||
/// is acked in one sweep. Pass the highest id you've actually
|
||||
/// seen/triaged — anything above it stays queued for later turns.
|
||||
pub up_to: i64,
|
||||
}
|
||||
|
||||
/// MCP tool args for `remind`. Exactly one of `delay_seconds` or
|
||||
/// `at_unix_timestamp` must be set; both / neither is a tool-side error.
|
||||
/// Hides the tagged `ReminderTiming` enum behind a flatter schema so the
|
||||
/// model picks one field instead of building `{"timing_type": "in_seconds",
|
||||
/// "seconds": 60}` shaped objects.
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RemindArgs {
|
||||
/// Body that lands in your inbox when the reminder fires (sender
|
||||
/// will appear as `reminder`). Soft cap at 4 KiB inline — anything
|
||||
/// larger gets auto-persisted to a file under
|
||||
/// `/agents/<you>/state/reminders/auto-<ts>.md` and the inbox
|
||||
/// message becomes a short pointer. Pass `file_path` if you want
|
||||
/// to control the destination yourself.
|
||||
pub message: String,
|
||||
/// Fire `delay_seconds` from now (relative). Set this OR
|
||||
/// `at_unix_timestamp`, not both.
|
||||
#[serde(default)]
|
||||
pub delay_seconds: Option<u64>,
|
||||
/// Fire at this absolute unix timestamp (seconds since epoch). Set
|
||||
/// this OR `delay_seconds`, not both.
|
||||
#[serde(default)]
|
||||
pub at_unix_timestamp: Option<i64>,
|
||||
/// Optional path to a file the scheduler should reference instead of
|
||||
/// inlining a long `message`. Use this for large payloads (research
|
||||
/// notes, file lists, intermediate state). Path must be reachable from
|
||||
/// the agent's container — typically under `/agents/<you>/state/`.
|
||||
#[serde(default)]
|
||||
pub file_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Unified MCP tool surface for both sub-agent and manager roles.
|
||||
///
|
||||
/// `AgentRequest = ManagerRequest = Request` and `AgentResponse =
|
||||
|
|
@ -1531,305 +1056,3 @@ pub async fn serve_http(socket: PathBuf, addr: std::net::SocketAddr) -> Result<(
|
|||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Privileged tool arg types (lifecycle, approvals, scheduling, diagnostics)
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RequestInitConfigArgs {
|
||||
/// New sub-agent name (≤9 chars). Queues an `InitConfig` approval; on
|
||||
/// approval hive-c0re seeds the proposed config repo at
|
||||
/// `/agents/<name>/config/agent.nix` with the default template. After
|
||||
/// the approval the manager edits + commits the config and calls
|
||||
/// `request_apply_commit` to pin the customised sha for the container's
|
||||
/// first build.
|
||||
pub name: String,
|
||||
/// Optional description shown on the dashboard approval card.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct KillArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct SetStatusArgs {
|
||||
/// Status text to display on the dashboard card. Pass an empty string to clear.
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct CreateRepoArgs {
|
||||
/// Repo name — a single segment of letters, digits, `-`, `_`, `.`
|
||||
/// (no leading `-`/`.`). The repo is created as `agents/<repo>`.
|
||||
pub repo: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct GetAgentMetaArgs {
|
||||
/// Logical name of the agent to query (e.g. `"iris"`, `"manager"`).
|
||||
/// Omit to query your own identity + status — replaces the
|
||||
/// previous `whoami` self-introspection tool.
|
||||
#[serde(default)]
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct StartArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RestartArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct UpdateArgs {
|
||||
/// Sub-agent name (without the `h-` container prefix).
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AskArgs {
|
||||
/// The question to surface.
|
||||
pub question: String,
|
||||
/// Optional fixed-choice answers. The dashboard renders these as
|
||||
/// chips alongside a free-text fallback ("Other…") so the operator
|
||||
/// is never trapped by an incomplete list; peer-agent recipients
|
||||
/// see the list in their inbox event and can return any string.
|
||||
#[serde(default)]
|
||||
pub options: Vec<String>,
|
||||
/// When true, options are rendered as checkboxes — the answerer
|
||||
/// can pick any subset. The answer comes back as a single string
|
||||
/// with selections joined by ", ". Ignored when `options` is empty.
|
||||
#[serde(default)]
|
||||
pub multi: bool,
|
||||
/// Optional auto-cancel after `ttl_seconds` (capped server-side at
|
||||
/// 6 hours). On expiry the question resolves with answer
|
||||
/// `[expired]` and the asker receives the usual
|
||||
/// `question_answered` system event (with `answerer:
|
||||
/// "ttl-watchdog"`). `None` (default) = wait indefinitely.
|
||||
#[serde(default)]
|
||||
pub ttl_seconds: Option<u64>,
|
||||
/// Recipient. Omit (or pass `"operator"`) to ask the human
|
||||
/// operator via the dashboard. Pass another agent's logical name
|
||||
/// to ask that peer — they receive a `question_asked` event in
|
||||
/// their inbox and answer via `mcp__hyperhive__answer`.
|
||||
#[serde(default)]
|
||||
pub to: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AnswerArgs {
|
||||
/// Id of the question being answered — comes from the
|
||||
/// `question_asked` event in your inbox.
|
||||
pub id: i64,
|
||||
/// Free-text answer body. Soft-capped at 4 KiB by the same
|
||||
/// `MESSAGE_MAX_BYTES` limit as `send`; keep it short or write the
|
||||
/// detail to a file and pass a path.
|
||||
pub answer: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct CancelLooseEndArgs {
|
||||
/// Which kind of thread to cancel — `"question"` for an open
|
||||
/// `ask` that's still waiting on an answer, `"reminder"` for a
|
||||
/// scheduled `remind` that hasn't fired yet. Use the `kind`
|
||||
/// field straight off the `get_loose_ends` row.
|
||||
pub kind: String,
|
||||
/// Row id from the matching `get_loose_ends` entry (or the
|
||||
/// `question_queued` reply when you submitted it).
|
||||
pub id: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct AgentGetLooseEndsArgs {
|
||||
/// Whose loose ends to list. Omit (or `null`) for your own. You may
|
||||
/// also pass a direct child agent's name without any extra capability.
|
||||
/// Pass any other agent name to inspect their threads — requires the
|
||||
/// `query_agent_state` capability; without it the request is rejected
|
||||
/// with an error. The `"*"` hive-wide value is not available on the
|
||||
/// agent socket.
|
||||
#[serde(default)]
|
||||
pub agent: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RequestApplyCommitArgs {
|
||||
/// Logical agent name whose config repo the commit lives in.
|
||||
pub agent: String,
|
||||
/// Commit sha (full or short, 7-40 hex chars) in that agent's
|
||||
/// proposed config repo. Must be a sha — a branch or tag name
|
||||
/// (e.g. `main`) is rejected; the approval pins the exact commit.
|
||||
pub commit_ref: String,
|
||||
/// Optional description shown on the dashboard approval card so the
|
||||
/// operator knows what the change does without opening the diff.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct UpdateMetaInputsArgs {
|
||||
/// Flake input names to update (e.g. `["bitburner-agent", "nixpkgs"]`).
|
||||
/// Pass an empty list to update ALL inputs.
|
||||
#[serde(default)]
|
||||
pub inputs: Vec<String>,
|
||||
/// Optional description shown on the dashboard approval card.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct RequestSchedulePromptArgs {
|
||||
/// Recipient agents — one schedule fires to many inboxes at the
|
||||
/// scheduled time. `operator` is a legitimate target (mara: "we
|
||||
/// want to get rid of the manager special case so yes manager
|
||||
/// can be recipient" — the operator slot follows the same rule).
|
||||
pub targets: Vec<String>,
|
||||
/// Message body delivered to each target's inbox at fire time.
|
||||
/// Same size budget as `send` bodies.
|
||||
pub body: String,
|
||||
/// Absolute unix timestamp (seconds) for the FIRST fire. For
|
||||
/// recurring schedules the worker re-arms in
|
||||
/// `interval_seconds` steps from this point on.
|
||||
pub first_fire_at_unix: i64,
|
||||
/// `None` / absent = one-shot. `Some(n > 0)` = recurring every
|
||||
/// `n` seconds. The worker clamps catch-up so a long downtime
|
||||
/// fires ONCE on resume (skipped-cycle count surfaces in the
|
||||
/// per-target `last_result`), not N delayed pulses in a row.
|
||||
#[serde(default)]
|
||||
pub interval_seconds: Option<u64>,
|
||||
/// Optional description shown on the dashboard approval card +
|
||||
/// preserved on the schedule row for later operator reference.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct FireScheduleNowArgs {
|
||||
/// Schedule id to fire out of band. Get this from a prior
|
||||
/// `list_schedules` call or the approval-resolved event for
|
||||
/// the originating `request_schedule_prompt`.
|
||||
pub id: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct CancelScheduleArgs {
|
||||
/// Schedule id from a prior `list_schedules` call or the
|
||||
/// approval-resolved event for a `request_schedule_prompt`.
|
||||
pub id: i64,
|
||||
/// Optional target list. `None` / empty = cancel the entire
|
||||
/// schedule. `Some(["alice", "bob"])` = cancel just those
|
||||
/// recipients (the schedule keeps firing for any remaining
|
||||
/// active targets, and auto-cancels its parent row when every
|
||||
/// target is gone).
|
||||
#[serde(default)]
|
||||
pub targets: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct EditScheduleArgs {
|
||||
/// Schedule id from a prior `list_schedules` call or the
|
||||
/// approval-resolved event for a `request_schedule_prompt`.
|
||||
pub id: i64,
|
||||
/// New body text. Omit to keep the existing one.
|
||||
#[serde(default)]
|
||||
pub body: Option<String>,
|
||||
/// New description. Omit to keep the existing one. (To CLEAR
|
||||
/// the description, use the dashboard PATCH endpoint
|
||||
/// directly — the agent surface intentionally keeps the args
|
||||
/// flat / non-nullable to dodge the doubly-wrapped Option
|
||||
/// schemars quirk; clearing fields is rare and operator-side.)
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
/// Recurring interval in seconds. Omit to keep the existing
|
||||
/// cadence; pass an explicit value to set a new one. Toggling
|
||||
/// recurring↔one-shot (clearing the interval) is operator-only
|
||||
/// for the same reason as `description` above.
|
||||
#[serde(default)]
|
||||
pub interval_seconds: Option<u64>,
|
||||
/// New absolute unix timestamp for the next fire. Omit to
|
||||
/// leave the schedule on its current cadence.
|
||||
#[serde(default)]
|
||||
pub next_fire_at_unix: Option<i64>,
|
||||
/// Names of new targets to add. Replace-on-conflict: re-adding
|
||||
/// a previously cancelled target resets its history (operator
|
||||
/// intent on re-add = "this target is active again").
|
||||
#[serde(default)]
|
||||
pub targets_add: Option<Vec<String>>,
|
||||
/// Names of targets to cancel. Tombstones preserve per-target
|
||||
/// audit; when no active targets remain the schedule
|
||||
/// auto-cancels.
|
||||
#[serde(default)]
|
||||
pub targets_remove: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct GetLogsArgs {
|
||||
/// Logical agent name to fetch logs for (e.g. `gui`, `iris`).
|
||||
/// hive-c0re maps it to the underlying machine name (`h-gui`)
|
||||
/// itself — pass the plain agent name, not the `h-` form.
|
||||
pub agent: String,
|
||||
/// How many journal lines to return (default: 50, max: 500).
|
||||
#[serde(default)]
|
||||
pub lines: Option<u32>,
|
||||
}
|
||||
|
||||
/// Arguments for `get_host_journal` (capability-gated: `read_host_journal`).
|
||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
||||
pub struct GetHostJournalArgs {
|
||||
/// Systemd unit to filter (e.g. `hive-c0re.service`). Omit for all units.
|
||||
#[serde(default)]
|
||||
pub unit: Option<String>,
|
||||
/// nspawn machine name verbatim (e.g. `h-iris`). Omit for host journal.
|
||||
/// Agent containers use the `h-` prefix (e.g. `h-iris`); infrastructure
|
||||
/// containers use their full name (e.g. `hive-ci`, `hive-forge`,
|
||||
/// `hive-matrix`, `hive-gateway`).
|
||||
#[serde(default)]
|
||||
pub container: Option<String>,
|
||||
/// Number of lines to return (default 30, max 100).
|
||||
#[serde(default)]
|
||||
pub lines: Option<u32>,
|
||||
/// Minimum syslog priority level.
|
||||
#[serde(default)]
|
||||
pub priority: Option<hive_sh4re::JournalPriority>,
|
||||
/// Regex to match against log message fields (journalctl --grep).
|
||||
#[serde(default)]
|
||||
pub grep: Option<String>,
|
||||
/// Show entries on or newer than this timestamp (e.g. `-1h`).
|
||||
#[serde(default)]
|
||||
pub since: Option<String>,
|
||||
/// Show entries on or older than this timestamp.
|
||||
#[serde(default)]
|
||||
pub until: Option<String>,
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{IDLE_WAIT_HINT, format_recv};
|
||||
|
||||
#[test]
|
||||
fn empty_recv_after_wait_appends_idle_hint() {
|
||||
let out = format_recv(
|
||||
Ok(hive_sh4re::Response::Messages { messages: vec![] }),
|
||||
true,
|
||||
);
|
||||
assert!(out.starts_with("(empty)"));
|
||||
assert!(out.contains(IDLE_WAIT_HINT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_recv_without_wait_has_no_hint() {
|
||||
let out = format_recv(
|
||||
Ok(hive_sh4re::Response::Messages { messages: vec![] }),
|
||||
false,
|
||||
);
|
||||
assert_eq!(out, "(empty)");
|
||||
}
|
||||
}
|
||||
448
hive-ag3nt/src/mcp/render.rs
Normal file
448
hive-ag3nt/src/mcp/render.rs
Normal file
|
|
@ -0,0 +1,448 @@
|
|||
//! Formatting / render helpers for the MCP tool surface: ack / recv /
|
||||
//! loose-end / agent-meta reply shaping plus the retry annotation.
|
||||
//! Stateless string builders, with one exception —
|
||||
//! [`matrix_unread_summary`] queries the local matrix daemon socket
|
||||
//! (best-effort) so `get_loose_ends` can prepend an unread-rooms entry.
|
||||
|
||||
/// Render the three identical failure arms every data-returning tool handler
|
||||
/// repeats: a broker `Err` → `"{tool} failed: {m}"`, an unexpected `Ok` variant
|
||||
/// → `"{tool} unexpected response: …"`, and a transport error → `"{tool}
|
||||
/// transport error: …"`. Handlers match their own happy-path variant and route
|
||||
/// everything else here via a catch-all arm (`other => reply_err(other, tool)`),
|
||||
/// so the triplet lives in exactly one place.
|
||||
pub(super) fn reply_err(resp: Result<hive_sh4re::Response, anyhow::Error>, tool: &str) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::Err { message }) => format!("{tool} failed: {message}"),
|
||||
Ok(other) => format!("{tool} unexpected response: {other:?}"),
|
||||
Err(e) => format!("{tool} transport error: {e:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format helper for "send-like" tools (anything that expects an `Ok`).
|
||||
/// `tool` and `ok_msg` only appear in the result string; they don't change
|
||||
/// behavior.
|
||||
#[must_use]
|
||||
pub fn format_ack(
|
||||
resp: Result<hive_sh4re::Response, anyhow::Error>,
|
||||
tool: &str,
|
||||
ok_msg: String,
|
||||
) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::Ok) => ok_msg,
|
||||
other => reply_err(other, tool),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format helper for `recv`: renders zero, one, or many popped
|
||||
/// messages. Empty list collapses to "(empty)" so claude doesn't go
|
||||
/// hunting for content; when `waited` is set (the call parked on a
|
||||
/// long-poll that timed out) the empty result also carries
|
||||
/// [`IDLE_WAIT_HINT`] nudging the model toward other work. A single
|
||||
/// message renders as the historical `from: X\n\nbody` block (banner
|
||||
/// first if `redelivered`). A multi-message batch renders with a
|
||||
/// `popped N message(s):` header and `---` separators between bodies
|
||||
/// so the model can tell where one ends and the next begins;
|
||||
/// per-message redelivery banners included.
|
||||
#[must_use]
|
||||
pub fn format_recv(resp: Result<hive_sh4re::Response, anyhow::Error>, waited: bool) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::Messages { messages }) => render_recv_messages(&messages, waited),
|
||||
// A graceful stop is pending — the inbox is fenced. Render a single
|
||||
// explicit directive (not an empty inbox, which claude's "park on recv"
|
||||
// habit would long-poll again, stalling the stop-checkpoint turn until
|
||||
// the drain wait times out into a hard stop) so every recv during the
|
||||
// stop unmissably tells claude to flush + end.
|
||||
Ok(hive_sh4re::Response::GracefulStop) => {
|
||||
render_recv_messages(&[graceful_stop_message()], waited)
|
||||
}
|
||||
other => reply_err(other, "recv"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The synthetic single-message directive rendered for a fenced (graceful-stop)
|
||||
/// inbox — see the `GracefulStop` arm of [`format_recv`].
|
||||
fn graceful_stop_message() -> hive_sh4re::DeliveredMessage {
|
||||
hive_sh4re::DeliveredMessage {
|
||||
from: "graceful-stop".into(),
|
||||
body: "⛔ GRACEFUL STOP IN PROGRESS — the container shuts down as soon as this turn \
|
||||
ends. Flush anything worth keeping to your durable /state files, then END \
|
||||
YOUR TURN now. Do NOT call recv again: the inbox is fenced and recv will \
|
||||
only keep returning this same notice."
|
||||
.into(),
|
||||
id: 0,
|
||||
redelivered: false,
|
||||
in_reply_to: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Render the popped-message payload of a successful `recv` (see `format_recv`
|
||||
/// for the empty/single/batch shapes).
|
||||
fn render_recv_messages(messages: &[hive_sh4re::DeliveredMessage], waited: bool) -> String {
|
||||
use std::fmt::Write as _;
|
||||
if messages.is_empty() {
|
||||
return if waited {
|
||||
format!("(empty){IDLE_WAIT_HINT}")
|
||||
} else {
|
||||
"(empty)".to_owned()
|
||||
};
|
||||
}
|
||||
if messages.len() == 1 {
|
||||
let m = &messages[0];
|
||||
let banner = if m.redelivered { REDELIVERY_HINT } else { "" };
|
||||
return format!("{banner}{}from: {}\n\n{}", msg_id_tag(m.id), m.from, m.body);
|
||||
}
|
||||
let n = messages.len();
|
||||
let mut out = format!("popped {n} message(s):\n\n");
|
||||
for (i, m) in messages.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push_str("\n---\n\n");
|
||||
}
|
||||
let banner = if m.redelivered { REDELIVERY_HINT } else { "" };
|
||||
let _ = write!(
|
||||
out,
|
||||
"{banner}{}from: {}\n\n{}",
|
||||
msg_id_tag(m.id),
|
||||
m.from,
|
||||
m.body
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `[msg #<id>] ` marker prefixed to each recv row so the agent knows
|
||||
/// what to pass to `ack_until` when bulk-triaging a backlog. Transient
|
||||
/// pings carry the sentinel id 0 (in-memory only, nothing in the
|
||||
/// broker to ack) and render without the marker.
|
||||
fn msg_id_tag(id: i64) -> String {
|
||||
if id > 0 {
|
||||
format!("[msg #{id}] ")
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Header prepended to message bodies that were popped by a prior
|
||||
/// harness session, never acked (turn crash / OOM / restart), and
|
||||
/// resurfaced by `RequeueInflight` on this session's boot. Same
|
||||
/// string surfaces in the wake prompt (see the bin loops) and the
|
||||
/// in-turn `recv` tool result so claude sees the warning either way.
|
||||
pub const REDELIVERY_HINT: &str = "[redelivered after harness restart — may already be handled]\n";
|
||||
|
||||
/// Appended to the `recv` empty result when the agent parked on a
|
||||
/// long-poll (`wait_seconds > 0`) that timed out with nothing new.
|
||||
/// Nudges the model to spend the idle time on other useful work
|
||||
/// instead of immediately re-blocking on `recv`.
|
||||
pub const IDLE_WAIT_HINT: &str = " — nothing arrived before the wait timed out. \
|
||||
If you have other useful work (assigned issues, in-flight PRs, a docs sweep, \
|
||||
notes to update), do that now rather than immediately parking on recv again.";
|
||||
|
||||
/// Inner renderer for a `Vec<LooseEnd>` already extracted from the socket
|
||||
/// reply. Called by the `get_loose_ends` handler, which injects the
|
||||
/// `UnreadMatrix` entry before formatting.
|
||||
pub(super) fn render_loose_ends(loose_ends: &[hive_sh4re::LooseEnd]) -> String {
|
||||
use std::fmt::Write as _;
|
||||
if loose_ends.is_empty() {
|
||||
return "(no loose ends)".to_owned();
|
||||
}
|
||||
let mut out = format!("{} loose end(s):\n", loose_ends.len());
|
||||
for t in loose_ends {
|
||||
match t {
|
||||
hive_sh4re::LooseEnd::Approval {
|
||||
id,
|
||||
agent,
|
||||
commit_ref,
|
||||
description,
|
||||
age_seconds,
|
||||
} => {
|
||||
let desc = description
|
||||
.as_deref()
|
||||
.map(|d| format!(" — {d}"))
|
||||
.unwrap_or_default();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- approval #{id} ({agent} @ {commit_ref}, {age_seconds}s old){desc}"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::Question {
|
||||
id,
|
||||
asker,
|
||||
target,
|
||||
question,
|
||||
age_seconds,
|
||||
} => {
|
||||
let to = target.as_deref().unwrap_or("operator");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- question #{id} ({asker} → {to}, {age_seconds}s old): {question}"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::Reminder {
|
||||
id,
|
||||
owner,
|
||||
message,
|
||||
due_at,
|
||||
age_seconds,
|
||||
} => {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- reminder #{id} ({owner}, scheduled {age_seconds}s ago, due_at={due_at}): {message}"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::PendingMessages { count } => {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"- {count} pending inbox message(s) — drain with recv (recv(max: {count}) to batch)"
|
||||
);
|
||||
}
|
||||
hive_sh4re::LooseEnd::UnreadMatrix { rooms, summary } => {
|
||||
let _ = write!(out, "- unread matrix messages in {rooms} room(s)");
|
||||
if summary.is_empty() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" — use list_rooms + read_room to view, mark_read to clear"
|
||||
);
|
||||
} else {
|
||||
let _ = writeln!(out, ":");
|
||||
for line in summary.lines() {
|
||||
let _ = writeln!(out, " {line}");
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" use list_rooms + read_room to view, mark_read to clear"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Per-room unread entry returned by `matrix_unread_summary`. Mirrors
|
||||
/// `hive-matrix-mcp`'s `RoomUnread` but defined locally to avoid a
|
||||
/// cross-crate dep on the matrix-sdk crate tree.
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
pub(super) struct MatrixRoomUnread {
|
||||
label: String,
|
||||
count: u32,
|
||||
last_body: Option<String>,
|
||||
last_sender: Option<String>,
|
||||
}
|
||||
|
||||
/// Query the local matrix daemon for per-room unread summaries. Returns
|
||||
/// `None` if the daemon socket is absent or the query fails. Best-effort:
|
||||
/// agents without matrix configured are not penalised.
|
||||
pub(super) async fn matrix_unread_summary() -> Option<Vec<MatrixRoomUnread>> {
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::UnixStream;
|
||||
let socket = std::env::var_os("HIVE_MATRIX_SOCKET").map_or_else(
|
||||
|| std::path::PathBuf::from("/run/hive-matrix/socket"),
|
||||
std::path::PathBuf::from,
|
||||
);
|
||||
if !socket.exists() {
|
||||
return None;
|
||||
}
|
||||
let mut stream = UnixStream::connect(&socket).await.ok()?;
|
||||
stream
|
||||
.write_all(b"{\"method\":\"unread_summary\"}\n")
|
||||
.await
|
||||
.ok()?;
|
||||
let mut lines = BufReader::new(stream).lines();
|
||||
let line = lines.next_line().await.ok()??;
|
||||
let val: serde_json::Value = serde_json::from_str(&line).ok()?;
|
||||
// Response: {"kind":"ok","payload":[{label, count, last_body?, last_sender?}]}
|
||||
let arr = val.get("payload")?.as_array()?;
|
||||
serde_json::from_value(serde_json::Value::Array(arr.clone())).ok()
|
||||
}
|
||||
|
||||
/// Format a `Vec<MatrixRoomUnread>` into a per-room summary string.
|
||||
/// Single room / single message collapses to one line; multi-room
|
||||
/// expands to a bulleted list. Returns an empty string for empty input.
|
||||
pub(super) fn format_matrix_summary(rooms: &[MatrixRoomUnread]) -> String {
|
||||
use std::fmt::Write as _;
|
||||
if rooms.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for r in rooms {
|
||||
if r.count == 1
|
||||
&& let (Some(body), Some(sender)) = (&r.last_body, &r.last_sender)
|
||||
{
|
||||
let _ = writeln!(out, "- {}: {sender}: {body}", r.label);
|
||||
continue;
|
||||
}
|
||||
let _ = writeln!(out, "- {}: {} unread", r.label, r.count);
|
||||
}
|
||||
// Remove trailing newline.
|
||||
if out.ends_with('\n') {
|
||||
out.pop();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the user-facing `kind` string for `cancel_loose_end` into the
|
||||
/// wire enum. Accepts a small alias set so claude doesn't have to
|
||||
/// remember the exact spelling (`"q"` / `"r"` shorthand falls out
|
||||
/// for free).
|
||||
pub(super) fn parse_loose_end_kind(raw: &str) -> Result<hive_sh4re::CancelLooseEndKind, String> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
"question" | "q" => Ok(hive_sh4re::CancelLooseEndKind::Question),
|
||||
"reminder" | "r" => Ok(hive_sh4re::CancelLooseEndKind::Reminder),
|
||||
"approval" | "a" => Ok(hive_sh4re::CancelLooseEndKind::Approval),
|
||||
other => Err(format!(
|
||||
"cancel_loose_end: unknown kind '{other}' \
|
||||
(expected \"question\", \"reminder\", or \"approval\")"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical user-facing label for a `CancelLooseEndKind` — used in
|
||||
/// the success ack so the caller always sees `"question"` /
|
||||
/// `"reminder"` instead of whatever alias they passed in (`"q"` /
|
||||
/// `"r"`).
|
||||
pub(super) fn loose_end_kind_label(kind: hive_sh4re::CancelLooseEndKind) -> &'static str {
|
||||
match kind {
|
||||
hive_sh4re::CancelLooseEndKind::Question => "question",
|
||||
hive_sh4re::CancelLooseEndKind::Reminder => "reminder",
|
||||
hive_sh4re::CancelLooseEndKind::Approval => "approval",
|
||||
}
|
||||
}
|
||||
|
||||
/// Format helper for `get_agent_meta`: renders an agent's identity +
|
||||
/// current status as a short human-readable block. `name`,
|
||||
/// `hyperhive_rev`, and `running` are always shown; `status` only
|
||||
/// appears when one is set, otherwise the line reads `status: <none>`.
|
||||
/// When `running` is false the host has already cleared `status_text`
|
||||
/// (it would be a stale snapshot from before the stop) so the status
|
||||
/// line is implicitly `<none>` in that case — but the explicit
|
||||
/// `running: no` line tells the caller WHY. See
|
||||
/// `docs/turn-loop/mcp.md::Core tools` (`get_agent_meta`).
|
||||
#[must_use]
|
||||
pub fn format_agent_meta(resp: Result<hive_sh4re::Response, anyhow::Error>) -> String {
|
||||
match resp {
|
||||
Ok(hive_sh4re::Response::AgentMeta {
|
||||
name,
|
||||
running,
|
||||
hyperhive_rev,
|
||||
status_text,
|
||||
status_set_at,
|
||||
hive_name,
|
||||
swarm_name,
|
||||
matrix_accounts,
|
||||
}) => {
|
||||
let rev = hyperhive_rev.as_deref().unwrap_or("<unknown>");
|
||||
let run = if running { "yes" } else { "no" };
|
||||
let mut out = format!("name: {name}\nhyperhive_rev: {rev}\nrunning: {run}");
|
||||
// Surface hive + swarm display names only when set, so
|
||||
// single-hive deployments don't see noisy `<none>` lines.
|
||||
if let Some(hn) = hive_name.as_deref() {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(out, "\nhive_name: {hn}");
|
||||
}
|
||||
if let Some(sn) = swarm_name.as_deref() {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(out, "\nswarm_name: {sn}");
|
||||
}
|
||||
match status_text {
|
||||
None => out.push_str("\nstatus: <none>"),
|
||||
Some(s) => {
|
||||
use std::fmt::Write as _;
|
||||
let age = status_set_at.and_then(|ts| {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?
|
||||
.as_secs();
|
||||
// `ts` is a unix epoch second the agent itself
|
||||
// sourced from `SystemTime` — always positive
|
||||
// in normal operation. Clamp the negative
|
||||
// (clock-skew) edge to 0 before the unsigned
|
||||
// cast so the cast loses no real precision.
|
||||
let ts_secs = u64::try_from(ts).unwrap_or(0);
|
||||
let secs = now.saturating_sub(ts_secs);
|
||||
Some(format_age_secs(secs))
|
||||
});
|
||||
// `write!` into the buffer instead of `push_str(&format!(…))` —
|
||||
// avoids the intermediate allocation clippy::format_push_string
|
||||
// flags. The infallible `String` writer makes this safe to
|
||||
// `let _ =`-ignore.
|
||||
match age {
|
||||
Some(a) => {
|
||||
let _ = write!(out, "\nstatus: {s} (set {a} ago)");
|
||||
}
|
||||
None => {
|
||||
let _ = write!(out, "\nstatus: {s}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Matrix identities the agent can act as (the `account` arg on
|
||||
// the matrix tools). Listed only when matrix is provisioned, so
|
||||
// non-matrix agents don't see an empty line.
|
||||
if !matrix_accounts.is_empty() {
|
||||
use std::fmt::Write as _;
|
||||
out.push_str("\nmatrix_accounts:");
|
||||
for acct in &matrix_accounts {
|
||||
let uid = acct.user_id.as_deref().unwrap_or("?");
|
||||
let _ = write!(out, "\n {} ({uid}) on {}", acct.name, acct.homeserver);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
other => reply_err(other, "get_agent_meta"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a duration in seconds as a human-readable age string.
|
||||
fn format_age_secs(secs: u64) -> String {
|
||||
if secs < 60 {
|
||||
format!("{secs}s")
|
||||
} else if secs < 3600 {
|
||||
format!("{}m", secs / 60)
|
||||
} else if secs < 86400 {
|
||||
format!("{}h", secs / 3600)
|
||||
} else {
|
||||
format!("{}d", secs / 86400)
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a short note to a tool result when the underlying socket call
|
||||
/// took retries to land. Lets claude distinguish "my request was wrong"
|
||||
/// from "c0re flickered and the harness rode it out" — without the
|
||||
/// hint, a tool result that took 30s to come back looks identical to a
|
||||
/// content failure and the model would burn a turn retrying it.
|
||||
#[must_use]
|
||||
pub fn annotate_retries(mut s: String, retries: u32) -> String {
|
||||
if retries > 0 {
|
||||
use std::fmt::Write as _;
|
||||
let suffix = if retries == 1 { "retry" } else { "retries" };
|
||||
let _ = write!(
|
||||
s,
|
||||
"\n\n(note: hive socket connect needed {retries} {suffix} — c0re likely \
|
||||
restarted. Your request did succeed on the final attempt; no action needed.)"
|
||||
);
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{IDLE_WAIT_HINT, format_recv};
|
||||
|
||||
#[test]
|
||||
fn empty_recv_after_wait_appends_idle_hint() {
|
||||
let out = format_recv(
|
||||
Ok(hive_sh4re::Response::Messages { messages: vec![] }),
|
||||
true,
|
||||
);
|
||||
assert!(out.starts_with("(empty)"));
|
||||
assert!(out.contains(IDLE_WAIT_HINT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_recv_without_wait_has_no_hint() {
|
||||
let out = format_recv(
|
||||
Ok(hive_sh4re::Response::Messages { messages: vec![] }),
|
||||
false,
|
||||
);
|
||||
assert_eq!(out, "(empty)");
|
||||
}
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ use crate::events::Bus;
|
|||
use crate::mcp::REDELIVERY_HINT;
|
||||
use crate::turn::{TurnError, TurnOutcome};
|
||||
use crate::turn_stats::TurnStatRow;
|
||||
pub use hive_sh4re::wire_time::now_unix;
|
||||
|
||||
/// Assemble the per-turn wake prompt string. The role/tools/etc. live in the
|
||||
/// system prompt; this is just the wake signal body. `id` is the broker row
|
||||
|
|
@ -44,16 +45,6 @@ pub fn format_wake_prompt(
|
|||
format!("{banner}{tag}Incoming message from `{from}`:\n---\n{body}\n---{pending}")
|
||||
}
|
||||
|
||||
/// Current time as a Unix timestamp (seconds). Returns 0 on any error.
|
||||
#[must_use]
|
||||
pub fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Field-named args for [`build_row`]. Mirrors the turn-stats row
|
||||
/// columns; `outcome` and `bus` borrow for the duration of the call.
|
||||
pub struct TurnRowArgs<'a> {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ use rusqlite::{Connection, OpenFlags};
|
|||
use serde::Serialize;
|
||||
|
||||
use hive_sh4re::ReminderStats;
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
|
||||
/// Window param accepted by `/api/stats?window=`. Each maps to a
|
||||
/// total span + the bucket width used to roll up trend series.
|
||||
|
|
@ -208,7 +209,7 @@ fn default_path() -> PathBuf {
|
|||
}
|
||||
|
||||
fn empty_snapshot(window: Window) -> Snapshot {
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
let from = now - window.span_secs();
|
||||
let buckets = fill_buckets(from, now, window.bucket_secs(), &HashMap::new());
|
||||
Snapshot {
|
||||
|
|
@ -239,7 +240,7 @@ fn snapshot(path: &Path, window: Window) -> Result<Snapshot> {
|
|||
// matches hive-c0re's host-side reader (`hive_stats::read_agent`).
|
||||
conn.busy_timeout(std::time::Duration::from_millis(500))
|
||||
.with_context(|| format!("set busy_timeout on {}", path.display()))?;
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
// Fixed windows look back a constant span; `all` starts at the earliest
|
||||
// recorded turn (`MIN(started_at)`, falling back to `now` on an empty
|
||||
// table) and sizes its buckets adaptively from that span.
|
||||
|
|
@ -567,12 +568,6 @@ fn u64_from_i64(v: i64) -> u64 {
|
|||
u64::try_from(v).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn now_secs() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -638,7 +633,7 @@ mod tests {
|
|||
fn snapshot_aggregates_rows() {
|
||||
let db = tmp_db();
|
||||
let _ = std::fs::remove_file(&db);
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
seed_db(
|
||||
&db,
|
||||
&[
|
||||
|
|
@ -727,7 +722,7 @@ mod tests {
|
|||
fn bash_breakdown_empty_without_table() {
|
||||
let db = tmp_db();
|
||||
let _ = std::fs::remove_file(&db);
|
||||
seed_db(&db, &[(now_secs() - 100, 1000, "opus", "recv", "ok", "{}")]);
|
||||
seed_db(&db, &[(now_unix() - 100, 1000, "opus", "recv", "ok", "{}")]);
|
||||
let s = snapshot(&db, Window::Day).unwrap();
|
||||
assert!(s.bash_breakdown.is_empty());
|
||||
}
|
||||
|
|
@ -739,7 +734,7 @@ mod tests {
|
|||
let db = tmp_db();
|
||||
let _ = std::fs::remove_file(&db);
|
||||
seed_db(&db, &[]);
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
let conn = Connection::open(&db).unwrap();
|
||||
conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@
|
|||
//! the honest fix is to clean them up where they live.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::Duration;
|
||||
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, Result, params};
|
||||
|
||||
/// How often the sweep runs.
|
||||
|
|
@ -130,11 +131,3 @@ fn vacuum_events(path: &Path) -> Result<u64> {
|
|||
)?;
|
||||
Ok(u64::try_from(removed).unwrap_or(0))
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,13 +90,7 @@ fn signal_group(pgid: Option<i32>, sig: i32) {
|
|||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.cast_signed()
|
||||
}
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
|
||||
/// Generate a task ID: `<timestamp_hex><seq_hex>`.
|
||||
#[must_use]
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ clap-markdown = "0.1"
|
|||
hive-sh4re.workspace = true
|
||||
libc.workspace = true
|
||||
listenfd = "1"
|
||||
petgraph.workspace = true
|
||||
rusqlite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -13,19 +13,20 @@ use crate::lifecycle;
|
|||
|
||||
/// Approve a pending request. Marks the approval row durably, then
|
||||
/// either runs the work inline (`InitConfig`, sub-second git ops) or
|
||||
/// enqueues it into `rebuild_queue` so the dashboard POST returns
|
||||
/// submits it to the job queue so the dashboard POST returns
|
||||
/// immediately while the long-running pipeline runs off-thread
|
||||
/// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`).
|
||||
///
|
||||
/// Dispatch:
|
||||
/// - `ApplyCommit` → `QueueKind::Rebuild` (~30-90s wall time)
|
||||
/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s)
|
||||
/// - `Spawn` → `QueueKind::Spawn` (~30-90s)
|
||||
/// - `ApplyCommit` / `MergeConfigPr` → a single-node `ApprovalDeploy`
|
||||
/// DAG (the two-phase meta deploy stays opaque in v1; ~30-90s)
|
||||
/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
|
||||
/// - `Spawn` → a `Spawn` DAG (`Create → WriteDropin → Reconcile`)
|
||||
/// - `InitConfig` → inline (<1s; queue card would be noise)
|
||||
///
|
||||
/// The queue worker re-fetches the approval row on dispatch, runs
|
||||
/// the kind-specific pipeline, and fires `ApprovalResolved` /
|
||||
/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`.
|
||||
/// `ApprovalDeploy` resolves the approval inside its pipeline; the
|
||||
/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`]
|
||||
/// when their DAG settles terminal.
|
||||
pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
||||
let approval = coord.approvals.mark_approved(id)?;
|
||||
tracing::info!(
|
||||
|
|
@ -56,54 +57,41 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
|||
}
|
||||
ApprovalKind::UpdateMetaInputs => {
|
||||
// Inputs JSON-encoded into commit_ref by the manager's
|
||||
// submit path — surface them on the queue entry so the
|
||||
// dashboard can show *which* inputs are about to bump.
|
||||
// submit path — surface them on the DAG so the dashboard
|
||||
// can show *which* inputs are about to bump. The cascade
|
||||
// rebuilds fan out when the lock bump lands (so they build
|
||||
// against the post-bump lock, and a failed bump fans out
|
||||
// nothing).
|
||||
let inputs: Vec<String> =
|
||||
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
||||
let parent_id = coord
|
||||
.rebuild_queue
|
||||
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
||||
kind: crate::rebuild_queue::QueueKind::MetaUpdate,
|
||||
agent: approval.agent.clone(),
|
||||
source: crate::rebuild_queue::QueueSource::Approval,
|
||||
reason: format!("approval #{id} meta input update"),
|
||||
parent_id: None,
|
||||
inputs: inputs.clone(),
|
||||
approval_id: Some(id),
|
||||
perm_payload: None,
|
||||
depends_on: Vec::new(),
|
||||
});
|
||||
// Pre-enqueue cascade rebuilds in topological order so
|
||||
// agents depending on updated inputs are rebuilt after the
|
||||
// lock bump, matching the dashboard post_meta_update path.
|
||||
let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await;
|
||||
let cascade_reason = format!("approval #{id} meta input cascade");
|
||||
for name in cascade_agents {
|
||||
coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Rebuild,
|
||||
name,
|
||||
crate::rebuild_queue::QueueSource::MetaUpdate,
|
||||
cascade_reason.clone(),
|
||||
Some(parent_id),
|
||||
);
|
||||
let submitted = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::meta_update(
|
||||
inputs,
|
||||
crate::job_queue::Source::Approval,
|
||||
format!("approval #{id} meta input update"),
|
||||
Some(id),
|
||||
));
|
||||
if let Err(e) = submitted {
|
||||
return Err(e.context("submit meta-update dag"));
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
ApprovalKind::Spawn => {
|
||||
coord
|
||||
.rebuild_queue
|
||||
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
||||
kind: crate::rebuild_queue::QueueKind::Spawn,
|
||||
agent: approval.agent.clone(),
|
||||
source: crate::rebuild_queue::QueueSource::Approval,
|
||||
reason: format!("approval #{id} spawn"),
|
||||
parent_id: None,
|
||||
inputs: Vec::new(),
|
||||
approval_id: Some(id),
|
||||
perm_payload: None,
|
||||
depends_on: Vec::new(),
|
||||
});
|
||||
// The spawn's tail `Reconcile` starts the container, so the
|
||||
// new agent's power intent is `Up` from the outset.
|
||||
if let Err(e) = coord.power.set(&approval.agent, crate::power::Wanted::Up) {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
|
||||
}
|
||||
let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn(
|
||||
&approval.agent,
|
||||
id,
|
||||
format!("approval #{id} spawn"),
|
||||
));
|
||||
if let Err(e) = submitted {
|
||||
return Err(e.context("submit spawn dag"));
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -137,29 +125,27 @@ pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Enqueue a `Rebuild` queue entry tied to an approval id. Shared by the
|
||||
/// `ApplyCommit` and `MergeConfigPr` dispatch arms — both end in a container
|
||||
/// rebuild routed through the queue, differing only in the queue `reason`.
|
||||
/// The queue worker branches on the approval's kind to pick the right handler.
|
||||
/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id.
|
||||
/// Shared by the `ApplyCommit` and `MergeConfigPr` dispatch arms — both
|
||||
/// end in a container rebuild routed through the queue, differing only
|
||||
/// in the `reason`. The node executor branches on the approval's kind
|
||||
/// to pick the right handler.
|
||||
fn enqueue_approval_rebuild(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
approval_id: i64,
|
||||
reason: String,
|
||||
) {
|
||||
coord
|
||||
.rebuild_queue
|
||||
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
||||
kind: crate::rebuild_queue::QueueKind::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source: crate::rebuild_queue::QueueSource::Approval,
|
||||
if let Err(e) = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::approval_deploy(
|
||||
agent,
|
||||
approval_id,
|
||||
reason,
|
||||
parent_id: None,
|
||||
inputs: Vec::new(),
|
||||
approval_id: Some(approval_id),
|
||||
perm_payload: None,
|
||||
depends_on: Vec::new(),
|
||||
});
|
||||
))
|
||||
{
|
||||
tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed");
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
|
|
@ -375,69 +361,60 @@ async fn run_approval_schedule_prompt(
|
|||
finish_approval(coord, &approval, result, None, false)
|
||||
}
|
||||
|
||||
/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue
|
||||
/// entries. Inputs come from the approval row's `commit_ref` field
|
||||
/// (JSON-encoded by the manager submit path), not the queue entry's
|
||||
/// `inputs` — the queue copy is for dashboard display only.
|
||||
pub async fn run_approval_update_meta_inputs(
|
||||
/// Terminal hook for approval-carrying DAGs — the job queue's
|
||||
/// scheduler calls this exactly once when such a DAG settles terminal.
|
||||
/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is
|
||||
/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves
|
||||
/// *inside* its node, so its DAG is skipped — unless it was cancelled
|
||||
/// while still queued, in which case the node never ran and the row
|
||||
/// would otherwise dangle forever.
|
||||
pub(crate) async fn resolve_approval_dag(
|
||||
coord: &Arc<Coordinator>,
|
||||
queue_entry_id: Option<u64>,
|
||||
approval_id: i64,
|
||||
) -> Result<()> {
|
||||
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?;
|
||||
let inputs: Vec<String> = serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
||||
coord.set_queue_step(queue_entry_id, "nix flake update");
|
||||
let result = crate::meta::lock_update(&inputs).await;
|
||||
finish_approval(coord, &approval, result, None, false)
|
||||
}
|
||||
|
||||
/// Worker entry point for `ApprovalKind::Spawn` queue entries.
|
||||
/// Differs from `run_approval_apply_commit` only in routing through
|
||||
/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous
|
||||
/// in the queue worker — the previous `tokio::spawn` wrapper is gone
|
||||
/// (the queue worker itself is the async task).
|
||||
pub async fn run_approval_spawn(
|
||||
coord: &Arc<Coordinator>,
|
||||
queue_entry_id: Option<u64>,
|
||||
approval_id: i64,
|
||||
) -> Result<()> {
|
||||
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?;
|
||||
let agent_dir = coord.ensure_runtime(&approval.agent)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(&approval.agent, agent_dir);
|
||||
// Transient guard keeps the per-container "Spawning" pill lit while
|
||||
// the worker is doing the actual nixos-container create. Auto-clears
|
||||
// on the function's scope exit (success or panic).
|
||||
let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning);
|
||||
coord.set_queue_step(queue_entry_id, "lifecycle::spawn");
|
||||
let result = lifecycle::spawn(&approval.agent, &hive, &paths).await;
|
||||
if result.is_ok() {
|
||||
coord.set_queue_step(queue_entry_id, "forge user");
|
||||
if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed");
|
||||
terminal: &crate::job_queue::TerminalDag,
|
||||
) {
|
||||
use crate::job_queue::{State, Template};
|
||||
let Some(approval_id) = terminal.approval_id else {
|
||||
return;
|
||||
};
|
||||
if terminal.template == Template::Rebuild && terminal.state != State::Cancelled {
|
||||
return; // ApprovalDeploy resolved inside the node.
|
||||
}
|
||||
let approval = match coord.approvals.get(approval_id) {
|
||||
Ok(Some(a)) => a,
|
||||
Ok(None) => {
|
||||
tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
|
||||
return;
|
||||
}
|
||||
coord.set_queue_step(queue_entry_id, "forge config repo");
|
||||
if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed");
|
||||
Err(e) => {
|
||||
tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
|
||||
return;
|
||||
}
|
||||
coord.set_queue_step(queue_entry_id, "forge push");
|
||||
if let Err(e) = crate::forge::push_config(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed");
|
||||
}
|
||||
coord.set_queue_step(queue_entry_id, "forge meta access");
|
||||
if let Some(core_token) = crate::forge::core_token()
|
||||
&& let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await
|
||||
{
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed");
|
||||
}
|
||||
if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed");
|
||||
};
|
||||
let result: Result<()> = match terminal.state {
|
||||
State::Done => Ok(()),
|
||||
State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")),
|
||||
_ => Err(anyhow::anyhow!(
|
||||
"{}",
|
||||
terminal
|
||||
.error
|
||||
.clone()
|
||||
.unwrap_or_else(|| "job dag failed".to_owned())
|
||||
)),
|
||||
};
|
||||
if approval.kind == ApprovalKind::Spawn {
|
||||
// Post-spawn forge bookkeeping (user, config repo mirror, meta
|
||||
// access) — warn-only, then the resolution events + a rescan so
|
||||
// the dashboard reflects the post-spawn state either way.
|
||||
if result.is_ok() {
|
||||
forge_after_first_spawn(coord, &approval.agent).await;
|
||||
} else {
|
||||
coord.rescan_containers_and_emit().await;
|
||||
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
||||
}
|
||||
}
|
||||
let final_result = finish_approval(coord, &approval, result, None, false);
|
||||
coord.rescan_containers_and_emit().await;
|
||||
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
||||
final_result
|
||||
if let Err(e) = finish_approval(coord, &approval, result, None, false) {
|
||||
tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-fetch an approval row from sqlite for a queue-worker dispatch.
|
||||
|
|
@ -563,14 +540,7 @@ fn finish_approval(
|
|||
// snapshot refetch. `approved` rows that succeed get the
|
||||
// approval's logged resolved_at indirectly via `now_unix()`;
|
||||
// failures already wrote it via mark_failed above.
|
||||
let approval_kind = match approval.kind {
|
||||
ApprovalKind::Spawn => "spawn",
|
||||
ApprovalKind::ApplyCommit => "apply_commit",
|
||||
ApprovalKind::InitConfig => "init_config",
|
||||
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
||||
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
||||
ApprovalKind::MergeConfigPr => "merge_config_pr",
|
||||
};
|
||||
let approval_kind = approval.kind.as_str();
|
||||
let sha_short = approval
|
||||
.fetched_sha
|
||||
.as_deref()
|
||||
|
|
@ -867,13 +837,7 @@ async fn deploy_applied_target(
|
|||
// part of this entry rather than a deferred fast-lane follow-up.
|
||||
false,
|
||||
&|step| coord.set_queue_step(queue_entry_id, step),
|
||||
&|log_id| {
|
||||
if let Some(qid) = queue_entry_id
|
||||
&& coord.rebuild_queue.set_build_log_id(qid, log_id)
|
||||
{
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
},
|
||||
&|log_id| coord.set_queue_build_log(queue_entry_id, log_id),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -984,6 +948,11 @@ pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Resul
|
|||
"agent destroyed"
|
||||
},
|
||||
);
|
||||
// Drop the durable power intent — a future agent of the same name
|
||||
// seeds fresh from its observed state.
|
||||
if let Err(e) = coord.power.remove(name) {
|
||||
tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed");
|
||||
}
|
||||
drop(guard);
|
||||
coord.notify_manager(&HelperEvent::Destroyed {
|
||||
agent: name.to_owned(),
|
||||
|
|
@ -1045,14 +1014,7 @@ pub async fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()
|
|||
tracing::warn!(%id, agent = %a.agent, error = ?e, "forge: push_config after deny failed");
|
||||
}
|
||||
}
|
||||
let approval_kind = match a.kind {
|
||||
ApprovalKind::Spawn => "spawn",
|
||||
ApprovalKind::ApplyCommit => "apply_commit",
|
||||
ApprovalKind::InitConfig => "init_config",
|
||||
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
||||
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
||||
ApprovalKind::MergeConfigPr => "merge_config_pr",
|
||||
};
|
||||
let approval_kind = a.kind.as_str();
|
||||
let sha_short = sha.as_deref().map(|s| s[..s.len().min(12)].to_owned());
|
||||
let description = a.description.clone();
|
||||
let agent_owned = a.agent.clone();
|
||||
|
|
|
|||
10
hive-c0re/src/agent_config/mod.rs
Normal file
10
hive-c0re/src/agent_config/mod.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
//! Per-agent configuration registries: tool groups, capabilities,
|
||||
//! topology (all JSON files under `/var/lib/hyperhive/meta/`) and the
|
||||
//! shared wire-protocol size limits. Each submodule is re-exported at
|
||||
//! the crate root, so `crate::topology::…` etc. keep working
|
||||
//! unchanged.
|
||||
|
||||
pub mod capabilities;
|
||||
pub mod limits;
|
||||
pub mod tool_groups;
|
||||
pub mod topology;
|
||||
|
|
@ -1,418 +0,0 @@
|
|||
//! Startup auto-update: on `hive-c0re serve` boot, rebuild containers that
|
||||
//! actually need it. Two skip rules keep boot-time work minimal:
|
||||
//!
|
||||
//! 1. **Stopped containers** are deferred — they will be rebuilt the first
|
||||
//! time the operator starts them (see `rebuild_queue::run_start` and
|
||||
//! `socket_server::handle_start`).
|
||||
//! 2. **Running containers whose rev marker matches** the current hyperhive
|
||||
//! flake path are skipped — nothing changed, no nix work to do.
|
||||
//!
|
||||
//! See `docs/coordinator.md::Auto-update sweep`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
|
||||
|
||||
/// Marker file recording the hyperhive rev a sub-agent's container was last
|
||||
/// built against. Sibling of `applied/<name>/` (rather than inside it) to
|
||||
/// keep it out of the applied repo's git history. Uses a leading dot so a
|
||||
/// glob over `applied/*` doesn't include it.
|
||||
pub fn rev_marker_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev"))
|
||||
}
|
||||
|
||||
/// Resolve the current rev of `hyperhive_flake`. For a path on disk we
|
||||
/// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/...
|
||||
/// update yields a different string. For anything else we return None.
|
||||
#[must_use]
|
||||
pub fn current_flake_rev(hyperhive_flake: &str) -> Option<String> {
|
||||
let path = Path::new(hyperhive_flake);
|
||||
if !path.exists() {
|
||||
return None;
|
||||
}
|
||||
std::fs::canonicalize(path)
|
||||
.ok()
|
||||
.map(|p| p.display().to_string())
|
||||
}
|
||||
|
||||
/// Returns true when the applied repo has commits that have not yet been
|
||||
/// deployed (i.e. the applied HEAD differs from the sha currently locked in
|
||||
/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip
|
||||
/// conveys: "there is a config change ready to apply via rebuild."
|
||||
///
|
||||
/// Async on purpose: this runs per agent inside `container_view::build_all`,
|
||||
/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and
|
||||
/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous
|
||||
/// `git` fork here blocks a tokio worker for the whole exec — under
|
||||
/// nix-build disk saturation that's long enough that concurrent sweeps
|
||||
/// starved the runtime and stalled the per-agent sockets.
|
||||
pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
|
||||
let applied_head = tokio::process::Command::new("git")
|
||||
.args([
|
||||
"-C",
|
||||
&format!("/var/lib/hyperhive/applied/{name}"),
|
||||
"rev-parse",
|
||||
"HEAD",
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.ok()
|
||||
.filter(|o| o.status.success())
|
||||
.and_then(|o| String::from_utf8(o.stdout).ok())
|
||||
.map(|s| s.trim().to_owned());
|
||||
|
||||
match (applied_head.as_deref(), deployed_sha) {
|
||||
(Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild one sub-agent and refresh its marker. Used by both the startup
|
||||
/// scanner and the dashboard's manual "update" button so the two paths
|
||||
/// can't diverge.
|
||||
///
|
||||
/// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from
|
||||
/// the `rebuild_queue` worker (lets the function annotate its phase via
|
||||
/// `coord.set_queue_step`) and `None` when called directly (e.g. the
|
||||
/// root-agent migration nudge in `ensure_root_agent`).
|
||||
///
|
||||
/// `relock` bumps the agent's meta input to `applied/<n>/main` before
|
||||
/// the container rebuild. Pass `false` only for meta-update cascade
|
||||
/// rebuilds, where re-locking would revert the bump the cascade just
|
||||
/// committed (see `lifecycle::rebuild`).
|
||||
///
|
||||
/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds:
|
||||
/// instead of holding the serialized build lane through the container
|
||||
/// boot, the start-after-rebuild is enqueued as a fast-lane `Start`
|
||||
/// entry (grouped under this rebuild via `parent_id`, same split as the
|
||||
/// graceful-stop follow-up). Pass `None` for direct callers to keep the
|
||||
/// start inline.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`.
|
||||
pub async fn rebuild_agent(
|
||||
coord: &Arc<Coordinator>,
|
||||
name: &str,
|
||||
current_rev: &str,
|
||||
queue_entry_id: Option<u64>,
|
||||
relock: bool,
|
||||
defer_start_source: Option<crate::rebuild_queue::QueueSource>,
|
||||
) -> Result<()> {
|
||||
tracing::info!(%name, rev = %current_rev, "rebuild agent");
|
||||
let agent_dir = coord
|
||||
.ensure_runtime(name)
|
||||
.with_context(|| format!("ensure_runtime {name}"))?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
// Suppress crash_watch during the stop+start window inside
|
||||
// lifecycle::rebuild. Dashboard rebuilds already do this via
|
||||
// lifecycle_action; this catches the auto-update scan + any
|
||||
// other direct caller.
|
||||
let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding);
|
||||
let result = lifecycle::rebuild(
|
||||
name,
|
||||
&hive,
|
||||
&paths,
|
||||
relock,
|
||||
defer_start_source.is_some(),
|
||||
&|step| coord.set_queue_step(queue_entry_id, step),
|
||||
&|log_id| {
|
||||
if let Some(qid) = queue_entry_id
|
||||
&& coord.rebuild_queue.set_build_log_id(qid, log_id)
|
||||
{
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
drop(guard);
|
||||
match &result {
|
||||
Ok(needs_start) => {
|
||||
if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) {
|
||||
tracing::warn!(%name, error = ?e, "write rev marker failed");
|
||||
}
|
||||
// Deferred start: hand the container boot to the fast lane so
|
||||
// this build-lane entry completes now and the next queued
|
||||
// rebuild's nix build overlaps with the boot. `parent_id`
|
||||
// groups the follow-up under this rebuild on the dashboard —
|
||||
// same split the graceful-stop path uses for its container
|
||||
// stop. A start failure surfaces on the Start entry (with
|
||||
// the cold-start fallback) instead of failing the rebuild.
|
||||
if *needs_start && let Some(source) = defer_start_source {
|
||||
coord
|
||||
.rebuild_queue
|
||||
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
||||
kind: crate::rebuild_queue::QueueKind::Start,
|
||||
agent: name.to_owned(),
|
||||
source,
|
||||
reason: format!("start after rebuild of {name}"),
|
||||
parent_id: queue_entry_id,
|
||||
inputs: Vec::new(),
|
||||
approval_id: None,
|
||||
perm_payload: None,
|
||||
depends_on: Vec::new(),
|
||||
});
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: name.to_owned(),
|
||||
ok: true,
|
||||
note: None,
|
||||
sha: None,
|
||||
tag: None,
|
||||
});
|
||||
coord.set_queue_step(queue_entry_id, "forge sync");
|
||||
// Run the full forge sync on every successful rebuild so
|
||||
// the rebuild path is equivalent to the hive-c0re startup
|
||||
// sweep: token, config-repo mirror, meta read access, and
|
||||
// meta remote are all kept in sync. Recovers missing tokens
|
||||
// (e.g. first-spawn seeding failed transiently) without
|
||||
// requiring a full hive-c0re restart.
|
||||
crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
|
||||
// Mirror the matrix side of the startup sweep: if hive-matrix
|
||||
// is present, ensure this agent has a registered user + token.
|
||||
// Idempotent (skips if token file already exists). Keeps the
|
||||
// rebuild path equivalent to the startup sweep for newly-spawned
|
||||
// agents that missed ensure_all().
|
||||
crate::matrix::sync_agent_standalone(name).await;
|
||||
// Wake the agent on its next turn so claude sees a
|
||||
// "you were rebuilt — check /state/ for notes, --continue
|
||||
// session intact" hint. Covers dashboard rebuild, admin
|
||||
// CLI rebuild, auto-update startup scan, and the
|
||||
// dashboard's meta-input update path — all of which
|
||||
// route through rebuild_agent.
|
||||
coord.kick_agent(name, "container rebuilt");
|
||||
// Container state (needs_update, deployed_sha) may have
|
||||
// shifted — rescan so dashboards drop the "needs update"
|
||||
// chip without waiting for the next /api/state poll.
|
||||
coord.rescan_containers_and_emit().await;
|
||||
// Lock bump → meta-inputs panel needs to re-render.
|
||||
crate::dashboard::emit_meta_inputs_snapshot(coord);
|
||||
}
|
||||
Err(e) => {
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: name.to_owned(),
|
||||
ok: false,
|
||||
note: Some(format!("{e:#}")),
|
||||
sha: None,
|
||||
tag: None,
|
||||
});
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
}
|
||||
result.map(|_| ())
|
||||
}
|
||||
|
||||
/// Whether this hive is "ruthless" — running with no root/manager agent at
|
||||
/// all (no ruth). When true, hive-c0re skips the root-agent create/start
|
||||
/// sweep entirely. Controlled by the host option
|
||||
/// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS`
|
||||
/// env var. Defaults to `false` when the var is unset (back-compat: the
|
||||
/// root agent was always auto-managed before this opt-out existed); only
|
||||
/// an explicit `true` / `1` / `yes` enables ruthless mode.
|
||||
fn ruthless() -> bool {
|
||||
match std::env::var("HYPERHIVE_RUTHLESS") {
|
||||
Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Auto-create the manager container on startup if it isn't already there.
|
||||
/// hive-c0re manages the manager end-to-end: operators no longer declare
|
||||
/// `containers.h-ruth` in their host NixOS config. Bypasses the approval
|
||||
/// queue — the root/manager is auto-managed by default. Operators who
|
||||
/// don't want a root agent at all set `services.hyperhive.ruthless = true`,
|
||||
/// which short-circuits this whole function. Idempotent.
|
||||
pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
|
||||
if ruthless() {
|
||||
tracing::info!(
|
||||
"ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let existing = lifecycle::list().await.unwrap_or_default();
|
||||
let current_rev = current_flake_rev(&coord.hyperhive_flake);
|
||||
if existing
|
||||
.iter()
|
||||
.any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME))
|
||||
{
|
||||
// Container exists already. If it predates the unified lifecycle
|
||||
// (no applied flake on disk) we must rebuild — otherwise it's
|
||||
// running whatever the host-declarative config was at create
|
||||
// time, with a wrong systemd unit and port.
|
||||
let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
|
||||
if !applied_flake.exists()
|
||||
&& let Some(rev) = current_rev.as_ref()
|
||||
{
|
||||
tracing::warn!(
|
||||
"manager container exists but no applied flake — forcing rebuild to migrate"
|
||||
);
|
||||
let coord_clone = coord.clone();
|
||||
if let Err(e) =
|
||||
rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await
|
||||
{
|
||||
tracing::warn!(error = ?e, "manager migration rebuild failed");
|
||||
}
|
||||
} else {
|
||||
tracing::debug!("manager container already present");
|
||||
}
|
||||
// hive-c0re auto-manages the root/manager by default, so a
|
||||
// present-but-stopped root (e.g. a first-start failure on a fresh
|
||||
// install) is brought back up here: the startup sweep's rebuild only
|
||||
// restarts a container that was already running, so without this it
|
||||
// stays down until a manual `nixos-container start`. The sub-agent
|
||||
// `was_running` guard is intentionally left untouched. (Operators
|
||||
// opt out of this whole auto-management with
|
||||
// `services.hyperhive.ruthless = true`, gated at the top of
|
||||
// this function.)
|
||||
if !lifecycle::is_running(MANAGER_NAME).await {
|
||||
tracing::info!("manager container present but not running — starting");
|
||||
if let Err(e) = lifecycle::start(MANAGER_NAME).await {
|
||||
tracing::warn!(error = ?e, "manager start failed");
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!("manager container missing — spawning");
|
||||
let runtime = coord.ensure_runtime(MANAGER_NAME)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
|
||||
lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
|
||||
if let Some(rev) = current_rev {
|
||||
let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sort `names` in-place so parents precede their children in the topology.
|
||||
/// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last,
|
||||
/// alphabetically within their tier. Stable within each depth tier.
|
||||
pub fn topology_sort(
|
||||
names: &mut [String],
|
||||
topo: &std::collections::BTreeMap<String, Option<String>>,
|
||||
) {
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
// Build depth map using owned clones so the borrow on `names` is released
|
||||
// before the sort_by mutable borrow.
|
||||
let name_set: Vec<String> = names.to_vec();
|
||||
let mut depth: HashMap<String, usize> = HashMap::new();
|
||||
let mut queue: VecDeque<String> = VecDeque::new();
|
||||
// Seed roots: entries with no parent, or names not present in topo at all.
|
||||
for name in &name_set {
|
||||
if topo.get(name).is_none_or(Option::is_none) {
|
||||
depth.insert(name.clone(), 0);
|
||||
queue.push_back(name.clone());
|
||||
}
|
||||
}
|
||||
// BFS to assign depths to children.
|
||||
while let Some(parent) = queue.pop_front() {
|
||||
let d = depth[&parent] + 1;
|
||||
for name in &name_set {
|
||||
let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str());
|
||||
if is_child && !depth.contains_key(name) {
|
||||
depth.insert(name.clone(), d);
|
||||
queue.push_back(name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
names.sort_by(|a, b| {
|
||||
let da = depth.get(a).copied().unwrap_or(usize::MAX);
|
||||
let db = depth.get(b).copied().unwrap_or(usize::MAX);
|
||||
da.cmp(&db).then(a.cmp(b))
|
||||
});
|
||||
}
|
||||
|
||||
/// Rebuild containers that need it on startup. Skips:
|
||||
/// - **Stopped containers**: deferred to on-start (`run_start` / `handle_start`
|
||||
/// upgrades a plain start to rebuild+start when the rev marker is stale).
|
||||
/// - **Running containers with a matching rev marker**: no nix work needed.
|
||||
///
|
||||
/// Enqueues a `StartupSweep` parent entry followed by per-agent `Rebuild`
|
||||
/// children linked via `parent_id`. Returns Ok even if some rebuilds failed.
|
||||
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||
let containers = match lifecycle::list().await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "auto-update: nixos-container list failed");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let current_rev = current_flake_rev(&coord.hyperhive_flake);
|
||||
|
||||
// Resolve container names to logical agent names, then sort by
|
||||
// topology depth so parents are always rebuilt before their
|
||||
// children. Root agents (depth 0) go first; agents absent from
|
||||
// the topology file sort last (stable, alphabetical within tier).
|
||||
let mut logical_names: Vec<String> = containers
|
||||
.iter()
|
||||
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
|
||||
.collect();
|
||||
let topo = crate::topology::read();
|
||||
topology_sort(&mut logical_names, &topo);
|
||||
|
||||
// Pre-classify: decide which agents need a rebuild now vs can be skipped.
|
||||
let mut to_rebuild: Vec<String> = Vec::new();
|
||||
let mut n_deferred = 0usize;
|
||||
let mut n_skipped = 0usize;
|
||||
for name in &logical_names {
|
||||
// Idea 2: stopped containers are deferred — rebuild happens the first
|
||||
// time the operator starts them.
|
||||
if !lifecycle::is_running(name).await {
|
||||
n_deferred += 1;
|
||||
tracing::debug!(%name, "startup sweep: stopped — deferring rebuild to on-start");
|
||||
continue;
|
||||
}
|
||||
// Idea 1: running containers with a matching rev marker need no rebuild.
|
||||
if let Some(ref rev) = current_rev {
|
||||
let stored = std::fs::read_to_string(rev_marker_path(name)).ok();
|
||||
if stored.as_deref() == Some(rev.as_str()) {
|
||||
n_skipped += 1;
|
||||
tracing::debug!(%name, "startup sweep: rev unchanged — skipping rebuild");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
to_rebuild.push(name.clone());
|
||||
}
|
||||
|
||||
// Enqueue the parent sweep entry. The worker processes it trivially
|
||||
// (no-op dispatch); its purpose is to give the dashboard a "why" header.
|
||||
let sweep_id = coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::StartupSweep,
|
||||
"hyperhive".to_owned(),
|
||||
crate::rebuild_queue::QueueSource::AutoUpdate,
|
||||
format!(
|
||||
"startup sweep: {} rebuild(s), {} deferred (stopped), {} skipped (up-to-date)",
|
||||
to_rebuild.len(),
|
||||
n_deferred,
|
||||
n_skipped,
|
||||
),
|
||||
None,
|
||||
);
|
||||
|
||||
tracing::info!(
|
||||
total = containers.len(),
|
||||
rebuilds = to_rebuild.len(),
|
||||
deferred = n_deferred,
|
||||
skipped = n_skipped,
|
||||
sweep_id,
|
||||
"auto-update: startup sweep"
|
||||
);
|
||||
|
||||
for name in to_rebuild {
|
||||
coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Rebuild,
|
||||
name,
|
||||
crate::rebuild_queue::QueueSource::StartupSweep,
|
||||
"startup sweep".to_owned(),
|
||||
Some(sweep_id),
|
||||
);
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -139,13 +139,17 @@ enum Cmd {
|
|||
#[command(flatten)]
|
||||
scope: ScopeArgs,
|
||||
/// Gracefully quiesce each agent before stopping, instead of a
|
||||
/// hard stop. Each agent is enqueued as a `GracefulStop` on the
|
||||
/// rebuild queue: the harness is signalled, runs one
|
||||
/// stop-checkpoint turn to flush durable `/state`, drains, then
|
||||
/// the container is stopped (bounded by a 3-min timeout that
|
||||
/// falls back to a hard stop). Applies to agents only.
|
||||
/// hard stop. Each agent gets a graceful-stop DAG on the job
|
||||
/// queue: the harness is signalled, runs one stop-checkpoint
|
||||
/// turn to flush durable `/state`, drains, then the container
|
||||
/// is stopped (bounded by a 3-min timeout that falls back to a
|
||||
/// hard stop). All drains overlap. Applies to agents only.
|
||||
#[arg(long)]
|
||||
graceful: bool,
|
||||
/// Return immediately after the stop DAGs are queued instead of
|
||||
/// waiting for them with live per-node progress.
|
||||
#[arg(long)]
|
||||
no_wait: bool,
|
||||
},
|
||||
/// Start containers hive-wide — the inverse of `hivectl stop`. Bare
|
||||
/// `hivectl start` starts everything back up; the same scope flags as
|
||||
|
|
@ -154,6 +158,10 @@ enum Cmd {
|
|||
Start {
|
||||
#[command(flatten)]
|
||||
scope: ScopeArgs,
|
||||
/// Return immediately after the start DAGs are queued instead
|
||||
/// of waiting for them with live per-node progress.
|
||||
#[arg(long)]
|
||||
no_wait: bool,
|
||||
},
|
||||
/// Restart containers hive-wide — `stop` then `start` over the same
|
||||
/// scope. Bare `hivectl restart` restarts **everything** (all sub-agents
|
||||
|
|
@ -531,15 +539,23 @@ enum AgentsCmd {
|
|||
/// Stop and start a single agent container without rebuilding config.
|
||||
/// Useful for "kick the container" when the process is stuck or the
|
||||
/// container needs a clean restart without changing the NixOS config.
|
||||
/// Rides the job queue (serialized against in-flight rebuilds for
|
||||
/// the same agent); waits with live progress unless `--no-wait`.
|
||||
Restart {
|
||||
/// Agent name (e.g. `damocles`, `ruth`).
|
||||
name: String,
|
||||
/// Return immediately after the restart DAG is queued.
|
||||
#[arg(long)]
|
||||
no_wait: bool,
|
||||
},
|
||||
/// Restart ALL managed agent containers via one restart DAG each —
|
||||
/// unrelated agents overlap, each serializes on its own lease.
|
||||
/// Waits for the whole set with live progress unless `--no-wait`.
|
||||
RestartAll {
|
||||
/// Return immediately after the restart DAGs are queued.
|
||||
#[arg(long)]
|
||||
no_wait: bool,
|
||||
},
|
||||
/// Stop and restart ALL managed agent containers in sequence.
|
||||
/// Iterates the live container list and restarts each one. Any per-agent
|
||||
/// failure is reported at the end rather than stopping mid-run, so all
|
||||
/// containers get a restart attempt.
|
||||
RestartAll,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
|
|
@ -602,8 +618,8 @@ async fn main() -> Result<()> {
|
|||
},
|
||||
Cmd::Agents { cmd } => match cmd {
|
||||
AgentsCmd::List { json } => agents_list(&socket, json).await,
|
||||
AgentsCmd::Restart { name } => agents_restart(&socket, &name).await,
|
||||
AgentsCmd::RestartAll => agents_restart_all(&socket).await,
|
||||
AgentsCmd::Restart { name, no_wait } => agents_restart(&socket, &name, no_wait).await,
|
||||
AgentsCmd::RestartAll { no_wait } => agents_restart_all(&socket, no_wait).await,
|
||||
},
|
||||
Cmd::Wg { cmd } => match cmd {
|
||||
WgCmd::Init { address } => wg_init(&socket, address.as_deref()).await,
|
||||
|
|
@ -626,8 +642,12 @@ async fn main() -> Result<()> {
|
|||
peer_config(&domain, wg_address.as_deref(), wg_endpoint.as_deref());
|
||||
Ok(())
|
||||
}
|
||||
Cmd::Stop { scope, graceful } => stop(&socket, scope.to_scope(), graceful).await,
|
||||
Cmd::Start { scope } => start(&socket, scope.to_scope()).await,
|
||||
Cmd::Stop {
|
||||
scope,
|
||||
graceful,
|
||||
no_wait,
|
||||
} => stop(&socket, scope.to_scope(), graceful, no_wait).await,
|
||||
Cmd::Start { scope, no_wait } => start(&socket, scope.to_scope(), no_wait).await,
|
||||
Cmd::Restart { scope, graceful } => restart(&socket, scope.to_scope(), graceful).await,
|
||||
Cmd::Subvol { cmd } => match cmd {
|
||||
SubvolCmd::Upgrade { name, yes } => subvol_upgrade(&socket, &name, yes).await,
|
||||
|
|
@ -1406,7 +1426,7 @@ fn gateway_list_users(file: &Path) -> Result<()> {
|
|||
// Agent management helpers (require daemon via host admin socket)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn agents_restart(socket: &Path, name: &str) -> Result<()> {
|
||||
async fn agents_restart(socket: &Path, name: &str, no_wait: bool) -> Result<()> {
|
||||
let resp = hive_c0re::client::request(
|
||||
socket,
|
||||
hive_sh4re::HostRequest::Restart {
|
||||
|
|
@ -1416,8 +1436,8 @@ async fn agents_restart(socket: &Path, name: &str) -> Result<()> {
|
|||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
if resp.ok {
|
||||
println!("restarted: {name}");
|
||||
Ok(())
|
||||
println!("restart queued: {name}");
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
|
||||
} else {
|
||||
bail!(
|
||||
"restart {name}: {}",
|
||||
|
|
@ -1426,6 +1446,107 @@ async fn agents_restart(socket: &Path, name: &str) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job-queue wait/progress loop — shared by every verb that submits DAGs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Poll the submitted DAG ids (`HostRequest::QueueDag`, ~1s interval)
|
||||
/// and print a progress line whenever a DAG's rendered state changes —
|
||||
/// including fan-out children that appear under a polled parent. Exits
|
||||
/// non-zero when any DAG (or child) ends `failed`; a `cancelled` DAG
|
||||
/// terminates the wait but is an operator action, not an error.
|
||||
async fn wait_for_dags(socket: &Path, ids: Vec<u64>, no_wait: bool) -> Result<()> {
|
||||
if no_wait || ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut pending: std::collections::BTreeSet<u64> = ids.into_iter().collect();
|
||||
let mut last: std::collections::HashMap<u64, String> = std::collections::HashMap::new();
|
||||
let mut failed: Vec<String> = Vec::new();
|
||||
while !pending.is_empty() {
|
||||
for id in pending.clone() {
|
||||
let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::QueueDag { id })
|
||||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
let dags = resp.dags.unwrap_or_default();
|
||||
if dags.is_empty() {
|
||||
// Evicted from the queue's history tail — it finished a
|
||||
// while ago; nothing left to report on.
|
||||
println!("job #{id}: gone from queue history");
|
||||
pending.remove(&id);
|
||||
continue;
|
||||
}
|
||||
let mut all_terminal = true;
|
||||
for d in &dags {
|
||||
let line = render_dag_line(d);
|
||||
if last.get(&d.id) != Some(&line) {
|
||||
println!("{line}");
|
||||
last.insert(d.id, line);
|
||||
}
|
||||
// Node-level terminality, not the roll-up: a DAG rolls
|
||||
// up `failed` the moment one node fails while its
|
||||
// after-any recovery node (rebuild's tail Reconcile)
|
||||
// may still be running — keep watching so the operator
|
||||
// sees whether the agent came back.
|
||||
if !d.nodes.iter().all(|n| n.state.is_terminal()) {
|
||||
all_terminal = false;
|
||||
} else if d.state == hive_sh4re::jobs::State::Failed {
|
||||
failed.push(format!("{} {}", d.kind.as_str(), d.agent));
|
||||
}
|
||||
}
|
||||
if all_terminal {
|
||||
pending.remove(&id);
|
||||
}
|
||||
}
|
||||
if !pending.is_empty() {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
if failed.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
failed.sort();
|
||||
failed.dedup();
|
||||
bail!("queued job(s) failed: {}", failed.join(", "))
|
||||
}
|
||||
}
|
||||
|
||||
fn state_glyph(state: hive_sh4re::jobs::State) -> &'static str {
|
||||
match state {
|
||||
hive_sh4re::jobs::State::Queued => "⏸",
|
||||
hive_sh4re::jobs::State::Running => "▶",
|
||||
hive_sh4re::jobs::State::Done => "✔",
|
||||
hive_sh4re::jobs::State::Failed => "✖",
|
||||
hive_sh4re::jobs::State::Cancelled => "⊘",
|
||||
}
|
||||
}
|
||||
|
||||
/// One progress line for a DAG: roll-up glyph, template, agent, then
|
||||
/// the node chain with the running node's live step label — the CLI
|
||||
/// twin of the dashboard's queue card.
|
||||
fn render_dag_line(d: &hive_sh4re::jobs::DagView) -> String {
|
||||
use std::fmt::Write as _;
|
||||
let mut out = format!(
|
||||
"{} {} {:<12}",
|
||||
state_glyph(d.state),
|
||||
d.kind.as_str(),
|
||||
d.agent
|
||||
);
|
||||
for (i, n) in d.nodes.iter().enumerate() {
|
||||
let sep = if i == 0 { " " } else { " → " };
|
||||
let _ = write!(out, "{sep}{} {}", state_glyph(n.state), n.kind);
|
||||
if n.state == hive_sh4re::jobs::State::Running
|
||||
&& let Some(step) = &n.step
|
||||
{
|
||||
let _ = write!(out, " ({step})");
|
||||
}
|
||||
}
|
||||
if let Some(err) = d.nodes.iter().find_map(|n| n.error.as_deref()) {
|
||||
let short: String = err.chars().take(120).collect();
|
||||
let _ = write!(out, " — {short}");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `hivectl agents list` — fetch the per-agent status roster from the
|
||||
/// daemon (`HostRequest::AgentStatus`) and render it as a padded table,
|
||||
/// or the raw JSON rows with `--json`. Reuses the dashboard's
|
||||
|
|
@ -1502,7 +1623,7 @@ async fn agents_list(socket: &Path, json: bool) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
async fn agents_restart_all(socket: &Path) -> Result<()> {
|
||||
async fn agents_restart_all(socket: &Path, no_wait: bool) -> Result<()> {
|
||||
let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::RestartAll)
|
||||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
|
|
@ -1511,7 +1632,7 @@ async fn agents_restart_all(socket: &Path) -> Result<()> {
|
|||
println!("restart-all: no managed containers found");
|
||||
} else {
|
||||
for a in agents {
|
||||
println!("restarted: {a}");
|
||||
println!("restart queued: {a}");
|
||||
}
|
||||
}
|
||||
if !resp.ok {
|
||||
|
|
@ -1520,22 +1641,34 @@ async fn agents_restart_all(socket: &Path) -> Result<()> {
|
|||
resp.error.as_deref().unwrap_or("unknown error")
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
|
||||
}
|
||||
|
||||
async fn stop(socket: &Path, scope: hive_sh4re::LifecycleScope, graceful: bool) -> Result<()> {
|
||||
async fn stop(
|
||||
socket: &Path,
|
||||
scope: hive_sh4re::LifecycleScope,
|
||||
graceful: bool,
|
||||
no_wait: bool,
|
||||
) -> Result<()> {
|
||||
let resp =
|
||||
hive_c0re::client::request(socket, hive_sh4re::HostRequest::Stop { scope, graceful })
|
||||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
render_lifecycle(&resp, "stopped")
|
||||
// Render first, but even when an infra failure makes it bail,
|
||||
// watch the already-queued agent DAGs before surfacing the error —
|
||||
// they run regardless.
|
||||
let rendered = render_lifecycle(&resp, "stop queued");
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?;
|
||||
rendered
|
||||
}
|
||||
|
||||
async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope) -> Result<()> {
|
||||
async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope, no_wait: bool) -> Result<()> {
|
||||
let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::Start { scope })
|
||||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
render_lifecycle(&resp, "started")
|
||||
let rendered = render_lifecycle(&resp, "start queued");
|
||||
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), no_wait).await?;
|
||||
rendered
|
||||
}
|
||||
|
||||
/// Restart = `stop` then `start` over the same scope, composed client-side
|
||||
|
|
@ -1543,9 +1676,13 @@ async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope) -> Result<()> {
|
|||
/// `--graceful`; if it reports a failure (`stop` returns `Err`) the `?`
|
||||
/// short-circuits before the start phase, so a half-stopped hive isn't
|
||||
/// blindly started over — the operator sees the stop errors and can recover.
|
||||
///
|
||||
/// No `--no-wait` here on purpose: the stop DAGs must complete before
|
||||
/// the start submits, otherwise the start's `wanted = Up` write would
|
||||
/// land before the queued stops execute and turn them into noops.
|
||||
async fn restart(socket: &Path, scope: hive_sh4re::LifecycleScope, graceful: bool) -> Result<()> {
|
||||
stop(socket, scope.clone(), graceful).await?;
|
||||
start(socket, scope).await
|
||||
stop(socket, scope.clone(), graceful, false).await?;
|
||||
start(socket, scope, false).await
|
||||
}
|
||||
|
||||
/// A [`LifecycleScope`](hive_sh4re::LifecycleScope) targeting exactly one
|
||||
|
|
@ -1594,6 +1731,12 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> {
|
|||
stop_resp.error.as_deref().unwrap_or("unknown error")
|
||||
);
|
||||
}
|
||||
// The stop is a queued DAG now — the migration below snapshots +
|
||||
// swaps the state dir and MUST NOT run under a live bind mount, so
|
||||
// wait for the stop to actually execute before touching anything.
|
||||
wait_for_dags(socket, stop_resp.queued_dags.unwrap_or_default(), false)
|
||||
.await
|
||||
.with_context(|| format!("waiting for {name} to stop before the migration"))?;
|
||||
|
||||
println!("migrating {name} state dir to a btrfs subvolume…");
|
||||
let upgrade = hive_c0re::priv_client::upgrade_agent_subvolume(name).await;
|
||||
|
|
@ -1633,6 +1776,14 @@ async fn subvol_upgrade(socket: &Path, name: &str, yes: bool) -> Result<()> {
|
|||
start_resp.error.as_deref().unwrap_or("unknown error")
|
||||
);
|
||||
}
|
||||
wait_for_dags(socket, start_resp.queued_dags.unwrap_or_default(), false)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"{name} migrated to a btrfs subvolume, but its restart job failed — run \
|
||||
`hivectl start --agent {name}` to retry"
|
||||
)
|
||||
})?;
|
||||
println!("upgraded {name} to a btrfs subvolume and restarted it");
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1673,3 +1824,82 @@ fn validate_htpasswd_username(username: &str) -> Result<()> {
|
|||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use hive_sh4re::jobs::{DagView, NodeView, Source, State, Template};
|
||||
|
||||
use super::render_dag_line;
|
||||
|
||||
fn node(id: u32, kind: &str, state: State, step: Option<&str>) -> NodeView {
|
||||
NodeView {
|
||||
id,
|
||||
kind: kind.to_owned(),
|
||||
deps: if id == 0 { vec![] } else { vec![id - 1] },
|
||||
state,
|
||||
step: step.map(str::to_owned),
|
||||
build_log_id: None,
|
||||
started_at: None,
|
||||
finished_at: None,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_dag_line_shows_chain_and_running_step() {
|
||||
let dag = DagView {
|
||||
id: 7,
|
||||
agent: "alice".to_owned(),
|
||||
kind: Template::Rebuild,
|
||||
state: State::Running,
|
||||
source: Source::Manual,
|
||||
parent_id: None,
|
||||
reason: "manual".to_owned(),
|
||||
enqueued_at: 0,
|
||||
started_at: Some(1),
|
||||
finished_at: None,
|
||||
inputs: vec![],
|
||||
approval_id: None,
|
||||
perm_payload: None,
|
||||
nodes: vec![
|
||||
node(0, "prebuild", State::Done, None),
|
||||
node(1, "stop_for_update", State::Done, None),
|
||||
node(2, "swap", State::Running, Some("nixos-container update")),
|
||||
node(3, "reconcile", State::Queued, None),
|
||||
],
|
||||
};
|
||||
let line = render_dag_line(&dag);
|
||||
assert!(line.starts_with("▶ rebuild alice"), "{line}");
|
||||
assert!(
|
||||
line.contains(
|
||||
"✔ prebuild → ✔ stop_for_update → ▶ swap (nixos-container update) → ⏸ reconcile"
|
||||
),
|
||||
"{line}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_dag_line_surfaces_first_node_error() {
|
||||
let mut failed = node(0, "prebuild", State::Failed, None);
|
||||
failed.error = Some("nix build exploded".to_owned());
|
||||
let dag = DagView {
|
||||
id: 8,
|
||||
agent: "bob".to_owned(),
|
||||
kind: Template::Rebuild,
|
||||
state: State::Failed,
|
||||
source: Source::Manual,
|
||||
parent_id: None,
|
||||
reason: "manual".to_owned(),
|
||||
enqueued_at: 0,
|
||||
started_at: Some(1),
|
||||
finished_at: Some(2),
|
||||
inputs: vec![],
|
||||
approval_id: None,
|
||||
perm_payload: None,
|
||||
nodes: vec![failed],
|
||||
};
|
||||
let line = render_dag_line(&dag);
|
||||
assert!(line.contains("✖ rebuild"), "{line}");
|
||||
assert!(line.contains("— nix build exploded"), "{line}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -173,12 +173,17 @@ pub struct Coordinator {
|
|||
/// tokio mutex so the rescan can `await` `lifecycle::list` /
|
||||
/// `is_running` without blocking other coordinator paths.
|
||||
last_containers: tokio::sync::Mutex<HashMap<String, ContainerView>>,
|
||||
/// Global rebuild queue. Every long-running container/meta op
|
||||
/// (rebuild, meta-update, first-spawn) goes through this queue so
|
||||
/// hive-c0re runs at most one at a time and the dashboard can
|
||||
/// render a single ordered view of pending + running work. See
|
||||
/// `rebuild_queue.rs` for the dedup rules + history retention.
|
||||
pub rebuild_queue: Arc<crate::rebuild_queue::RebuildQueue>,
|
||||
/// Global job-DAG queue. Every container/meta op (rebuild,
|
||||
/// meta-update, first-spawn, power changes) is submitted as a DAG
|
||||
/// of primitive nodes; a single scheduler drives them with
|
||||
/// build-slot + per-agent-lease gating so the dashboard renders one
|
||||
/// ordered view of pending + running work. See `job_queue/` for
|
||||
/// the dedup rules, resource classes, and history retention.
|
||||
pub job_queue: Arc<crate::job_queue::JobQueue>,
|
||||
/// Durable per-agent power intent (`wanted: Up | Offline`) — the
|
||||
/// spec half of desired-state reconciliation; the queue's
|
||||
/// `Reconcile` nodes converge observed state to it.
|
||||
pub power: Arc<crate::power::PowerStore>,
|
||||
/// Shutdown signal broadcast to all background tasks. Sending
|
||||
/// `true` asks every loop to exit after its current work item.
|
||||
/// Use `shutdown_rx()` to subscribe; `request_shutdown()` to fire.
|
||||
|
|
@ -244,12 +249,27 @@ impl Default for HiveEnv {
|
|||
/// instead of every host-level setting as its own JSON-blob argument.
|
||||
/// `#[serde(default)]` lets any field be omitted and fall back to its
|
||||
/// canonical default.
|
||||
#[derive(Clone, Debug, Default, serde::Deserialize)]
|
||||
#[derive(Clone, Debug, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ServeConfig {
|
||||
#[serde(flatten)]
|
||||
pub env: HiveEnv,
|
||||
pub model_prices: crate::hive_stats::PriceTable,
|
||||
/// Number of concurrent nix-heavy job-queue nodes (prebuild /
|
||||
/// profile-swap / create / meta lock). hive-c0re-local like
|
||||
/// `model_prices` — never injected into containers. Set via
|
||||
/// `services.hyperhive.c0re.buildSlots`.
|
||||
pub build_slots: usize,
|
||||
}
|
||||
|
||||
impl Default for ServeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
env: HiveEnv::default(),
|
||||
model_prices: crate::hive_stats::PriceTable::default(),
|
||||
build_slots: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -433,6 +453,7 @@ impl Coordinator {
|
|||
db_path: &Path,
|
||||
env: HiveEnv,
|
||||
model_prices: crate::hive_stats::PriceTable,
|
||||
build_slots: usize,
|
||||
) -> Result<Self> {
|
||||
let HiveEnv {
|
||||
hyperhive_flake,
|
||||
|
|
@ -469,6 +490,7 @@ impl Coordinator {
|
|||
let audit_log =
|
||||
Arc::new(crate::audit_log::AuditLog::open(build_logs_dir).context("open audit_log")?);
|
||||
crate::audit_log::install(audit_log.clone());
|
||||
let power = Arc::new(crate::power::PowerStore::open(db_path).context("open agent_power")?);
|
||||
let (dashboard_events, _) = broadcast::channel(DASHBOARD_CHANNEL);
|
||||
let (shutdown_tx, _) = watch::channel(false);
|
||||
Ok(Self {
|
||||
|
|
@ -497,7 +519,8 @@ impl Coordinator {
|
|||
event_seq: AtomicU64::new(0),
|
||||
meta_updates_active: AtomicU64::new(0),
|
||||
last_containers: tokio::sync::Mutex::new(HashMap::new()),
|
||||
rebuild_queue: Arc::new(crate::rebuild_queue::RebuildQueue::new()),
|
||||
job_queue: Arc::new(crate::job_queue::JobQueue::new(build_slots)),
|
||||
power,
|
||||
shutdown_tx,
|
||||
})
|
||||
}
|
||||
|
|
@ -543,7 +566,7 @@ impl Coordinator {
|
|||
/// wrappers below) and the worker so every state transition
|
||||
/// surfaces on the dashboard without extra plumbing.
|
||||
pub fn emit_rebuild_queue_snapshot(self: &Arc<Self>) {
|
||||
let queue = self.rebuild_queue.snapshot();
|
||||
let queue = self.job_queue.snapshot();
|
||||
self.emit_dashboard_event(DashboardEvent::RebuildQueueChanged {
|
||||
seq: self.next_seq(),
|
||||
queue,
|
||||
|
|
@ -665,15 +688,27 @@ impl Coordinator {
|
|||
});
|
||||
}
|
||||
|
||||
/// Update the `step` label on a running queue entry and (if it
|
||||
/// actually changed) re-emit the queue snapshot so the dashboard
|
||||
/// renders the new phase. Returns `true` when the label was new
|
||||
/// and an emit fired, mostly for tracing/logging callers; safe to
|
||||
/// ignore. No-op when `id` is `None` (e.g. callers that aren't
|
||||
/// running from the queue worker) or when the row isn't `Running`.
|
||||
/// Update the `step` label on the currently-running node of DAG
|
||||
/// `id` and (if it actually changed) re-emit the queue snapshot so
|
||||
/// the dashboard renders the new phase. DAG-id-only surface for
|
||||
/// the opaque approval pipeline in `actions.rs`, whose callbacks
|
||||
/// don't know node ids (its DAGs are single-node, so the lookup is
|
||||
/// exact); queue executors use the precise per-node sink in
|
||||
/// `job_queue::exec` instead. No-op when `id` is `None` (callers
|
||||
/// not running from the queue) or when nothing is `Running`.
|
||||
pub fn set_queue_step(self: &Arc<Self>, id: Option<u64>, step: &str) {
|
||||
let Some(id) = id else { return };
|
||||
if self.rebuild_queue.set_step(id, step) {
|
||||
if self.job_queue.set_step_running(id, step) {
|
||||
self.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
|
||||
/// Link a `build_logs` row to the currently-running node of DAG
|
||||
/// `id` and re-emit the snapshot. Same DAG-id-only compatibility
|
||||
/// surface as [`Self::set_queue_step`].
|
||||
pub fn set_queue_build_log(self: &Arc<Self>, id: Option<u64>, log_id: i64) {
|
||||
let Some(id) = id else { return };
|
||||
if self.job_queue.set_build_log_id_running(id, log_id) {
|
||||
self.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,10 +1,12 @@
|
|||
//! Container lifecycle endpoints for the dashboard.
|
||||
//!
|
||||
//! Rebuild / restart / start / stop (hard + graceful) / update-all all
|
||||
//! enqueue onto the rebuild queue, so each shows a visible queued→running
|
||||
//! transient on the dashboard — a direct sub-second start/stop only flashed
|
||||
//! the badge; destroy delegates to `actions::destroy` (optionally
|
||||
//! purging).
|
||||
//! submit DAGs to the job queue (`job_queue::submit`), so each shows a
|
||||
//! visible queued→running transient on the dashboard — a direct
|
||||
//! sub-second start/stop only flashed the badge. Start/stop also
|
||||
//! persist the agent's `wanted` power intent before submitting; the
|
||||
//! DAG's `Reconcile` converges to it. Destroy delegates to
|
||||
//! `actions::destroy` (optionally purging).
|
||||
|
||||
use axum::{
|
||||
extract::{Form, Path as AxumPath, Query, State},
|
||||
|
|
@ -23,6 +25,7 @@ pub(super) struct KillParams {
|
|||
}
|
||||
|
||||
use super::{AppState, error_response, guard_agent_name, strip_container_prefix};
|
||||
use crate::job_queue::{Source, submit};
|
||||
use crate::{actions, lifecycle};
|
||||
|
||||
pub(super) async fn post_rebuild(
|
||||
|
|
@ -33,14 +36,12 @@ pub(super) async fn post_rebuild(
|
|||
if let Some(reject) = guard_agent_name(&state, &logical).await {
|
||||
return reject;
|
||||
}
|
||||
state.coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Rebuild,
|
||||
logical,
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
submit::rebuild(
|
||||
&state.coord,
|
||||
&logical,
|
||||
Source::Manual,
|
||||
"manual via dashboard ↻ R3BU1LD button".to_owned(),
|
||||
None,
|
||||
);
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
|
|
@ -54,19 +55,17 @@ pub(super) async fn post_kill(
|
|||
return reject;
|
||||
}
|
||||
if params.graceful {
|
||||
// Graceful stop: enqueue the quiesce orchestration (signal the harness
|
||||
// → one stop-checkpoint turn → drain → container stop, with a timeout
|
||||
// fallback to a hard stop). Serialised through the rebuild queue so it
|
||||
// can't race an in-flight rebuild for the same agent, and its per-step
|
||||
// progress surfaces on the queue snapshot + build log.
|
||||
state.coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::GracefulStop,
|
||||
logical,
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
// Graceful stop: submit the quiesce DAG (signal the harness →
|
||||
// one stop-checkpoint turn → drain → container stop, with a
|
||||
// timeout fallback to a hard stop). The agent's lifecycle
|
||||
// lease keeps it from racing an in-flight rebuild for the same
|
||||
// agent, and per-node progress surfaces on the queue snapshot.
|
||||
submit::graceful_stop(
|
||||
&state.coord,
|
||||
&logical,
|
||||
Source::Manual,
|
||||
"manual via dashboard graceful stop".to_owned(),
|
||||
None,
|
||||
);
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
return (StatusCode::OK, "ok").into_response();
|
||||
}
|
||||
// Manager is stoppable from the dashboard like any other
|
||||
|
|
@ -79,14 +78,12 @@ pub(super) async fn post_kill(
|
|||
// `socket_server.rs::ManagerRequest::Kill` stays in place: a
|
||||
// manager calling Kill on its own container is self-suicide
|
||||
// mid-call, not a legitimate operator action.
|
||||
state.coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Stop,
|
||||
logical,
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
submit::stop(
|
||||
&state.coord,
|
||||
&logical,
|
||||
Source::Manual,
|
||||
"manual via dashboard stop".to_owned(),
|
||||
None,
|
||||
);
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
|
|
@ -98,14 +95,12 @@ pub(super) async fn post_restart(
|
|||
if let Some(reject) = guard_agent_name(&state, &logical).await {
|
||||
return reject;
|
||||
}
|
||||
state.coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Restart,
|
||||
logical,
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
submit::restart(
|
||||
&state.coord,
|
||||
&logical,
|
||||
Source::Manual,
|
||||
"manual via dashboard ↺ R3START button".to_owned(),
|
||||
None,
|
||||
);
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
|
|
@ -117,14 +112,12 @@ pub(super) async fn post_start(
|
|||
if let Some(reject) = guard_agent_name(&state, &logical).await {
|
||||
return reject;
|
||||
}
|
||||
state.coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Start,
|
||||
logical,
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
submit::start(
|
||||
&state.coord,
|
||||
&logical,
|
||||
Source::Manual,
|
||||
"manual via dashboard start".to_owned(),
|
||||
None,
|
||||
);
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
|
|
@ -137,15 +130,13 @@ pub(super) async fn post_update_all(State(state): State<AppState>) -> Response {
|
|||
else {
|
||||
continue;
|
||||
};
|
||||
state.coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::Rebuild,
|
||||
logical,
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
submit::rebuild(
|
||||
&state.coord,
|
||||
&logical,
|
||||
Source::Manual,
|
||||
"manual via dashboard 🌀 UPDATE ALL".to_owned(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
|
|
|
|||
279
hive-c0re/src/dashboard/meta_inputs.rs
Normal file
279
hive-c0re/src/dashboard/meta_inputs.rs
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
//! META INPUTS panel backend: walks `meta/flake.lock` into
|
||||
//! `MetaInputView` rows for the snapshot, emits the `MetaInputsChanged`
|
||||
//! event after lock bumps, and handles `POST /api/meta-update` (bulk
|
||||
//! flake-input update + rebuild ripple via the job queue).
|
||||
|
||||
use axum::{
|
||||
extract::{Form, State},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
use super::{AppState, error_response};
|
||||
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
pub struct MetaInputView {
|
||||
/// Input key in meta's `flake.nix` — `hyperhive`, `agent-<n>`, etc.
|
||||
pub name: String,
|
||||
/// Full locked sha. Not displayed verbatim; the dashboard
|
||||
/// truncates to the first 12 chars for the chip.
|
||||
pub rev: String,
|
||||
/// Unix seconds — `locked.lastModified`. Drives the relative
|
||||
/// "2h ago" timestamp on each input row.
|
||||
pub last_modified: i64,
|
||||
/// `original.url` if available, for the tooltip / row meta text.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub url: Option<String>,
|
||||
}
|
||||
|
||||
/// Walk `flake.lock`'s `nodes` graph from `root` and emit one
|
||||
/// `MetaInputView` per fetched input, at **every** depth. That
|
||||
/// surfaces the direct meta inputs (`hyperhive`, `agent-<n>`), the
|
||||
/// agent flakes' own inputs (`agent-dmatrix/mcp-matrix`,
|
||||
/// `hyperhive/nixpkgs`), and any deeper transitive inputs — so the
|
||||
/// operator can bump any of them individually. Names are
|
||||
/// slash-separated paths from root, the syntax `nix flake update`
|
||||
/// accepts for transitive inputs.
|
||||
///
|
||||
/// Filtering:
|
||||
/// - Inputs that resolve via a `follows` chain (lock value is an
|
||||
/// array) are skipped — they alias another node, not their own
|
||||
/// fetched derivation, so updating them does nothing.
|
||||
/// - A node is emitted only when it carries a `locked.rev`.
|
||||
/// - Each fetched node is walked exactly once (a `visited` set):
|
||||
/// the lock graph shares nodes (many flakes reference one
|
||||
/// nixpkgs), so without this a shared subtree re-walks per parent
|
||||
/// and a cycle would recurse forever. The result is a spanning
|
||||
/// tree — every input shown once, at its shallowest path.
|
||||
pub(super) fn read_meta_inputs() -> Vec<MetaInputView> {
|
||||
let mut out = Vec::new();
|
||||
let Ok(raw) = std::fs::read_to_string("/var/lib/hyperhive/meta/flake.lock") else {
|
||||
return out;
|
||||
};
|
||||
let Ok(json) = serde_json::from_str::<serde_json::Value>(&raw) else {
|
||||
return out;
|
||||
};
|
||||
let Some(nodes) = json.get("nodes").and_then(|v| v.as_object()) else {
|
||||
return out;
|
||||
};
|
||||
let Some(root_name) = json.get("root").and_then(|v| v.as_str()) else {
|
||||
return out;
|
||||
};
|
||||
let mut visited = std::collections::HashSet::new();
|
||||
visited.insert(root_name.to_owned());
|
||||
walk_meta_inputs(nodes, root_name, "", &mut visited, &mut out);
|
||||
// hyperhive first, then alphabetical. String-sorting the
|
||||
// slash-paths puts every node directly above its own children
|
||||
// (`agent-foo`, `agent-foo/bar`, `agent-foo/bar/baz`), so the
|
||||
// result is a pre-order traversal the tree renderer can consume.
|
||||
out.sort_by(|a, b| match (a.name.as_str(), b.name.as_str()) {
|
||||
("hyperhive", _) => std::cmp::Ordering::Less,
|
||||
(_, "hyperhive") => std::cmp::Ordering::Greater,
|
||||
_ => a.name.cmp(&b.name),
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
fn walk_meta_inputs(
|
||||
nodes: &serde_json::Map<String, serde_json::Value>,
|
||||
node_name: &str,
|
||||
prefix: &str,
|
||||
visited: &mut std::collections::HashSet<String>,
|
||||
out: &mut Vec<MetaInputView>,
|
||||
) {
|
||||
let Some(node) = nodes.get(node_name) else {
|
||||
return;
|
||||
};
|
||||
let Some(inputs_map) = node.get("inputs").and_then(|v| v.as_object()) else {
|
||||
return;
|
||||
};
|
||||
// Two passes: claim (and emit) every direct input of this node
|
||||
// before descending into any of them. A shallow input that a
|
||||
// deeper flake also references then keeps its shallow path
|
||||
// rather than being captured first by the deep walk.
|
||||
let mut to_recurse: Vec<(String, String)> = Vec::new();
|
||||
for (alias, target) in inputs_map {
|
||||
// Inputs map value is either a string (node name) or an
|
||||
// array (a `follows` chain). The latter just aliases another
|
||||
// node — we can't `nix flake update` it directly, so skip.
|
||||
let serde_json::Value::String(target_name) = target else {
|
||||
continue;
|
||||
};
|
||||
// Walk each fetched node once — guards shared subtrees and
|
||||
// cycles, and keeps the panel free of duplicate rows.
|
||||
if !visited.insert(target_name.clone()) {
|
||||
continue;
|
||||
}
|
||||
let Some(target_node) = nodes.get(target_name) else {
|
||||
continue;
|
||||
};
|
||||
let path = if prefix.is_empty() {
|
||||
alias.clone()
|
||||
} else {
|
||||
format!("{prefix}/{alias}")
|
||||
};
|
||||
if let Some(rev) = target_node
|
||||
.get("locked")
|
||||
.and_then(|v| v.get("rev"))
|
||||
.and_then(|v| v.as_str())
|
||||
{
|
||||
let last_modified = target_node
|
||||
.get("locked")
|
||||
.and_then(|v| v.get("lastModified"))
|
||||
.and_then(serde_json::Value::as_i64)
|
||||
.unwrap_or(0);
|
||||
let url = target_node
|
||||
.get("original")
|
||||
.and_then(|v| v.get("url"))
|
||||
.and_then(|v| v.as_str())
|
||||
.map(str::to_owned);
|
||||
out.push(MetaInputView {
|
||||
name: path.clone(),
|
||||
rev: rev.to_owned(),
|
||||
last_modified,
|
||||
url,
|
||||
});
|
||||
}
|
||||
to_recurse.push((target_name.clone(), path));
|
||||
}
|
||||
// Recurse hyperhive's subtree before any agent's — without this,
|
||||
// when meta's top-level `nixpkgs` is a `follows` alias the
|
||||
// `String` check above skips it, and the alphabetical BTreeMap
|
||||
// iteration descends into `agent-*` first. The agent walk then
|
||||
// claims `nixpkgs` at `agent-X/nixpkgs` instead of
|
||||
// `hyperhive/nixpkgs`, which is where the operator expects it.
|
||||
// Sort by the same "hyperhive first, then alpha"
|
||||
// priority `read_meta_inputs` uses for the final output.
|
||||
to_recurse.sort_by(|(a, _), (b, _)| match (a.as_str(), b.as_str()) {
|
||||
("hyperhive", _) => std::cmp::Ordering::Less,
|
||||
(_, "hyperhive") => std::cmp::Ordering::Greater,
|
||||
_ => a.cmp(b),
|
||||
});
|
||||
for (target_name, path) in to_recurse {
|
||||
walk_meta_inputs(nodes, &target_name, &path, visited, out);
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot meta/flake.lock's root inputs + emit
|
||||
/// `MetaInputsChanged`. Call after any mutation that bumps a lock
|
||||
/// (`run_meta_update`, `auto_update::rebuild_agent`).
|
||||
pub(crate) fn emit_meta_inputs_snapshot(coord: &Coordinator) {
|
||||
let inputs = read_meta_inputs();
|
||||
coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::MetaInputsChanged {
|
||||
seq: coord.next_seq(),
|
||||
inputs,
|
||||
});
|
||||
}
|
||||
|
||||
/// Form for `POST /meta-update`. Inputs ride in as a comma-separated
|
||||
/// list under the `inputs` field — the JS submitter joins the
|
||||
/// checked boxes since axum's `Form` extractor doesn't natively
|
||||
/// decode repeated keys without a helper.
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct MetaUpdateForm {
|
||||
inputs: String,
|
||||
}
|
||||
|
||||
/// Bulk-update selected meta flake inputs, then rebuild the affected
|
||||
/// agents in the background. Idempotent w.r.t. selection — choosing
|
||||
/// an input that's already at the latest sha is a no-op (no commit,
|
||||
/// no rebuild ripple). Returns immediately after queueing the work;
|
||||
/// dashboard polls for progress via container `pending` spinners +
|
||||
/// the meta-inputs row sha update.
|
||||
pub(super) async fn post_meta_update(
|
||||
State(state): State<AppState>,
|
||||
Form(form): Form<MetaUpdateForm>,
|
||||
) -> Response {
|
||||
let inputs: Vec<String> = form
|
||||
.inputs
|
||||
.split(',')
|
||||
.map(|s| s.trim().to_owned())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect();
|
||||
if inputs.is_empty() {
|
||||
return error_response("meta-update: no inputs selected");
|
||||
}
|
||||
let inputs_label = inputs.join(", ");
|
||||
// Cascade rebuild children fan out from the MetaLock node when the
|
||||
// lock bump lands — appended by the scheduler so they build against
|
||||
// the post-bump lock, and a failed bump simply fans out nothing.
|
||||
crate::job_queue::submit::meta_update(
|
||||
&state.coord,
|
||||
inputs,
|
||||
crate::job_queue::Source::Manual,
|
||||
format!("meta-update via dashboard ({inputs_label})"),
|
||||
);
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn walk_meta_inputs_keeps_nixpkgs_under_hyperhive_post_follows_refactor() {
|
||||
// Reproduce the shape where meta has
|
||||
// `nixpkgs.follows = "hyperhive/nixpkgs"` at the top level
|
||||
// (rendered as an array — `["hyperhive" "nixpkgs"]` — which
|
||||
// walk_meta_inputs skips because we can't `nix flake update`
|
||||
// a follows alias). The remaining top-level inputs are
|
||||
// `hyperhive` (string) and `agent-z` (string). Without the
|
||||
// hyperhive-first recursion sort, the BTreeMap alphabetical
|
||||
// order descends into `agent-z` first and claims
|
||||
// `nixpkgs` at `agent-z/nixpkgs`.
|
||||
let raw = r#"{
|
||||
"root": "root",
|
||||
"version": 7,
|
||||
"nodes": {
|
||||
"root": {
|
||||
"inputs": {
|
||||
"hyperhive": "hyperhive",
|
||||
"nixpkgs": ["hyperhive", "nixpkgs"],
|
||||
"agent-z": "agent-z"
|
||||
}
|
||||
},
|
||||
"hyperhive": {
|
||||
"inputs": { "nixpkgs": "nixpkgs" },
|
||||
"locked": {"rev": "hhrev", "lastModified": 1},
|
||||
"original": {"url": "git+file:///tmp/hyperhive"}
|
||||
},
|
||||
"agent-z": {
|
||||
"inputs": { "nixpkgs": "nixpkgs" },
|
||||
"locked": {"rev": "azrev", "lastModified": 2},
|
||||
"original": {"url": "git+file:///tmp/agent-z"}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {"rev": "npkrev", "lastModified": 3},
|
||||
"original": {"url": "github:NixOS/nixpkgs/nixos-26.05"}
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
let json: serde_json::Value = serde_json::from_str(raw).unwrap();
|
||||
let nodes = json.get("nodes").unwrap().as_object().unwrap();
|
||||
let root_name = json.get("root").unwrap().as_str().unwrap();
|
||||
let mut visited = std::collections::HashSet::new();
|
||||
visited.insert(root_name.to_owned());
|
||||
let mut out = Vec::new();
|
||||
walk_meta_inputs(nodes, root_name, "", &mut visited, &mut out);
|
||||
|
||||
let nixpkgs = out
|
||||
.iter()
|
||||
.find(|v| v.rev == "npkrev")
|
||||
.expect("nixpkgs node should be emitted exactly once");
|
||||
assert_eq!(
|
||||
nixpkgs.name, "hyperhive/nixpkgs",
|
||||
"nixpkgs should be claimed under hyperhive, not under agent-z. \
|
||||
got: {:?}",
|
||||
nixpkgs.name
|
||||
);
|
||||
// And the agent-z path should NOT also carry a nixpkgs entry —
|
||||
// the spanning-tree visited set guarantees it's claimed once.
|
||||
assert!(
|
||||
!out.iter().any(|v| v.name == "agent-z/nixpkgs"),
|
||||
"agent-z/nixpkgs should not be emitted (already claimed under hyperhive)"
|
||||
);
|
||||
}
|
||||
}
|
||||
220
hive-c0re/src/dashboard/misc_api.rs
Normal file
220
hive-c0re/src/dashboard/misc_api.rs
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
//! Remaining single-endpoint dashboard handlers: the operator inbox
|
||||
//! (`Y3R C4LL`) + mark-all-read, operator compose (`op-send`),
|
||||
//! spawn-request, hive-wide turn stats, container resources, and the
|
||||
//! audit log.
|
||||
|
||||
use axum::{
|
||||
extract::{Form, Path as AxumPath, State},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{AppState, error_response, scan_validated_paths, validate_agent_name};
|
||||
|
||||
/// Unread operator-directed messages for the dashboard's Y3R C4LL inbox.
|
||||
/// Returns messages addressed to `"operator"` that haven't been
|
||||
/// acked yet (the operator clears them via the existing
|
||||
/// `POST /api/agent/operator/mark-all-read`). Newest-first; path-shaped
|
||||
/// tokens are validated so the client renders file links like the
|
||||
/// terminal does. Shape: `{ "messages": [{ id, from, body, at,
|
||||
/// in_reply_to, file_refs }] }`.
|
||||
pub(super) async fn api_operator_inbox(State(state): State<AppState>) -> Response {
|
||||
const INBOX_LIMIT: u64 = 100;
|
||||
match state
|
||||
.coord
|
||||
.broker
|
||||
.unread_for_recipient("operator", INBOX_LIMIT)
|
||||
{
|
||||
Ok(messages) => {
|
||||
let items: Vec<serde_json::Value> = messages
|
||||
.into_iter()
|
||||
.filter_map(|m| {
|
||||
let crate::broker::MessageEvent::Sent {
|
||||
id,
|
||||
from,
|
||||
body,
|
||||
at,
|
||||
in_reply_to,
|
||||
..
|
||||
} = m
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
let file_refs = scan_validated_paths(&body);
|
||||
Some(serde_json::json!({
|
||||
"id": id,
|
||||
"from": from,
|
||||
"body": body,
|
||||
"at": hive_sh4re::wire_time::from_secs(at),
|
||||
"in_reply_to": in_reply_to,
|
||||
"file_refs": file_refs,
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
axum::Json(serde_json::json!({ "messages": items })).into_response()
|
||||
}
|
||||
Err(e) => error_response(&format!("operator-inbox failed: {e:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct StatsHiveQuery {
|
||||
window: Option<String>,
|
||||
}
|
||||
|
||||
/// Hive-wide turn-stats rollup for the dashboard swarm-stats view.
|
||||
/// Aggregates every agent's `hyperhive-turn-stats.sqlite` read-only
|
||||
/// (skips missing/unreadable ones). Window defaults to `24h`.
|
||||
pub(super) async fn api_stats_hive(
|
||||
State(state): State<AppState>,
|
||||
axum::extract::Query(q): axum::extract::Query<StatsHiveQuery>,
|
||||
) -> Response {
|
||||
let window = crate::hive_stats::Window::parse(q.window.as_deref().unwrap_or("24h"));
|
||||
axum::Json(crate::hive_stats::hive_snapshot(
|
||||
window,
|
||||
&state.coord.model_prices,
|
||||
))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// Live per-agent-container CPU + memory load from cgroup v2. Samples
|
||||
/// CPU over a short interval (~200 ms), so this call briefly awaits.
|
||||
pub(super) async fn api_container_resources() -> Response {
|
||||
axum::Json(crate::container_stats::gather().await).into_response()
|
||||
}
|
||||
|
||||
/// `GET /api/audit-log` — most-recent agent-initiated privileged-action
|
||||
/// audit entries, newest first (server-clamped to 500). Backs the
|
||||
/// operator dashboard's audit view. Returns
|
||||
/// `{ "entries": [AuditEntry…], "total": N }` so the UI can show
|
||||
/// "latest 500 of N" rather than silently capping. `ts_unix` is in
|
||||
/// **seconds**.
|
||||
pub(super) async fn api_audit_log(State(state): State<AppState>) -> Response {
|
||||
const LIMIT: usize = 500;
|
||||
let entries = match state.coord.audit_log.list_recent(LIMIT) {
|
||||
Ok(rows) => rows,
|
||||
Err(e) => return error_response(&format!("audit-log: {e:#}")),
|
||||
};
|
||||
let total = match state.coord.audit_log.count_total() {
|
||||
Ok(n) => n,
|
||||
Err(e) => return error_response(&format!("audit-log count: {e:#}")),
|
||||
};
|
||||
axum::Json(serde_json::json!({ "entries": entries, "total": total })).into_response()
|
||||
}
|
||||
|
||||
/// Operator-driven "clear this agent's inbox" — backs the side-panel
|
||||
/// "mark all read" button. Marks every message addressed to the
|
||||
/// agent as acked (backfilling `delivered_at` for any still-pending
|
||||
/// rows so vacuum can collect them). Returns `{ "marked": N }` so the
|
||||
/// frontend can show "cleared N messages" feedback without an extra
|
||||
/// fetch.
|
||||
pub(super) async fn post_mark_all_read(
|
||||
State(state): State<AppState>,
|
||||
AxumPath(name): AxumPath<String>,
|
||||
) -> Response {
|
||||
if let Some(reason) = validate_agent_name(&name) {
|
||||
return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response();
|
||||
}
|
||||
match state.coord.broker.mark_all_read(&name) {
|
||||
Ok(n) => {
|
||||
tracing::info!(%name, marked = n, "operator marked all messages read");
|
||||
axum::Json(serde_json::json!({ "marked": n })).into_response()
|
||||
}
|
||||
Err(e) => error_response(&format!("mark-all-read {name} failed: {e:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Operator-side compose form on the dashboard terminal. Drops a
|
||||
/// message into the broker as `{from: "operator", to, body}`. Same
|
||||
/// shape that per-agent web UIs use via `OperatorMsg`, but here the
|
||||
/// operator picks the recipient explicitly with `@name`. No
|
||||
/// validation that `to` resolves to a known agent — broker accepts
|
||||
/// arbitrary recipients (and the agent's inbox grows whether or not
|
||||
/// they exist, which is fine for spawn-then-greet flows).
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct OpSendForm {
|
||||
to: String,
|
||||
body: String,
|
||||
}
|
||||
|
||||
pub(super) async fn post_op_send(
|
||||
State(state): State<AppState>,
|
||||
Form(form): Form<OpSendForm>,
|
||||
) -> Response {
|
||||
let to = form.to.trim().to_owned();
|
||||
let body = form.body.trim().to_owned();
|
||||
if to.is_empty() {
|
||||
return error_response("op-send: `to` required");
|
||||
}
|
||||
if body.is_empty() {
|
||||
return error_response("op-send: `body` required");
|
||||
}
|
||||
if to == "*" {
|
||||
let errors = state
|
||||
.coord
|
||||
.broadcast_send(hive_sh4re::OPERATOR_RECIPIENT, &body);
|
||||
if !errors.is_empty() {
|
||||
return error_response(&format!(
|
||||
"op-send broadcast partial fail: {}",
|
||||
errors.join("; ")
|
||||
));
|
||||
}
|
||||
} else if let Err(e) = state.coord.broker.send(&hive_sh4re::Message {
|
||||
from: hive_sh4re::OPERATOR_RECIPIENT.to_owned(),
|
||||
to: to.clone(),
|
||||
body,
|
||||
in_reply_to: None,
|
||||
}) {
|
||||
return error_response(&format!("op-send to {to} failed: {e:#}"));
|
||||
}
|
||||
// 200 instead of 303 → the client doesn't refetch /api/state. The
|
||||
// broker `send` already emitted a `MessageEvent` which the
|
||||
// dashboard channel forwarder mirrors as `DashboardEvent::Sent`,
|
||||
// and the page's terminal + inbox derive from that stream — so the
|
||||
// operator's send shows up the same way an agent's send does, with
|
||||
// no full-state refresh in between.
|
||||
(axum::http::StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct RequestSpawnForm {
|
||||
name: String,
|
||||
}
|
||||
|
||||
pub(super) async fn post_request_spawn(
|
||||
State(state): State<AppState>,
|
||||
Form(form): Form<RequestSpawnForm>,
|
||||
) -> Response {
|
||||
let name = form.name.trim().to_owned();
|
||||
if name.is_empty() {
|
||||
return error_response("spawn: `name` required");
|
||||
}
|
||||
match state.coord.approvals.submit_kind(
|
||||
&name,
|
||||
hive_sh4re::ApprovalKind::Spawn,
|
||||
"",
|
||||
None,
|
||||
"operator",
|
||||
) {
|
||||
Ok(id) => {
|
||||
tracing::info!(%id, %name, "operator: spawn approval queued via dashboard");
|
||||
// Phase 5b: notify the dashboard event channel so live
|
||||
// subscribers can append the row without a snapshot
|
||||
// refetch. Spawn approvals carry no diff/sha.
|
||||
state
|
||||
.coord
|
||||
.emit_approval_added(crate::coordinator::ApprovalAdded {
|
||||
id,
|
||||
agent: &name,
|
||||
approval_kind: "spawn",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description: None,
|
||||
pr_number: None,
|
||||
});
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
Err(e) => error_response(&format!("request-spawn {name} failed: {e:#}")),
|
||||
}
|
||||
}
|
||||
418
hive-c0re/src/dashboard/mod.rs
Normal file
418
hive-c0re/src/dashboard/mod.rs
Normal file
|
|
@ -0,0 +1,418 @@
|
|||
//! Hyperhive dashboard. Lists managed containers (with deep-links to each
|
||||
//! container's web UI), pending approvals (with unified diff vs the applied
|
||||
//! repo, plus approve/deny buttons), and the manager.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use axum::{
|
||||
Router,
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
routing::{get, post},
|
||||
};
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle;
|
||||
|
||||
mod approvals;
|
||||
mod build_logs;
|
||||
mod journal;
|
||||
mod lifecycle_ops;
|
||||
mod matrix_accounts;
|
||||
mod meta_inputs;
|
||||
mod misc_api;
|
||||
pub(crate) mod permissions;
|
||||
mod questions;
|
||||
mod reminders;
|
||||
mod schedules;
|
||||
mod state_files;
|
||||
mod state_snapshot;
|
||||
mod tombstones;
|
||||
mod topology;
|
||||
mod webhook;
|
||||
|
||||
// Pre-computed at approval-submit time by the manager-socket handler
|
||||
// (`socket_server.rs`) and embedded in the `ApprovalAdded` event, so
|
||||
// re-exported at the module root to preserve the `crate::dashboard::approval_diff`
|
||||
// path across the submodule split.
|
||||
pub(crate) use approvals::approval_diff;
|
||||
// Run after lock bumps by the job queue (`job_queue/exec.rs`); the view
|
||||
// type feeds `DashboardEvent::MetaInputsChanged` (`dashboard_events.rs`).
|
||||
// Re-exported to preserve the `crate::dashboard::*` paths across the split.
|
||||
pub use meta_inputs::MetaInputView;
|
||||
pub(crate) use meta_inputs::emit_meta_inputs_snapshot;
|
||||
// Run at broker-message ingest by the coordinator + the operator-msg path
|
||||
// (`main.rs`); re-exported to preserve the `crate::dashboard::scan_validated_paths`
|
||||
// path across the split.
|
||||
pub use state_files::scan_validated_paths;
|
||||
// Called after destroy/purge/spawn finalisation (`actions.rs`); the view
|
||||
// type feeds `DashboardEvent::TombstonesChanged` (`dashboard_events.rs`).
|
||||
// Re-exported to preserve the `crate::dashboard::*` paths across the split.
|
||||
pub use tombstones::TombstoneView;
|
||||
pub(crate) use tombstones::emit_tombstones_snapshot;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AppState {
|
||||
coord: Arc<Coordinator>,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::too_many_lines,
|
||||
reason = "the body is dominated by the flat axum route table — one line \
|
||||
per endpoint mapping a URL to its (now per-concern submodule) \
|
||||
handler; splitting that exhaustive list across helpers would \
|
||||
obscure the route map for no readability gain"
|
||||
)]
|
||||
pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
|
||||
// API-only: the gateway static-serves the dashboard dist and proxies
|
||||
// non-static requests here (see hive-gateway.nix). Unmatched paths 404.
|
||||
let app = Router::new()
|
||||
.route("/api/state", get(state_snapshot::api_state))
|
||||
.route("/api/journal/{name}", get(journal::get_journal))
|
||||
.route("/api/journal-host", get(journal::get_journal_host))
|
||||
.route("/api/approval-diff/{id}", get(approvals::get_approval_diff))
|
||||
.route("/api/state-file", get(state_files::get_state_file))
|
||||
.route(
|
||||
"/api/matrix-accounts",
|
||||
get(matrix_accounts::get_matrix_accounts),
|
||||
)
|
||||
.route("/api/reminders", get(reminders::api_reminders))
|
||||
.route("/api/operator-inbox", get(misc_api::api_operator_inbox))
|
||||
.route("/api/stats-hive", get(misc_api::api_stats_hive))
|
||||
.route(
|
||||
"/api/container-resources",
|
||||
get(misc_api::api_container_resources),
|
||||
)
|
||||
.route("/api/audit-log", get(misc_api::api_audit_log))
|
||||
.route("/api/build-logs", get(build_logs::get_build_logs_all))
|
||||
.route(
|
||||
"/api/build-logs/{agent}",
|
||||
get(build_logs::get_build_logs_agent),
|
||||
)
|
||||
.route(
|
||||
"/api/build-logs/id/{id}",
|
||||
get(build_logs::get_build_log_full),
|
||||
)
|
||||
.route(
|
||||
"/api/build-logs/id/{id}/stream",
|
||||
get(build_logs::get_build_log_stream),
|
||||
)
|
||||
.route(
|
||||
"/api/build-logs/id/{id}/raw",
|
||||
get(build_logs::get_build_log_raw),
|
||||
)
|
||||
.route(
|
||||
"/api/agent/{name}/mark-all-read",
|
||||
post(misc_api::post_mark_all_read),
|
||||
)
|
||||
.route("/api/topology/set-parent", post(topology::post_set_parent))
|
||||
.route(
|
||||
"/api/topology/set-parent-bulk",
|
||||
post(topology::post_set_parent_bulk),
|
||||
)
|
||||
.route("/api/tool-groups", get(permissions::get_tool_groups))
|
||||
.route(
|
||||
"/api/tool-groups/{agent}",
|
||||
post(permissions::post_tool_groups),
|
||||
)
|
||||
.route("/api/capabilities", get(permissions::get_capabilities))
|
||||
.route(
|
||||
"/api/capabilities/{agent}",
|
||||
post(permissions::post_capabilities),
|
||||
)
|
||||
.route("/api/permissions", post(permissions::post_permissions))
|
||||
.route(
|
||||
"/api/permissions/stale",
|
||||
get(permissions::get_stale_permissions),
|
||||
)
|
||||
.route(
|
||||
"/api/permissions/{agent}",
|
||||
axum::routing::delete(permissions::delete_agent_permissions),
|
||||
)
|
||||
.route(
|
||||
"/api/schedules",
|
||||
get(schedules::api_schedules).post(schedules::post_schedule_new),
|
||||
)
|
||||
.route(
|
||||
"/api/schedules/{id}",
|
||||
axum::routing::patch(schedules::patch_schedule),
|
||||
)
|
||||
.route(
|
||||
"/api/schedules/{id}/cancel",
|
||||
post(schedules::post_schedule_cancel),
|
||||
)
|
||||
.route(
|
||||
"/api/schedules/{id}/pause",
|
||||
post(schedules::post_schedule_pause),
|
||||
)
|
||||
.route(
|
||||
"/api/schedules/{id}/resume",
|
||||
post(schedules::post_schedule_resume),
|
||||
)
|
||||
.route(
|
||||
"/api/schedules/{id}/fire-now",
|
||||
post(schedules::post_schedule_fire_now),
|
||||
)
|
||||
.route(
|
||||
"/api/rebuild-queue/{id}/cancel",
|
||||
post(schedules::post_rebuild_queue_cancel),
|
||||
)
|
||||
.route("/webhook/knowledge", post(webhook::post_webhook_knowledge))
|
||||
// Backend routes — the frontend calls these `/api/` paths. The
|
||||
// transitional bare top-level aliases were removed once the
|
||||
// frontend migrated. `/webhook/knowledge` keeps its own prefix
|
||||
// (forge-driven, not the SPA).
|
||||
.route("/api/approve/{id}", post(approvals::post_approve))
|
||||
.route("/api/deny/{id}", post(approvals::post_deny))
|
||||
.route("/api/destroy/{name}", post(lifecycle_ops::post_destroy))
|
||||
.route("/api/kill/{name}", post(lifecycle_ops::post_kill))
|
||||
.route("/api/restart/{name}", post(lifecycle_ops::post_restart))
|
||||
.route("/api/start/{name}", post(lifecycle_ops::post_start))
|
||||
.route("/api/rebuild/{name}", post(lifecycle_ops::post_rebuild))
|
||||
.route("/api/update-all", post(lifecycle_ops::post_update_all))
|
||||
.route(
|
||||
"/api/answer-question/{id}",
|
||||
post(questions::post_answer_question),
|
||||
)
|
||||
.route(
|
||||
"/api/cancel-question/{id}",
|
||||
post(questions::post_cancel_question),
|
||||
)
|
||||
.route(
|
||||
"/api/purge-tombstone/{name}",
|
||||
post(tombstones::post_purge_tombstone),
|
||||
)
|
||||
.route(
|
||||
"/api/matrix-account-login",
|
||||
post(matrix_accounts::post_matrix_account_login),
|
||||
)
|
||||
.route(
|
||||
"/api/cancel-reminder/{id}",
|
||||
post(reminders::post_cancel_reminder),
|
||||
)
|
||||
.route(
|
||||
"/api/retry-reminder/{id}",
|
||||
post(reminders::post_retry_reminder),
|
||||
)
|
||||
.route("/api/request-spawn", post(misc_api::post_request_spawn))
|
||||
.route("/api/op-send", post(misc_api::post_op_send))
|
||||
.route("/api/meta-update", post(meta_inputs::post_meta_update))
|
||||
.route(
|
||||
"/api/dashboard/stream",
|
||||
get(state_snapshot::dashboard_stream),
|
||||
)
|
||||
.route(
|
||||
"/api/dashboard/history",
|
||||
get(state_snapshot::dashboard_history),
|
||||
)
|
||||
// No static fallback — the gateway owns the dist; unmatched paths 404.
|
||||
.with_state(AppState { coord });
|
||||
// Binds loopback-only; external access via gateway.
|
||||
// Rationale: docs/gateway.md::Firewall posture.
|
||||
let addr = SocketAddr::from(([127, 0, 0, 1], port));
|
||||
let listener = bind_with_retry(addr).await?;
|
||||
tracing::info!(%addr, "dashboard listening");
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// SPA shape + SSE channels: docs/web-ui/shape.md.
|
||||
|
||||
/// `SO_REUSEADDR` bind with retry. Retry mechanics, attempt-cap
|
||||
/// rationale, and log-level cadence: `docs/web-ui/shape.md::Listener bind`.
|
||||
async fn bind_with_retry(addr: SocketAddr) -> Result<tokio::net::TcpListener> {
|
||||
let mut delay_ms = 250u64;
|
||||
let mut attempts = 0u32;
|
||||
loop {
|
||||
match try_bind(addr) {
|
||||
Ok(l) => {
|
||||
if attempts > 0 {
|
||||
tracing::info!(
|
||||
%addr, attempts,
|
||||
"dashboard: bind succeeded after retry"
|
||||
);
|
||||
}
|
||||
return Ok(l);
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
|
||||
let attempt = attempts + 1;
|
||||
if attempt <= 12 {
|
||||
tracing::warn!(
|
||||
%addr, attempt,
|
||||
"dashboard: AddrInUse, retrying in {delay_ms}ms"
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
%addr, attempt,
|
||||
"dashboard: AddrInUse still holding, retrying in {delay_ms}ms"
|
||||
);
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
attempts += 1;
|
||||
delay_ms = (delay_ms * 2).min(2000);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e).with_context(|| format!("bind dashboard on {addr}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_bind(addr: SocketAddr) -> std::io::Result<tokio::net::TcpListener> {
|
||||
let sock = match addr {
|
||||
SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4()?,
|
||||
SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6()?,
|
||||
};
|
||||
sock.set_reuseaddr(true)?;
|
||||
sock.bind(addr)?;
|
||||
sock.listen(1024)
|
||||
}
|
||||
|
||||
/// Validate that a path-param agent name conforms to the hyperhive
|
||||
/// naming whitelist: 1-63 chars of `[a-z0-9_-]`. Rejects empty,
|
||||
/// uppercase, slashes, dots, and any non-ASCII (incl. unicode
|
||||
/// homoglyphs of dash/underscore). Returns `None` on accept, `Some(reason)`
|
||||
/// on reject — caller wraps the reason in a 400 response. Conservative
|
||||
/// whitelist matching `nixos-container` basename rules and the existing
|
||||
/// agent-name convention across the codebase.
|
||||
pub(crate) fn validate_agent_name(name: &str) -> Option<&'static str> {
|
||||
if name.is_empty() {
|
||||
return Some("agent name must not be empty");
|
||||
}
|
||||
if name.len() > 63 {
|
||||
return Some("agent name must be 63 characters or fewer");
|
||||
}
|
||||
if !name
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_')
|
||||
{
|
||||
return Some("agent name must contain only [a-z0-9_-]");
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Two-axis path-param guard for write routes. Combines:
|
||||
///
|
||||
/// 1. **format validation** (`validate_agent_name`) — rejects path
|
||||
/// traversal / unicode homoglyphs / empty + too-long names with
|
||||
/// HTTP 400.
|
||||
/// 2. **existence check** — looks up `name` in the coordinator's
|
||||
/// container snapshot; unknown name → HTTP 404 with a clear
|
||||
/// "no such agent" message. catches the operator-typo case where
|
||||
/// a destructive POST would otherwise hit silently (mark-all-read
|
||||
/// returning 0) or hit downstream lifecycle code that fails with
|
||||
/// a confusing nspawn error.
|
||||
///
|
||||
/// Returns `None` when both checks pass (caller proceeds), `Some(Response)`
|
||||
/// when the request should be rejected. Use at the top of every write
|
||||
/// handler taking a name path-param. Read-only GET handlers and
|
||||
/// handlers that legitimately operate on tombstoned agents (e.g.
|
||||
/// `mark-all-read` on broker rows for a destroyed agent) call
|
||||
/// `validate_agent_name` directly and skip the existence check.
|
||||
async fn guard_agent_name(state: &AppState, name: &str) -> Option<Response> {
|
||||
if let Some(reason) = validate_agent_name(name) {
|
||||
return Some(
|
||||
(StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(),
|
||||
);
|
||||
}
|
||||
let snapshot = state.coord.containers_snapshot().await;
|
||||
if !snapshot.iter().any(|c| c.name == name) {
|
||||
return Some((StatusCode::NOT_FOUND, format!("no such agent: {name}")).into_response());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Convert either a logical name or a container name back to the logical
|
||||
/// name. Sub-agents are `h-foo` → `foo`; manager stays `root`.
|
||||
fn strip_container_prefix(name: &str) -> String {
|
||||
name.strip_prefix(lifecycle::AGENT_PREFIX)
|
||||
.unwrap_or(name)
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
/// The common internal-error case as a `ProblemDetails`: a 500 RFC 9457
|
||||
/// (`application/problem+json`) value via the `problem_details` crate.
|
||||
/// `from_status_code` sets `status` + `title` (the canonical reason phrase)
|
||||
/// and leaves `type` as the default `about:blank`; `with_detail` carries the
|
||||
/// caller message; the crate's axum `IntoResponse` emits the
|
||||
/// `application/problem+json` body the frontend parses (it reads `detail`).
|
||||
/// Handlers that surface client failures return `Result<_, ProblemDetails>`
|
||||
/// and hand this (or an inline `from_status_code(4xx)`) straight to `Err` —
|
||||
/// no manual `.into_response()`.
|
||||
fn error_problem(message: &str) -> problem_details::ProblemDetails {
|
||||
problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.with_detail(message)
|
||||
}
|
||||
|
||||
/// `Response` wrapper around [`error_problem`] for the many handlers typed
|
||||
/// `-> Response` whose only failure mode is a 500 — they funnel errors
|
||||
/// through here rather than threading a `Result` return type.
|
||||
fn error_response(message: &str) -> Response {
|
||||
error_problem(message).into_response()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn problem_details_carry_rfc9457_status_and_detail() {
|
||||
// Contract the frontend depends on: the problem_details crate
|
||||
// serialises the RFC 9457 members we rely on — `status` (numeric)
|
||||
// and `detail` (the caller message; the FE reads `.detail`).
|
||||
let pd = problem_details::ProblemDetails::from_status_code(StatusCode::BAD_REQUEST)
|
||||
.with_detail("bad input");
|
||||
let v = serde_json::to_value(&pd).expect("problem details serialise");
|
||||
assert_eq!(v["status"], 400);
|
||||
assert_eq!(v["detail"], "bad input");
|
||||
// The 500 wrapper path carries the internal-error status.
|
||||
let five =
|
||||
problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.with_detail("boom");
|
||||
let fv = serde_json::to_value(&five).expect("problem details serialise");
|
||||
assert_eq!(fv["status"], 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_agent_name_accepts_canonical_shapes() {
|
||||
assert!(validate_agent_name("damocles").is_none());
|
||||
assert!(validate_agent_name("hm1nd").is_none());
|
||||
assert!(validate_agent_name("agent-with-dashes").is_none());
|
||||
assert!(validate_agent_name("snake_case").is_none());
|
||||
assert!(validate_agent_name("mixed_2-3").is_none());
|
||||
let max = "a".repeat(63);
|
||||
assert!(
|
||||
validate_agent_name(&max).is_none(),
|
||||
"63-char name should pass"
|
||||
);
|
||||
}
|
||||
|
||||
// The two-axis guard (`guard_agent_name`) wires `validate_agent_name`
|
||||
// + an async coordinator lookup. The lookup needs a populated
|
||||
// `Coordinator`, which needs sqlite + tokio runtime; rather than
|
||||
// build that scaffolding for an integration-flavoured test we cover
|
||||
// the format axis here (the existence axis is enforced by the
|
||||
// shared `containers_snapshot` API, tested in `coordinator.rs`'s
|
||||
// own suite). 9 cases below cover the boundary-length case and
|
||||
// other expected rejects to make the contract explicit.
|
||||
#[test]
|
||||
fn validate_agent_name_rejects_bad_input() {
|
||||
assert!(validate_agent_name("").is_some());
|
||||
let too_long = "a".repeat(64);
|
||||
assert!(validate_agent_name(&too_long).is_some());
|
||||
// Path-traversal attempts.
|
||||
assert!(validate_agent_name("../etc/passwd").is_some());
|
||||
assert!(validate_agent_name("alice/bob").is_some());
|
||||
// Uppercase rejected — canonical lowercase convention.
|
||||
assert!(validate_agent_name("Alice").is_some());
|
||||
// No spaces, dots, special chars.
|
||||
assert!(validate_agent_name("alice bob").is_some());
|
||||
assert!(validate_agent_name("alice.bob").is_some());
|
||||
assert!(validate_agent_name("alice;DROP TABLE messages").is_some());
|
||||
// Non-ASCII (incl. unicode homoglyphs of ASCII dash).
|
||||
assert!(validate_agent_name("damóclès").is_some());
|
||||
assert!(validate_agent_name("alice\u{2013}bob").is_some()); // en-dash
|
||||
}
|
||||
}
|
||||
|
|
@ -128,18 +128,19 @@ pub(super) async fn post_tool_groups(
|
|||
return Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST)
|
||||
.with_detail(format!("invalid tool-groups for {logical}: {e}")));
|
||||
}
|
||||
// Enqueue a PermChange so the JSON file write is serialised through
|
||||
// the FIFO worker. Prevents concurrent batch-apply actions for
|
||||
// different agents from racing on the shared tool-groups.json.
|
||||
state.coord.rebuild_queue.enqueue_with_perm(
|
||||
logical.clone(),
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
// Submit a PermChange DAG: the JSON file write commits under
|
||||
// META_LOCK inside the WritePermFile node, so concurrent
|
||||
// batch-apply actions for different agents never race on the
|
||||
// shared tool-groups.json.
|
||||
crate::job_queue::submit::perm_change(
|
||||
&state.coord,
|
||||
&logical,
|
||||
crate::job_queue::Source::Manual,
|
||||
"tool-group change via permissions UI".to_owned(),
|
||||
crate::rebuild_queue::PermPayload::ToolGroups {
|
||||
crate::job_queue::PermPayload::ToolGroups {
|
||||
groups: body.groups.clone(),
|
||||
},
|
||||
);
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard");
|
||||
Ok((StatusCode::OK, "ok").into_response())
|
||||
}
|
||||
|
|
@ -214,18 +215,19 @@ pub(super) async fn post_capabilities(
|
|||
.with_detail(format!("unknown capability: {cap}")));
|
||||
}
|
||||
}
|
||||
// Enqueue a PermChange so the JSON file write is serialised through
|
||||
// the FIFO worker. Prevents concurrent batch-apply actions for
|
||||
// different agents from racing on the shared capabilities.json.
|
||||
state.coord.rebuild_queue.enqueue_with_perm(
|
||||
logical.clone(),
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
// Submit a PermChange DAG: the JSON file write commits under
|
||||
// META_LOCK inside the WritePermFile node, so concurrent
|
||||
// batch-apply actions for different agents never race on the
|
||||
// shared capabilities.json.
|
||||
crate::job_queue::submit::perm_change(
|
||||
&state.coord,
|
||||
&logical,
|
||||
crate::job_queue::Source::Manual,
|
||||
"capability change via dashboard".to_owned(),
|
||||
crate::rebuild_queue::PermPayload::Capabilities {
|
||||
crate::job_queue::PermPayload::Capabilities {
|
||||
caps: body.caps.clone(),
|
||||
},
|
||||
);
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard");
|
||||
Ok((StatusCode::OK, "ok").into_response())
|
||||
}
|
||||
|
|
@ -298,17 +300,17 @@ pub(super) async fn post_permissions(
|
|||
));
|
||||
}
|
||||
}
|
||||
// Phase 2 — enqueue one combined PermChange per affected agent.
|
||||
// Phase 2 — submit one combined PermChange DAG per affected agent.
|
||||
for (logical, groups, caps) in staged {
|
||||
state.coord.rebuild_queue.enqueue_with_perm(
|
||||
logical.clone(),
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
crate::job_queue::submit::perm_change(
|
||||
&state.coord,
|
||||
&logical,
|
||||
crate::job_queue::Source::Manual,
|
||||
"batch permission change via permissions UI".to_owned(),
|
||||
crate::rebuild_queue::PermPayload::Combined { groups, caps },
|
||||
crate::job_queue::PermPayload::Combined { groups, caps },
|
||||
);
|
||||
tracing::info!(agent = %logical, "operator: batch perm change via dashboard");
|
||||
}
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
Ok((StatusCode::OK, "ok").into_response())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -115,20 +115,19 @@ pub(super) async fn post_schedule_fire_now(
|
|||
}
|
||||
}
|
||||
|
||||
/// `POST /api/rebuild-queue/{id}/cancel` — drop a `Queued` entry
|
||||
/// from the rebuild queue. Refuses `Running` / terminal
|
||||
/// entries: an in-flight rebuild owns the agent's nix store +
|
||||
/// nixos-container update lock and can't be safely interrupted
|
||||
/// from the queue side. Always returns 200; the body is
|
||||
/// `{"cancelled": true}` on a successful flip from Queued →
|
||||
/// Cancelled, `{"cancelled": false}` when the row was Running /
|
||||
/// terminal / gone. On success a fresh `RebuildQueueChanged`
|
||||
/// snapshot fires so the row's state flip surfaces live.
|
||||
/// `POST /api/rebuild-queue/{id}/cancel` — drop a still-fully-queued
|
||||
/// DAG from the job queue. Refuses `Running` / terminal DAGs: an
|
||||
/// in-flight node owns the agent's nix store + nixos-container update
|
||||
/// lock and can't be safely interrupted from the queue side. Always
|
||||
/// returns 200; the body is `{"cancelled": true}` on a successful
|
||||
/// flip to Cancelled, `{"cancelled": false}` when the DAG was
|
||||
/// Running / terminal / gone. On success a fresh `RebuildQueueChanged`
|
||||
/// snapshot fires so the state flip surfaces live.
|
||||
pub(super) async fn post_rebuild_queue_cancel(
|
||||
State(state): State<AppState>,
|
||||
AxumPath(id): AxumPath<u64>,
|
||||
) -> Response {
|
||||
let cancelled = state.coord.rebuild_queue.cancel(id);
|
||||
let cancelled = state.coord.job_queue.cancel(id);
|
||||
if cancelled {
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
axum::Json(serde_json::json!({"cancelled": true})).into_response()
|
||||
|
|
|
|||
750
hive-c0re/src/dashboard/state_snapshot.rs
Normal file
750
hive-c0re/src/dashboard/state_snapshot.rs
Normal file
|
|
@ -0,0 +1,750 @@
|
|||
//! `/api/state` cold-load snapshot plus the dashboard's live read side:
|
||||
//! the `StateSnapshot` shape and its view builders, the
|
||||
//! `/api/dashboard/stream` SSE channel, and the `/api/dashboard/history`
|
||||
//! backfill. SPA shape + SSE channels: docs/web-ui/shape.md.
|
||||
|
||||
use std::convert::Infallible;
|
||||
|
||||
use axum::{
|
||||
extract::State,
|
||||
http::HeaderMap,
|
||||
response::{
|
||||
IntoResponse, Response,
|
||||
sse::{Event, KeepAlive, Sse},
|
||||
},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use hive_sh4re::Approval;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio_stream::wrappers::BroadcastStream;
|
||||
use tokio_stream::{Stream, StreamExt};
|
||||
|
||||
use crate::container_view::ContainerView;
|
||||
|
||||
use super::meta_inputs::{MetaInputView, read_meta_inputs};
|
||||
use super::tombstones::{TombstoneView, build_tombstone_views};
|
||||
use super::{AppState, approval_diff, approvals, error_response, scan_validated_paths};
|
||||
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct StateSnapshot {
|
||||
/// Broker seq at the moment this snapshot was assembled. Clients
|
||||
/// dedupe their buffered SSE traffic against this value: any
|
||||
/// `MessageEvent` with `seq <= snapshot.seq` is already reflected in
|
||||
/// the snapshot (or pre-dates it); anything with `seq > snapshot.seq`
|
||||
/// is post-snapshot and should be applied. Set to 0 in the
|
||||
/// pre-emit case (no events ever fired) — clients treat that as
|
||||
/// "apply everything you've buffered".
|
||||
seq: u64,
|
||||
hostname: String,
|
||||
any_stale: bool,
|
||||
containers: Vec<ContainerView>,
|
||||
transients: Vec<TransientView>,
|
||||
approvals: Vec<ApprovalView>,
|
||||
/// Last 30 resolved approvals (approved / denied / failed), newest-
|
||||
/// first. Drives the "history" tab on the approvals section.
|
||||
approval_history: Vec<ApprovalHistoryView>,
|
||||
/// Pending operator-targeted questions (`target IS NULL`). Any
|
||||
/// agent can `ask` the operator and `ask` returns immediately with
|
||||
/// the id; on `/answer-question` we mark the row answered and
|
||||
/// fire `HelperEvent::QuestionAnswered` back into the asker's
|
||||
/// inbox. Peer-to-peer questions live in the same table but never
|
||||
/// surface here (see `OperatorQuestions::pending`).
|
||||
questions: Vec<QuestionView>,
|
||||
/// Last 20 answered questions, newest-first.
|
||||
question_history: Vec<QuestionView>,
|
||||
/// State dirs (config history + claude creds + /state/ notes) that
|
||||
/// survive after a destroy-without-purge. The operator can re-spawn
|
||||
/// with the same name to resume, or PURG3 to wipe them.
|
||||
tombstones: Vec<TombstoneView>,
|
||||
/// Sub-agents whose FNV-1a hashed web UI port collides with at
|
||||
/// least one other agent. Operator resolves by renaming. The
|
||||
/// dashboard renders a banner at the top listing each cluster.
|
||||
port_conflicts: Vec<PortConflict>,
|
||||
/// Inputs in `meta/flake.lock` the operator can selectively
|
||||
/// `nix flake update`. Hyperhive first, then `agent-<n>` rows.
|
||||
meta_inputs: Vec<MetaInputView>,
|
||||
/// True while a dashboard-triggered `meta-update` (flake lock bump +
|
||||
/// agent rebuild ripple) is running in the background. Lets a
|
||||
/// client that cold-loads mid-update render the META INPUTS panel's
|
||||
/// disabled "updating…" state; live transitions arrive via the
|
||||
/// `MetaUpdateRunning` event.
|
||||
meta_update_running: bool,
|
||||
/// Current state of the global job queue — pending + running DAGs
|
||||
/// (rebuild / meta-update / spawn / power ops) with their per-node
|
||||
/// breakdowns, plus the most recent few terminal DAGs the queue
|
||||
/// retains for history. Live transitions arrive via the
|
||||
/// `RebuildQueueChanged` event. See `job_queue/`. Field name kept
|
||||
/// from the old flat queue for wire compatibility.
|
||||
rebuild_queue: Vec<crate::job_queue::DagView>,
|
||||
/// Whether the hive-forge container is up. When true the dashboard
|
||||
/// links each container's config + each approval's commit into the
|
||||
/// forge's `agent-configs` repos.
|
||||
forge_present: bool,
|
||||
/// Whether the matrix GUI is reachable at `/matrix/`. Sourced from
|
||||
/// `HIVE_MATRIX_GUI_ENABLED` env var (set by the c0re NixOS module
|
||||
/// when `services.hyperhive.matrix.gui.enable` is on). The gateway
|
||||
/// (hive-gateway.nix) does the actual `/matrix/` static serving;
|
||||
/// this flag is just an availability signal for iris's dashboard
|
||||
/// chrome so the `M4TR1X →` tab doesn't flash when the GUI is off.
|
||||
matrix_gui_enabled: bool,
|
||||
/// Whether `hive-gateway` is in front of this dashboard. Sourced
|
||||
/// from the `HIVE_GATEWAY_ENABLED` env var, which the c0re NixOS
|
||||
/// module now always sets (the gateway runs unconditionally
|
||||
/// alongside hyperhive), so this is effectively always true: the
|
||||
/// dashboard frontend builds same-origin `/agent/<name>/` links to
|
||||
/// the per-agent web UI (the gateway routes them via the
|
||||
/// runtime-generated `agents.conf` include file — see
|
||||
/// `gateway_nginx.rs`). The `false` branch (direct
|
||||
/// `http://<hostname>:<port>/` TCP links) is retained as a defensive
|
||||
/// fallback for the env being unset. See `docs/gateway.md::Vhost map`.
|
||||
gateway_enabled: bool,
|
||||
/// Public URL of the forge vhost served by hive-gateway (e.g.
|
||||
/// `"https://forge.pr1ma.darkest.space"`). Sourced from the
|
||||
/// `HIVE_FORGE_PUBLIC_URL` env var, which the c0re NixOS module
|
||||
/// sets when `forge.behindGateway = true`. `None` when absent —
|
||||
/// the frontend falls back to `http://<hostname>:3000`.
|
||||
forge_public_url: Option<String>,
|
||||
/// Human name of this single-host hive instance (e.g. `"pr1ma"`).
|
||||
/// Sourced from `HYPERHIVE_HIVE_NAME` env var, set by the c0re
|
||||
/// NixOS module from `services.hyperhive.hiveName`. `None` when
|
||||
/// the option is unset — chrome falls back to `hostname`.
|
||||
hive_name: Option<String>,
|
||||
/// Human name of the wider swarm this hive belongs to (e.g.
|
||||
/// `"constellat1on"`). Sourced from `HYPERHIVE_SWARM_NAME` env
|
||||
/// var, set from `services.hyperhive.swarmName`. `None` when
|
||||
/// unset — chrome omits the swarm segment of the breadcrumb.
|
||||
swarm_name: Option<String>,
|
||||
/// Peer hives in the same swarm. Parsed from `HYPERHIVE_PEERS`
|
||||
/// (JSON array of `{domain,cert_fingerprint}` objects, emitted by
|
||||
/// the c0re NixOS module from `services.hyperhive.swarm.peers`).
|
||||
/// Empty on single-hive deploys. Feeds the P33RS dashboard tab.
|
||||
peer_hives: Vec<PeerHiveView>,
|
||||
/// Server-level warnings for the dashboard's top-of-page banner
|
||||
/// (currently host disk-pressure; more producers can be added
|
||||
/// backend-side). Empty when all clear. Built by
|
||||
/// `host_stats::server_warnings`; the frontend renders this list
|
||||
/// generically, so new warning kinds need no frontend change.
|
||||
server_warnings: Vec<crate::host_stats::ServerWarning>,
|
||||
}
|
||||
|
||||
/// One peer hive for the P33RS dashboard tab. Derived from
|
||||
/// `HYPERHIVE_PEERS` env; `url` is the peer's HTTPS dashboard root.
|
||||
/// `cert_fingerprint` is `Some("sha256:<hex64>")` when the peer uses a
|
||||
/// self-signed cert and the operator pinned its fingerprint in
|
||||
/// `services.hyperhive.swarm.peers`.
|
||||
#[derive(Serialize)]
|
||||
struct PeerHiveView {
|
||||
name: String,
|
||||
url: String,
|
||||
cert_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
/// `OpQuestion` + computed `question_refs` / `answer_refs`. Built
|
||||
/// from the snapshot read; the live channel attaches the same
|
||||
/// fields directly on `QuestionAdded` / `QuestionResolved`.
|
||||
#[derive(Serialize)]
|
||||
struct QuestionView {
|
||||
#[serde(flatten)]
|
||||
inner: crate::operator_questions::OpQuestion,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
question_refs: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
answer_refs: Vec<String>,
|
||||
}
|
||||
|
||||
impl QuestionView {
|
||||
fn from_question(q: crate::operator_questions::OpQuestion) -> Self {
|
||||
let question_refs = scan_validated_paths(&q.question);
|
||||
let answer_refs = q
|
||||
.answer
|
||||
.as_deref()
|
||||
.map(scan_validated_paths)
|
||||
.unwrap_or_default();
|
||||
Self {
|
||||
inner: q,
|
||||
question_refs,
|
||||
answer_refs,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PortConflict {
|
||||
port: u16,
|
||||
/// All agent names sharing this port (sorted, ≥2 entries).
|
||||
agents: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct TransientView {
|
||||
name: String,
|
||||
kind: &'static str,
|
||||
secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ApprovalHistoryView {
|
||||
id: i64,
|
||||
agent: String,
|
||||
kind: &'static str,
|
||||
/// First 12 chars of the canonical sha (preferred) or
|
||||
/// manager-supplied ref. None for resolved spawn approvals.
|
||||
sha_short: Option<String>,
|
||||
/// `approved` / `denied` / `failed`.
|
||||
status: &'static str,
|
||||
/// RFC 3339 UTC. Renders as a relative time on the dashboard.
|
||||
resolved_at: DateTime<Utc>,
|
||||
/// Operator-supplied deny reason (for `denied`) or build error
|
||||
/// (for `failed`). None on `approved`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
note: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ApprovalView {
|
||||
id: i64,
|
||||
agent: String,
|
||||
kind: &'static str,
|
||||
/// First 12 chars of the `commit_ref`, for `ApplyCommit` only.
|
||||
sha_short: Option<String>,
|
||||
/// Raw unified diff text, for `ApplyCommit` only. The client splits
|
||||
/// on `\n` and per-line classifies (`+` / `-` / `@@` / `--- ` / `+++ `
|
||||
/// → diff-add / diff-del / diff-hunk / diff-file). Shipping raw
|
||||
/// instead of pre-rendered HTML saves bytes on the wire (no
|
||||
/// per-line `<span>` markup) and removes the only HTML-escape
|
||||
/// surface from the snapshot.
|
||||
diff: Option<String>,
|
||||
/// Manager-supplied description shown on the approval card.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
description: Option<String>,
|
||||
/// Forge PR number, for `MergeConfigPr` only. Lets the frontend
|
||||
/// build a "review PR on forge" link
|
||||
/// (`{forgeBase}/agent-configs/{agent}/pulls/{pr_number}`) the same
|
||||
/// way it builds the `apply_commit` "commit on forge" link from the
|
||||
/// sha. `None` for every other kind.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pr_number: Option<u64>,
|
||||
/// Raw `commit_ref` payload for `UpdateMetaInputs` (JSON-encoded
|
||||
/// `Vec<String>` of input names; `"[]"` = all inputs) and
|
||||
/// `SchedulePrompt` (JSON-encoded `SchedulePromptPayload`). The
|
||||
/// frontend parses this to render a human-readable card body.
|
||||
/// `None` for every other kind.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
commit_ref: Option<String>,
|
||||
/// RFC 3339 UTC time the approval was queued. Rendered as a
|
||||
/// relative time on the card so the operator can spot a stale
|
||||
/// request.
|
||||
requested_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Replace silent `.unwrap_or_default()` on the data sources behind
|
||||
/// `/api/state` so that whichever query degrades surfaces in journald
|
||||
/// instead of leaving the operator staring at an empty list. The
|
||||
/// dashboard still degrades to a sensible default value; the warn
|
||||
/// is just the diagnostic breadcrumb the old code swallowed.
|
||||
fn log_default<T, E>(what: &str, result: std::result::Result<T, E>) -> T
|
||||
where
|
||||
T: Default,
|
||||
E: std::fmt::Debug,
|
||||
{
|
||||
match result {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(target: "api_state", source = %what, error = ?e, "snapshot source failed; using default");
|
||||
T::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Window over which container crashes count toward the `agents_crashing`
|
||||
/// banner warning. Wide enough that a crash-looping container (restarted
|
||||
/// by `Restart=on-failure` every few seconds) keeps the warning lit
|
||||
/// between flaps, short enough that a single recovered crash clears within
|
||||
/// minutes.
|
||||
const CRASH_WARNING_WINDOW: std::time::Duration = std::time::Duration::from_mins(10);
|
||||
|
||||
pub(super) async fn api_state(
|
||||
headers: HeaderMap,
|
||||
State(state): State<AppState>,
|
||||
) -> axum::Json<StateSnapshot> {
|
||||
let host = headers
|
||||
.get("host")
|
||||
.and_then(|h| h.to_str().ok())
|
||||
.unwrap_or("localhost");
|
||||
let hostname = host.split(':').next().unwrap_or(host).to_owned();
|
||||
|
||||
// Capture the unified dashboard-channel seq *before* any read so the
|
||||
// dedupe contract is "events with seq > snapshot.seq are
|
||||
// post-snapshot, never missed." An event landing during snapshot
|
||||
// construction may be doubly applied (snapshot caught the write +
|
||||
// client also applies the SSE frame) — that's a renderer's problem
|
||||
// to make idempotent, not ours to avoid here.
|
||||
let seq = state.coord.current_seq();
|
||||
|
||||
// Refresh the coordinator's cached container snapshot before
|
||||
// reading. Cold-load clients then see whatever the latest rescan
|
||||
// produced; live clients converge via the matching
|
||||
// `ContainerStateChanged` / `ContainerRemoved` events the rescan
|
||||
// emits.
|
||||
//
|
||||
// Bound the rescan: it shells out (`nixos-container list` etc.), so a
|
||||
// saturated/wedged build backend — e.g. hive-c0re mid-startup-sweep
|
||||
// hammering slow `nixos-container update` subprocesses — can stall it
|
||||
// long enough that `/api/state` hangs for the whole request (the
|
||||
// ~minute-long /state reported in the field). On timeout we skip the
|
||||
// fresh rescan and serve the last cached snapshot instead; live
|
||||
// clients still converge via the SSE events a later successful rescan
|
||||
// emits, and the next /state call retries the refresh. Introspection
|
||||
// stays responsive regardless of the build backend's health.
|
||||
if tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
state.coord.rescan_containers_and_emit(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
"api_state: container rescan exceeded 3s (build backend likely saturated); \
|
||||
serving last cached snapshot"
|
||||
);
|
||||
}
|
||||
let containers = state.coord.containers_snapshot().await;
|
||||
let any_stale = containers.iter().any(|c| c.needs_update);
|
||||
let transient_snapshot = state.coord.transient_snapshot();
|
||||
let pending_approvals = approvals::gc_orphans(
|
||||
&state.coord,
|
||||
log_default("approvals.pending", state.coord.approvals.pending()),
|
||||
);
|
||||
let transients = build_transient_views(&containers, &transient_snapshot);
|
||||
let approvals = build_approval_views(pending_approvals).await;
|
||||
let approval_history = log_default(
|
||||
"approvals.recent_resolved",
|
||||
state.coord.approvals.recent_resolved(30),
|
||||
)
|
||||
.into_iter()
|
||||
.map(history_view)
|
||||
.collect();
|
||||
let tombstones = build_tombstone_views(&state.coord, &containers, &transient_snapshot);
|
||||
let port_conflicts = build_port_conflicts(&containers);
|
||||
|
||||
// Both operator-targeted and peer threads surface on the dashboard
|
||||
// (the client filters by target). Each row is wrapped in QuestionView
|
||||
// so the snapshot carries the same file_refs the live event variants
|
||||
// attach.
|
||||
let questions: Vec<QuestionView> =
|
||||
log_default("questions.pending_all", state.coord.questions.pending_all())
|
||||
.into_iter()
|
||||
.map(QuestionView::from_question)
|
||||
.collect();
|
||||
let question_history: Vec<QuestionView> = log_default(
|
||||
"questions.recent_answered_all",
|
||||
state.coord.questions.recent_answered_all(20),
|
||||
)
|
||||
.into_iter()
|
||||
.map(QuestionView::from_question)
|
||||
.collect();
|
||||
|
||||
// Banner warnings: host probes (disk) + agent-state (pending logins,
|
||||
// crashing agents). Built before the response struct because the
|
||||
// agent-state producer borrows `containers`, which moves in below.
|
||||
let server_warnings = {
|
||||
let mut w = crate::host_stats::server_warnings();
|
||||
w.extend(crate::host_stats::agent_state_warnings(
|
||||
&containers,
|
||||
&state.coord.recent_crash_counts(CRASH_WARNING_WINDOW),
|
||||
));
|
||||
w
|
||||
};
|
||||
|
||||
axum::Json(StateSnapshot {
|
||||
seq,
|
||||
hostname,
|
||||
any_stale,
|
||||
containers,
|
||||
transients,
|
||||
approvals,
|
||||
approval_history,
|
||||
meta_inputs: read_meta_inputs(),
|
||||
meta_update_running: state.coord.meta_update_in_progress(),
|
||||
questions,
|
||||
question_history,
|
||||
tombstones,
|
||||
port_conflicts,
|
||||
rebuild_queue: state.coord.job_queue.snapshot(),
|
||||
forge_present: crate::forge::is_present().await,
|
||||
matrix_gui_enabled: std::env::var_os("HIVE_MATRIX_GUI_ENABLED").is_some_and(|v| {
|
||||
// Accept any truthy string ("1", "true", "yes") since the
|
||||
// env var is set by NixOS module wiring with the literal
|
||||
// "1"; defensive parse so manual overrides also work.
|
||||
let s = v.to_string_lossy().to_ascii_lowercase();
|
||||
matches!(s.as_str(), "1" | "true" | "yes")
|
||||
}),
|
||||
gateway_enabled: std::env::var_os("HIVE_GATEWAY_ENABLED").is_some_and(|v| {
|
||||
// Same truthy-string parse as `matrix_gui_enabled`; the
|
||||
// env var is set by the c0re NixOS module to the literal
|
||||
// "1" — the gateway always runs alongside hyperhive.
|
||||
let s = v.to_string_lossy().to_ascii_lowercase();
|
||||
matches!(s.as_str(), "1" | "true" | "yes")
|
||||
}),
|
||||
forge_public_url: std::env::var("HIVE_FORGE_PUBLIC_URL")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty()),
|
||||
hive_name: std::env::var("HYPERHIVE_HIVE_NAME")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty()),
|
||||
swarm_name: std::env::var("HYPERHIVE_SWARM_NAME")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty()),
|
||||
peer_hives: parse_peer_hives(),
|
||||
server_warnings,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse `HYPERHIVE_PEERS` env var into dashboard-ready `PeerHiveView`
|
||||
/// entries. The env var is a JSON array of `{domain, cert_fingerprint}`
|
||||
/// objects emitted by the c0re NixOS module from
|
||||
/// `services.hyperhive.swarm.peers`. Each entry becomes
|
||||
/// `{ name: domain, url: "https://domain/" }` for the P33RS tab.
|
||||
/// Returns empty vec when unset (single-hive deploy).
|
||||
fn parse_peer_hives() -> Vec<PeerHiveView> {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Raw {
|
||||
domain: String,
|
||||
cert_fingerprint: Option<String>,
|
||||
}
|
||||
let Ok(json) = std::env::var("HYPERHIVE_PEERS") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(raw): Result<Vec<Raw>, _> = serde_json::from_str(&json) else {
|
||||
tracing::warn!("HYPERHIVE_PEERS is not valid JSON; ignoring");
|
||||
return Vec::new();
|
||||
};
|
||||
raw.into_iter()
|
||||
.map(|r| {
|
||||
let cert_fingerprint = r.cert_fingerprint.and_then(|fp| {
|
||||
if validate_cert_fingerprint(&fp) {
|
||||
Some(fp)
|
||||
} else {
|
||||
tracing::warn!(
|
||||
domain = %r.domain,
|
||||
fingerprint = %fp,
|
||||
"HYPERHIVE_PEERS: invalid cert_fingerprint format \
|
||||
(expected `sha256:<64 hex chars>`); ignoring fingerprint"
|
||||
);
|
||||
None
|
||||
}
|
||||
});
|
||||
PeerHiveView {
|
||||
name: r.domain.clone(),
|
||||
url: format!("https://{}/", r.domain),
|
||||
cert_fingerprint,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Validate a TLS certificate fingerprint string from `HYPERHIVE_PEERS`.
|
||||
/// Accepts `sha256:<64 hex chars>` (upper or lower case).
|
||||
fn validate_cert_fingerprint(fp: &str) -> bool {
|
||||
let Some(hex) = fp.strip_prefix("sha256:") else {
|
||||
return false;
|
||||
};
|
||||
hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
/// Group live containers by their assigned web UI port; clusters with
|
||||
/// more than one member are port-hash collisions the operator needs
|
||||
/// to resolve by renaming. Manager (fixed at 8000) and sub-agents
|
||||
/// (8100..8999) can't collide with each other — collisions are
|
||||
/// strictly between sub-agents.
|
||||
fn build_port_conflicts(containers: &[ContainerView]) -> Vec<PortConflict> {
|
||||
let mut by_port: std::collections::BTreeMap<u16, Vec<String>> =
|
||||
std::collections::BTreeMap::new();
|
||||
for c in containers {
|
||||
by_port.entry(c.port).or_default().push(c.name.clone());
|
||||
}
|
||||
by_port
|
||||
.into_iter()
|
||||
.filter(|(_, agents)| agents.len() > 1)
|
||||
.map(|(port, mut agents)| {
|
||||
agents.sort();
|
||||
PortConflict { port, agents }
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Transient state for agents whose container does NOT yet exist
|
||||
/// (`Spawning`). Lifecycle ops on existing containers surface as
|
||||
/// `ContainerView.pending` inline; this list only catches pre-creation.
|
||||
fn build_transient_views(
|
||||
containers: &[ContainerView],
|
||||
transient_snapshot: &std::collections::HashMap<String, crate::coordinator::TransientState>,
|
||||
) -> Vec<TransientView> {
|
||||
transient_snapshot
|
||||
.iter()
|
||||
.filter(|(name, _)| !containers.iter().any(|c| &c.name == *name))
|
||||
.map(|(name, st)| TransientView {
|
||||
name: name.clone(),
|
||||
kind: transient_label(st.kind),
|
||||
secs: st.since.elapsed().as_secs(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn transient_label(k: crate::coordinator::TransientKind) -> &'static str {
|
||||
use crate::coordinator::TransientKind::{
|
||||
Destroying, Rebuilding, Restarting, Spawning, Starting, Stopping,
|
||||
};
|
||||
match k {
|
||||
Spawning => "spawning",
|
||||
Starting => "starting",
|
||||
Stopping => "stopping",
|
||||
Restarting => "restarting",
|
||||
Rebuilding => "rebuilding",
|
||||
Destroying => "destroying",
|
||||
}
|
||||
}
|
||||
|
||||
/// Render each pending approval into its dashboard view (short sha +
|
||||
/// unified diff for `ApplyCommit`, just the name for `Spawn`).
|
||||
/// Project a resolved sqlite row into the lean shape the dashboard
|
||||
/// history tab consumes — no `diff_html` (rendering 30 of them
|
||||
/// per /api/state poll would mean 30 git diffs per refresh).
|
||||
fn history_view(a: Approval) -> ApprovalHistoryView {
|
||||
let displayed = a.fetched_sha.as_deref().unwrap_or(&a.commit_ref);
|
||||
let sha_short = if displayed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(displayed[..displayed.len().min(12)].to_owned())
|
||||
};
|
||||
let status = match a.status {
|
||||
hive_sh4re::ApprovalStatus::Approved => "approved",
|
||||
hive_sh4re::ApprovalStatus::Denied => "denied",
|
||||
hive_sh4re::ApprovalStatus::Failed => "failed",
|
||||
hive_sh4re::ApprovalStatus::Cancelled => "cancelled",
|
||||
// Pending shouldn't appear in recent_resolved, but be defensive.
|
||||
hive_sh4re::ApprovalStatus::Pending => "pending",
|
||||
};
|
||||
let kind = a.kind.as_str();
|
||||
ApprovalHistoryView {
|
||||
id: a.id,
|
||||
agent: a.agent,
|
||||
kind,
|
||||
sha_short,
|
||||
status,
|
||||
resolved_at: a.resolved_at.unwrap_or_default(),
|
||||
note: a.note,
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_approval_views(approvals: Vec<Approval>) -> Vec<ApprovalView> {
|
||||
let mut out = Vec::with_capacity(approvals.len());
|
||||
for a in approvals {
|
||||
out.push(match a.kind {
|
||||
hive_sh4re::ApprovalKind::ApplyCommit => {
|
||||
// Prefer the canonical fetched sha from applied;
|
||||
// commit_ref is only the manager's claim and may be
|
||||
// amended out from under us.
|
||||
let displayed = a.fetched_sha.as_deref().unwrap_or(&a.commit_ref);
|
||||
let sha = displayed[..displayed.len().min(12)].to_owned();
|
||||
let diff = approval_diff(&a.agent, a.id).await;
|
||||
ApprovalView {
|
||||
id: a.id,
|
||||
agent: a.agent.clone(),
|
||||
kind: "apply_commit",
|
||||
sha_short: Some(sha),
|
||||
diff: Some(diff),
|
||||
description: a.description,
|
||||
pr_number: None,
|
||||
commit_ref: None,
|
||||
requested_at: a.requested_at,
|
||||
}
|
||||
}
|
||||
hive_sh4re::ApprovalKind::Spawn => ApprovalView {
|
||||
id: a.id,
|
||||
agent: a.agent,
|
||||
kind: "spawn",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description: a.description,
|
||||
pr_number: None,
|
||||
commit_ref: None,
|
||||
requested_at: a.requested_at,
|
||||
},
|
||||
hive_sh4re::ApprovalKind::InitConfig => ApprovalView {
|
||||
id: a.id,
|
||||
agent: a.agent,
|
||||
kind: "init_config",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description: a.description,
|
||||
pr_number: None,
|
||||
commit_ref: None,
|
||||
requested_at: a.requested_at,
|
||||
},
|
||||
hive_sh4re::ApprovalKind::UpdateMetaInputs => ApprovalView {
|
||||
id: a.id,
|
||||
agent: a.agent,
|
||||
kind: "update_meta_inputs",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description: a.description,
|
||||
pr_number: None,
|
||||
commit_ref: Some(a.commit_ref),
|
||||
requested_at: a.requested_at,
|
||||
},
|
||||
hive_sh4re::ApprovalKind::SchedulePrompt => ApprovalView {
|
||||
id: a.id,
|
||||
agent: a.agent,
|
||||
kind: "schedule_prompt",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description: a.description,
|
||||
pr_number: None,
|
||||
commit_ref: Some(a.commit_ref),
|
||||
requested_at: a.requested_at,
|
||||
},
|
||||
hive_sh4re::ApprovalKind::MergeConfigPr => {
|
||||
// commit_ref = PR number; fetched_sha = the reviewed PR
|
||||
// head. Show the head sha; the forge PR diff surface is
|
||||
// a later phase of the PR-based config flow — None for now.
|
||||
let sha = a
|
||||
.fetched_sha
|
||||
.as_deref()
|
||||
.map(|s| s[..s.len().min(12)].to_owned());
|
||||
// Surface the PR number so the frontend can link to the
|
||||
// PR on the forge. commit_ref holds the number as text.
|
||||
let pr_number = a.commit_ref.parse::<u64>().ok();
|
||||
ApprovalView {
|
||||
id: a.id,
|
||||
agent: a.agent,
|
||||
kind: "merge_config_pr",
|
||||
sha_short: sha,
|
||||
diff: None,
|
||||
description: a.description,
|
||||
pr_number,
|
||||
commit_ref: None,
|
||||
requested_at: a.requested_at,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(super) async fn dashboard_history(State(state): State<AppState>) -> Response {
|
||||
// Backfill source for the dashboard terminal. Returns up to ~200
|
||||
// historical broker messages (no other event kinds are persisted)
|
||||
// converted to `DashboardEvent::Sent` JSON so the client can replay
|
||||
// through the same dispatch path as live frames. Wrapped in
|
||||
// `{ seq, events }`: the seq is the dashboard channel's high-water
|
||||
// mark at fetch time. Clients use it to dedupe their buffered live
|
||||
// SSE traffic (drop anything with `seq <= history_seq`) so a frame
|
||||
// that lands between SSE-subscribe and history-fetch isn't shown
|
||||
// twice and isn't lost. Historical rows carry `seq = 0`; the
|
||||
// boundary seq is what closes the dedupe window.
|
||||
const HISTORY_LIMIT: u64 = 200;
|
||||
let seq = state.coord.current_seq();
|
||||
match state.coord.broker.recent_all(HISTORY_LIMIT) {
|
||||
Ok(mut messages) => {
|
||||
messages.reverse();
|
||||
let events: Vec<crate::dashboard_events::DashboardEvent> = messages
|
||||
.into_iter()
|
||||
.map(|m| match m {
|
||||
crate::broker::MessageEvent::Sent {
|
||||
id,
|
||||
from,
|
||||
to,
|
||||
body,
|
||||
at,
|
||||
in_reply_to,
|
||||
} => {
|
||||
let file_refs = scan_validated_paths(&body);
|
||||
crate::dashboard_events::DashboardEvent::Sent {
|
||||
seq: 0,
|
||||
id,
|
||||
from,
|
||||
to,
|
||||
body,
|
||||
at: hive_sh4re::wire_time::from_secs(at),
|
||||
in_reply_to,
|
||||
file_refs,
|
||||
}
|
||||
}
|
||||
crate::broker::MessageEvent::Delivered {
|
||||
id,
|
||||
from,
|
||||
to,
|
||||
body,
|
||||
at,
|
||||
in_reply_to,
|
||||
} => {
|
||||
let file_refs = scan_validated_paths(&body);
|
||||
crate::dashboard_events::DashboardEvent::Delivered {
|
||||
seq: 0,
|
||||
id,
|
||||
from,
|
||||
to,
|
||||
body,
|
||||
at: hive_sh4re::wire_time::from_secs(at),
|
||||
in_reply_to,
|
||||
file_refs,
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
axum::Json(serde_json::json!({ "seq": seq, "events": events })).into_response()
|
||||
}
|
||||
Err(e) => error_response(&format!("dashboard/history failed: {e:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// `/dashboard/stream` query string. Today's only field is `kinds`:
|
||||
/// a comma-separated allow-list of event-`kind` strings.
|
||||
/// Empty / absent ⇒ no filter (current behaviour, all variants
|
||||
/// forwarded). Set ⇒ only the named kinds reach the subscriber,
|
||||
/// non-matches are skipped before the JSON serialise cost.
|
||||
///
|
||||
/// Useful for narrow pages (e.g. `flow.js` only cares about `sent`
|
||||
/// / `delivered` / `container_state_changed` / `container_removed`)
|
||||
/// that want to drop the dispatch overhead on every unrelated mutation.
|
||||
#[derive(Deserialize, Default)]
|
||||
pub(super) struct DashboardStreamQuery {
|
||||
/// Comma-separated event kinds to forward. Each token is
|
||||
/// trimmed; unknown kinds are silently ignored on lookup
|
||||
/// (subscriber sees nothing instead of an error).
|
||||
kinds: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) async fn dashboard_stream(
|
||||
State(state): State<AppState>,
|
||||
axum::extract::Query(q): axum::extract::Query<DashboardStreamQuery>,
|
||||
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
|
||||
let rx = state.coord.dashboard_subscribe();
|
||||
// Pre-parse the allow-list once at subscription time, so the
|
||||
// per-event hot path is just a `HashSet::contains` on a
|
||||
// `&'static str` — no string churn per frame.
|
||||
let kind_filter: Option<std::collections::HashSet<String>> = q.kinds.and_then(|raw| {
|
||||
let set: std::collections::HashSet<String> = raw
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
if set.is_empty() { None } else { Some(set) }
|
||||
});
|
||||
let stream = BroadcastStream::new(rx).filter_map(move |res| {
|
||||
// Drop lagged frames. Browsers reconnect; the seq dedupe on
|
||||
// reconnect skips any frame already reflected in the snapshot.
|
||||
let event = res.ok()?;
|
||||
if let Some(filter) = kind_filter.as_ref()
|
||||
&& !filter.contains(event.kind_tag())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let json = serde_json::to_string(&event).ok()?;
|
||||
Some(Ok(Event::default().data(json)))
|
||||
});
|
||||
Sse::new(stream).keep_alive(KeepAlive::default())
|
||||
}
|
||||
159
hive-c0re/src/dashboard/tombstones.rs
Normal file
159
hive-c0re/src/dashboard/tombstones.rs
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
//! Tombstone rows for the dashboard: state dirs surviving a
|
||||
//! destroy-without-purge. Builds `TombstoneView`s for the snapshot,
|
||||
//! emits the `TombstonesChanged` event after mutations, and handles
|
||||
//! `POST /api/purge-tombstone/{name}`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
extract::{Path as AxumPath, State},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::container_view::{ContainerView, claude_has_session};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle;
|
||||
|
||||
use super::{AppState, error_response, validate_agent_name};
|
||||
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
pub struct TombstoneView {
|
||||
pub name: String,
|
||||
/// Bytes used by the state dir tree. Cheap-ish to compute; let the
|
||||
/// operator know how much they're holding onto.
|
||||
pub state_bytes: u64,
|
||||
/// Mtime (unix seconds) of the state dir; rough "last seen".
|
||||
pub last_seen: i64,
|
||||
pub has_creds: bool,
|
||||
}
|
||||
|
||||
/// State-dir names that don't appear in the live container list. Each
|
||||
/// one surfaces in the dashboard as a row with R3V1V3 + PURG3 actions.
|
||||
pub(super) fn build_tombstone_views(
|
||||
coord: &Coordinator,
|
||||
containers: &[ContainerView],
|
||||
transient_snapshot: &std::collections::HashMap<String, crate::coordinator::TransientState>,
|
||||
) -> Vec<TombstoneView> {
|
||||
let _ = coord; // kept_state_names is a free fn but takes &self by future plan
|
||||
let live: std::collections::HashSet<&str> = containers
|
||||
.iter()
|
||||
.map(|c| c.name.as_str())
|
||||
.chain(transient_snapshot.keys().map(String::as_str))
|
||||
.collect();
|
||||
Coordinator::kept_state_names()
|
||||
.into_iter()
|
||||
.filter(|name| !live.contains(name.as_str()))
|
||||
.map(|name| {
|
||||
let root = Coordinator::agent_state_root(&name);
|
||||
let state_bytes = dir_size_bytes(&root);
|
||||
let last_seen = std::fs::metadata(&root)
|
||||
.and_then(|m| m.modified())
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0);
|
||||
let has_creds = claude_has_session(&Coordinator::agent_claude_dir(&name));
|
||||
TombstoneView {
|
||||
name,
|
||||
state_bytes,
|
||||
last_seen,
|
||||
has_creds,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Sum the byte size of every regular file under `root`. Cheap to compute
|
||||
/// for typical agent state (config repo + claude creds + notes file —
|
||||
/// usually a few MB); fine to do inline on each /api/state. Returns 0 on
|
||||
/// any error.
|
||||
fn dir_size_bytes(root: &Path) -> u64 {
|
||||
fn walk(p: &Path, acc: &mut u64) {
|
||||
let Ok(rd) = std::fs::read_dir(p) else { return };
|
||||
for entry in rd.flatten() {
|
||||
let Ok(ft) = entry.file_type() else { continue };
|
||||
if ft.is_dir() {
|
||||
walk(&entry.path(), acc);
|
||||
} else if ft.is_file()
|
||||
&& let Ok(meta) = entry.metadata()
|
||||
{
|
||||
*acc += meta.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut total = 0u64;
|
||||
walk(root, &mut total);
|
||||
total
|
||||
}
|
||||
|
||||
/// Snapshot the current tombstone list and emit a
|
||||
/// `TombstonesChanged` event. Call after any mutation that could
|
||||
/// add or remove a tombstone (`actions::destroy`,
|
||||
/// `post_purge_tombstone`, spawn finalisation). Cheap — the list
|
||||
/// is tiny.
|
||||
pub(crate) async fn emit_tombstones_snapshot(coord: &Arc<Coordinator>) {
|
||||
let containers = coord.containers_snapshot().await;
|
||||
let transient_snapshot = coord.transient_snapshot();
|
||||
let tombstones = build_tombstone_views(coord, &containers, &transient_snapshot);
|
||||
coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::TombstonesChanged {
|
||||
seq: coord.next_seq(),
|
||||
tombstones,
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) async fn post_purge_tombstone(
|
||||
State(state): State<AppState>,
|
||||
AxumPath(name): AxumPath<String>,
|
||||
) -> Response {
|
||||
// Format guard FIRST so a name like `..` can't traverse into the
|
||||
// parent of `/var/lib/hyperhive/agents/{name}` and have
|
||||
// `remove_dir_all` wipe `/var/lib/hyperhive/` itself. Existing
|
||||
// manager + live-container checks below don't catch `..` — only
|
||||
// the whitelist does. Existence check via
|
||||
// `containers_snapshot()` is deliberately NOT used here:
|
||||
// tombstoned agents are gone from the snapshot by design; that's
|
||||
// the whole point of this endpoint.
|
||||
if let Some(reason) = validate_agent_name(&name) {
|
||||
return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response();
|
||||
}
|
||||
// Sanity: refuse to purge if a live container still exists with this
|
||||
// name. The dashboard already filters tombstones to non-live names,
|
||||
// but the operator could send a stale POST.
|
||||
let live = lifecycle::list().await.unwrap_or_default();
|
||||
if live
|
||||
.iter()
|
||||
.any(|c| c == &format!("{}{name}", lifecycle::AGENT_PREFIX) || c == &name)
|
||||
{
|
||||
return error_response(&format!(
|
||||
"refusing to purge {name}: container still exists — use DESTR0Y first"
|
||||
));
|
||||
}
|
||||
let mut errors = Vec::new();
|
||||
for dir in [
|
||||
Coordinator::agent_state_root(&name),
|
||||
Coordinator::agent_applied_dir(&name),
|
||||
] {
|
||||
if dir.exists()
|
||||
&& let Err(e) = std::fs::remove_dir_all(&dir)
|
||||
{
|
||||
errors.push(format!("{}: {e}", dir.display()));
|
||||
}
|
||||
}
|
||||
let _ = state
|
||||
.coord
|
||||
.approvals
|
||||
.fail_pending_for_agent(&name, "agent state purged");
|
||||
if errors.is_empty() {
|
||||
tracing::info!(%name, "tombstone purged");
|
||||
// Fire the post-purge tombstones snapshot so dashboards
|
||||
// drop the row live; matching form carries
|
||||
// `data-no-refresh`.
|
||||
emit_tombstones_snapshot(&state.coord).await;
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
} else {
|
||||
error_response(&format!("purge {name} partial: {}", errors.join(", ")))
|
||||
}
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@ use serde::Serialize;
|
|||
|
||||
use crate::container_view::ContainerView;
|
||||
use crate::dashboard::{MetaInputView, TombstoneView};
|
||||
use crate::rebuild_queue::QueueEntry;
|
||||
use crate::job_queue::DagView;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
|
|
@ -210,7 +210,7 @@ pub enum DashboardEvent {
|
|||
/// the add/remove races a per-row event would have, and the
|
||||
/// dashboard's grouping (`parent_id`) is most naturally re-derived
|
||||
/// from the full list.
|
||||
RebuildQueueChanged { seq: u64, queue: Vec<QueueEntry> },
|
||||
RebuildQueueChanged { seq: u64, queue: Vec<DagView> },
|
||||
/// Full snapshot of all scheduled prompts. Emitted after every
|
||||
/// operator mutation (new / edit / cancel / fire-now) and after the
|
||||
/// worker fires or rearms a row. Same snapshot-shape rationale as
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
280
hive-c0re/src/forge/mod.rs
Normal file
280
hive-c0re/src/forge/mod.rs
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
//! Optional Forgejo wiring — per-agent user + token provisioning,
|
||||
//! config-repo mirroring, meta read-access grants. Also seeds
|
||||
//! `internal/docs` — a private repo every agent gets read-only
|
||||
//! collaborator access to for operator-curated shared content.
|
||||
//! No-op when `hive-forge` isn't running. Full design: `docs/forge.md`.
|
||||
|
||||
mod pr_merge;
|
||||
mod repos;
|
||||
mod users;
|
||||
|
||||
pub use pr_merge::{
|
||||
ForgeMergeError, config_repo, fetch_pr_head_into_applied, ff_push_to_main, mark_pr_merged,
|
||||
pr_head_sha,
|
||||
};
|
||||
pub use repos::{
|
||||
create_agent_repo, ensure_config_repo, ensure_knowledge_repo, ensure_meta_remote, ensure_repo,
|
||||
ensure_shared_docs_repo, meta_read_access, push_config, push_meta, shared_docs_access,
|
||||
};
|
||||
pub use users::{core_token, ensure_user_for, provision_user_token};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use repos::{ensure_mirrors, ensure_operators_team, ensure_org};
|
||||
use users::{
|
||||
ensure_config_org_avatar, ensure_core_avatar, ensure_core_user_and_token,
|
||||
ensure_repo_creation_disabled, ensure_user_email,
|
||||
};
|
||||
|
||||
const FORGE_CONTAINER: &str = "hive-forge";
|
||||
pub(crate) const FORGE_HTTP: &str = "http://localhost:3000";
|
||||
/// Forgejo org grouping every agent's config repo. Core is a site admin
|
||||
/// and reads + writes every repo here. As of the agent-config-PR flow each
|
||||
/// agent is a **write collaborator on its own** `agent-configs/<name>` repo —
|
||||
/// the editable PR surface it pushes config-change branches to — but `main` is
|
||||
/// branch-protected core-only, so only hive-c0re's verify-and-ff-push merge
|
||||
/// handler lands on it (operator approval required; the agent can't push
|
||||
/// `main` or self-merge). The repos remain private, so an agent still can't
|
||||
/// reach *another* agent's config. `main` is fast-forward-only — hive-c0re
|
||||
/// never force-pushes; the `push_config` mirror runs best-effort until the
|
||||
/// PR-merge flow retires it.
|
||||
const CONFIG_ORG: &str = "agent-configs";
|
||||
/// Forgejo org hosting the operator-curated shared docs/skills repo
|
||||
/// that every agent gets read-only access to. Agents use it as a
|
||||
/// common reference without the operator having to bake content into
|
||||
/// the system prompt or rely on `/shared`. Only the manager + operator
|
||||
/// (i.e. `core` user) can push.
|
||||
const SHARED_ORG: &str = "internal";
|
||||
/// The shared docs repo inside `SHARED_ORG`. Cloneable by every agent
|
||||
/// at `{FORGE_HTTP}/internal/docs.git`.
|
||||
const SHARED_DOCS_REPO: &str = "docs";
|
||||
/// The hive-wide knowledge repo inside `SHARED_ORG`. Public — agents
|
||||
/// can fork it and open PRs without explicit collaborator grants.
|
||||
/// Bind-mounted read-only into every container at `/knowledge`.
|
||||
/// See `hive-c0re/src/knowledge.rs`.
|
||||
const KNOWLEDGE_REPO: &str = crate::knowledge::REPO;
|
||||
/// Forgejo org that owns agent-created repos. Agents can't create
|
||||
/// repos with their own token (`max_repo_creation = 0`); instead hive-c0re
|
||||
/// creates them here and adds the requesting agent as a **write** member
|
||||
/// (not owner/admin). Because the org — not the agent — owns the repo,
|
||||
/// perms stay c0re-managed and branch protection (referencing
|
||||
/// [`OPERATORS_TEAM`]) can block the author from merging their own PR. This
|
||||
/// is the "agents namespace" repos land in by default.
|
||||
const AGENTS_ORG: &str = "agents";
|
||||
/// Operator merge-gate team inside [`AGENTS_ORG`]. Provisioned **empty** by
|
||||
/// hive-c0re (so perms can be set before anyone joins); the operator adds
|
||||
/// herself via the forge UI / hivectl. Branch protection on agents-org repos
|
||||
/// references this team by name for the merge/approval whitelist, so the
|
||||
/// rule never hardcodes a specific reviewer agent (which may not exist).
|
||||
const OPERATORS_TEAM: &str = "operators";
|
||||
/// Hive-managed Forgejo namespaces that agent-initiated repo creation must
|
||||
/// never target. `internal` is operator-curated shared content;
|
||||
/// `agent-configs` + `core` are hive-c0re-internal mirror/meta namespaces.
|
||||
/// (`hyperhive` is NOT managed — it's just a repo that happens to be built
|
||||
/// by this hive.) hive-c0re's create path forces [`AGENTS_ORG`], so this is
|
||||
/// a defensive guard against any future caller passing an explicit owner.
|
||||
const HIVE_MANAGED_NAMESPACES: &[&str] = &[SHARED_ORG, CONFIG_ORG, "core"];
|
||||
/// Forgejo orgs hive-c0re ensures on startup. The meta repo lives at
|
||||
/// `core/meta` (the `core` user's own namespace — no org needed).
|
||||
const SEEDED_ORGS: &[&str] = &[CONFIG_ORG, SHARED_ORG, AGENTS_ORG];
|
||||
|
||||
/// Probe whether `hive-forge` exists as a nixos-container. Cheap —
|
||||
/// `nixos-container list` is just a directory scan in /etc. Routed
|
||||
/// through hive-priv: `nixos-container` needs root, and hive-c0re runs
|
||||
/// unprivileged (privsep).
|
||||
pub async fn is_present() -> bool {
|
||||
let Ok(stdout) = crate::priv_client::list_containers().await else {
|
||||
return false;
|
||||
};
|
||||
stdout.lines().any(|l| l.trim() == FORGE_CONTAINER)
|
||||
}
|
||||
|
||||
/// Run `forgejo admin <args>` inside the hive-forge container as the
|
||||
/// forgejo user (the only uid with write access to the state dir).
|
||||
/// Returns stdout on success; bails with stderr context on failure.
|
||||
async fn forge_admin(args: &[&str]) -> Result<String> {
|
||||
// Route through hive-priv (root helper) because `nixos-container run`
|
||||
// uses nsenter to enter the container's namespaces, which requires root.
|
||||
// hive-c0re runs as the unprivileged `hive-core` user and cannot call
|
||||
// nsenter directly — doing so produces:
|
||||
// nsenter: stat of /proc/<pid>/ns/user failed: Permission denied
|
||||
let (stdout, _stderr) = crate::priv_client::run_forge_admin(args)
|
||||
.await
|
||||
.with_context(|| format!("forgejo admin {} (via hive-priv)", args.join(" ")))?;
|
||||
Ok(stdout)
|
||||
}
|
||||
|
||||
/// Thin Forgejo REST helper. Sends `method` to `url` with a JSON body
|
||||
/// and `Authorization: token <token>`, returns the HTTP status code.
|
||||
/// All Forgejo API calls that don't shell out to `forgejo admin` go
|
||||
/// through here — one place for auth header, content-type, error
|
||||
/// propagation, and the shared reqwest Client.
|
||||
/// Returns the response status **and body**. The body lets callers log
|
||||
/// *why* Forgejo rejected a request (e.g. the validation message on a
|
||||
/// 422); status-only callers just bind `(status, _)`. Body read is
|
||||
/// best-effort — a read error yields an empty string rather than
|
||||
/// failing the whole call.
|
||||
async fn forge_http(
|
||||
method: reqwest::Method,
|
||||
url: &str,
|
||||
token: &str,
|
||||
body: &str,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.request(method, url)
|
||||
.header("Authorization", format!("token {token}"))
|
||||
.header("Content-Type", "application/json")
|
||||
.body(body.to_owned())
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("forge HTTP request to {url}"))?;
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
Ok((status, text))
|
||||
}
|
||||
|
||||
/// Whether `ns` is a hive-managed Forgejo namespace that agent-initiated
|
||||
/// repo creation must never target — `internal` (operator-curated
|
||||
/// shared content) + `agent-configs` / `core` (hive-c0re-internal). The
|
||||
/// create path forces [`AGENTS_ORG`], so this guards a future surface that
|
||||
/// might accept an explicit owner.
|
||||
#[must_use]
|
||||
pub fn is_hive_managed_namespace(ns: &str) -> bool {
|
||||
HIVE_MANAGED_NAMESPACES.contains(&ns)
|
||||
}
|
||||
|
||||
/// Per-agent forge sync: ensure the agent has a forgejo user + token,
|
||||
/// a mirrored config repo, read access to `core/meta`, and the `meta`
|
||||
/// remote in its proposed repo. All operations are idempotent; failures
|
||||
/// are logged as warnings but don't abort the caller.
|
||||
///
|
||||
/// `core_token` is `core_token()` — passed in so callers that already
|
||||
/// fetched it don't re-read the file. Pass `None` to skip the
|
||||
/// `meta_read_access` step (safe: the access grant is best-effort).
|
||||
///
|
||||
/// Called by both `ensure_all()` (startup sweep) and `rebuild_agent`
|
||||
/// (per-rebuild) so the two paths stay equivalent.
|
||||
pub async fn sync_agent(name: &str, core_token: Option<&str>) {
|
||||
if let Err(e) = ensure_user_for(name).await {
|
||||
tracing::warn!(%name, error = ?e, "forge: ensure_user failed");
|
||||
}
|
||||
// Align email to match the git user.email set by meta::render_flake
|
||||
// so commits link to the agent's Forgejo profile. Best-effort;
|
||||
// also patches up agents created before this fix (old @hive.local).
|
||||
ensure_user_email(name).await;
|
||||
// Block direct agent-initiated repo creation: agents create
|
||||
// repos through hive-c0re, never with their own token. Idempotent +
|
||||
// marker-guarded; also covers agents provisioned before this landed.
|
||||
ensure_repo_creation_disabled(name).await;
|
||||
// Mirror the agent's applied config repo into agent-configs.
|
||||
// ensure_config_repo is idempotent; push_config catches any
|
||||
// drift since the last run — e.g. the startup migration just
|
||||
// relocated `deployed/0`, or a deploy landed while the forge
|
||||
// was down.
|
||||
if let Err(e) = ensure_config_repo(name).await {
|
||||
tracing::warn!(%name, error = ?e, "forge: ensure_config_repo failed");
|
||||
}
|
||||
if let Err(e) = push_config(name).await {
|
||||
tracing::warn!(%name, error = ?e, "forge: push_config failed");
|
||||
}
|
||||
// Grant read-only access to core/meta and wire the `meta` remote
|
||||
// into the proposed repo so agents can fetch their deployment context.
|
||||
if let Some(token) = core_token
|
||||
&& let Err(e) = meta_read_access(name, token).await
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "forge: ensure_meta_read_access failed");
|
||||
}
|
||||
if let Err(e) = ensure_meta_remote(name).await {
|
||||
tracing::warn!(%name, error = ?e, "forge: ensure_meta_remote failed");
|
||||
}
|
||||
// Grant read-only access to internal/docs so the agent can clone
|
||||
// the operator-curated shared skills/runbook repo. Best-effort.
|
||||
if let Some(token) = core_token
|
||||
&& let Err(e) = shared_docs_access(name, token).await
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "forge: shared_docs_access failed");
|
||||
}
|
||||
// internal/knowledge is public — no per-agent collaborator grant needed.
|
||||
}
|
||||
|
||||
/// Sweep every existing container (manager + sub-agents) and ensure
|
||||
/// each has a forgejo user + token, plus an `agent-configs/<name>`
|
||||
/// repo mirroring its applied config. Also seeds the `core` admin
|
||||
/// user (hive-c0re's own identity for pushing the meta repo + driving
|
||||
/// the API), the `agent-configs` org, and the `core/meta` repo.
|
||||
/// Called once at hive-c0re startup. Per-step failures are logged
|
||||
/// but don't abort the sweep.
|
||||
pub async fn ensure_all() {
|
||||
if !is_present().await {
|
||||
tracing::debug!("forge: hive-forge container absent, skipping user sweep");
|
||||
return;
|
||||
}
|
||||
let core_token = match ensure_core_user_and_token().await {
|
||||
Ok(t) => Some(t),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "forge: ensure_core_user_and_token failed");
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(token) = core_token.as_deref() {
|
||||
for org in SEEDED_ORGS {
|
||||
if let Err(e) = ensure_org(org, token).await {
|
||||
tracing::warn!(%org, error = ?e, "forge: ensure_org failed");
|
||||
}
|
||||
}
|
||||
// Seed the operator-declared pull-mirrors (nix `forge.mirrors` +
|
||||
// the CI-auto `actions/checkout`, forwarded via the
|
||||
// `HYPERHIVE_FORGE_MIRRORS` env). Each ensures its own dest org, so
|
||||
// this is independent of the SEEDED_ORGS loop above.
|
||||
ensure_mirrors(token).await;
|
||||
// Provision the operator merge-gate team (empty) inside BOTH the
|
||||
// agents org and the agent-configs org so branch protection in each
|
||||
// can reference it before anyone joins. Gitea teams are org-scoped —
|
||||
// missing the agent-configs copy 422'd every config-repo protection
|
||||
// apply, leaving those repos unprotected and letting operator-merged
|
||||
// config PRs bypass the deploy pipeline. The operator adds herself as
|
||||
// a member out-of-band.
|
||||
for org in [AGENTS_ORG, CONFIG_ORG] {
|
||||
if let Err(e) = ensure_operators_team(org, token).await {
|
||||
tracing::warn!(%org, error = ?e, "forge: ensure_operators_team failed");
|
||||
}
|
||||
}
|
||||
// Meta repo lives at core/meta — pushed from git_commit in
|
||||
// meta.rs on every deploy/lock-update. Make sure it exists
|
||||
// before the first push hits a 404.
|
||||
if let Err(e) = ensure_repo("meta", token).await {
|
||||
tracing::warn!(error = ?e, "forge: ensure_repo core/meta failed");
|
||||
}
|
||||
// Seed the shared docs repo. internal is already in
|
||||
// SEEDED_ORGS above so the org exists; ensure the repo itself.
|
||||
if let Err(e) = ensure_shared_docs_repo(token).await {
|
||||
tracing::warn!(error = ?e, "forge: ensure_shared_docs_repo failed");
|
||||
}
|
||||
// Seed the hive-wide knowledge repo.
|
||||
if let Err(e) = ensure_knowledge_repo(token).await {
|
||||
tracing::warn!(error = ?e, "forge: ensure_knowledge_repo failed");
|
||||
}
|
||||
// Clone knowledge repo locally so it can be bind-mounted into agents.
|
||||
if let Err(e) = crate::knowledge::ensure_local_clone(token).await {
|
||||
tracing::warn!(error = ?e, "knowledge: ensure_local_clone failed");
|
||||
}
|
||||
if let Err(e) = ensure_core_avatar(token).await {
|
||||
tracing::warn!(error = ?e, "forge: ensure_core_avatar failed");
|
||||
}
|
||||
if let Err(e) = ensure_config_org_avatar(token).await {
|
||||
tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed");
|
||||
}
|
||||
}
|
||||
let Ok(containers) = crate::lifecycle::list().await else {
|
||||
tracing::warn!("forge: nixos-container list failed; skipping user sweep");
|
||||
return;
|
||||
};
|
||||
for c in containers {
|
||||
let Some(name) = c.strip_prefix(crate::lifecycle::AGENT_PREFIX) else {
|
||||
continue;
|
||||
};
|
||||
sync_agent(name, core_token.as_deref()).await;
|
||||
}
|
||||
}
|
||||
295
hive-c0re/src/forge/pr_merge.rs
Normal file
295
hive-c0re/src/forge/pr_merge.rs
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
//! PR-based config-flow merge primitives — the forge-side mechanics
|
||||
//! hive-c0re's approve-handler (`run_merge_config_pr`) orchestrates to
|
||||
//! land an operator-approved config PR. Part of the operator trust
|
||||
//! boundary; moved verbatim from the `forge` module root.
|
||||
|
||||
use anyhow::Context;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
use super::{CONFIG_ORG, FORGE_HTTP, core_token, forge_http};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PR-based config-flow merge primitives (part of the
|
||||
// dashboard-approve-driven config-change flow).
|
||||
//
|
||||
// The dashboard-approve-driven flow has hive-c0re verify an operator-approved
|
||||
// config PR, then land it: ff-push the verified sha to the protected default
|
||||
// branch (= the merge) and mark the PR merged manually. These three fns are
|
||||
// the forge-side mechanics the c0re approve-handler (`run_merge_config_pr`)
|
||||
// orchestrates; the orchestration fetches the verified sha into the agent's
|
||||
// applied repo before calling `ff_push_to_main`. The core token is sourced
|
||||
// internally (`core_token`), never passed in. `repo` is the agent's editable
|
||||
// forge config repo in `owner/name` form (e.g. `agent-configs/<agent>`).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Typed failure for the merge primitives so the c0re approve-handler can
|
||||
/// `match` recoverable drift (refresh the request sha + re-verify) against a
|
||||
/// hard failure (fail the approval). The two drift variants carry the observed
|
||||
/// sha so the handler can re-pin to it; `Other` is everything else (transport,
|
||||
/// API, unexpected) and is not auto-retried.
|
||||
#[derive(Debug)]
|
||||
pub enum ForgeMergeError {
|
||||
/// The PR head moved off `expected` (now at `actual`), seen at
|
||||
/// mark-merged time. The handler's pre-merge `pr_head_sha` re-read is the
|
||||
/// primary drift gate; this is the belt-and-suspenders race catch.
|
||||
HeadDrift { expected: String, actual: String },
|
||||
/// `main` (`actual_head`) is not a descendant of `expected_ancestor`, so
|
||||
/// pushing the verified sha would not be a fast-forward — `main` raced.
|
||||
NotFastForward {
|
||||
expected_ancestor: String,
|
||||
actual_head: String,
|
||||
},
|
||||
/// Transport / API / unexpected failure — hard-fail, no auto-retry.
|
||||
Other(anyhow::Error),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ForgeMergeError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::HeadDrift { expected, actual } => {
|
||||
write!(f, "PR head drifted: expected {expected}, found {actual}")
|
||||
}
|
||||
Self::NotFastForward {
|
||||
expected_ancestor,
|
||||
actual_head,
|
||||
} => write!(
|
||||
f,
|
||||
"not a fast-forward: main {actual_head} is not a descendant of {expected_ancestor}"
|
||||
),
|
||||
Self::Other(e) => write!(f, "{e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ForgeMergeError {}
|
||||
|
||||
impl From<anyhow::Error> for ForgeMergeError {
|
||||
fn from(e: anyhow::Error) -> Self {
|
||||
Self::Other(e)
|
||||
}
|
||||
}
|
||||
|
||||
/// Agent name from an `owner/name` forge repo string (the trailing segment).
|
||||
fn repo_agent_name(repo: &str) -> &str {
|
||||
repo.rsplit('/').next().unwrap_or(repo)
|
||||
}
|
||||
|
||||
/// Token-in-URL clone/push URL for a forge repo (`owner/name`). Mirrors
|
||||
/// `push_config`'s pattern; the token is passed straight to git and never
|
||||
/// stored as a named remote.
|
||||
fn tokenised_repo_url(repo: &str, token: &str) -> String {
|
||||
format!("http://core:{token}@localhost:3000/{repo}.git")
|
||||
}
|
||||
|
||||
/// Resolve a PR's head sha via `git ls-remote <repo> refs/pull/<pr>/head`
|
||||
/// (Forgejo exposes PR heads there). Pure read, no mutation — the handler's
|
||||
/// primary drift gate (compare against the approved sha), and `mark_pr_merged`
|
||||
/// uses it to detect head-drift on failure.
|
||||
///
|
||||
/// # Errors
|
||||
/// `Other` on transport failure or an empty/missing ref.
|
||||
pub async fn pr_head_sha(repo: &str, pr: u64) -> Result<String, ForgeMergeError> {
|
||||
let token = core_token()
|
||||
.ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?;
|
||||
let url = tokenised_repo_url(repo, &token);
|
||||
let refspec = format!("refs/pull/{pr}/head");
|
||||
let out = crate::lifecycle::git_command()
|
||||
.args(["ls-remote", &url, &refspec])
|
||||
.output()
|
||||
.await
|
||||
.context("git ls-remote PR head")?;
|
||||
if !out.status.success() {
|
||||
return Err(ForgeMergeError::Other(anyhow::anyhow!(
|
||||
"git ls-remote {repo} {refspec} failed ({}): {}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
)));
|
||||
}
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
let sha = stdout
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.filter(|s| !s.is_empty())
|
||||
.ok_or_else(|| {
|
||||
ForgeMergeError::Other(anyhow::anyhow!(
|
||||
"no head ref for PR #{pr} in {repo} (ls-remote empty)"
|
||||
))
|
||||
})?;
|
||||
Ok(sha.to_string())
|
||||
}
|
||||
|
||||
/// Full `owner/name` path of an agent's config repo on the forge — the
|
||||
/// `agent-configs` org mirror that the PR-merge flow reads + fast-forwards.
|
||||
pub fn config_repo(agent: &str) -> String {
|
||||
format!("{CONFIG_ORG}/{agent}")
|
||||
}
|
||||
|
||||
/// Fetch PR #`pr`'s head into the agent's applied repo via
|
||||
/// `refs/pull/<pr>/head` (which Forgejo always serves — a bare-sha fetch can
|
||||
/// be refused by uploadpack policy). This makes the reviewed head an object
|
||||
/// in the applied repo so the ancestor check in [`ff_push_to_main`], the
|
||||
/// `git_update_ref(main, …)` in the deploy tail, and the eval-verify all
|
||||
/// resolve it locally before the irreversible push.
|
||||
///
|
||||
/// # Errors
|
||||
/// `Other` on transport failure or a non-zero git exit.
|
||||
pub async fn fetch_pr_head_into_applied(repo: &str, pr: u64) -> Result<(), ForgeMergeError> {
|
||||
let token = core_token()
|
||||
.ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?;
|
||||
let url = tokenised_repo_url(repo, &token);
|
||||
let applied = Coordinator::agent_applied_dir(repo_agent_name(repo));
|
||||
let refspec = format!("refs/pull/{pr}/head");
|
||||
let out = crate::lifecycle::git_command()
|
||||
.current_dir(&applied)
|
||||
.args(["fetch", "--no-tags", &url, &refspec])
|
||||
.output()
|
||||
.await
|
||||
.context("git fetch PR head into applied")?;
|
||||
if !out.status.success() {
|
||||
return Err(ForgeMergeError::Other(anyhow::anyhow!(
|
||||
"git fetch {repo} {refspec} into applied failed ({}): {}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fast-forward the forge repo's `main` to `sha` — THE merge in the PR flow.
|
||||
/// Reads `main`'s current sha (`git ls-remote … refs/heads/main`), verifies it
|
||||
/// is a strict ancestor of `sha` (`git merge-base --is-ancestor`, run in the
|
||||
/// agent's applied repo where the orchestration has already fetched `sha`),
|
||||
/// then does a **non-force** `git push <sha>:refs/heads/main`. The ancestor
|
||||
/// pre-check and the non-force push are two independent guards: either catches
|
||||
/// a raced `main` (→ `NotFastForward`) rather than clobbering reviewed history.
|
||||
///
|
||||
/// # Errors
|
||||
/// `NotFastForward` if `main` raced ahead of `sha`; `Other` on transport/other.
|
||||
pub async fn ff_push_to_main(repo: &str, sha: &str) -> Result<(), ForgeMergeError> {
|
||||
let token = core_token()
|
||||
.ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?;
|
||||
let url = tokenised_repo_url(repo, &token);
|
||||
let applied = Coordinator::agent_applied_dir(repo_agent_name(repo));
|
||||
|
||||
// Current `main` on the forge repo.
|
||||
let ls = crate::lifecycle::git_command()
|
||||
.args(["ls-remote", &url, "refs/heads/main"])
|
||||
.output()
|
||||
.await
|
||||
.context("git ls-remote main")?;
|
||||
if !ls.status.success() {
|
||||
return Err(ForgeMergeError::Other(anyhow::anyhow!(
|
||||
"git ls-remote {repo} refs/heads/main failed ({}): {}",
|
||||
ls.status,
|
||||
String::from_utf8_lossy(&ls.stderr).trim()
|
||||
)));
|
||||
}
|
||||
let ls_out = String::from_utf8_lossy(&ls.stdout);
|
||||
let main_sha = ls_out.split_whitespace().next().unwrap_or("").to_string();
|
||||
|
||||
// Strict-ancestor check: `main` must be an ancestor of `sha` for a true
|
||||
// fast-forward. Skip when `main` is unborn (empty) — the push creates it.
|
||||
if !main_sha.is_empty() {
|
||||
let anc = crate::lifecycle::git_command()
|
||||
.current_dir(&applied)
|
||||
.args(["merge-base", "--is-ancestor", &main_sha, sha])
|
||||
.output()
|
||||
.await
|
||||
.context("git merge-base --is-ancestor")?;
|
||||
match anc.status.code() {
|
||||
Some(0) => {} // ancestor → fast-forward safe
|
||||
Some(1) => {
|
||||
return Err(ForgeMergeError::NotFastForward {
|
||||
expected_ancestor: main_sha,
|
||||
actual_head: sha.to_string(),
|
||||
});
|
||||
}
|
||||
_ => {
|
||||
return Err(ForgeMergeError::Other(anyhow::anyhow!(
|
||||
"git merge-base --is-ancestor errored ({}): {}",
|
||||
anc.status,
|
||||
String::from_utf8_lossy(&anc.stderr).trim()
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Non-force push `sha` → `main`. Without `--force`, git rejects a
|
||||
// non-fast-forward (a race between the check above and now), surfaced as
|
||||
// `NotFastForward` rather than clobbering the remote.
|
||||
let push = crate::lifecycle::git_command()
|
||||
.current_dir(&applied)
|
||||
.args(["push", &url, &format!("{sha}:refs/heads/main")])
|
||||
.output()
|
||||
.await
|
||||
.context("git push sha:main")?;
|
||||
if !push.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&push.stderr);
|
||||
if stderr.contains("non-fast-forward") || stderr.contains("fetch first") {
|
||||
return Err(ForgeMergeError::NotFastForward {
|
||||
expected_ancestor: main_sha,
|
||||
actual_head: sha.to_string(),
|
||||
});
|
||||
}
|
||||
return Err(ForgeMergeError::Other(anyhow::anyhow!(
|
||||
"git push {repo} {sha}:main failed ({}): {}",
|
||||
push.status,
|
||||
stderr.trim()
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark PR `pr` as **manually merged** at `sha` (Forgejo
|
||||
/// `POST …/pulls/{pr}/merge` with `Do=manually-merged`, `MergeCommitID=sha`).
|
||||
/// `ff_push_to_main` must have already set `main` to `sha` (Forgejo requires
|
||||
/// the branch already be at the merge commit). On a non-2xx, re-reads the PR
|
||||
/// head to distinguish drift (`HeadDrift`) from a generic failure (`Other`) —
|
||||
/// best-effort, since the handler's pre-merge head re-read is the real gate.
|
||||
///
|
||||
/// # Errors
|
||||
/// `HeadDrift` if the PR head no longer matches `sha`; `Other` otherwise.
|
||||
pub async fn mark_pr_merged(repo: &str, pr: u64, sha: &str) -> Result<(), ForgeMergeError> {
|
||||
let token = core_token()
|
||||
.ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?;
|
||||
let url = format!("{FORGE_HTTP}/api/v1/repos/{repo}/pulls/{pr}/merge");
|
||||
let body = format!(r#"{{"Do":"manually-merged","MergeCommitID":"{sha}"}}"#);
|
||||
let (status, _) = forge_http(reqwest::Method::POST, &url, &token, &body)
|
||||
.await
|
||||
.context("POST pulls/<pr>/merge (manually-merged)")?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
// Best-effort drift detection: if the live head no longer matches `sha`,
|
||||
// that's a head-drift race; otherwise surface as a hard failure.
|
||||
match pr_head_sha(repo, pr).await {
|
||||
Ok(actual) if actual != sha => Err(ForgeMergeError::HeadDrift {
|
||||
expected: sha.to_string(),
|
||||
actual,
|
||||
}),
|
||||
_ => Err(ForgeMergeError::Other(anyhow::anyhow!(
|
||||
"mark PR #{pr} in {repo} manually-merged at {sha} failed: HTTP {status}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{repo_agent_name, tokenised_repo_url};
|
||||
|
||||
#[test]
|
||||
fn repo_agent_name_takes_trailing_segment() {
|
||||
assert_eq!(repo_agent_name("agent-configs/atlas"), "atlas");
|
||||
assert_eq!(repo_agent_name("atlas"), "atlas");
|
||||
assert_eq!(repo_agent_name("a/b/c"), "c");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenised_repo_url_shape() {
|
||||
assert_eq!(
|
||||
tokenised_repo_url("agent-configs/iris", "tok"),
|
||||
"http://core:tok@localhost:3000/agent-configs/iris.git"
|
||||
);
|
||||
}
|
||||
}
|
||||
621
hive-c0re/src/forge/repos.rs
Normal file
621
hive-c0re/src/forge/repos.rs
Normal file
|
|
@ -0,0 +1,621 @@
|
|||
//! Repo + org plumbing on the local Forgejo: org / repo creation,
|
||||
//! the meta + shared-docs + knowledge repos, per-agent config-repo
|
||||
//! mirroring (`push_config` / `push_meta`), collaborator grants,
|
||||
//! pull-mirrors, and branch-protection rules. Shared HTTP helpers +
|
||||
//! org-name constants live in the module root (`super`).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use tokio::process::Command;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
use super::{
|
||||
AGENTS_ORG, CONFIG_ORG, FORGE_HTTP, KNOWLEDGE_REPO, OPERATORS_TEAM, SHARED_DOCS_REPO,
|
||||
SHARED_ORG, core_token, forge_http, is_present,
|
||||
};
|
||||
|
||||
/// JSON body for a private, empty repo defaulting to `main`.
|
||||
fn repo_body(name: &str) -> String {
|
||||
format!(r#"{{"name":"{name}","auto_init":false,"private":true,"default_branch":"main"}}"#)
|
||||
}
|
||||
|
||||
/// JSON body for a public, empty repo defaulting to `main`.
|
||||
fn repo_body_public(name: &str) -> String {
|
||||
format!(r#"{{"name":"{name}","auto_init":false,"private":false,"default_branch":"main"}}"#)
|
||||
}
|
||||
|
||||
/// Set an existing repo to public visibility. No-op if the repo is
|
||||
/// already public. Used for `internal/knowledge` which may have been
|
||||
/// created as private on an older deployment.
|
||||
async fn set_repo_public(owner: &str, repo: &str, token: &str) -> Result<()> {
|
||||
let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}");
|
||||
let (status, _) =
|
||||
forge_http(reqwest::Method::PATCH, &url, token, r#"{"private":false}"#).await?;
|
||||
match status.as_u16() {
|
||||
200 => {
|
||||
tracing::debug!(%owner, %repo, "forge: repo set to public");
|
||||
Ok(())
|
||||
}
|
||||
other => anyhow::bail!("PATCH {owner}/{repo} (set public) returned HTTP {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create `name` inside org `org` as a public repo. Idempotent.
|
||||
async fn ensure_org_repo_public(org: &str, name: &str, token: &str) -> Result<()> {
|
||||
create_repo(
|
||||
&format!("{FORGE_HTTP}/api/v1/orgs/{org}/repos"),
|
||||
&repo_body_public(name),
|
||||
token,
|
||||
&format!("{org}/{name}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// POST a repo-creation request to `url` and fold "already exists"
|
||||
/// (HTTP 409 / 422) into success. `label` is `<owner>/<name>` — purely
|
||||
/// for log + error context.
|
||||
async fn create_repo(url: &str, body: &str, token: &str, label: &str) -> Result<()> {
|
||||
let (status, _) = forge_http(reqwest::Method::POST, url, token, body).await?;
|
||||
match status.as_u16() {
|
||||
201 => {
|
||||
tracing::info!(%label, "forge: created repo");
|
||||
Ok(())
|
||||
}
|
||||
409 | 422 => {
|
||||
tracing::debug!(%label, "forge: repo already exists");
|
||||
Ok(())
|
||||
}
|
||||
other => anyhow::bail!("POST {url} ({label}) returned HTTP {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a repo in the token-owner's own namespace. `token` belongs
|
||||
/// to the user we want the repo owned by (we use `core`'s token for
|
||||
/// `core/meta`). Idempotent.
|
||||
pub async fn ensure_repo(name: &str, token: &str) -> Result<()> {
|
||||
create_repo(
|
||||
&format!("{FORGE_HTTP}/api/v1/user/repos"),
|
||||
&repo_body(name),
|
||||
token,
|
||||
&format!("core/{name}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Create `name` inside org `org` (used for `agent-configs/<agent>`).
|
||||
/// Idempotent.
|
||||
async fn ensure_org_repo(org: &str, name: &str, token: &str) -> Result<()> {
|
||||
create_repo(
|
||||
&format!("{FORGE_HTTP}/api/v1/orgs/{org}/repos"),
|
||||
&repo_body(name),
|
||||
token,
|
||||
&format!("{org}/{name}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Push `dir` (the meta repo) to `core/meta` on the local forge.
|
||||
/// Best-effort: returns Err which callers log + ignore. No-op when
|
||||
/// the core token isn't present yet (forge container not provisioned).
|
||||
pub async fn push_meta(dir: &Path) -> Result<()> {
|
||||
let Some(token) = core_token() else {
|
||||
return Ok(());
|
||||
};
|
||||
// Token-in-URL push. Forgejo accepts `oauth2:<token>` or just
|
||||
// any-username:<token>; using `core` matches the owner so the
|
||||
// remote name is self-describing.
|
||||
let url = format!("http://core:{token}@localhost:3000/core/meta.git");
|
||||
let out = Command::new("git")
|
||||
.current_dir(dir)
|
||||
.args(["push", "--force", &url, "HEAD:main"])
|
||||
.output()
|
||||
.await
|
||||
.context("invoke git push core/meta")?;
|
||||
if !out.status.success() {
|
||||
anyhow::bail!(
|
||||
"git push core/meta failed ({}): {}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
tracing::info!("forge: pushed meta to core/meta");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure the `agent-configs/<name>` repo exists so the first
|
||||
/// `push_config` doesn't 404, and wire it as the agent-editable PR surface:
|
||||
/// the agent is a **write** collaborator (can push feature branches +
|
||||
/// open config PRs) and `main` is branch-protected core-only (only hive-c0re's
|
||||
/// merge handler lands on it; operator approval required). No-op when the forge
|
||||
/// isn't running or the core token isn't minted yet. Safe to call on every
|
||||
/// spawn and on every startup (all steps idempotent).
|
||||
pub async fn ensure_config_repo(name: &str) -> Result<()> {
|
||||
if !is_present().await {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(token) = core_token() else {
|
||||
return Ok(());
|
||||
};
|
||||
ensure_org_repo(CONFIG_ORG, name, &token).await?;
|
||||
// Agent = write collaborator: it can push config-PR branches + open PRs,
|
||||
// but the branch protection below keeps it off `main` directly.
|
||||
add_collaborator(CONFIG_ORG, name, name, "write", &token).await?;
|
||||
// Protect `main` core-only, fast-forward-only (no auto force-push).
|
||||
apply_config_repo_branch_protection(name, &token).await
|
||||
}
|
||||
|
||||
/// Ensure the `internal/docs` repo exists. Called once at startup
|
||||
/// after `ensure_org(SHARED_ORG)`. Idempotent — `ensure_org_repo`
|
||||
/// treats 409 as success.
|
||||
pub async fn ensure_shared_docs_repo(core_token: &str) -> Result<()> {
|
||||
ensure_org_repo(SHARED_ORG, SHARED_DOCS_REPO, core_token).await
|
||||
}
|
||||
|
||||
/// Grant agent `name` read-only collaborator access to `internal/docs`.
|
||||
/// Idempotent: HTTP 204 (already a collaborator) is treated as success.
|
||||
/// Mirrors `meta_read_access` so agents can clone the shared docs repo
|
||||
/// without authentication hassle.
|
||||
pub async fn shared_docs_access(name: &str, core_token: &str) -> Result<()> {
|
||||
let url =
|
||||
format!("{FORGE_HTTP}/api/v1/repos/{SHARED_ORG}/{SHARED_DOCS_REPO}/collaborators/{name}");
|
||||
let body = r#"{"permission":"read"}"#;
|
||||
let out = Command::new("curl")
|
||||
.args([
|
||||
"-sS",
|
||||
"-o",
|
||||
"/dev/null",
|
||||
"-w",
|
||||
"%{http_code}",
|
||||
"-X",
|
||||
"PUT",
|
||||
"-H",
|
||||
"Content-Type: application/json",
|
||||
"-H",
|
||||
&format!("Authorization: token {core_token}"),
|
||||
"-d",
|
||||
body,
|
||||
&url,
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.context("invoke curl PUT internal/docs/collaborators")?;
|
||||
let code = String::from_utf8_lossy(&out.stdout).trim().to_owned();
|
||||
match code.as_str() {
|
||||
"204" => {
|
||||
tracing::info!(%name, "forge: granted shared-docs read access");
|
||||
Ok(())
|
||||
}
|
||||
other => anyhow::bail!(
|
||||
"PUT {SHARED_ORG}/{SHARED_DOCS_REPO}/collaborators/{name} returned HTTP {other}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Ensure the `internal/knowledge` repo exists and is public.
|
||||
/// Called once at startup after `ensure_org(SHARED_ORG)`. Idempotent.
|
||||
///
|
||||
/// The repo is created as public so any agent with a forge account can
|
||||
/// fork it and open PRs to contribute. Existing deployments that ended
|
||||
/// up with a private repo are patched to public on the next hive-c0re
|
||||
/// startup via `set_repo_public`.
|
||||
pub async fn ensure_knowledge_repo(core_token: &str) -> Result<()> {
|
||||
ensure_org_repo_public(SHARED_ORG, KNOWLEDGE_REPO, core_token).await?;
|
||||
// Ensure public even if the repo already existed as private (older deployment).
|
||||
set_repo_public(SHARED_ORG, KNOWLEDGE_REPO, core_token).await
|
||||
}
|
||||
|
||||
/// Grant agent `name` read-only collaborator access to `core/meta` on
|
||||
/// the forge so the agent can clone/fetch the meta flake. Idempotent:
|
||||
/// HTTP 204 (already a collaborator) is treated as success.
|
||||
pub async fn meta_read_access(name: &str, core_token: &str) -> Result<()> {
|
||||
let url = format!("{FORGE_HTTP}/api/v1/repos/core/meta/collaborators/{name}");
|
||||
let body = r#"{"permission":"read"}"#;
|
||||
let out = Command::new("curl")
|
||||
.args([
|
||||
"-sS",
|
||||
"-o",
|
||||
"/dev/null",
|
||||
"-w",
|
||||
"%{http_code}",
|
||||
"-X",
|
||||
"PUT",
|
||||
"-H",
|
||||
"Content-Type: application/json",
|
||||
"-H",
|
||||
&format!("Authorization: token {core_token}"),
|
||||
"-d",
|
||||
body,
|
||||
&url,
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.context("invoke curl PUT core/meta/collaborators")?;
|
||||
let code = String::from_utf8_lossy(&out.stdout).trim().to_owned();
|
||||
match code.as_str() {
|
||||
"204" => {
|
||||
tracing::info!(%name, "forge: granted meta read access");
|
||||
Ok(())
|
||||
}
|
||||
other => anyhow::bail!("PUT core/meta/collaborators/{name} returned HTTP {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add `http://localhost:3000/core/meta.git` as the `meta` remote in
|
||||
/// the agent's proposed config repo so the agent (and the manager) can
|
||||
/// fetch the meta flake from the forge. Idempotent: no-op when the
|
||||
/// remote already points at the right URL, or when the proposed repo
|
||||
/// does not exist yet. No-op when the forge is not running.
|
||||
pub async fn ensure_meta_remote(name: &str) -> Result<()> {
|
||||
if !is_present().await {
|
||||
return Ok(());
|
||||
}
|
||||
let proposed_dir = Coordinator::agent_proposed_dir(name);
|
||||
if !proposed_dir.join(".git").exists() {
|
||||
return Ok(());
|
||||
}
|
||||
let want = format!("{FORGE_HTTP}/core/meta.git");
|
||||
let existing = crate::lifecycle::git_command()
|
||||
.current_dir(&proposed_dir)
|
||||
.args(["remote", "get-url", "meta"])
|
||||
.output()
|
||||
.await
|
||||
.context("git remote get-url meta")?;
|
||||
if existing.status.success() {
|
||||
let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned();
|
||||
if current == want {
|
||||
return Ok(());
|
||||
}
|
||||
crate::lifecycle::git(&proposed_dir, &["remote", "set-url", "meta", &want]).await
|
||||
} else {
|
||||
crate::lifecycle::git(&proposed_dir, &["remote", "add", "meta", &want]).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirror agent `name`'s applied config repo — `main` plus every tag
|
||||
/// (`proposal` / `approved` / `building` / `deployed` / `failed` /
|
||||
/// `denied`) — to `agent-configs/<name>` on the local forge.
|
||||
/// Best-effort: returns Err which callers log + ignore. No-op when the
|
||||
/// forge isn't seeded or the applied repo doesn't exist yet.
|
||||
///
|
||||
/// Call this after every hive-c0re mutation of an applied repo's refs
|
||||
/// so the forge copy always reflects what core actually did.
|
||||
///
|
||||
/// Never force-pushes. The status tags are id-suffixed
|
||||
/// (`proposal/<id>`, `deployed/<id>`, …) and therefore add-only, and
|
||||
/// `main` is published history — after a failed deploy rolls the LOCAL
|
||||
/// applied `main` back to last-good, the forge `main` may legitimately
|
||||
/// be ahead (e.g. an operator-merged config PR whose rebuild failed).
|
||||
/// Rewinding it would erase that merged commit from the forge, which
|
||||
/// is exactly the incident this guards against: the local repo tracks
|
||||
/// "what last built", the forge tracks "what was approved", and the
|
||||
/// `failed/<id>` tag records the divergence. A non-fast-forward
|
||||
/// rejection of `main` is therefore expected + logged at info; the
|
||||
/// tags in the same push still land (git pushes refspecs
|
||||
/// independently). Any other failure is a real error.
|
||||
///
|
||||
/// The tokenised URL is passed straight to `git push` and deliberately
|
||||
/// never stored as a named remote: the applied repo is bind-mounted
|
||||
/// READ-ONLY into the manager container (`/applied`), so a token in
|
||||
/// `.git/config` would leak core's admin credential to an agent.
|
||||
pub async fn push_config(name: &str) -> Result<()> {
|
||||
let Some(token) = core_token() else {
|
||||
return Ok(());
|
||||
};
|
||||
let dir = Coordinator::agent_applied_dir(name);
|
||||
if !dir.join(".git").exists() {
|
||||
return Ok(());
|
||||
}
|
||||
let url = format!("http://core:{token}@localhost:3000/{CONFIG_ORG}/{name}.git");
|
||||
let out = crate::lifecycle::git_command()
|
||||
.current_dir(&dir)
|
||||
.args([
|
||||
"push",
|
||||
&url,
|
||||
"refs/heads/main:refs/heads/main",
|
||||
"refs/tags/*:refs/tags/*",
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.context("invoke git push agent-configs")?;
|
||||
if !out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
if stderr.contains("non-fast-forward") {
|
||||
tracing::info!(
|
||||
%name,
|
||||
"forge: mirror push of main rejected (non-fast-forward) — forge main is \
|
||||
ahead of local applied main (rolled-back deploy); leaving forge history intact"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
anyhow::bail!(
|
||||
"git push {CONFIG_ORG}/{name} failed ({}): {}",
|
||||
out.status,
|
||||
stderr.trim()
|
||||
);
|
||||
}
|
||||
tracing::info!(%name, "forge: mirrored applied config to agent-configs");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// POST `/api/v1/orgs` to create an org named `name`. Idempotent:
|
||||
/// HTTP 422 ("user already exists") is treated as success.
|
||||
pub(super) async fn ensure_org(name: &str, admin_token: &str) -> Result<()> {
|
||||
let body = format!(r#"{{"username":"{name}"}}"#);
|
||||
let url = format!("{FORGE_HTTP}/api/v1/orgs");
|
||||
let (status, _) = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?;
|
||||
match status.as_u16() {
|
||||
201 => {
|
||||
tracing::info!(%name, "forge: created org");
|
||||
Ok(())
|
||||
}
|
||||
422 | 409 => {
|
||||
tracing::debug!(%name, "forge: org already exists");
|
||||
Ok(())
|
||||
}
|
||||
other => anyhow::bail!("POST /api/v1/orgs name={name} returned HTTP {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// One operator-declared pull-mirror, forwarded from the nix
|
||||
/// `services.hyperhive.forge.mirrors` option as JSON in
|
||||
/// `HYPERHIVE_FORGE_MIRRORS`.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Mirror {
|
||||
/// Upstream clone URL to mirror from (e.g. `https://github.com/actions/checkout`).
|
||||
upstream: String,
|
||||
/// Local `<owner>/<repo>` the mirror is created at.
|
||||
dest: String,
|
||||
}
|
||||
|
||||
/// Ensure each `HYPERHIVE_FORGE_MIRRORS` entry exists as a real Forgejo
|
||||
/// pull-mirror. The env carries the JSON-encoded nix `forge.mirrors` list
|
||||
/// (plus the CI-auto `actions/checkout` entry). Absent/empty env = no-op.
|
||||
/// Per-mirror failures warn and continue — never abort the startup sweep.
|
||||
pub(super) async fn ensure_mirrors(admin_token: &str) {
|
||||
let raw = match std::env::var("HYPERHIVE_FORGE_MIRRORS") {
|
||||
Ok(s) if !s.trim().is_empty() => s,
|
||||
_ => return,
|
||||
};
|
||||
let mirrors: Vec<Mirror> = match serde_json::from_str(&raw) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "forge: HYPERHIVE_FORGE_MIRRORS is not valid JSON; skipping mirror seed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for m in mirrors {
|
||||
let Some((owner, repo)) = m.dest.split_once('/') else {
|
||||
tracing::warn!(dest = %m.dest, "forge: mirror dest is not <owner>/<repo>; skipping");
|
||||
continue;
|
||||
};
|
||||
// Create the dest org first (idempotent); the mirror can't land
|
||||
// without its owner existing.
|
||||
if let Err(e) = ensure_org(owner, admin_token).await {
|
||||
tracing::warn!(%owner, error = ?e, "forge: ensure_org for mirror failed");
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = ensure_mirror_repo(&m.upstream, owner, repo, admin_token).await {
|
||||
tracing::warn!(dest = %m.dest, error = ?e, "forge: ensure_mirror_repo failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Periodic sync interval for pull-mirrors. Forgejo syncs mirrors
|
||||
/// on-access by default, which re-introduces external DNS latency on
|
||||
/// every `git clone` (the hive-ci runner shares the host netns and is
|
||||
/// therefore affected by host resolver blips). A fixed periodic interval
|
||||
/// isolates CI from transient DNS failures — a stale mirror is
|
||||
/// acceptable; a broken clone because of a momentary DNS blip is not.
|
||||
const MIRROR_INTERVAL: &str = "8h0m0s";
|
||||
|
||||
/// Create `owner/repo` as a pull-mirror of `upstream` via the migrate API.
|
||||
/// Idempotent: if the repo already exists this function patches its
|
||||
/// `mirror_interval` to ensure it matches (covers mirrors that were
|
||||
/// created before the interval was introduced). A 409 on the migrate
|
||||
/// POST (a race between the GET check and the POST) is also success.
|
||||
async fn ensure_mirror_repo(
|
||||
upstream: &str,
|
||||
owner: &str,
|
||||
repo: &str,
|
||||
admin_token: &str,
|
||||
) -> Result<()> {
|
||||
let repo_url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}");
|
||||
let (status, _) = forge_http(reqwest::Method::GET, &repo_url, admin_token, "").await?;
|
||||
if status.is_success() {
|
||||
// Mirror already present. Patch interval so mirrors seeded before
|
||||
// this field was introduced (or with a different value) converge.
|
||||
let patch_body = serde_json::json!({ "mirror_interval": MIRROR_INTERVAL }).to_string();
|
||||
let (patch_status, patch_text) =
|
||||
forge_http(reqwest::Method::PATCH, &repo_url, admin_token, &patch_body).await?;
|
||||
if patch_status.is_success() {
|
||||
tracing::debug!(%owner, %repo, interval = MIRROR_INTERVAL, "forge: pull-mirror interval updated");
|
||||
} else {
|
||||
tracing::warn!(
|
||||
%owner, %repo, status = %patch_status, body = %patch_text,
|
||||
"forge: failed to set mirror_interval on existing pull-mirror"
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
// serde_json::json! → the upstream URL is escaped safely (no string
|
||||
// interpolation into the JSON body).
|
||||
let body = serde_json::json!({
|
||||
"clone_addr": upstream,
|
||||
"repo_owner": owner,
|
||||
"repo_name": repo,
|
||||
"mirror": true,
|
||||
// Periodic refresh instead of on-access sync — keeps CI isolated
|
||||
// from external DNS failures at clone time.
|
||||
"interval": MIRROR_INTERVAL,
|
||||
"service": "git",
|
||||
"private": false,
|
||||
})
|
||||
.to_string();
|
||||
let url = format!("{FORGE_HTTP}/api/v1/repos/migrate");
|
||||
let (status, text) = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?;
|
||||
match status.as_u16() {
|
||||
201 => {
|
||||
tracing::info!(%owner, %repo, %upstream, interval = MIRROR_INTERVAL, "forge: created pull-mirror");
|
||||
Ok(())
|
||||
}
|
||||
// 409 = a race created it between our GET check and here (the GET
|
||||
// is the real idempotency guard). NOT 422: for the migrate endpoint
|
||||
// 422 is a validation error (bad clone_addr / service), so it must
|
||||
// surface via the bail arm, not be swallowed as "already exists".
|
||||
409 => {
|
||||
tracing::debug!(%owner, %repo, "forge: pull-mirror already exists (race)");
|
||||
Ok(())
|
||||
}
|
||||
other => {
|
||||
anyhow::bail!("POST /api/v1/repos/migrate {owner}/{repo} returned HTTP {other}: {text}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Provision the [`OPERATORS_TEAM`] inside `org` as an **empty** team.
|
||||
/// Branch protection on that org's repos references it as the
|
||||
/// merge/approval whitelist; the operator adds herself as a member via the
|
||||
/// forge UI / hivectl. `includes_all_repositories` so the gate applies to
|
||||
/// every repo in the org; `write` is enough to approve + merge. hive-c0re
|
||||
/// never manages membership. Idempotent (422/409 = already exists).
|
||||
///
|
||||
/// Must run for BOTH [`AGENTS_ORG`] and [`CONFIG_ORG`]: Gitea teams are
|
||||
/// org-scoped, so a config-repo branch-protection rule referencing
|
||||
/// `operators` needs the team to exist in `agent-configs` too. Missing it
|
||||
/// there 422'd every `apply_config_repo_branch_protection`, leaving config
|
||||
/// repos unprotected — operator-merged config PRs then bypassed the deploy
|
||||
/// pipeline and silently didn't apply.
|
||||
pub(super) async fn ensure_operators_team(org: &str, token: &str) -> Result<()> {
|
||||
let url = format!("{FORGE_HTTP}/api/v1/orgs/{org}/teams");
|
||||
let body = format!(
|
||||
r#"{{"name":"{OPERATORS_TEAM}","description":"hyperhive operators — merge gate for agent repos","permission":"write","includes_all_repositories":true,"can_create_org_repo":false}}"#
|
||||
);
|
||||
let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?;
|
||||
match status.as_u16() {
|
||||
201 => {
|
||||
tracing::info!(%org, "forge: created {OPERATORS_TEAM} team");
|
||||
Ok(())
|
||||
}
|
||||
409 | 422 => {
|
||||
tracing::debug!(%org, "forge: {OPERATORS_TEAM} team already exists");
|
||||
Ok(())
|
||||
}
|
||||
other => {
|
||||
anyhow::bail!("POST /orgs/{org}/teams ({OPERATORS_TEAM}) returned HTTP {other}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Add `user` as a collaborator on `owner/repo` at `permission`
|
||||
/// (`read` / `write` / `admin`). Idempotent: 201 (added) and 204 (already a
|
||||
/// collaborator / permission updated) both count as success.
|
||||
async fn add_collaborator(
|
||||
owner: &str,
|
||||
repo: &str,
|
||||
user: &str,
|
||||
permission: &str,
|
||||
token: &str,
|
||||
) -> Result<()> {
|
||||
let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}/collaborators/{user}");
|
||||
let body = format!(r#"{{"permission":"{permission}"}}"#);
|
||||
let (status, _) = forge_http(reqwest::Method::PUT, &url, token, &body).await?;
|
||||
match status.as_u16() {
|
||||
201 | 204 => {
|
||||
tracing::debug!(%owner, %repo, %user, %permission, "forge: collaborator set");
|
||||
Ok(())
|
||||
}
|
||||
other => {
|
||||
anyhow::bail!("PUT {owner}/{repo}/collaborators/{user} returned HTTP {other}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply the operator merge-gate branch protection to `repo`'s default
|
||||
/// branch: only [`OPERATORS_TEAM`] members can merge, and an
|
||||
/// approving review from that team is required — so the author (a write-level
|
||||
/// agent, not in the team) cannot merge its own PR. Idempotent: an existing
|
||||
/// rule for the branch (200/409/422) is treated as success.
|
||||
async fn apply_operator_branch_protection(repo: &str, token: &str) -> Result<()> {
|
||||
let url = format!("{FORGE_HTTP}/api/v1/repos/{AGENTS_ORG}/{repo}/branch_protections");
|
||||
let body = format!(
|
||||
r#"{{"branch_name":"main","enable_merge_whitelist":true,"merge_whitelist_teams":["{OPERATORS_TEAM}"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true}}"#
|
||||
);
|
||||
let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?;
|
||||
match status.as_u16() {
|
||||
201 => {
|
||||
tracing::info!(%repo, "forge: applied operator branch protection");
|
||||
Ok(())
|
||||
}
|
||||
200 | 409 | 422 => {
|
||||
tracing::debug!(%repo, "forge: branch protection already present");
|
||||
Ok(())
|
||||
}
|
||||
other => anyhow::bail!("POST {AGENTS_ORG}/{repo}/branch_protections returned HTTP {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply branch protection to an `agent-configs/<name>` repo's `main` so it
|
||||
/// can serve as the agent-editable, PR-merge config surface:
|
||||
/// - **push + merge whitelists are `core`-only** — the agent (a write
|
||||
/// collaborator) can push feature branches and open config PRs, but only
|
||||
/// hive-c0re lands on `main`, via its verify-and-ff-push merge handler
|
||||
/// (`run_merge_config_pr`). The agent can never push `main` directly.
|
||||
/// - **operator-team approval is required** to merge, and the author (not in
|
||||
/// the team) cannot self-approve.
|
||||
/// - **`enable_force_push` is `false`** — `main` only ever advances by
|
||||
/// fast-forward. The merge handler's `ff_push_to_main` is already a
|
||||
/// non-force push, so it lands fine. The legacy `push_config` mirror DOES
|
||||
/// force-push (it re-points status tags and rewinds `main` on a failed-build
|
||||
/// rollback), so the protection now rejects those non-ff updates — that
|
||||
/// mirror runs best-effort until the agent-opened PR-merge flow retires it.
|
||||
/// (Auto force-push is intentionally not allowed: per operator directive a
|
||||
/// silent force-push is a bug, not a feature.)
|
||||
///
|
||||
/// Idempotent: an existing rule for the branch (200/409/422) is success.
|
||||
async fn apply_config_repo_branch_protection(repo: &str, token: &str) -> Result<()> {
|
||||
let url = format!("{FORGE_HTTP}/api/v1/repos/{CONFIG_ORG}/{repo}/branch_protections");
|
||||
let body = format!(
|
||||
r#"{{"branch_name":"main","enable_push_whitelist":true,"push_whitelist_usernames":["core"],"enable_merge_whitelist":true,"merge_whitelist_usernames":["core"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true,"allow_manual_merge":true,"enable_force_push":false}}"#
|
||||
);
|
||||
let (status, resp_body) = forge_http(reqwest::Method::POST, &url, token, &body).await?;
|
||||
if status.as_u16() == 201 {
|
||||
tracing::info!(%repo, "forge: applied config-repo branch protection");
|
||||
return Ok(());
|
||||
}
|
||||
// Non-201 is ambiguous: it can mean "rule already exists" (idempotent
|
||||
// success) OR a silent rejection — e.g. a 422 where Forgejo refused
|
||||
// the request and created NO rule. The old code treated 200/409/422
|
||||
// all as success, so a rejected POST left the repo unprotected with
|
||||
// no error (the reported case: a new agent's config repo had no
|
||||
// `main` rule and nothing was logged). Don't trust the status code:
|
||||
// verify the `main` rule actually exists, and on failure surface the
|
||||
// POST's response body so the real reason is in the journal.
|
||||
let main_url = format!("{url}/main");
|
||||
let (check, _) = forge_http(reqwest::Method::GET, &main_url, token, "").await?;
|
||||
if check.as_u16() == 200 {
|
||||
tracing::debug!(%repo, %status, "forge: config-repo branch protection already present");
|
||||
Ok(())
|
||||
} else {
|
||||
anyhow::bail!(
|
||||
"branch protection for {CONFIG_ORG}/{repo} not applied: POST -> HTTP {status} \
|
||||
(body: {body}); GET main -> HTTP {check}, no `main` rule present",
|
||||
body = resp_body.trim(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a repo for `agent` in the c0re-owned [`AGENTS_ORG`] and wire the
|
||||
/// perms: the org owns it (perms stay c0re-managed), the agent is added
|
||||
/// as a **write** collaborator (not owner — can push + open PRs but can't
|
||||
/// bypass branch protection), and the default branch gets the operator
|
||||
/// merge gate. This is the sanctioned create path now that agents can't
|
||||
/// create repos directly (`max_repo_creation = 0`). Idempotent.
|
||||
pub async fn create_agent_repo(agent: &str, repo: &str, core_token: &str) -> Result<String> {
|
||||
ensure_org_repo(AGENTS_ORG, repo, core_token).await?;
|
||||
add_collaborator(AGENTS_ORG, repo, agent, "write", core_token).await?;
|
||||
apply_operator_branch_protection(repo, core_token).await?;
|
||||
tracing::info!(%agent, %repo, "forge: created agent repo in {AGENTS_ORG} with operator merge gate");
|
||||
Ok(format!("{AGENTS_ORG}/{repo}"))
|
||||
}
|
||||
534
hive-c0re/src/forge/users.rs
Normal file
534
hive-c0re/src/forge/users.rs
Normal file
|
|
@ -0,0 +1,534 @@
|
|||
//! Per-agent Forgejo user + access-token provisioning, account
|
||||
//! policy (email alignment, repo-creation lockdown), avatar uploads,
|
||||
//! and the bootstrap `core` admin user + token lifecycle. Shared
|
||||
//! HTTP / `forgejo admin` helpers live in the module root (`super`).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use base64::Engine;
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use super::{CONFIG_ORG, FORGE_HTTP, forge_admin, forge_http, is_present};
|
||||
|
||||
const TOKEN_NAME_PREFIX: &str = "hyperhive";
|
||||
/// Where the host-side `core` admin token lives. Used by hive-c0re
|
||||
/// itself to push the meta repo + drive admin API calls (org
|
||||
/// creation, future webhook setup, etc.). Root-only.
|
||||
const CORE_TOKEN_PATH: &str = "/var/lib/hyperhive/forge-core-token";
|
||||
// Forge provisioning markers (`forge/core-avatar-set`,
|
||||
// `forge/agent-configs-avatar-set`, `forge/email-aligned-<name>`) live
|
||||
// in `crate::paths` — one-shot guards: the upload/align runs once, the
|
||||
// marker is written, subsequent startups skip. Delete one to force its
|
||||
// step to re-run.
|
||||
// Avatar PNGs are loaded at runtime from
|
||||
// `$HIVE_ASSETS_DIR/branding/{hyperhive,agent-configs}.png` via the
|
||||
// helpers in `hive_sh4re::assets`. The `agent-configs.png` is
|
||||
// rendered from its SVG during the `hyperhive-assets` derivation's
|
||||
// build.
|
||||
/// Per-agent token scopes (broad-but-not-admin). See
|
||||
/// `docs/forge.md::Token scopes` for the per-scope rationale.
|
||||
const TOKEN_SCOPES: &str = "read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc";
|
||||
|
||||
/// Bootstrap `core` token scopes — adds `read:admin,write:admin` on
|
||||
/// top of `TOKEN_SCOPES` so the host daemon can drive
|
||||
/// `/api/v1/admin/*`. Site-admin membership alone isn't enough: the
|
||||
/// token's own scope gate runs before the user-permission check.
|
||||
/// See `docs/forge.md::Token scopes`.
|
||||
const CORE_TOKEN_SCOPES: &str = "read:admin,write:admin,read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc";
|
||||
|
||||
/// Pull the access token out of forgejo's success message. Format
|
||||
/// has shifted across versions (table form vs. "Access token was
|
||||
/// successfully created: <hex>"), so just hunt the output for the
|
||||
/// first long hex-looking word.
|
||||
fn extract_token(output: &str) -> Option<String> {
|
||||
output
|
||||
.split(|c: char| c.is_whitespace() || c == ',' || c == ':')
|
||||
.find(|w| w.len() >= 32 && w.chars().all(|c| c.is_ascii_hexdigit()))
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
/// Canonical email address for a hive agent's Forgejo account.
|
||||
/// Must match the `user.email` set by `meta::render_flake` so commits
|
||||
/// by the agent link back to their Forgejo profile page.
|
||||
fn agent_email(name: &str) -> String {
|
||||
format!("{name}@hyperhive.local")
|
||||
}
|
||||
|
||||
/// Ensure a forgejo user named `name` exists. Idempotent: forgejo
|
||||
/// returns a "user already exists" error which we treat as success.
|
||||
/// `admin` adds `--admin` (site admin) — used for the bootstrap
|
||||
/// `core` user that drives the API. `password` picks the initial
|
||||
/// account password: `None` uses `--random-password` (the existing
|
||||
/// agent provisioning shape — the password is never read, agents auth
|
||||
/// by token); `Some(pw)` uses `--password <pw>` so the operator path
|
||||
/// in `hivectl` can set a real password for matrix-style web-UI login.
|
||||
async fn ensure_user_exists(name: &str, admin: bool, password: Option<&str>) -> Result<()> {
|
||||
let email = agent_email(name);
|
||||
let mut args = vec!["user", "create", "--username", name, "--email", &email];
|
||||
match password {
|
||||
Some(pw) => args.extend(["--password", pw, "--must-change-password=false"]),
|
||||
None => args.extend(["--random-password", "--must-change-password=false"]),
|
||||
}
|
||||
if admin {
|
||||
args.push("--admin");
|
||||
}
|
||||
let result = forge_admin(&args).await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
tracing::info!(%name, "forge: created user");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
// Forgejo's "already exists" error wording varies; just
|
||||
// try the next step and let token issuance surface a
|
||||
// real failure if the user truly isn't there.
|
||||
let msg = format!("{e:#}");
|
||||
if msg.contains("already exists") || msg.contains("user already") {
|
||||
tracing::debug!(%name, "forge: user already exists");
|
||||
Ok(())
|
||||
} else {
|
||||
tracing::warn!(%name, error = %msg, "forge: user create unclear; trying token anyway");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the forgejo password for an existing user. Used by the operator
|
||||
/// path in `hivectl forge create-user --password` so re-running on an
|
||||
/// already-created account still updates the password (covers the
|
||||
/// "I forgot the password I set last week" case + the "argus retried
|
||||
/// the verb to verify the fix" case — `forgejo admin user create`
|
||||
/// silently skips a password change once the account exists). Idempotent
|
||||
/// from the operator's point of view: same password input → same final
|
||||
/// account state.
|
||||
async fn change_user_password(name: &str, password: &str) -> Result<()> {
|
||||
let args = [
|
||||
"user",
|
||||
"change-password",
|
||||
"--username",
|
||||
name,
|
||||
"--password",
|
||||
password,
|
||||
];
|
||||
forge_admin(&args)
|
||||
.await
|
||||
.with_context(|| format!("forgejo admin user change-password {name}"))?;
|
||||
tracing::info!(%name, "forge: changed user password");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Idempotently align the Forgejo account email to `agent_email(name)`.
|
||||
/// Existing agents were created with `{name}@hive.local`; this corrects
|
||||
/// that so git commits (which use `{name}@hyperhive`) link to profiles.
|
||||
/// Best-effort: failures are warned, not propagated.
|
||||
///
|
||||
/// Marker-guarded: writes `EMAIL_ALIGNED_MARKER_PREFIX{name}` on first
|
||||
/// success and skips the PATCH on all subsequent calls. This prevents
|
||||
/// Forgejo's admin-user-edit endpoint from resetting `use_custom_avatar`
|
||||
/// on every `sync_agent` tick. Delete the marker to force re-alignment.
|
||||
///
|
||||
/// Uses the admin REST API (`PATCH /api/v1/admin/users/{name}`) rather
|
||||
/// than `forgejo admin user edit` because the CLI dropped the `edit`
|
||||
/// subcommand somewhere between forgejo 8 and current. Body includes
|
||||
/// `login_name` (required by Forgejo's `EditUserOption` validator) and
|
||||
/// `source_id = 0` (local auth, the default for users hive-c0re creates).
|
||||
pub(super) async fn ensure_user_email(name: &str) {
|
||||
let marker = crate::paths::forge_email_aligned_marker(name);
|
||||
if marker.exists() {
|
||||
return;
|
||||
}
|
||||
let Some(token) = core_token() else {
|
||||
tracing::debug!(%name, "forge: skipping ensure_user_email — no core token yet");
|
||||
return;
|
||||
};
|
||||
let email = agent_email(name);
|
||||
// `login_name` is required by Forgejo's EditUserOption validator.
|
||||
// Omitting it caused Forgejo to reset use_custom_avatar on each call.
|
||||
let body = format!(r#"{{"email":"{email}","login_name":"{name}","source_id":0}}"#);
|
||||
let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}");
|
||||
match forge_http(reqwest::Method::PATCH, &url, &token, &body).await {
|
||||
Ok((status, _)) if status.is_success() => {
|
||||
if let Some(parent) = marker.parent() {
|
||||
std::fs::create_dir_all(parent).ok();
|
||||
}
|
||||
std::fs::write(&marker, "").ok();
|
||||
tracing::info!(%name, %email, "forge: user email aligned");
|
||||
}
|
||||
Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => {
|
||||
// Core token missing admin scope — see
|
||||
// `docs/forge.md::Token scopes` migration note.
|
||||
tracing::warn!(
|
||||
%name, %email, %status,
|
||||
"forge: PATCH user email forbidden — core token likely missing admin scope. \
|
||||
Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes."
|
||||
);
|
||||
}
|
||||
Ok((status, _)) => {
|
||||
tracing::warn!(%name, %email, %status, "forge: PATCH user email returned non-success");
|
||||
}
|
||||
Err(e) => tracing::warn!(%name, error = %e, "forge: PATCH user email transport error"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disable direct repo creation for agent `name` by setting
|
||||
/// `max_repo_creation = 0` on its Forgejo account. Agents must
|
||||
/// create repos *through hive-c0re* (which owns the perms), never with
|
||||
/// their own token — a write-scoped token can otherwise create + own
|
||||
/// repos and self-merge, bypassing the operator-only-merge policy.
|
||||
///
|
||||
/// `max_repo_creation = 0` means `CanCreateRepo()` is false for any
|
||||
/// count (Forgejo: `MaxRepoCreation >= 0 && NumRepos >= MaxRepoCreation`),
|
||||
/// so creation is refused while push / PR / clone stay intact. **Existing
|
||||
/// repos are untouched** — this only blocks *new* direct creation.
|
||||
///
|
||||
/// Marker-guarded like [`ensure_user_email`]: the PATCH runs once per
|
||||
/// agent (delete the marker to re-apply). Body carries `login_name` +
|
||||
/// `source_id` for the same reason `ensure_user_email` does — omitting
|
||||
/// `login_name` makes Forgejo's `EditUserOption` validator reset
|
||||
/// `use_custom_avatar`. Best-effort: failures warn, don't propagate.
|
||||
pub(super) async fn ensure_repo_creation_disabled(name: &str) {
|
||||
let marker = crate::paths::forge_repo_creation_disabled_marker(name);
|
||||
if marker.exists() {
|
||||
return;
|
||||
}
|
||||
let Some(token) = core_token() else {
|
||||
tracing::debug!(%name, "forge: skipping ensure_repo_creation_disabled — no core token yet");
|
||||
return;
|
||||
};
|
||||
let body = format!(r#"{{"login_name":"{name}","source_id":0,"max_repo_creation":0}}"#);
|
||||
let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}");
|
||||
match forge_http(reqwest::Method::PATCH, &url, &token, &body).await {
|
||||
Ok((status, _)) if status.is_success() => {
|
||||
if let Some(parent) = marker.parent() {
|
||||
std::fs::create_dir_all(parent).ok();
|
||||
}
|
||||
std::fs::write(&marker, "").ok();
|
||||
tracing::info!(%name, "forge: disabled direct repo creation (max_repo_creation=0)");
|
||||
}
|
||||
Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => {
|
||||
tracing::warn!(
|
||||
%name, %status,
|
||||
"forge: PATCH max_repo_creation forbidden — core token likely missing admin scope. \
|
||||
Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes."
|
||||
);
|
||||
}
|
||||
Ok((status, _)) => {
|
||||
tracing::warn!(%name, %status, "forge: PATCH max_repo_creation returned non-success");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(%name, error = %e, "forge: PATCH max_repo_creation transport error");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mint a fresh access token for `name`. Token name is suffixed with
|
||||
/// a monotonic clock so re-issuing doesn't collide with an existing
|
||||
/// token of the same name in the DB. `scopes` is the scope string
|
||||
/// passed to `forgejo admin user generate-access-token --scopes`;
|
||||
/// use `TOKEN_SCOPES` for agents, `CORE_TOKEN_SCOPES` for the
|
||||
/// bootstrap `core` user.
|
||||
async fn mint_token(name: &str, scopes: &str) -> Result<String> {
|
||||
let token_name = format!(
|
||||
"{TOKEN_NAME_PREFIX}-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_secs())
|
||||
);
|
||||
let stdout = forge_admin(&[
|
||||
"user",
|
||||
"generate-access-token",
|
||||
"--username",
|
||||
name,
|
||||
"--token-name",
|
||||
&token_name,
|
||||
"--scopes",
|
||||
scopes,
|
||||
])
|
||||
.await?;
|
||||
let token = extract_token(&stdout)
|
||||
.with_context(|| format!("parse token from forgejo output: {stdout:?}"))?;
|
||||
tracing::debug!(%name, %token_name, "forge: minted access token");
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Mint a fresh Forgejo access token for an agent and write it to the
|
||||
/// agent's state dir via hive-priv. hive-c0re runs unprivileged and
|
||||
/// cannot write to agent-owned (0755) state directories directly.
|
||||
async fn mint_and_persist_agent_token(name: &str) -> Result<()> {
|
||||
let token = mint_token(name, TOKEN_SCOPES).await?;
|
||||
crate::priv_client::write_agent_forge_token(name, &token)
|
||||
.await
|
||||
.with_context(|| format!("write forge-token for {name} via hive-priv"))
|
||||
}
|
||||
|
||||
/// Mint a fresh Forgejo access token for the `core` admin user and
|
||||
/// write it directly to `path`. Unlike agent tokens this path is owned
|
||||
/// by hive-c0re itself (under `/var/lib/hyperhive/`), so a direct
|
||||
/// write is both correct and necessary (no priv round-trip).
|
||||
async fn mint_and_persist_core_token(path: &Path) -> Result<()> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let token = mint_token("core", CORE_TOKEN_SCOPES).await?;
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).ok();
|
||||
}
|
||||
std::fs::write(path, format!("{token}\n"))
|
||||
.with_context(|| format!("write core token to {}", path.display()))?;
|
||||
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600));
|
||||
tracing::info!(path = %path.display(), "forge: persisted core access token");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure `name` has a forgejo user + token file. Always re-mints the
|
||||
/// token so the on-disk file always reflects the current `TOKEN_SCOPES`.
|
||||
/// Safe to call on every spawn and on every hive-c0re startup.
|
||||
pub async fn ensure_user_for(name: &str) -> Result<()> {
|
||||
if !is_present().await {
|
||||
return Ok(());
|
||||
}
|
||||
ensure_user_exists(name, false, None).await?;
|
||||
ensure_user_email(name).await;
|
||||
mint_and_persist_agent_token(name).await
|
||||
}
|
||||
|
||||
/// Provision a forgejo user for `name` and return the freshly-minted
|
||||
/// token. Unlike [`ensure_user_for`], the token is **not** persisted to
|
||||
/// disk — the caller is responsible for storing it. Used by `hivectl
|
||||
/// forge create-user` for human (non-agent) accounts so we don't create
|
||||
/// stray `/var/lib/hyperhive/agents/<name>/` directories for users that
|
||||
/// aren't agents.
|
||||
///
|
||||
/// `password` picks the account password. `None` keeps the existing
|
||||
/// random-throwaway shape (caller doesn't need web UI access — token
|
||||
/// alone is enough). `Some(pw)` sets `pw` as the password, including
|
||||
/// running `forgejo admin user change-password` if the account already
|
||||
/// exists, so the operator can log into the forge web UI afterwards.
|
||||
/// Idempotent: re-running with the same `Some(pw)` lands on the same
|
||||
/// final state.
|
||||
pub async fn provision_user_token(name: &str, password: Option<&str>) -> Result<String> {
|
||||
if !is_present().await {
|
||||
anyhow::bail!(
|
||||
"hive-forge container not running — wait for hive-c0re to start it before provisioning forge users"
|
||||
);
|
||||
}
|
||||
ensure_user_exists(name, false, password).await?;
|
||||
if let Some(pw) = password {
|
||||
// `user create` silently no-ops on an existing account, so
|
||||
// we run change-password unconditionally when the caller
|
||||
// asked for a specific password — keeps the verb idempotent
|
||||
// for "set or reset" use.
|
||||
change_user_password(name, pw).await?;
|
||||
}
|
||||
ensure_user_email(name).await;
|
||||
mint_token(name, TOKEN_SCOPES).await
|
||||
}
|
||||
|
||||
/// Set `core`'s Forgejo avatar to the hyperhive logo once, then
|
||||
/// remember it so subsequent startups don't re-upload. Best-effort
|
||||
/// — any non-2xx is logged at the caller; the project runs fine
|
||||
/// with the default hash identicon.
|
||||
pub(super) async fn ensure_core_avatar(token: &str) -> Result<()> {
|
||||
let marker = crate::paths::forge_core_avatar_marker();
|
||||
if marker.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
let png_path = hive_sh4re::assets::core_avatar_png();
|
||||
let png_bytes = tokio::fs::read(&png_path)
|
||||
.await
|
||||
.with_context(|| format!("read core avatar PNG from {}", png_path.display()))?;
|
||||
let body = format!(
|
||||
r#"{{"image":"{}"}}"#,
|
||||
base64::engine::general_purpose::STANDARD.encode(&png_bytes),
|
||||
);
|
||||
let url = format!("{FORGE_HTTP}/api/v1/admin/users/core/avatar");
|
||||
let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?;
|
||||
if !status.is_success() {
|
||||
anyhow::bail!("set core avatar: HTTP {status}");
|
||||
}
|
||||
if let Some(parent) = marker.parent() {
|
||||
std::fs::create_dir_all(parent).ok();
|
||||
}
|
||||
std::fs::write(marker, "").ok();
|
||||
tracing::info!("forge: set core user avatar to hyperhive logo");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the `agent-configs` org's Forgejo avatar to the
|
||||
/// configs-stack glyph once. Sibling to `ensure_core_avatar`:
|
||||
/// one-shot, marker-guarded, best-effort. Forgejo's per-org avatar
|
||||
/// endpoint is `POST /api/v1/orgs/{org}/avatar` with a base64-PNG
|
||||
/// JSON body — same shape as the admin user endpoint above.
|
||||
pub(super) async fn ensure_config_org_avatar(token: &str) -> Result<()> {
|
||||
let marker = crate::paths::forge_config_org_avatar_marker();
|
||||
if marker.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
let png_path = hive_sh4re::assets::config_org_avatar_png();
|
||||
let png_bytes = tokio::fs::read(&png_path)
|
||||
.await
|
||||
.with_context(|| format!("read {CONFIG_ORG} avatar PNG from {}", png_path.display()))?;
|
||||
let body = format!(
|
||||
r#"{{"image":"{}"}}"#,
|
||||
base64::engine::general_purpose::STANDARD.encode(&png_bytes),
|
||||
);
|
||||
let url = format!("{FORGE_HTTP}/api/v1/orgs/{CONFIG_ORG}/avatar");
|
||||
let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?;
|
||||
if !status.is_success() {
|
||||
anyhow::bail!("set {CONFIG_ORG} avatar: HTTP {status}");
|
||||
}
|
||||
if let Some(parent) = marker.parent() {
|
||||
std::fs::create_dir_all(parent).ok();
|
||||
}
|
||||
std::fs::write(marker, "").ok();
|
||||
tracing::info!(
|
||||
org = CONFIG_ORG,
|
||||
"forge: set org avatar to configs-stack logo"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outcome of probing whether the persisted core token still works
|
||||
/// against the *current* forge. Existence on disk is not validity: a
|
||||
/// token minted before a forge rebuild / re-provision is unknown to the
|
||||
/// new forge's DB and 401s on every call — which silently breaks the
|
||||
/// hive-ci runner-registration prefetch (it reads this same token to
|
||||
/// fetch a runner registration token).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum CoreTokenCheck {
|
||||
/// Token authenticated successfully — keep using it.
|
||||
Valid,
|
||||
/// Forge explicitly rejected the token (401/403) — re-mint.
|
||||
Invalid,
|
||||
/// Couldn't determine (forge unreachable / 5xx). Don't re-mint on a
|
||||
/// transient: keep the existing token and let a later ensure pass
|
||||
/// re-check once the forge is responsive. Re-minting here would both
|
||||
/// fail (mint needs the forge too) and churn tokens needlessly.
|
||||
Indeterminate,
|
||||
}
|
||||
|
||||
/// Map the HTTP status of the token-probe call to a [`CoreTokenCheck`].
|
||||
/// Pure so the decision logic is unit-testable without a live forge.
|
||||
fn classify_core_token_status(status: StatusCode) -> CoreTokenCheck {
|
||||
if status.is_success() {
|
||||
CoreTokenCheck::Valid
|
||||
} else if status == StatusCode::UNAUTHORIZED || status == StatusCode::FORBIDDEN {
|
||||
CoreTokenCheck::Invalid
|
||||
} else {
|
||||
CoreTokenCheck::Indeterminate
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe whether `token` is still accepted by the current forge with a
|
||||
/// cheap authenticated `GET /api/v1/user` (covered by the core token's
|
||||
/// `read:user` scope). See [`CoreTokenCheck`] for how the outcome is
|
||||
/// interpreted.
|
||||
async fn check_core_token(token: &str) -> CoreTokenCheck {
|
||||
let url = format!("{FORGE_HTTP}/api/v1/user");
|
||||
match forge_http(reqwest::Method::GET, &url, token, "").await {
|
||||
Ok((status, _)) => classify_core_token_status(status),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
error = %e,
|
||||
"forge: core-token probe could not reach forge; treating as indeterminate"
|
||||
);
|
||||
CoreTokenCheck::Indeterminate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Ensure the bootstrap `core` admin user + a token at
|
||||
/// `CORE_TOKEN_PATH`. The token is what hive-c0re uses for forgejo
|
||||
/// API calls (org creation, meta-repo push, and the hive-ci
|
||||
/// runner-registration prefetch). Returns the token.
|
||||
///
|
||||
/// Idempotent, but validity-aware: when a token file is already present
|
||||
/// it is **probed against the current forge** before being trusted. A
|
||||
/// token persisted before a forge rebuild / re-provision is stale (the
|
||||
/// new forge DB doesn't know it) and would 401 every caller — so on a
|
||||
/// definitive rejection the token is re-minted. A merely-unreachable
|
||||
/// forge leaves the existing token in place (a later ensure pass
|
||||
/// re-checks) rather than churning tokens on a transient.
|
||||
pub(super) async fn ensure_core_user_and_token() -> Result<String> {
|
||||
let path = std::path::Path::new(CORE_TOKEN_PATH);
|
||||
if let Ok(existing) = std::fs::read_to_string(path) {
|
||||
let trimmed = existing.trim().to_owned();
|
||||
if !trimmed.is_empty() {
|
||||
match check_core_token(&trimmed).await {
|
||||
CoreTokenCheck::Valid | CoreTokenCheck::Indeterminate => return Ok(trimmed),
|
||||
CoreTokenCheck::Invalid => {
|
||||
tracing::warn!(
|
||||
path = %path.display(),
|
||||
"forge: persisted core token rejected by forge (stale after rebuild?); \
|
||||
re-minting"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ensure_user_exists("core", true, None).await?;
|
||||
mint_and_persist_core_token(path).await?;
|
||||
let raw = std::fs::read_to_string(path)
|
||||
.with_context(|| format!("read {CORE_TOKEN_PATH} after mint"))?;
|
||||
Ok(raw.trim().to_owned())
|
||||
}
|
||||
|
||||
/// Read the persisted core token, or None when the forge isn't
|
||||
/// seeded yet. Cheap — just a file read.
|
||||
pub fn core_token() -> Option<String> {
|
||||
std::fs::read_to_string(CORE_TOKEN_PATH)
|
||||
.ok()
|
||||
.map(|s| s.trim().to_owned())
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CoreTokenCheck, classify_core_token_status};
|
||||
use reqwest::StatusCode;
|
||||
|
||||
#[test]
|
||||
fn success_statuses_are_valid() {
|
||||
assert_eq!(
|
||||
classify_core_token_status(StatusCode::OK),
|
||||
CoreTokenCheck::Valid
|
||||
);
|
||||
assert_eq!(
|
||||
classify_core_token_status(StatusCode::NO_CONTENT),
|
||||
CoreTokenCheck::Valid
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_rejection_statuses_are_invalid() {
|
||||
// The whole point: a stale token (forge rebuilt out from under it)
|
||||
// 401s, and 401/403 are the only outcomes that trigger a re-mint.
|
||||
assert_eq!(
|
||||
classify_core_token_status(StatusCode::UNAUTHORIZED),
|
||||
CoreTokenCheck::Invalid
|
||||
);
|
||||
assert_eq!(
|
||||
classify_core_token_status(StatusCode::FORBIDDEN),
|
||||
CoreTokenCheck::Invalid
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transient_and_unexpected_statuses_are_indeterminate() {
|
||||
// Never re-mint on a transient — minting needs the forge too, and
|
||||
// churning tokens on a blip is worse than keeping the existing one.
|
||||
for s in [
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
StatusCode::NOT_FOUND,
|
||||
] {
|
||||
assert_eq!(
|
||||
classify_core_token_status(s),
|
||||
CoreTokenCheck::Indeterminate,
|
||||
"status {s} should be indeterminate"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
491
hive-c0re/src/job_queue/exec.rs
Normal file
491
hive-c0re/src/job_queue/exec.rs
Normal file
|
|
@ -0,0 +1,491 @@
|
|||
//! Node executors — one async fn per [`NodeKind`], each a thin wrapper
|
||||
//! over existing `lifecycle.rs` / `meta.rs` / `actions.rs` code. Node
|
||||
//! executors keep their own internal error handling where it exists
|
||||
//! today (cold-start fallback inside `Reconcile`, non-fatal boot-time
|
||||
//! lock bump inside the sweep `MetaLock`, warn-only forge sync in the
|
||||
//! `Swap` tail); DAG-level failure handling is cancel-downstream in
|
||||
//! the queue.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
|
||||
use super::model::{NodeKind, State, Template};
|
||||
use super::{Claim, TerminalDag};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::power::{ReconcileAction, reconcile_action};
|
||||
|
||||
/// Max time `Drain` waits for the harness to run its stop-checkpoint
|
||||
/// turn before falling back to the hard stop. Generous — a checkpoint
|
||||
/// turn can take a while — but bounded so a wedged agent never blocks
|
||||
/// the stop indefinitely. Drains hold no build slot, so a whole-hive
|
||||
/// graceful stop overlaps every agent's drain instead of serialising
|
||||
/// N × this timeout.
|
||||
pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
|
||||
|
||||
/// Extra signal an executor hands back to the scheduler alongside
|
||||
/// success.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct NodeOutput {
|
||||
/// Agents to fan child `Rebuild` DAGs out for (`MetaLock` only).
|
||||
pub fanout: Vec<String>,
|
||||
}
|
||||
|
||||
/// Step-label + build-log sink for one claimed node.
|
||||
struct Ctx<'a> {
|
||||
coord: &'a Arc<Coordinator>,
|
||||
dag_id: u64,
|
||||
node_id: super::NodeId,
|
||||
}
|
||||
|
||||
impl Ctx<'_> {
|
||||
fn step(&self, step: &str) {
|
||||
if self
|
||||
.coord
|
||||
.job_queue
|
||||
.set_step(self.dag_id, self.node_id, step)
|
||||
{
|
||||
self.coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
|
||||
fn build_log(&self, log_id: i64) {
|
||||
if self
|
||||
.coord
|
||||
.job_queue
|
||||
.set_build_log_id(self.dag_id, self.node_id, log_id)
|
||||
{
|
||||
self.coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one claimed node to completion. Called from a task the
|
||||
/// scheduler spawns per claim; the `Result` (stringified) becomes the
|
||||
/// node's terminal state.
|
||||
pub(super) async fn run_node(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
|
||||
let ctx = Ctx {
|
||||
coord,
|
||||
dag_id: claim.dag_id,
|
||||
node_id: claim.node_id,
|
||||
};
|
||||
match &claim.kind {
|
||||
NodeKind::Prebuild { relock } => run_prebuild(coord, claim, &ctx, *relock).await,
|
||||
NodeKind::Swap => run_swap(coord, claim, &ctx).await,
|
||||
NodeKind::Create => run_create(coord, claim, &ctx).await,
|
||||
NodeKind::MetaLock { sweep, fanout } => {
|
||||
run_meta_lock(coord, claim, &ctx, *sweep, fanout.clone()).await
|
||||
}
|
||||
NodeKind::Reconcile => run_reconcile(coord, claim, &ctx).await,
|
||||
NodeKind::StopForUpdate => run_stop_for_update(coord, claim, &ctx).await,
|
||||
NodeKind::Signal => Ok(run_signal(coord, claim, &ctx)),
|
||||
NodeKind::Drain => run_drain(coord, claim, &ctx).await,
|
||||
NodeKind::WriteDropin => run_write_dropin(coord, claim).await,
|
||||
NodeKind::WritePermFile => run_write_perm_file(coord, claim, &ctx).await,
|
||||
NodeKind::ApprovalDeploy => run_approval_deploy(coord, claim).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Out-of-band toplevel build while the container keeps serving: meta
|
||||
/// sync + optional per-agent relock, then warm
|
||||
/// `system.build.toplevel` so the later `Swap` hits cache and skips
|
||||
/// straight to the profile-swap.
|
||||
async fn run_prebuild(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
relock: bool,
|
||||
) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord
|
||||
.ensure_runtime(name)
|
||||
.with_context(|| format!("ensure_runtime {name}"))?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
crate::lifecycle::prepare_rebuild_dirs(name, &paths).await?;
|
||||
// Idempotent meta sync so a manual rebuild can also recover from a
|
||||
// divergent meta repo; then bump just this agent's input. `relock =
|
||||
// false` only for meta-update cascade children, where re-locking
|
||||
// would revert the bump the cascade just committed. Both run under
|
||||
// the deploy-window gate so they can never land inside another
|
||||
// node's staged prepare→finalize window; the gate drops before the
|
||||
// (long) toplevel build, which only reads the store.
|
||||
{
|
||||
let _window = crate::meta::exclusive().await;
|
||||
let agents = crate::lifecycle::agents_for_meta_listing().await?;
|
||||
crate::meta::sync_agents(&hive, &agents).await?;
|
||||
if relock {
|
||||
crate::meta::lock_update_for_rebuild(name).await?;
|
||||
}
|
||||
}
|
||||
ctx.step("nix build");
|
||||
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
||||
crate::lifecycle::prebuild_toplevel(name, &flake_ref, &|log_id| ctx.build_log(log_id)).await?;
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
|
||||
/// `nixos-container update`, then the post-rebuild bookkeeping tail
|
||||
/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan).
|
||||
/// The recovery-start on failure is NOT here — the DAG's tail
|
||||
/// `Reconcile` runs after this node terminal ok *or* fail.
|
||||
async fn run_swap(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord.ensure_runtime(name)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
let result =
|
||||
crate::lifecycle::swap_update(name, &hive, &paths, &|step| ctx.step(step), &|log_id| {
|
||||
ctx.build_log(log_id);
|
||||
})
|
||||
.await;
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
if let Some(rev) = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
|
||||
&& let Err(e) = std::fs::write(crate::auto_update::rev_marker_path(name), rev)
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "write rev marker failed");
|
||||
}
|
||||
// The `Rebuilt` manager event fires exactly once per DAG
|
||||
// from the terminal hook — emitting ok here and letting a
|
||||
// failed tail `Reconcile` add a contradictory !ok would
|
||||
// double-report the same rebuild.
|
||||
ctx.step("forge sync");
|
||||
// Full forge + matrix sync on every successful rebuild so
|
||||
// the rebuild path is equivalent to the startup sweep:
|
||||
// tokens, config-repo mirror, meta access all recover
|
||||
// without a hive-c0re restart.
|
||||
crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
|
||||
crate::matrix::sync_agent_standalone(name).await;
|
||||
// Wake the agent on its next turn so claude sees a "you
|
||||
// were rebuilt" hint; rescan so dashboards drop the
|
||||
// "needs update" chip; lock bump → meta-inputs re-render.
|
||||
coord.kick_agent(name, "container rebuilt");
|
||||
coord.rescan_containers_and_emit().await;
|
||||
crate::dashboard::emit_meta_inputs_snapshot(coord);
|
||||
}
|
||||
Err(_) => {
|
||||
// The `Rebuilt { ok: false }` manager event fires once per
|
||||
// DAG from the terminal hook (any node may be the one that
|
||||
// failed); here only refresh the observed state.
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
}
|
||||
result.map(|()| NodeOutput::default())
|
||||
}
|
||||
|
||||
/// First-spawn provisioning + `nixos-container create` (atomic
|
||||
/// build+create — no prebuild needed).
|
||||
async fn run_create(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord.ensure_runtime(name)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
ctx.step("nixos-container create");
|
||||
// create_container registers the new agent in the meta flake
|
||||
// (sync_agents commit) before `nixos-container create` — hold the
|
||||
// deploy-window gate so that commit can't land inside another
|
||||
// node's staged deploy window.
|
||||
let _window = crate::meta::exclusive().await;
|
||||
crate::lifecycle::create_container(name, &hive, &paths).await?;
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Meta flake lock bump. Boot-sweep flavour is non-fatal (a failed
|
||||
/// bump must not cancel the fan-out rebuilds — they proceed against
|
||||
/// the current lock, exactly like today's sweep); the meta-update
|
||||
/// flavour propagates errors, and a failed bump fans out nothing.
|
||||
async fn run_meta_lock(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
sweep: bool,
|
||||
fanout: Option<Vec<String>>,
|
||||
) -> Result<NodeOutput> {
|
||||
if sweep {
|
||||
ctx.step("nix flake update hyperhive");
|
||||
let _window = crate::meta::exclusive().await;
|
||||
if let Err(e) = crate::meta::lock_update_hyperhive().await {
|
||||
tracing::warn!(error = ?e, "startup sweep: meta lock_update_hyperhive failed");
|
||||
}
|
||||
return Ok(NodeOutput {
|
||||
fanout: fanout.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
let _progress = coord.meta_update_guard();
|
||||
ctx.step("nix flake update");
|
||||
{
|
||||
let _window = crate::meta::exclusive().await;
|
||||
crate::meta::lock_update(&claim.inputs).await?;
|
||||
}
|
||||
// Lock file changed — meta-inputs panel re-renders.
|
||||
crate::dashboard::emit_meta_inputs_snapshot(coord);
|
||||
let cascade = match fanout {
|
||||
Some(list) => list,
|
||||
None => meta_update_cascade_agents(&claim.inputs).await,
|
||||
};
|
||||
Ok(NodeOutput { fanout: cascade })
|
||||
}
|
||||
|
||||
/// Idempotent power converge: `wanted` (durable intent) vs observed.
|
||||
async fn run_reconcile(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let running = crate::lifecycle::is_running(name).await;
|
||||
let wanted = coord.power.get_or_seed(name, running)?;
|
||||
match reconcile_action(wanted, running) {
|
||||
ReconcileAction::Start => {
|
||||
// Node-local transient only when the DAG holds none (the
|
||||
// boot-reconcile template); lease-window guards otherwise
|
||||
// already cover this node.
|
||||
let _guard = claim
|
||||
.transient
|
||||
.is_none()
|
||||
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Starting));
|
||||
ctx.step("nixos-container start");
|
||||
crate::lifecycle::start_with_fallback(name).await?;
|
||||
coord.kick_agent(name, "container started");
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
ReconcileAction::Stop => {
|
||||
let _guard = claim
|
||||
.transient
|
||||
.is_none()
|
||||
.then(|| coord.transient_guard(name, crate::coordinator::TransientKind::Stopping));
|
||||
ctx.step("nixos-container stop");
|
||||
crate::lifecycle::kill(name).await?;
|
||||
coord.unregister_agent(name);
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
|
||||
agent: name.clone(),
|
||||
});
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
ReconcileAction::Noop => {
|
||||
tracing::debug!(%name, wanted = wanted.as_str(), running, "reconcile: noop");
|
||||
}
|
||||
}
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Mechanical stop for the profile swap. Never *changes* `wanted`;
|
||||
/// noop when already stopped.
|
||||
async fn run_stop_for_update(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
if crate::lifecycle::is_running(name).await {
|
||||
// Seed a missing agent_power row from the PRE-stop observation
|
||||
// — the DAG's tail `Reconcile` observes only the mechanically
|
||||
// stopped state and would otherwise seed a running-but-unknown
|
||||
// agent as `Offline`, stranding it down after its own rebuild.
|
||||
if let Err(e) = coord.power.get_or_seed(name, true) {
|
||||
tracing::warn!(%name, error = ?e, "agent_power: pre-stop seed failed");
|
||||
}
|
||||
ctx.step("nixos-container stop");
|
||||
crate::lifecycle::kill(name).await?;
|
||||
coord.rescan_containers_and_emit().await;
|
||||
}
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Set the graceful fence + kick so the harness sees it promptly and
|
||||
/// runs its one stop-checkpoint turn.
|
||||
fn run_signal(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> NodeOutput {
|
||||
ctx.step("graceful stop: signalling agent");
|
||||
coord.mark_graceful_stop(&claim.agent);
|
||||
coord.kick_agent(&claim.agent, "graceful stop requested");
|
||||
NodeOutput::default()
|
||||
}
|
||||
|
||||
/// Await the harness clearing the fence (`GracefulStopComplete`) or
|
||||
/// the timeout — either way the downstream `Reconcile` proceeds with
|
||||
/// the actual stop.
|
||||
async fn run_drain(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
ctx.step("graceful stop: draining");
|
||||
let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
|
||||
while coord.is_graceful_stop_pending(name) {
|
||||
if std::time::Instant::now() >= deadline {
|
||||
tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping");
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
coord.clear_graceful_stop(name);
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
||||
async fn run_write_dropin(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
|
||||
let name = &claim.agent;
|
||||
let agent_dir = coord.ensure_runtime(name)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
crate::lifecycle::write_dropins(name, &hive, &paths).await?;
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Write + commit the perm file(s) (fused under `META_LOCK` so the
|
||||
/// working tree is never left dirty), then emit the P3RM1SS10NS-tab
|
||||
/// snapshots so the dashboard reflects the new assignment.
|
||||
async fn run_write_perm_file(
|
||||
coord: &Arc<Coordinator>,
|
||||
claim: &Claim,
|
||||
ctx: &Ctx<'_>,
|
||||
) -> Result<NodeOutput> {
|
||||
use super::model::PermPayload;
|
||||
let name = &claim.agent;
|
||||
ctx.step("writing + committing perm file");
|
||||
// Deploy-window gate: a perm commit landing inside another node's
|
||||
// staged prepare→finalize window would sweep the staged deploy
|
||||
// lock into its commit (the commits are also path-limited in
|
||||
// meta.rs — belt and braces).
|
||||
let _window = crate::meta::exclusive().await;
|
||||
match &claim.perm_payload {
|
||||
Some(PermPayload::ToolGroups { groups }) => {
|
||||
crate::meta::commit_tool_groups(name, groups)
|
||||
.await
|
||||
.with_context(|| format!("commit tool-groups for {name}"))?;
|
||||
coord.emit_tool_groups_snapshot();
|
||||
}
|
||||
Some(PermPayload::Capabilities { caps }) => {
|
||||
crate::meta::commit_capabilities(name, caps)
|
||||
.await
|
||||
.with_context(|| format!("commit capabilities for {name}"))?;
|
||||
coord.emit_capabilities_snapshot();
|
||||
}
|
||||
Some(PermPayload::Combined { groups, caps }) => {
|
||||
crate::meta::commit_perms(name, groups.as_deref(), caps.as_deref())
|
||||
.await
|
||||
.with_context(|| format!("commit perms for {name}"))?;
|
||||
if groups.is_some() {
|
||||
coord.emit_tool_groups_snapshot();
|
||||
}
|
||||
if caps.is_some() {
|
||||
coord.emit_capabilities_snapshot();
|
||||
}
|
||||
}
|
||||
None => anyhow::bail!(
|
||||
"perm_change dag {} for {name} is missing perm_payload",
|
||||
claim.dag_id
|
||||
),
|
||||
}
|
||||
Ok(NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Opaque approval deploy pipeline: `ApplyCommit` and `MergeConfigPr`
|
||||
/// both end in a container rebuild; branch on the approval row's kind
|
||||
/// (the authoritative source). The two-phase prepare/finalize/abort
|
||||
/// meta deploy — and the approval resolution — stay inside
|
||||
/// `actions.rs` in v1 (design doc §9).
|
||||
async fn run_approval_deploy(coord: &Arc<Coordinator>, claim: &Claim) -> Result<NodeOutput> {
|
||||
let approval_id = claim
|
||||
.approval_id
|
||||
.with_context(|| format!("approval_deploy dag {} has no approval_id", claim.dag_id))?;
|
||||
// Hold the deploy-window gate for the whole prepare→finalize span:
|
||||
// `prepare_deploy` stages `flake.lock` uncommitted for the entire
|
||||
// container build, and no other meta mutation may land inside that
|
||||
// window (it would sweep the staged lock and neuter `abort_deploy`).
|
||||
let _window = crate::meta::exclusive().await;
|
||||
let kind = coord
|
||||
.approvals
|
||||
.get(approval_id)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|a| a.kind);
|
||||
let result = if kind == Some(hive_sh4re::ApprovalKind::MergeConfigPr) {
|
||||
crate::actions::run_approval_merge_config_pr(coord, Some(claim.dag_id), approval_id).await
|
||||
} else {
|
||||
crate::actions::run_approval_apply_commit(coord, Some(claim.dag_id), approval_id).await
|
||||
};
|
||||
result.map(|()| NodeOutput::default())
|
||||
}
|
||||
|
||||
/// Terminal-roll-up hook, fired exactly once per DAG (node completion
|
||||
/// and cancel paths alike — the queue buffers roll-ups and the
|
||||
/// scheduler drains them). Three concerns:
|
||||
/// - approval DAGs resolve their approval row (except the opaque
|
||||
/// deploy pipeline, which resolves inside its node — unless it was
|
||||
/// cancelled while still queued and the node never ran);
|
||||
/// - non-approval rebuild-shaped DAGs emit exactly one `Rebuilt`
|
||||
/// manager event: ok on `Done`, !ok on `Failed`, none on cancel;
|
||||
/// - a cancelled power-op DAG reverts the `wanted` intent its submit
|
||||
/// wrote: the operator's cancel means "don't do it", so intent
|
||||
/// snaps back to the observed state instead of the flip executing
|
||||
/// as a surprise side effect of some later reconcile.
|
||||
pub(super) async fn on_dag_terminal(coord: &Arc<Coordinator>, terminal: &TerminalDag) {
|
||||
if terminal.state == State::Cancelled
|
||||
&& matches!(
|
||||
terminal.template,
|
||||
Template::Start | Template::Stop | Template::GracefulStop | Template::Restart
|
||||
)
|
||||
{
|
||||
let running = crate::lifecycle::is_running(&terminal.agent).await;
|
||||
if let Err(e) = coord
|
||||
.power
|
||||
.set(&terminal.agent, crate::power::Wanted::from_running(running))
|
||||
{
|
||||
tracing::warn!(agent = %terminal.agent, error = ?e, "agent_power: cancel revert failed");
|
||||
}
|
||||
}
|
||||
if terminal.approval_id.is_some() {
|
||||
crate::actions::resolve_approval_dag(coord, terminal).await;
|
||||
return;
|
||||
}
|
||||
if matches!(terminal.template, Template::Rebuild | Template::PermChange) {
|
||||
match terminal.state {
|
||||
State::Done => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: terminal.agent.clone(),
|
||||
ok: true,
|
||||
note: None,
|
||||
sha: None,
|
||||
tag: None,
|
||||
}),
|
||||
State::Failed => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: terminal.agent.clone(),
|
||||
ok: false,
|
||||
note: terminal.error.clone(),
|
||||
sha: None,
|
||||
tag: None,
|
||||
}),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute which agents a `nix flake update <inputs>` on the meta
|
||||
/// flake affects — the fan-out set for `MetaUpdate` DAGs. Empty
|
||||
/// `inputs` or any input under `hyperhive` → every container;
|
||||
/// otherwise just the agents named by `agent-<name>` inputs.
|
||||
/// Topology-sorted so parents rebuild before their children.
|
||||
pub async fn meta_update_cascade_agents(inputs: &[String]) -> Vec<String> {
|
||||
let touched_hyperhive = inputs
|
||||
.iter()
|
||||
.any(|i| i == "hyperhive" || i.starts_with("hyperhive/"));
|
||||
let touched_agents: Vec<String> = inputs
|
||||
.iter()
|
||||
.filter_map(|i| i.strip_prefix("agent-"))
|
||||
.map(|rest| rest.split('/').next().unwrap_or(rest).to_owned())
|
||||
.collect();
|
||||
let mut names = if touched_hyperhive || inputs.is_empty() {
|
||||
crate::lifecycle::list()
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|c| {
|
||||
c.strip_prefix(crate::lifecycle::AGENT_PREFIX)
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
touched_agents
|
||||
};
|
||||
let topo = crate::topology::read();
|
||||
crate::auto_update::topology_sort(&mut names, &topo);
|
||||
names
|
||||
}
|
||||
592
hive-c0re/src/job_queue/mod.rs
Normal file
592
hive-c0re/src/job_queue/mod.rs
Normal file
|
|
@ -0,0 +1,592 @@
|
|||
//! Generic job-DAG queue + desired-state reconciliation — replaces the
|
||||
//! old flat `rebuild_queue`. Jobs are nodes in per-request DAGs (see
|
||||
//! [`templates`]); the special cases (graceful-stop watcher thread,
|
||||
//! deferred-start follow-up, meta-update cascade) collapse into DAG
|
||||
//! *shapes* over a shared set of primitive nodes ([`model::NodeKind`]).
|
||||
//!
|
||||
//! Concurrency is gated by two resource classes:
|
||||
//! 1. **Build slots** — N permits (`services.hyperhive.c0re.buildSlots`,
|
||||
//! default 1) held by nix-heavy nodes for the node's duration.
|
||||
//! 2. **Per-agent lifecycle lease** — DAG-scoped: acquired before the
|
||||
//! DAG's first container-affecting node runs, held until the DAG is
|
||||
//! terminal, so two lifecycle DAGs for one agent never interleave
|
||||
//! their container ops.
|
||||
//!
|
||||
//! The meta *repo* is serialized by `meta::META_LOCK` inside the
|
||||
//! executors themselves. Per-agent power *intent* (`wanted`) lives in
|
||||
//! the durable [`crate::power`] store; the DAGs are the reconcile
|
||||
//! mechanism. Design + rationale: `docs/coordinator.md::Job queue`.
|
||||
|
||||
pub mod exec;
|
||||
pub mod model;
|
||||
pub mod scheduler;
|
||||
pub mod submit;
|
||||
pub mod templates;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
pub use model::{
|
||||
Dag, DagSpec, DagView, DepWhen, Node, NodeId, NodeKind, PermPayload, Source, State, Template,
|
||||
};
|
||||
|
||||
/// How many terminal DAGs (`Done` / `Failed` / `Cancelled`) to retain
|
||||
/// per template in the snapshot, matching the old per-kind history cap.
|
||||
const MAX_HISTORY_PER_TEMPLATE: usize = 5;
|
||||
|
||||
/// Terminal DAGs younger than this are exempt from the per-template
|
||||
/// history cap. A broad `hivectl stop`/`start` submits many
|
||||
/// same-template DAGs that can all settle within one poll interval —
|
||||
/// without the grace, the cap would evict some before the ~1s
|
||||
/// `QueueDag` poller ever observes their terminal state, silently
|
||||
/// swallowing failures.
|
||||
const HISTORY_GRACE_SECS: i64 = 300;
|
||||
|
||||
/// Cap on stored node error strings.
|
||||
const MAX_ERROR_LEN: usize = 2_000;
|
||||
|
||||
/// A node claimed for execution — everything the executor needs,
|
||||
/// snapshotted at claim time.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Claim {
|
||||
pub dag_id: u64,
|
||||
pub node_id: NodeId,
|
||||
pub kind: NodeKind,
|
||||
pub agent: String,
|
||||
pub template: Template,
|
||||
pub source: Source,
|
||||
pub approval_id: Option<i64>,
|
||||
pub inputs: Vec<String>,
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
/// True when claiming this node acquired the DAG's agent lease —
|
||||
/// the scheduler creates the DAG-scoped transient guard on this
|
||||
/// edge.
|
||||
pub lease_acquired: bool,
|
||||
/// Transient pill kind for the lease window (from the spec).
|
||||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
}
|
||||
|
||||
/// Summary of a DAG that just reached its terminal roll-up state —
|
||||
/// input to the approval-resolution hook and the lease/transient
|
||||
/// release.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TerminalDag {
|
||||
pub dag_id: u64,
|
||||
pub template: Template,
|
||||
pub agent: String,
|
||||
pub approval_id: Option<i64>,
|
||||
pub state: State,
|
||||
/// First failed node's error when `state == Failed`.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Inner {
|
||||
dags: VecDeque<Dag>,
|
||||
next_id: u64,
|
||||
build_slots: usize,
|
||||
slots_used: usize,
|
||||
/// agent → dag id currently holding that agent's lifecycle lease.
|
||||
leases: HashMap<String, u64>,
|
||||
/// Terminal roll-ups not yet consumed by the scheduler
|
||||
/// ([`JobQueue::drain_terminal`]). Fed by every path that settles
|
||||
/// state — node completion AND the cancel surfaces — so the
|
||||
/// terminal hooks (approval resolution, intent revert, transient
|
||||
/// release) fire exactly once per DAG no matter how it ended.
|
||||
pending_terminal: Vec<TerminalDag>,
|
||||
}
|
||||
|
||||
/// The queue. Lives on `Coordinator` (one per hive-c0re process); a
|
||||
/// single scheduler task ([`scheduler::run_worker`]) drives it —
|
||||
/// concurrency comes from the build-slot count, not multiple workers.
|
||||
#[derive(Debug)]
|
||||
pub struct JobQueue {
|
||||
inner: Mutex<Inner>,
|
||||
/// Wakes the scheduler when something new arrives or state changed.
|
||||
pub(crate) notify: Notify,
|
||||
}
|
||||
|
||||
impl Default for JobQueue {
|
||||
fn default() -> Self {
|
||||
Self::new(1)
|
||||
}
|
||||
}
|
||||
|
||||
impl JobQueue {
|
||||
pub fn new(build_slots: usize) -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(Inner {
|
||||
build_slots: build_slots.max(1),
|
||||
..Inner::default()
|
||||
}),
|
||||
notify: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit a DAG. Validates the spec (cycle rejection) and dedups
|
||||
/// against non-started DAGs; returns the DAG id (newly-allocated,
|
||||
/// or the existing DAG's id with the new reason appended).
|
||||
///
|
||||
/// Dedup: a DAG whose roll-up is still `Queued` (no node started)
|
||||
/// with the same `(template, agent, parent_id, approval_id)` — plus
|
||||
/// `inputs` for `MetaUpdate` and the perm-type discriminant for
|
||||
/// `PermChange` — swallows the repeat. `parent_id` is part of the
|
||||
/// key so a meta-update cascade rebuild never collapses into a
|
||||
/// standalone or sweep rebuild. Running / terminal DAGs never
|
||||
/// dedup — operators are free to re-queue.
|
||||
pub fn submit(&self, spec: DagSpec) -> anyhow::Result<u64> {
|
||||
templates::validate(&spec)?;
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
if let Some(existing) = Self::dedup_target(&mut inner, &spec) {
|
||||
if !existing.reason.contains(&spec.reason) {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(existing.reason, "\nalso requested by: {}", spec.reason);
|
||||
}
|
||||
return Ok(existing.id);
|
||||
}
|
||||
let id = Self::push_dag(&mut inner, spec);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Append fan-out children under a parent DAG (meta-update / sweep
|
||||
/// cascade). Applies the same dedup as [`Self::submit`]; returns
|
||||
/// the child ids actually created or coalesced into.
|
||||
pub fn append_children(&self, specs: Vec<DagSpec>) -> Vec<u64> {
|
||||
let mut ids = Vec::with_capacity(specs.len());
|
||||
for spec in specs {
|
||||
match self.submit(spec) {
|
||||
Ok(id) => ids.push(id),
|
||||
Err(e) => tracing::error!(error = ?e, "job_queue: invalid fan-out child spec"),
|
||||
}
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
fn dedup_target<'a>(inner: &'a mut Inner, spec: &DagSpec) -> Option<&'a mut Dag> {
|
||||
inner.dags.iter_mut().find(|d| {
|
||||
d.rollup() == State::Queued
|
||||
&& d.template == spec.template
|
||||
&& d.agent == spec.agent
|
||||
&& d.parent_id == spec.parent_id
|
||||
&& d.approval_id == spec.approval_id
|
||||
&& (d.template != Template::MetaUpdate || d.inputs == spec.inputs)
|
||||
&& model::perm_payload_same_type(
|
||||
d.perm_payload.as_ref(),
|
||||
spec.perm_payload.as_ref(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn push_dag(inner: &mut Inner, spec: DagSpec) -> u64 {
|
||||
inner.next_id += 1;
|
||||
let id = inner.next_id;
|
||||
let nodes = spec
|
||||
.nodes
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, n)| Node {
|
||||
id: u32::try_from(i).unwrap_or(u32::MAX),
|
||||
kind: n.kind,
|
||||
deps: n.deps,
|
||||
state: State::Queued,
|
||||
step: None,
|
||||
build_log_id: None,
|
||||
started_at: None,
|
||||
finished_at: None,
|
||||
error: None,
|
||||
})
|
||||
.collect();
|
||||
inner.dags.push_back(Dag {
|
||||
id,
|
||||
template: spec.template,
|
||||
agent: spec.agent,
|
||||
source: spec.source,
|
||||
reason: spec.reason,
|
||||
parent_id: spec.parent_id,
|
||||
approval_id: spec.approval_id,
|
||||
inputs: spec.inputs,
|
||||
perm_payload: spec.perm_payload,
|
||||
transient: spec.transient,
|
||||
created_at: now_unix(),
|
||||
nodes,
|
||||
terminal_reported: false,
|
||||
});
|
||||
id
|
||||
}
|
||||
|
||||
/// Claim every currently-ready node, acquiring resources, and mark
|
||||
/// them `Running`. A node is ready when it's `Queued`, every dep is
|
||||
/// satisfied (`AfterOk`: dep `Done`; `AfterAny`: dep terminal), and
|
||||
/// its resources are free (build slot; agent lease free or already
|
||||
/// held by this DAG). Iteration is in DAG-submit order, so
|
||||
/// simultaneously-ready nodes compete FIFO — bulk operations drain
|
||||
/// predictably.
|
||||
pub fn claim_ready(&self) -> Vec<Claim> {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
Self::propagate_cancellations(&mut inner);
|
||||
let mut claims = Vec::new();
|
||||
let inner = &mut *inner;
|
||||
for di in 0..inner.dags.len() {
|
||||
// Split-borrow dance: deps are checked against the same
|
||||
// DAG's other nodes, so snapshot the states first.
|
||||
let dag = &inner.dags[di];
|
||||
let dag_id = dag.id;
|
||||
let ready_ids: Vec<NodeId> = dag
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|n| n.state == State::Queued && Self::deps_satisfied(dag, n))
|
||||
.map(|n| n.id)
|
||||
.collect();
|
||||
for node_id in ready_ids {
|
||||
let dag = &inner.dags[di];
|
||||
let node = dag.node(node_id).expect("node id from same dag");
|
||||
let needs_slot = node.kind.needs_build_slot();
|
||||
if needs_slot && inner.slots_used >= inner.build_slots {
|
||||
continue;
|
||||
}
|
||||
let mut lease_acquired = false;
|
||||
if node.kind.needs_lease() {
|
||||
match inner.leases.get(dag.agent.as_str()) {
|
||||
Some(&holder) if holder != dag_id => continue,
|
||||
Some(_) => {}
|
||||
None => {
|
||||
inner.leases.insert(dag.agent.clone(), dag_id);
|
||||
lease_acquired = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if needs_slot {
|
||||
inner.slots_used += 1;
|
||||
}
|
||||
let dag = &mut inner.dags[di];
|
||||
let claim = Claim {
|
||||
dag_id,
|
||||
node_id,
|
||||
kind: dag.node(node_id).expect("node").kind.clone(),
|
||||
agent: dag.agent.clone(),
|
||||
template: dag.template,
|
||||
source: dag.source,
|
||||
approval_id: dag.approval_id,
|
||||
inputs: dag.inputs.clone(),
|
||||
perm_payload: dag.perm_payload.clone(),
|
||||
lease_acquired,
|
||||
transient: dag.transient,
|
||||
};
|
||||
let node = dag.node_mut(node_id).expect("node");
|
||||
node.state = State::Running;
|
||||
node.started_at = Some(now_unix());
|
||||
claims.push(claim);
|
||||
}
|
||||
}
|
||||
claims
|
||||
}
|
||||
|
||||
fn deps_satisfied(dag: &Dag, node: &Node) -> bool {
|
||||
node.deps.iter().all(|dep| {
|
||||
dag.node(dep.on).is_some_and(|d| match dep.when {
|
||||
DepWhen::AfterOk => d.state == State::Done,
|
||||
DepWhen::AfterAny => d.state.is_terminal(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Cancel-downstream: a `Queued` node with an `AfterOk` dep that
|
||||
/// `Failed` / `Cancelled` becomes `Cancelled` itself. Loops to a
|
||||
/// fixpoint so the cancellation cascades through chains.
|
||||
fn propagate_cancellations(inner: &mut Inner) {
|
||||
for dag in &mut inner.dags {
|
||||
loop {
|
||||
let doomed: Vec<NodeId> = dag
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|n| {
|
||||
n.state == State::Queued
|
||||
&& n.deps.iter().any(|dep| {
|
||||
dep.when == DepWhen::AfterOk
|
||||
&& dag.node(dep.on).is_some_and(|d| {
|
||||
matches!(d.state, State::Failed | State::Cancelled)
|
||||
})
|
||||
})
|
||||
})
|
||||
.map(|n| n.id)
|
||||
.collect();
|
||||
if doomed.is_empty() {
|
||||
break;
|
||||
}
|
||||
let now = now_unix();
|
||||
for id in doomed {
|
||||
if let Some(n) = dag.node_mut(id) {
|
||||
n.state = State::Cancelled;
|
||||
n.finished_at = Some(now);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark a claimed node terminal, release its build slot, cascade
|
||||
/// cancellations, and settle terminal DAGs (lease release + history
|
||||
/// trim; the terminal roll-up lands in the [`Self::drain_terminal`]
|
||||
/// buffer). `error` is stored (truncated) when `result` is `Err`.
|
||||
pub fn complete_node(&self, dag_id: u64, node_id: NodeId, result: Result<(), String>) {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
if let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id)
|
||||
&& let Some(node) = dag.node_mut(node_id)
|
||||
&& node.state == State::Running
|
||||
{
|
||||
let needs_slot = node.kind.needs_build_slot();
|
||||
node.finished_at = Some(now_unix());
|
||||
node.step = None;
|
||||
match result {
|
||||
Ok(()) => node.state = State::Done,
|
||||
Err(e) => {
|
||||
node.state = State::Failed;
|
||||
let mut msg = e;
|
||||
if msg.len() > MAX_ERROR_LEN {
|
||||
msg.truncate(
|
||||
(0..=MAX_ERROR_LEN)
|
||||
.rev()
|
||||
.find(|i| msg.is_char_boundary(*i))
|
||||
.unwrap_or(0),
|
||||
);
|
||||
msg.push('…');
|
||||
}
|
||||
node.error = Some(msg);
|
||||
}
|
||||
}
|
||||
if needs_slot {
|
||||
inner.slots_used = inner.slots_used.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
}
|
||||
|
||||
/// Take the terminal roll-ups accumulated since the last drain.
|
||||
/// The scheduler calls this after every wakeup and runs the
|
||||
/// terminal hooks on each entry.
|
||||
pub fn drain_terminal(&self) -> Vec<TerminalDag> {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
std::mem::take(&mut inner.pending_terminal)
|
||||
}
|
||||
|
||||
/// Propagate cancellations, release the leases of newly-terminal
|
||||
/// DAGs, buffer each terminal roll-up exactly once (the
|
||||
/// `terminal_reported` flag) for [`Self::drain_terminal`], and trim
|
||||
/// history.
|
||||
fn settle(inner: &mut Inner) {
|
||||
Self::propagate_cancellations(inner);
|
||||
let mut freed: Vec<String> = Vec::new();
|
||||
let mut reports: Vec<TerminalDag> = Vec::new();
|
||||
for dag in &mut inner.dags {
|
||||
if !dag.is_terminal() || dag.terminal_reported {
|
||||
continue;
|
||||
}
|
||||
dag.terminal_reported = true;
|
||||
if inner.leases.get(dag.agent.as_str()) == Some(&dag.id) {
|
||||
freed.push(dag.agent.clone());
|
||||
}
|
||||
reports.push(TerminalDag {
|
||||
dag_id: dag.id,
|
||||
template: dag.template,
|
||||
agent: dag.agent.clone(),
|
||||
approval_id: dag.approval_id,
|
||||
state: dag.rollup(),
|
||||
error: dag.first_error().map(str::to_owned),
|
||||
});
|
||||
}
|
||||
inner.pending_terminal.append(&mut reports);
|
||||
for agent in freed {
|
||||
inner.leases.remove(&agent);
|
||||
}
|
||||
Self::trim_history(inner, now_unix() - HISTORY_GRACE_SECS);
|
||||
}
|
||||
|
||||
/// Cancel a DAG that hasn't started yet (roll-up `Queued`): every
|
||||
/// node flips to `Cancelled`. No-op (returns `false`) once any node
|
||||
/// is running or terminal — an in-flight nix build isn't
|
||||
/// interruptible, matching the old queue's rule.
|
||||
pub fn cancel(&self, dag_id: u64) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(dag) = inner.dags.iter_mut().find(|d| d.id == dag_id) else {
|
||||
return false;
|
||||
};
|
||||
if dag.rollup() != State::Queued {
|
||||
return false;
|
||||
}
|
||||
let now = now_unix();
|
||||
for n in &mut dag.nodes {
|
||||
n.state = State::Cancelled;
|
||||
n.finished_at = Some(now);
|
||||
}
|
||||
// Settle buffers the terminal roll-up; the notify wakes the
|
||||
// scheduler, which drains it and fires the terminal hooks
|
||||
// (approval resolution, power-intent revert).
|
||||
Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
true
|
||||
}
|
||||
|
||||
/// Cancel every still-fully-queued child DAG of `parent`. Running
|
||||
/// children are left alone. Returns the count of cancelled DAGs.
|
||||
pub fn cancel_children(&self, parent: u64) -> usize {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let now = now_unix();
|
||||
let mut count = 0;
|
||||
for dag in &mut inner.dags {
|
||||
if dag.parent_id == Some(parent) && dag.rollup() == State::Queued {
|
||||
for n in &mut dag.nodes {
|
||||
n.state = State::Cancelled;
|
||||
n.finished_at = Some(now);
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
if count > 0 {
|
||||
Self::settle(&mut inner);
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
}
|
||||
count
|
||||
}
|
||||
|
||||
/// Set the step label on a `Running` node. Returns `true` when the
|
||||
/// label actually changed (callers emit a snapshot only then).
|
||||
pub fn set_step(&self, dag_id: u64, node_id: NodeId, step: &str) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.node_mut(node_id))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if node.state != State::Running || node.step.as_deref() == Some(step) {
|
||||
return false;
|
||||
}
|
||||
node.step = Some(step.to_owned());
|
||||
true
|
||||
}
|
||||
|
||||
/// Set the step label on the DAG's currently-running node —
|
||||
/// compatibility surface for the opaque approval pipeline, whose
|
||||
/// callbacks only know the DAG id. Single-node approval DAGs make
|
||||
/// this exact.
|
||||
pub fn set_step_running(&self, dag_id: u64, step: &str) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if node.step.as_deref() == Some(step) {
|
||||
return false;
|
||||
}
|
||||
node.step = Some(step.to_owned());
|
||||
true
|
||||
}
|
||||
|
||||
/// Link a `build_logs` row to a specific `Running` node.
|
||||
pub fn set_build_log_id(&self, dag_id: u64, node_id: NodeId, log_id: i64) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.node_mut(node_id))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if node.state != State::Running {
|
||||
return false;
|
||||
}
|
||||
node.build_log_id = Some(log_id);
|
||||
true
|
||||
}
|
||||
|
||||
/// Link a `build_logs` row to the DAG's currently-running node —
|
||||
/// DAG-id-only compatibility surface (approval pipeline callbacks).
|
||||
pub fn set_build_log_id_running(&self, dag_id: u64, log_id: i64) -> bool {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
let Some(node) = inner
|
||||
.dags
|
||||
.iter_mut()
|
||||
.find(|d| d.id == dag_id)
|
||||
.and_then(|d| d.nodes.iter_mut().find(|n| n.state == State::Running))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
node.build_log_id = Some(log_id);
|
||||
true
|
||||
}
|
||||
|
||||
/// Snapshot every DAG for `/api/state` + `RebuildQueueChanged`.
|
||||
pub fn snapshot(&self) -> Vec<DagView> {
|
||||
let inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
inner.dags.iter().map(Dag::view).collect()
|
||||
}
|
||||
|
||||
/// Number of live (non-terminal) DAGs — used by tests and
|
||||
/// diagnostics.
|
||||
#[cfg(test)]
|
||||
pub fn live_count(&self) -> usize {
|
||||
let inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
inner.dags.iter().filter(|d| !d.is_terminal()).count()
|
||||
}
|
||||
|
||||
/// Keep only the newest `MAX_HISTORY_PER_TEMPLATE` terminal DAGs
|
||||
/// per template. Never evicted: live DAGs; terminal parents with
|
||||
/// live children (a fan-out parent is terminal the moment its
|
||||
/// `MetaLock` completes — evicting it while cascade rebuilds run
|
||||
/// would orphan their dashboard group); and terminal DAGs that
|
||||
/// finished after `grace_cutoff` (see [`HISTORY_GRACE_SECS`]).
|
||||
fn trim_history(inner: &mut Inner, grace_cutoff: i64) {
|
||||
let live_parents: std::collections::HashSet<u64> = inner
|
||||
.dags
|
||||
.iter()
|
||||
.filter(|d| !d.is_terminal())
|
||||
.filter_map(|d| d.parent_id)
|
||||
.collect();
|
||||
let mut counts: HashMap<Template, usize> = HashMap::new();
|
||||
let kept: Vec<Dag> = inner
|
||||
.dags
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|d| {
|
||||
if !d.is_terminal() || live_parents.contains(&d.id) {
|
||||
return true;
|
||||
}
|
||||
let finished = d.nodes.iter().filter_map(|n| n.finished_at).max();
|
||||
if finished.is_none_or(|t| t > grace_cutoff) {
|
||||
return true;
|
||||
}
|
||||
let n = counts.entry(d.template).or_insert(0);
|
||||
*n += 1;
|
||||
*n <= MAX_HISTORY_PER_TEMPLATE
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
inner.dags = kept.into_iter().rev().collect();
|
||||
}
|
||||
|
||||
/// Test hook: trim with the grace window disabled, so eviction
|
||||
/// behavior is assertable without aging real timestamps.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn trim_ignoring_grace(&self) {
|
||||
let mut inner = self.inner.lock().expect("job_queue mutex poisoned");
|
||||
Self::trim_history(&mut inner, i64::MAX);
|
||||
}
|
||||
}
|
||||
332
hive-c0re/src/job_queue/model.rs
Normal file
332
hive-c0re/src/job_queue/model.rs
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
//! Data model for the generic job-DAG queue: node kinds (the primitive
|
||||
//! operations), dependency edges, and the runtime `Dag` / `Node` store.
|
||||
//! The serialized *views* — `DagView` / `NodeView` plus the `Template`
|
||||
//! / `Source` / `State` / `PermPayload` wire enums — live in
|
||||
//! `hive_sh4re::jobs` (wire types belong to the shared crate) and are
|
||||
//! re-exported here for the queue's internal use.
|
||||
//!
|
||||
//! Two levels: the **DAG** is the unit of dedup / cancel /
|
||||
//! approval-resolution and the dashboard group; the **node** is the
|
||||
//! unit of scheduling / execution / build-log / step label. See
|
||||
//! `docs/coordinator.md::Job queue` for the full design.
|
||||
|
||||
pub use hive_sh4re::jobs::{DagView, NodeId, NodeView, PermPayload, Source, State, Template};
|
||||
use serde::Serialize;
|
||||
|
||||
/// Dedup compares the perm *type*, not the value — a tool-groups
|
||||
/// change and a capabilities change for the same agent are distinct
|
||||
/// operations that must not collapse.
|
||||
pub(super) fn perm_payload_same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool {
|
||||
matches!(
|
||||
(a, b),
|
||||
(
|
||||
Some(PermPayload::ToolGroups { .. }),
|
||||
Some(PermPayload::ToolGroups { .. })
|
||||
) | (
|
||||
Some(PermPayload::Capabilities { .. }),
|
||||
Some(PermPayload::Capabilities { .. })
|
||||
) | (
|
||||
Some(PermPayload::Combined { .. }),
|
||||
Some(PermPayload::Combined { .. })
|
||||
) | (None, None)
|
||||
)
|
||||
}
|
||||
|
||||
/// When a dependency edge is considered satisfied.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum DepWhen {
|
||||
/// Dep must reach `Done`. A `Failed` / `Cancelled` dep cancels this
|
||||
/// node (cancel-downstream).
|
||||
AfterOk,
|
||||
/// Dep must merely reach a terminal state (ok *or* fail). Used only
|
||||
/// by `rebuild`'s tail `Reconcile` so the recovery-start runs even
|
||||
/// when `Swap` failed.
|
||||
AfterAny,
|
||||
}
|
||||
|
||||
/// A dependency edge (intra-DAG only — cross-DAG ordering comes from
|
||||
/// the per-agent lease + dedup, never from edges between DAGs).
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub struct Dep {
|
||||
pub on: NodeId,
|
||||
pub when: DepWhen,
|
||||
}
|
||||
|
||||
/// The primitive operations — each kind maps to one executor fn in
|
||||
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
|
||||
/// code. Concurrency is gated by two resource classes (see
|
||||
/// [`NodeKind::needs_build_slot`] / [`NodeKind::needs_lease`]); the
|
||||
/// meta *repo* is serialized by `meta::META_LOCK` inside the wrapped
|
||||
/// functions themselves, which is why there is no `GitCommit` node —
|
||||
/// a standalone commit node would open a dirty-working-tree window
|
||||
/// between nodes that the fused `meta.rs` ops deliberately close.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum NodeKind {
|
||||
/// Out-of-band toplevel build while the container keeps serving:
|
||||
/// meta `sync_agents`, optional per-agent relock, then
|
||||
/// `lifecycle::prebuild_toplevel`. `relock = false` only for
|
||||
/// meta-update cascade rebuilds (re-locking would revert the bump
|
||||
/// the cascade just committed).
|
||||
Prebuild { relock: bool },
|
||||
/// `nixos-container update` profile-swap (requires the container
|
||||
/// stopped). Re-applies nspawn flags + resource limits first —
|
||||
/// rebuild is the reconcile verb — and carries the post-rebuild
|
||||
/// bookkeeping tail (rev marker, forge/matrix sync, kick, rescan).
|
||||
Swap,
|
||||
/// First-spawn `nixos-container create` plus the pre-create
|
||||
/// provisioning (proposed/applied repos, state subvolume, meta
|
||||
/// registration).
|
||||
Create,
|
||||
/// Meta flake lock bump. `sweep = false`: `meta::lock_update`
|
||||
/// (commit fused, under `META_LOCK`) with the DAG's `inputs`;
|
||||
/// `sweep = true`: `meta::lock_update_hyperhive`, *non-fatal* (a
|
||||
/// failed boot-time bump must not cancel the fan-out rebuilds).
|
||||
/// On success the scheduler appends child `Rebuild` DAGs: the
|
||||
/// precomputed `fanout` list when present (boot sweep), else the
|
||||
/// post-bump affected set (`meta_update_cascade_agents`).
|
||||
MetaLock {
|
||||
sweep: bool,
|
||||
fanout: Option<Vec<String>>,
|
||||
},
|
||||
/// Idempotent power converge: read `wanted` + observed state;
|
||||
/// start if `Up` & down (with cold-start fallback), stop if
|
||||
/// `Offline` & up, else noop.
|
||||
Reconcile,
|
||||
/// Mechanical `nixos-container stop` for the profile swap. Never
|
||||
/// touches `wanted`. Noop if already stopped.
|
||||
StopForUpdate,
|
||||
/// Set the graceful-stop fence + kick the harness so it runs one
|
||||
/// stop-checkpoint turn.
|
||||
Signal,
|
||||
/// Await the harness clearing the fence, bounded by
|
||||
/// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
|
||||
/// downstream `Reconcile` performs the actual stop.
|
||||
Drain,
|
||||
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
||||
WriteDropin,
|
||||
/// Commit `tool-groups.json` / `capabilities.json` per the DAG's
|
||||
/// `perm_payload` (commit fused under `META_LOCK`).
|
||||
WritePermFile,
|
||||
/// Opaque approval deploy pipeline (`ApplyCommit` /
|
||||
/// `MergeConfigPr`): the two-phase prepare/finalize/abort meta
|
||||
/// deploy stays inside `actions.rs` in v1 — deliberately not
|
||||
/// modeled as scheduler nodes (see the design doc §9).
|
||||
ApprovalDeploy,
|
||||
}
|
||||
|
||||
impl NodeKind {
|
||||
/// Wire string for `NodeView.kind`.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
NodeKind::Prebuild { .. } => "prebuild",
|
||||
NodeKind::Swap => "swap",
|
||||
NodeKind::Create => "create",
|
||||
NodeKind::MetaLock { .. } => "meta_lock",
|
||||
NodeKind::Reconcile => "reconcile",
|
||||
NodeKind::StopForUpdate => "stop_for_update",
|
||||
NodeKind::Signal => "signal",
|
||||
NodeKind::Drain => "drain",
|
||||
NodeKind::WriteDropin => "write_dropin",
|
||||
NodeKind::WritePermFile => "write_perm_file",
|
||||
NodeKind::ApprovalDeploy => "approval_deploy",
|
||||
}
|
||||
}
|
||||
|
||||
/// Nix-heavy kinds hold one of the `buildSlots` semaphore permits
|
||||
/// for the node's duration.
|
||||
pub fn needs_build_slot(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
NodeKind::Prebuild { .. }
|
||||
| NodeKind::Swap
|
||||
| NodeKind::Create
|
||||
| NodeKind::MetaLock { .. }
|
||||
| NodeKind::ApprovalDeploy
|
||||
)
|
||||
}
|
||||
|
||||
/// Container-affecting kinds require the DAG to hold the agent's
|
||||
/// lifecycle lease (acquired at the first such node, held until the
|
||||
/// DAG is terminal). Lease-exempt kinds (`Prebuild`, `MetaLock`,
|
||||
/// `WritePermFile`) touch the store / meta repo, not the running
|
||||
/// container — which is exactly why a `Prebuild` can overlap
|
||||
/// another DAG's work on the same agent.
|
||||
pub fn needs_lease(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
NodeKind::Swap
|
||||
| NodeKind::Create
|
||||
| NodeKind::Reconcile
|
||||
| NodeKind::StopForUpdate
|
||||
| NodeKind::Signal
|
||||
| NodeKind::Drain
|
||||
| NodeKind::WriteDropin
|
||||
| NodeKind::ApprovalDeploy
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// One schedulable unit inside a DAG.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Node {
|
||||
pub id: NodeId,
|
||||
pub kind: NodeKind,
|
||||
pub deps: Vec<Dep>,
|
||||
pub state: State,
|
||||
/// Live sub-label while `Running` (kept for parity with the old
|
||||
/// per-entry `step`).
|
||||
pub step: Option<String>,
|
||||
/// Row id of the `build_logs` entry this node opened (`Prebuild` /
|
||||
/// `Swap` / `ApprovalDeploy`), for the dashboard's live-stream link.
|
||||
pub build_log_id: Option<i64>,
|
||||
pub started_at: Option<i64>,
|
||||
pub finished_at: Option<i64>,
|
||||
/// Populated when `state == Failed` (truncated by the queue).
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Submit-time spec for one node.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NodeSpec {
|
||||
pub kind: NodeKind,
|
||||
pub deps: Vec<Dep>,
|
||||
}
|
||||
|
||||
/// Submit-time spec for a whole DAG. Built by `templates.rs`; validated
|
||||
/// (cycle rejection) and dedup'd by `JobQueue::submit`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DagSpec {
|
||||
pub template: Template,
|
||||
/// Primary target agent, or `"hyperhive"` for meta-level DAGs.
|
||||
pub agent: String,
|
||||
pub source: Source,
|
||||
/// Free-form "why"; dedup appends "also requested by …" lines.
|
||||
pub reason: String,
|
||||
/// Cascade grouping (meta-update / sweep children).
|
||||
pub parent_id: Option<u64>,
|
||||
/// Fires the approval-resolution hook on DAG terminal.
|
||||
pub approval_id: Option<i64>,
|
||||
/// `MetaUpdate`-only: the inputs to bump (also part of the dedup
|
||||
/// key for that template). Display copy lives on the DAG.
|
||||
pub inputs: Vec<String>,
|
||||
/// `PermChange`-only payload.
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
/// Dashboard transient pill (and crash-watch suppression) held for
|
||||
/// the lease window — from lease acquisition to DAG terminal.
|
||||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
pub nodes: Vec<NodeSpec>,
|
||||
}
|
||||
|
||||
/// A live DAG in the queue.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Dag {
|
||||
pub id: u64,
|
||||
pub template: Template,
|
||||
pub agent: String,
|
||||
pub source: Source,
|
||||
pub reason: String,
|
||||
pub parent_id: Option<u64>,
|
||||
pub approval_id: Option<i64>,
|
||||
pub inputs: Vec<String>,
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
pub transient: Option<crate::coordinator::TransientKind>,
|
||||
pub created_at: i64,
|
||||
pub nodes: Vec<Node>,
|
||||
/// Terminal roll-up already reported to the scheduler's hooks
|
||||
/// (approval resolution, transient release). Internal bookkeeping,
|
||||
/// never serialized.
|
||||
pub terminal_reported: bool,
|
||||
}
|
||||
|
||||
impl Dag {
|
||||
/// Roll-up state: `Failed` if any node failed; else `Running` if
|
||||
/// any running; else `Queued` if any queued; else `Cancelled` if
|
||||
/// any cancelled; else `Done`.
|
||||
pub fn rollup(&self) -> State {
|
||||
let mut any_cancelled = false;
|
||||
let mut any_queued = false;
|
||||
let mut any_running = false;
|
||||
for n in &self.nodes {
|
||||
match n.state {
|
||||
State::Failed => return State::Failed,
|
||||
State::Running => any_running = true,
|
||||
State::Queued => any_queued = true,
|
||||
State::Cancelled => any_cancelled = true,
|
||||
State::Done => {}
|
||||
}
|
||||
}
|
||||
if any_running {
|
||||
State::Running
|
||||
} else if any_queued {
|
||||
State::Queued
|
||||
} else if any_cancelled {
|
||||
State::Cancelled
|
||||
} else {
|
||||
State::Done
|
||||
}
|
||||
}
|
||||
|
||||
/// True when every node is terminal.
|
||||
pub fn is_terminal(&self) -> bool {
|
||||
self.nodes.iter().all(|n| n.state.is_terminal())
|
||||
}
|
||||
|
||||
/// First failed node's error, for the roll-up `error` field.
|
||||
pub fn first_error(&self) -> Option<&str> {
|
||||
self.nodes
|
||||
.iter()
|
||||
.find(|n| n.state == State::Failed)
|
||||
.and_then(|n| n.error.as_deref())
|
||||
}
|
||||
|
||||
pub fn node(&self, id: NodeId) -> Option<&Node> {
|
||||
self.nodes.iter().find(|n| n.id == id)
|
||||
}
|
||||
|
||||
pub fn node_mut(&mut self, id: NodeId) -> Option<&mut Node> {
|
||||
self.nodes.iter_mut().find(|n| n.id == id)
|
||||
}
|
||||
}
|
||||
|
||||
impl Dag {
|
||||
pub fn view(&self) -> DagView {
|
||||
let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
|
||||
let finished_at = if self.is_terminal() {
|
||||
self.nodes.iter().filter_map(|n| n.finished_at).max()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
DagView {
|
||||
id: self.id,
|
||||
agent: self.agent.clone(),
|
||||
kind: self.template,
|
||||
state: self.rollup(),
|
||||
source: self.source,
|
||||
parent_id: self.parent_id,
|
||||
reason: self.reason.clone(),
|
||||
enqueued_at: self.created_at,
|
||||
started_at,
|
||||
finished_at,
|
||||
inputs: self.inputs.clone(),
|
||||
approval_id: self.approval_id,
|
||||
perm_payload: self.perm_payload.clone(),
|
||||
nodes: self
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|n| NodeView {
|
||||
id: n.id,
|
||||
kind: n.kind.as_str().to_owned(),
|
||||
deps: n.deps.iter().map(|d| d.on).collect(),
|
||||
state: n.state,
|
||||
step: n.step.clone(),
|
||||
build_log_id: n.build_log_id,
|
||||
started_at: n.started_at,
|
||||
finished_at: n.finished_at,
|
||||
error: n.error.clone(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
163
hive-c0re/src/job_queue/scheduler.rs
Normal file
163
hive-c0re/src/job_queue/scheduler.rs
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
//! The single scheduler task that drives all DAGs: claim every ready
|
||||
//! node (as many as the build slots / leases allow), spawn one
|
||||
//! executor task per claim, and on any completion re-evaluate.
|
||||
//! Concurrency comes from the build-slot count, not multiple workers.
|
||||
//!
|
||||
//! Also owns the two DAG-lifetime side channels the sync queue core
|
||||
//! can't hold itself:
|
||||
//! - the per-DAG transient guard (dashboard pill + crash-watch
|
||||
//! suppression), created when a DAG acquires its agent lease and
|
||||
//! dropped when the DAG settles terminal;
|
||||
//! - the `MetaLock` fan-out: appending child `Rebuild` DAGs once the
|
||||
//! lock bump lands, so children build against the post-bump lock
|
||||
//! (and a failed bump fans out nothing — replacing the old
|
||||
//! pre-enqueue + cancel-children dance).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::exec::{self, NodeOutput};
|
||||
use super::{Claim, Source, Template, templates};
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
struct NodeDone {
|
||||
claim: Claim,
|
||||
result: anyhow::Result<NodeOutput>,
|
||||
}
|
||||
|
||||
/// Scheduler loop. Spawned once at hive-c0re startup from `main.rs`.
|
||||
///
|
||||
/// Shutdown semantics: subscribes to `coord.shutdown_rx()`. On a true
|
||||
/// signal the loop exits immediately; already-running node tasks ride
|
||||
/// the runtime down with the process, and pending `Queued` DAGs are
|
||||
/// dropped — desired state is re-derived on next boot (boot sweep +
|
||||
/// reconcile), so the in-memory queue is deliberately not durable.
|
||||
pub async fn run_worker(coord: Arc<Coordinator>) {
|
||||
let mut shutdown = coord.shutdown_rx();
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<NodeDone>();
|
||||
// DAG id → transient guard held for the lease window.
|
||||
let mut transients: HashMap<u64, crate::coordinator::TransientGuard> = HashMap::new();
|
||||
loop {
|
||||
// Terminal roll-ups can appear without a node completion —
|
||||
// the cancel surfaces settle DAGs directly and wake this loop
|
||||
// via notify — so drain on every iteration, not just inside
|
||||
// handle_completion.
|
||||
process_terminals(&coord, &mut transients).await;
|
||||
let claims = coord.job_queue.claim_ready();
|
||||
if !claims.is_empty() {
|
||||
for claim in claims {
|
||||
if claim.lease_acquired
|
||||
&& let Some(kind) = claim.transient
|
||||
{
|
||||
transients.insert(claim.dag_id, coord.transient_guard(&claim.agent, kind));
|
||||
}
|
||||
tracing::info!(
|
||||
dag = claim.dag_id,
|
||||
node = claim.node_id,
|
||||
kind = claim.kind.as_str(),
|
||||
agent = %claim.agent,
|
||||
template = claim.template.as_str(),
|
||||
"job_queue: node running"
|
||||
);
|
||||
let coord = Arc::clone(&coord);
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = exec::run_node(&coord, &claim).await;
|
||||
// Send failure = scheduler gone (shutdown); drop.
|
||||
let _ = tx.send(NodeDone { claim, result });
|
||||
});
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
continue;
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
res = shutdown.changed() => {
|
||||
if res.is_err() || *shutdown.borrow() {
|
||||
tracing::info!("job_queue: scheduler exiting on shutdown");
|
||||
return;
|
||||
}
|
||||
}
|
||||
Some(done) = rx.recv() => {
|
||||
handle_completion(&coord, &mut transients, done).await;
|
||||
}
|
||||
() = coord.job_queue.notify.notified() => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_completion(
|
||||
coord: &Arc<Coordinator>,
|
||||
transients: &mut HashMap<u64, crate::coordinator::TransientGuard>,
|
||||
done: NodeDone,
|
||||
) {
|
||||
let NodeDone { claim, result } = done;
|
||||
let (queue_result, fanout) = match result {
|
||||
Ok(output) => {
|
||||
tracing::info!(
|
||||
dag = claim.dag_id,
|
||||
node = claim.node_id,
|
||||
"job_queue: node done"
|
||||
);
|
||||
(Ok(()), output.fanout)
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("{e:#}");
|
||||
tracing::warn!(
|
||||
dag = claim.dag_id,
|
||||
node = claim.node_id,
|
||||
kind = claim.kind.as_str(),
|
||||
agent = %claim.agent,
|
||||
error = %msg,
|
||||
"job_queue: node failed"
|
||||
);
|
||||
(Err(msg), Vec::new())
|
||||
}
|
||||
};
|
||||
coord
|
||||
.job_queue
|
||||
.complete_node(claim.dag_id, claim.node_id, queue_result);
|
||||
if !fanout.is_empty() {
|
||||
let specs = fanout_specs(&claim, fanout);
|
||||
coord.job_queue.append_children(specs);
|
||||
}
|
||||
process_terminals(coord, transients).await;
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
/// Drain buffered terminal roll-ups: drop each DAG's lease-window
|
||||
/// transient guard, then run the terminal hook (approval resolution,
|
||||
/// `Rebuilt` events, cancelled-power-op intent revert).
|
||||
async fn process_terminals(
|
||||
coord: &Arc<Coordinator>,
|
||||
transients: &mut HashMap<u64, crate::coordinator::TransientGuard>,
|
||||
) {
|
||||
for terminal in coord.job_queue.drain_terminal() {
|
||||
transients.remove(&terminal.dag_id);
|
||||
exec::on_dag_terminal(coord, &terminal).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Child `Rebuild` specs for a completed `MetaLock` fan-out, grouped
|
||||
/// under the parent via `parent_id`. Meta-update children skip the
|
||||
/// per-agent relock (it would revert the bump the parent just
|
||||
/// committed); sweep children relock like a manual rebuild.
|
||||
fn fanout_specs(claim: &Claim, agents: Vec<String>) -> Vec<super::DagSpec> {
|
||||
let sweep = claim.template == Template::StartupSweep;
|
||||
let (source, relock) = if sweep {
|
||||
(Source::StartupSweep, true)
|
||||
} else {
|
||||
(Source::MetaUpdate, false)
|
||||
};
|
||||
let reason = if sweep {
|
||||
"startup sweep".to_owned()
|
||||
} else if let Some(approval_id) = claim.approval_id {
|
||||
format!("approval #{approval_id} meta input cascade")
|
||||
} else {
|
||||
"meta-update cascade".to_owned()
|
||||
};
|
||||
agents
|
||||
.into_iter()
|
||||
.map(|agent| templates::rebuild(&agent, source, reason.clone(), Some(claim.dag_id), relock))
|
||||
.collect()
|
||||
}
|
||||
125
hive-c0re/src/job_queue/submit.rs
Normal file
125
hive-c0re/src/job_queue/submit.rs
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
//! Request-level submit API — the surface the dashboard POST handlers,
|
||||
//! the MCP socket handlers, and `hivectl` paths call. Owns the
|
||||
//! submit-time side effects the DAG templates deliberately don't:
|
||||
//! writing the durable `wanted` power intent (synchronously,
|
||||
//! last-writer-wins) before the DAG whose `Reconcile` reads it, and
|
||||
//! upgrading a stale start to a full rebuild. Every helper emits a
|
||||
//! fresh queue snapshot so the dashboard shows the new DAG immediately.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{Source, Template, templates};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::power::Wanted;
|
||||
|
||||
fn submit_and_emit(coord: &Arc<Coordinator>, spec: super::DagSpec) -> u64 {
|
||||
let id = coord
|
||||
.job_queue
|
||||
.submit(spec)
|
||||
.expect("template-built dag specs are acyclic");
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
id
|
||||
}
|
||||
|
||||
fn set_wanted(coord: &Arc<Coordinator>, agent: &str, wanted: Wanted) {
|
||||
if let Err(e) = coord.power.set(agent, wanted) {
|
||||
tracing::warn!(%agent, wanted = wanted.as_str(), error = ?e, "agent_power: set failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Manual/approval-independent rebuild (always relocks the agent's
|
||||
/// meta input — cascade children are built by the scheduler's fan-out
|
||||
/// instead of this surface).
|
||||
pub fn rebuild(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
submit_and_emit(coord, templates::rebuild(agent, source, reason, None, true))
|
||||
}
|
||||
|
||||
/// Restart: mechanical stop + converge to `wanted = Up`. The intent
|
||||
/// write matters when `wanted` drifted `Offline` under a running
|
||||
/// agent — the old `kill + start` always ended up, and an operator
|
||||
/// asking for a restart plainly wants it running, not a stop.
|
||||
pub fn restart(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Up);
|
||||
submit_and_emit(coord, templates::restart(agent, source, reason))
|
||||
}
|
||||
|
||||
/// Start: persist `wanted = Up`, then reconcile. A stale rev marker
|
||||
/// upgrades the start to a full rebuild (whose tail `Reconcile` does
|
||||
/// the start) so the container always comes up on current derivations
|
||||
/// — the old fast-lane `run_start` upgrade, moved to submit time.
|
||||
pub fn start(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Up);
|
||||
let stored = std::fs::read_to_string(crate::auto_update::rev_marker_path(agent)).ok();
|
||||
let stale = crate::auto_update::current_flake_rev(&coord.hyperhive_flake)
|
||||
.is_some_and(|rev| stored.as_deref() != Some(rev.as_str()));
|
||||
if stale {
|
||||
tracing::info!(%agent, "start: rev stale — upgrading to rebuild+start");
|
||||
return submit_and_emit(
|
||||
coord,
|
||||
templates::rebuild(
|
||||
agent,
|
||||
source,
|
||||
format!("{reason} (stale — rebuild+start)"),
|
||||
None,
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
agent,
|
||||
source,
|
||||
reason,
|
||||
Some(crate::coordinator::TransientKind::Starting),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Hard stop: persist `wanted = Offline`, then reconcile (kill +
|
||||
/// unregister + `Killed` event).
|
||||
pub fn stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Offline);
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
agent,
|
||||
source,
|
||||
reason,
|
||||
Some(crate::coordinator::TransientKind::Stopping),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Graceful stop: persist `wanted = Offline`, then signal → drain →
|
||||
/// reconcile (the actual stop).
|
||||
pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reason: String) -> u64 {
|
||||
set_wanted(coord, agent, Wanted::Offline);
|
||||
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
|
||||
}
|
||||
|
||||
/// Perm change: commit the JSON file(s) then rebuild.
|
||||
pub fn perm_change(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
payload: super::PermPayload,
|
||||
) -> u64 {
|
||||
submit_and_emit(
|
||||
coord,
|
||||
templates::perm_change(agent, source, reason, payload),
|
||||
)
|
||||
}
|
||||
|
||||
/// Meta-input lock bump; cascade rebuilds fan out on completion.
|
||||
pub fn meta_update(
|
||||
coord: &Arc<Coordinator>,
|
||||
inputs: Vec<String>,
|
||||
source: Source,
|
||||
reason: String,
|
||||
) -> u64 {
|
||||
submit_and_emit(coord, templates::meta_update(inputs, source, reason, None))
|
||||
}
|
||||
334
hive-c0re/src/job_queue/templates.rs
Normal file
334
hive-c0re/src/job_queue/templates.rs
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
//! DAG shape builders — every operation as a template over the shared
|
||||
//! node primitives — plus submit-time cycle validation (petgraph is
|
||||
//! confined to this validation; the runtime store stays the plain
|
||||
//! `Vec<Node>` + `deps`).
|
||||
//!
|
||||
//! ```text
|
||||
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
|
||||
//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
||||
//! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
||||
//! start(a): [wanted=Up] Reconcile(a)
|
||||
//! stop(a): [wanted=Offline] Reconcile(a)
|
||||
//! spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
|
||||
//! perm-change(a): WritePermFile(a) → «rebuild subgraph»
|
||||
//! meta-update(inp): MetaLock(inp) → «fan-out rebuild(a) per affected a»
|
||||
//! startup sweep: MetaLock(hyperhive, non-fatal) → «fan-out rebuild(stale a)»
|
||||
//! ```
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
|
||||
use super::model::{DagSpec, Dep, DepWhen, NodeKind, NodeSpec, PermPayload, Source, Template};
|
||||
use crate::coordinator::TransientKind;
|
||||
|
||||
/// After-ok edge on the previous node — the common chain link.
|
||||
fn after_ok(on: u32) -> Vec<Dep> {
|
||||
vec![Dep {
|
||||
on,
|
||||
when: DepWhen::AfterOk,
|
||||
}]
|
||||
}
|
||||
|
||||
/// The rebuild node chain. `Reconcile` deps on `Swap` with `AfterAny`:
|
||||
/// it must run even when the profile swap failed, so a previously-up
|
||||
/// agent comes back on its old config (today's recovery-start). This
|
||||
/// is the only `AfterAny` edge in v1.
|
||||
fn rebuild_nodes(relock: bool, base: u32) -> Vec<NodeSpec> {
|
||||
vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Prebuild { relock },
|
||||
deps: if base == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
after_ok(base - 1)
|
||||
},
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: after_ok(base),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Swap,
|
||||
deps: after_ok(base + 1),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: base + 2,
|
||||
when: DepWhen::AfterAny,
|
||||
}],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// One uniform rebuild shape — no `was_running` branch. `StopForUpdate`
|
||||
/// noops when already down; the tail `Reconcile` auto-noops the start
|
||||
/// when `wanted = Offline` (a rebuild of a deliberately-stopped agent
|
||||
/// leaves it stopped). `relock = false` only for meta-update cascade
|
||||
/// children.
|
||||
pub fn rebuild(
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
parent_id: Option<u64>,
|
||||
relock: bool,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes: rebuild_nodes(relock, 0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Approval-driven deploy (`ApplyCommit` / `MergeConfigPr`): the whole
|
||||
/// two-phase pipeline stays one opaque node in v1 (design doc §9) —
|
||||
/// wire-visible as a `rebuild` card like today.
|
||||
pub fn approval_deploy(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Rebuild,
|
||||
agent: agent.to_owned(),
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: Some(approval_id),
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::ApprovalDeploy,
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Graceful stop: cheap `Signal` fires immediately (no build slot), the
|
||||
/// `Drain` awaits the harness checkpoint (bounded), and the tail
|
||||
/// `Reconcile` performs the actual container stop — the caller sets
|
||||
/// `wanted = Offline` at submit time. A whole-hive graceful stop
|
||||
/// therefore signals every agent up front and overlaps every drain,
|
||||
/// replacing the old detached-watcher thread structurally.
|
||||
pub fn graceful_stop(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::GracefulStop,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Stopping),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Signal,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Drain,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(1),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Restart: mechanical stop, then converge to `wanted` — the submit
|
||||
/// layer writes `wanted = Up` first, so this is a stop + start like
|
||||
/// the old `lifecycle::restart` regardless of prior intent drift.
|
||||
pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Restart,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Restarting),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
|
||||
/// first) and the boot-time `Reconcile` converge (wanted untouched).
|
||||
pub fn reconcile_only(
|
||||
template: Template,
|
||||
agent: &str,
|
||||
source: Source,
|
||||
reason: String,
|
||||
transient: Option<TransientKind>,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// First-deploy spawn (approval-driven): pre-start provisioning +
|
||||
/// `nixos-container create`, drop-in write, then `Reconcile` starts the
|
||||
/// container (`wanted = Up` written at approve time).
|
||||
pub fn spawn(agent: &str, approval_id: i64, reason: String) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::Spawn,
|
||||
agent: agent.to_owned(),
|
||||
source: Source::Approval,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: Some(approval_id),
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: Some(TransientKind::Spawning),
|
||||
nodes: vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::Create,
|
||||
deps: Vec::new(),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::WriteDropin,
|
||||
deps: after_ok(0),
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: after_ok(1),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Perm change: commit the JSON file(s), then the rebuild subgraph so
|
||||
/// the updated `HIVE_TOOL_GROUPS` / `HIVE_CAPABILITIES` env var takes
|
||||
/// effect in the container.
|
||||
pub fn perm_change(agent: &str, source: Source, reason: String, payload: PermPayload) -> DagSpec {
|
||||
let mut nodes = vec![NodeSpec {
|
||||
kind: NodeKind::WritePermFile,
|
||||
deps: Vec::new(),
|
||||
}];
|
||||
nodes.extend(rebuild_nodes(true, 1));
|
||||
DagSpec {
|
||||
template: Template::PermChange,
|
||||
agent: agent.to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: Some(payload),
|
||||
transient: Some(TransientKind::Rebuilding),
|
||||
nodes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Meta-input lock bump. Child `Rebuild` DAGs fan out on completion —
|
||||
/// appended *after* the bump lands so their prebuilds run against the
|
||||
/// post-bump lock (and so a failed bump simply fans out nothing,
|
||||
/// replacing the old pre-enqueue + `cancel_children` dance).
|
||||
pub fn meta_update(
|
||||
inputs: Vec<String>,
|
||||
source: Source,
|
||||
reason: String,
|
||||
approval_id: Option<i64>,
|
||||
) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::MetaUpdate,
|
||||
agent: "hyperhive".to_owned(),
|
||||
source,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id,
|
||||
inputs,
|
||||
perm_payload: None,
|
||||
transient: None,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::MetaLock {
|
||||
sweep: false,
|
||||
fanout: None,
|
||||
},
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Boot-time sweep parent: bump meta's hyperhive input (non-fatal),
|
||||
/// then fan out `Rebuild` children for the precomputed stale agent
|
||||
/// list (topology-sorted by the caller).
|
||||
pub fn startup_sweep(reason: String, stale_agents: Vec<String>) -> DagSpec {
|
||||
DagSpec {
|
||||
template: Template::StartupSweep,
|
||||
agent: "hyperhive".to_owned(),
|
||||
source: Source::AutoUpdate,
|
||||
reason,
|
||||
parent_id: None,
|
||||
approval_id: None,
|
||||
inputs: Vec::new(),
|
||||
perm_payload: None,
|
||||
transient: None,
|
||||
nodes: vec![NodeSpec {
|
||||
kind: NodeKind::MetaLock {
|
||||
sweep: true,
|
||||
fanout: Some(stale_agents),
|
||||
},
|
||||
deps: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate a spec before it enters the queue: node ids are dense
|
||||
/// (index = id), deps reference existing nodes, and the dep graph is
|
||||
/// acyclic (petgraph `toposort`). Rejecting cycles here fixes the old
|
||||
/// queue's documented "circular dep silently deadlocks forever" caveat.
|
||||
pub fn validate(spec: &DagSpec) -> Result<()> {
|
||||
if spec.nodes.is_empty() {
|
||||
bail!("dag spec {:?} has no nodes", spec.template);
|
||||
}
|
||||
let n = spec.nodes.len();
|
||||
let mut graph = petgraph::graph::DiGraph::<u32, ()>::new();
|
||||
let idx: Vec<_> = (0..n)
|
||||
.map(|i| graph.add_node(u32::try_from(i).unwrap_or(u32::MAX)))
|
||||
.collect();
|
||||
for (i, node) in spec.nodes.iter().enumerate() {
|
||||
for dep in &node.deps {
|
||||
let Some(&dep_idx) = idx.get(dep.on as usize) else {
|
||||
bail!(
|
||||
"dag spec {:?} node {i} depends on unknown node {}",
|
||||
spec.template,
|
||||
dep.on
|
||||
);
|
||||
};
|
||||
graph.add_edge(dep_idx, idx[i], ());
|
||||
}
|
||||
}
|
||||
if petgraph::algo::toposort(&graph, None).is_err() {
|
||||
bail!("dag spec {:?} contains a dependency cycle", spec.template);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
920
hive-c0re/src/job_queue/tests.rs
Normal file
920
hive-c0re/src/job_queue/tests.rs
Normal file
|
|
@ -0,0 +1,920 @@
|
|||
//! Queue-core unit tests: dedup, cycle rejection, resource
|
||||
//! serialization (build slots / per-agent leases), lease-exempt
|
||||
//! overlap, FIFO fairness, cancel semantics, `AfterAny` failure
|
||||
//! routing, fan-out, and history retention. All synchronous — the
|
||||
//! scheduler's async loop is a thin claim/complete pump over the same
|
||||
//! methods exercised here.
|
||||
|
||||
use super::model::{Dep, DepWhen, NodeKind, NodeSpec};
|
||||
use super::*;
|
||||
|
||||
fn submit(q: &JobQueue, spec: DagSpec) -> u64 {
|
||||
q.submit(spec).expect("valid spec")
|
||||
}
|
||||
|
||||
fn rebuild(agent: &str, reason: &str) -> DagSpec {
|
||||
templates::rebuild(agent, Source::Manual, reason.to_owned(), None, true)
|
||||
}
|
||||
|
||||
/// Claim helper asserting exactly one node comes back.
|
||||
fn claim_one(q: &JobQueue) -> Claim {
|
||||
let mut claims = q.claim_ready();
|
||||
assert_eq!(
|
||||
claims.len(),
|
||||
1,
|
||||
"expected exactly one claim, got {claims:?}"
|
||||
);
|
||||
claims.pop().expect("one claim")
|
||||
}
|
||||
|
||||
fn state_of(q: &JobQueue, dag_id: u64) -> State {
|
||||
q.snapshot()
|
||||
.iter()
|
||||
.find(|d| d.id == dag_id)
|
||||
.expect("dag present")
|
||||
.state
|
||||
}
|
||||
|
||||
// ---- submit / dedup ----
|
||||
|
||||
#[test]
|
||||
fn submit_assigns_distinct_ids() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let b = submit(&q, rebuild("agent-b", "second"));
|
||||
assert_ne!(a, b);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_pending_same_template_and_agent() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let b = submit(&q, rebuild("agent-a", "auto sweep"));
|
||||
assert_eq!(a, b, "dedup should return existing id");
|
||||
let snap = q.snapshot();
|
||||
assert_eq!(snap.len(), 1);
|
||||
assert!(snap[0].reason.contains("first"));
|
||||
assert!(snap[0].reason.contains("auto sweep"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_does_not_apply_across_templates_or_agents() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
assert_ne!(a, b);
|
||||
assert_ne!(a, c);
|
||||
assert_eq!(q.snapshot().len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_skips_running_dags() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "first"));
|
||||
let claim = claim_one(&q); // Prebuild running
|
||||
assert_eq!(claim.dag_id, a);
|
||||
// While the original runs, re-submit is legitimate new work.
|
||||
let again = submit(&q, rebuild("agent-a", "config bumped during build"));
|
||||
assert_ne!(a, again);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_update_dedup_matches_inputs() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["nixpkgs".to_owned()],
|
||||
Source::Manual,
|
||||
"first".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let b = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["nixpkgs".to_owned()],
|
||||
Source::Manual,
|
||||
"duplicate click".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
assert_eq!(a, b, "identical-inputs meta-updates should dedup");
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec!["agent-bitburner/bitburner-agent".to_owned()],
|
||||
Source::Manual,
|
||||
"bump agent".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
assert_ne!(a, c, "different-inputs meta-updates must NOT dedup");
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn approval_dags_dedup_only_on_matching_id() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 1, "approval #1".to_owned()),
|
||||
);
|
||||
let b = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 2, "approval #2".to_owned()),
|
||||
);
|
||||
assert_ne!(a, b, "distinct approvals must not collapse");
|
||||
// Rapid double-click on the same approval IS a single op.
|
||||
let c = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 1, "approval #1 (dup)".to_owned()),
|
||||
);
|
||||
assert_eq!(a, c);
|
||||
assert_eq!(q.snapshot().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perm_change_dedup_respects_perm_type() {
|
||||
let q = JobQueue::new(1);
|
||||
let groups = templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"groups".to_owned(),
|
||||
PermPayload::ToolGroups { groups: vec![] },
|
||||
);
|
||||
let caps = templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"caps".to_owned(),
|
||||
PermPayload::Capabilities { caps: vec![] },
|
||||
);
|
||||
let a = submit(&q, groups.clone());
|
||||
let b = submit(&q, caps);
|
||||
assert_ne!(a, b, "tool-groups vs capabilities must not collapse");
|
||||
let c = submit(&q, groups);
|
||||
assert_eq!(a, c, "same perm type dedups");
|
||||
}
|
||||
|
||||
/// A `MetaUpdate` cascade `Rebuild` (with `parent_id = Some(meta_id)`)
|
||||
/// must NOT dedup into a queued `Rebuild` with a different
|
||||
/// `parent_id` (e.g. from a startup sweep) — without the guard the
|
||||
/// cascade child would be swallowed and the agent never rebuilt
|
||||
/// against the post-bump meta.
|
||||
#[test]
|
||||
fn dedup_respects_parent_id() {
|
||||
let q = JobQueue::new(1);
|
||||
let sweep = submit(&q, templates::startup_sweep("boot".to_owned(), vec![]));
|
||||
let sweep_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::StartupSweep,
|
||||
"startup sweep".to_owned(),
|
||||
Some(sweep),
|
||||
true,
|
||||
),
|
||||
);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let cascade_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"meta-update cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
assert_ne!(sweep_child, cascade_child);
|
||||
let rebuilds = q
|
||||
.snapshot()
|
||||
.iter()
|
||||
.filter(|d| d.kind == Template::Rebuild && d.agent == "alice")
|
||||
.count();
|
||||
assert_eq!(rebuilds, 2, "both rebuilds must be present");
|
||||
}
|
||||
|
||||
// ---- cycle rejection ----
|
||||
|
||||
#[test]
|
||||
fn cyclic_dag_is_rejected_at_submit() {
|
||||
let q = JobQueue::new(1);
|
||||
let mut spec = rebuild("agent-a", "cyclic");
|
||||
// 0 → 1 → 0 cycle.
|
||||
spec.nodes = vec![
|
||||
NodeSpec {
|
||||
kind: NodeKind::StopForUpdate,
|
||||
deps: vec![Dep {
|
||||
on: 1,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
},
|
||||
NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: 0,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
},
|
||||
];
|
||||
assert!(q.submit(spec).is_err(), "cyclic spec must be refused");
|
||||
assert!(q.snapshot().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dep_is_rejected_at_submit() {
|
||||
let q = JobQueue::new(1);
|
||||
let mut spec = rebuild("agent-a", "bad dep");
|
||||
spec.nodes = vec![NodeSpec {
|
||||
kind: NodeKind::Reconcile,
|
||||
deps: vec![Dep {
|
||||
on: 9,
|
||||
when: DepWhen::AfterOk,
|
||||
}],
|
||||
}];
|
||||
assert!(q.submit(spec).is_err());
|
||||
}
|
||||
|
||||
// ---- dependency order within a DAG ----
|
||||
|
||||
#[test]
|
||||
fn rebuild_chain_claims_in_dep_order() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
for expected in ["prebuild", "stop_for_update", "swap", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, id);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
assert!(
|
||||
q.claim_ready().is_empty(),
|
||||
"chain must serialize: nothing ready while {expected} runs"
|
||||
);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
|
||||
// ---- build slots ----
|
||||
|
||||
#[test]
|
||||
fn build_slot_serializes_nix_heavy_nodes() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let first = claim_one(&q); // a's Prebuild takes the only slot
|
||||
assert_eq!(first.dag_id, a);
|
||||
assert_eq!(first.kind.as_str(), "prebuild");
|
||||
q.complete_node(a, first.node_id, Ok(()));
|
||||
// With the slot free again, FIFO gives... a's StopForUpdate is
|
||||
// slot-free (lease) and b's Prebuild takes the slot — both run.
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<(u64, &str)> = claims.iter().map(|c| (c.dag_id, c.kind.as_str())).collect();
|
||||
assert!(kinds.contains(&(a, "stop_for_update")));
|
||||
assert!(kinds.contains(&(b, "prebuild")));
|
||||
assert_eq!(claims.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_build_slots_run_two_prebuilds() {
|
||||
let q = JobQueue::new(2);
|
||||
submit(&q, rebuild("agent-a", "r"));
|
||||
submit(&q, rebuild("agent-b", "r"));
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "two slots → two concurrent prebuilds");
|
||||
assert!(claims.iter().all(|c| c.kind.as_str() == "prebuild"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_fairness_for_the_slot() {
|
||||
let q = JobQueue::new(1);
|
||||
let a = submit(&q, rebuild("agent-a", "r"));
|
||||
let b = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = submit(&q, rebuild("agent-c", "r"));
|
||||
let first = claim_one(&q);
|
||||
assert_eq!(first.dag_id, a, "submit order wins the slot");
|
||||
q.complete_node(a, first.node_id, Ok(()));
|
||||
let next: Vec<u64> = q.claim_ready().iter().map(|cl| cl.dag_id).collect();
|
||||
assert!(next.contains(&b), "b's prebuild before c's");
|
||||
assert!(!next.contains(&c));
|
||||
}
|
||||
|
||||
// ---- per-agent lease ----
|
||||
|
||||
#[test]
|
||||
fn lease_serializes_two_lifecycle_dags_for_same_agent() {
|
||||
let q = JobQueue::new(4);
|
||||
let restart = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "restart".to_owned()),
|
||||
);
|
||||
let stop = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
// Restart's StopForUpdate acquires the lease; stop's Reconcile
|
||||
// must wait even though slots are free.
|
||||
let first = claim_one(&q);
|
||||
assert_eq!(first.dag_id, restart);
|
||||
assert!(first.lease_acquired);
|
||||
q.complete_node(restart, first.node_id, Ok(()));
|
||||
// Same DAG keeps the lease for its Reconcile.
|
||||
let second = claim_one(&q);
|
||||
assert_eq!(second.dag_id, restart);
|
||||
assert!(!second.lease_acquired, "lease already held by this DAG");
|
||||
q.complete_node(restart, second.node_id, Ok(()));
|
||||
// Restart terminal → lease released → stop's Reconcile runs.
|
||||
let third = claim_one(&q);
|
||||
assert_eq!(third.dag_id, stop);
|
||||
q.complete_node(stop, third.node_id, Ok(()));
|
||||
assert_eq!(state_of(&q, restart), State::Done);
|
||||
assert_eq!(state_of(&q, stop), State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lease_exempt_prebuild_overlaps_other_dag_on_same_agent() {
|
||||
let q = JobQueue::new(2);
|
||||
submit(&q, rebuild("agent-a", "rebuild"));
|
||||
let stop = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
// Prebuild is lease-exempt: the stop's Reconcile takes the lease
|
||||
// and runs concurrently with the rebuild's out-of-band nix build.
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||
assert!(kinds.contains(&"prebuild"));
|
||||
assert!(kinds.contains(&"reconcile"));
|
||||
// But the rebuild's StopForUpdate must then wait for the stop DAG
|
||||
// to finish (lease).
|
||||
let prebuild = claims
|
||||
.iter()
|
||||
.find(|c| c.kind.as_str() == "prebuild")
|
||||
.expect("prebuild claim")
|
||||
.clone();
|
||||
q.complete_node(prebuild.dag_id, prebuild.node_id, Ok(()));
|
||||
assert!(
|
||||
q.claim_ready().is_empty(),
|
||||
"StopForUpdate blocked while stop DAG holds the lease"
|
||||
);
|
||||
let reconcile = claims
|
||||
.iter()
|
||||
.find(|c| c.kind.as_str() == "reconcile")
|
||||
.expect("reconcile claim")
|
||||
.clone();
|
||||
q.complete_node(stop, reconcile.node_id, Ok(()));
|
||||
let next = claim_one(&q);
|
||||
assert_eq!(next.kind.as_str(), "stop_for_update");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agents_do_not_contend_on_each_others_leases() {
|
||||
let q = JobQueue::new(4);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::restart("agent-b", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
let claims = q.claim_ready();
|
||||
assert_eq!(claims.len(), 2, "different agents run concurrently");
|
||||
}
|
||||
|
||||
// ---- failure: cancel-downstream + AfterAny ----
|
||||
|
||||
#[test]
|
||||
fn failed_node_cancels_downstream_but_afterany_reconcile_runs() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let prebuild = claim_one(&q);
|
||||
q.complete_node(id, prebuild.node_id, Err("nix build exploded".to_owned()));
|
||||
// StopForUpdate + Swap are cancelled (AfterOk on a failed chain);
|
||||
// the AfterAny Reconcile still runs once Swap is terminal.
|
||||
let reconcile = claim_one(&q);
|
||||
assert_eq!(reconcile.kind.as_str(), "reconcile");
|
||||
q.complete_node(id, reconcile.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let dag = snap.iter().find(|d| d.id == id).expect("dag");
|
||||
assert_eq!(dag.state, State::Failed, "roll-up failed");
|
||||
let by_kind = |k: &str| {
|
||||
dag.nodes
|
||||
.iter()
|
||||
.find(|n| n.kind == k)
|
||||
.expect("node present")
|
||||
.state
|
||||
};
|
||||
assert_eq!(by_kind("prebuild"), State::Failed);
|
||||
assert_eq!(by_kind("stop_for_update"), State::Cancelled);
|
||||
assert_eq!(by_kind("swap"), State::Cancelled);
|
||||
assert_eq!(by_kind("reconcile"), State::Done);
|
||||
assert_eq!(
|
||||
dag.nodes
|
||||
.iter()
|
||||
.find(|n| n.kind == "prebuild")
|
||||
.and_then(|n| n.error.as_deref()),
|
||||
Some("nix build exploded")
|
||||
);
|
||||
}
|
||||
|
||||
/// The swap-failure recovery: `Swap` fails → the `AfterAny` edge still
|
||||
/// runs `Reconcile`, which brings a wanted-up agent back on its old
|
||||
/// config.
|
||||
#[test]
|
||||
fn swap_failure_still_runs_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
for _ in 0..2 {
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let swap = claim_one(&q);
|
||||
assert_eq!(swap.kind.as_str(), "swap");
|
||||
q.complete_node(id, swap.node_id, Err("update failed".to_owned()));
|
||||
let reconcile = claim_one(&q);
|
||||
assert_eq!(reconcile.kind.as_str(), "reconcile");
|
||||
q.complete_node(id, reconcile.node_id, Ok(()));
|
||||
assert_eq!(state_of(&q, id), State::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_reconcile_marks_dag_failed() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"start".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Err("start failed".to_owned()));
|
||||
assert_eq!(state_of(&q, id), State::Failed);
|
||||
}
|
||||
|
||||
// ---- cancel ----
|
||||
|
||||
#[test]
|
||||
fn cancel_clears_queued_dag() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(q.cancel(id));
|
||||
assert_eq!(state_of(&q, id), State::Cancelled);
|
||||
assert!(q.claim_ready().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_refuses_running_dag() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let _ = claim_one(&q);
|
||||
assert!(!q.cancel(id));
|
||||
assert_eq!(state_of(&q, id), State::Running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_marks_queued_children_only() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
// Parent's MetaLock is running while children exist.
|
||||
let lock = claim_one(&q);
|
||||
assert_eq!(lock.dag_id, meta);
|
||||
let child_a = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let child_b = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-b",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let unrelated = submit(&q, rebuild("agent-c", "operator queued"));
|
||||
// MetaLock holds the single build slot, so both children (and the
|
||||
// unrelated rebuild) are still fully queued here.
|
||||
let cancelled = q.cancel_children(meta);
|
||||
assert_eq!(cancelled, 2);
|
||||
assert_eq!(state_of(&q, child_a), State::Cancelled);
|
||||
assert_eq!(state_of(&q, child_b), State::Cancelled);
|
||||
assert_eq!(state_of(&q, unrelated), State::Queued);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_skips_running_child() {
|
||||
let q = JobQueue::new(2);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let lock = claim_one(&q);
|
||||
let running_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
let queued_child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-b",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
// Second slot lets running_child's prebuild start.
|
||||
let child_claim = claim_one(&q);
|
||||
assert_eq!(child_claim.dag_id, running_child);
|
||||
let n = q.cancel_children(meta);
|
||||
assert_eq!(n, 1);
|
||||
assert_eq!(state_of(&q, running_child), State::Running);
|
||||
assert_eq!(state_of(&q, queued_child), State::Cancelled);
|
||||
q.complete_node(meta, lock.node_id, Ok(()));
|
||||
}
|
||||
|
||||
// ---- fan-out ----
|
||||
|
||||
#[test]
|
||||
fn append_children_sets_parent_and_dedups() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let specs = vec![
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
templates::rebuild(
|
||||
"bob",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
// Duplicate — must coalesce into the first alice child.
|
||||
templates::rebuild(
|
||||
"alice",
|
||||
Source::MetaUpdate,
|
||||
"cascade again".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
];
|
||||
let ids = q.append_children(specs);
|
||||
assert_eq!(ids.len(), 3);
|
||||
assert_eq!(ids[0], ids[2], "duplicate child dedups");
|
||||
let snap = q.snapshot();
|
||||
let children: Vec<_> = snap.iter().filter(|d| d.parent_id == Some(meta)).collect();
|
||||
assert_eq!(children.len(), 2);
|
||||
}
|
||||
|
||||
// ---- terminal reporting + lease release ----
|
||||
|
||||
#[test]
|
||||
fn terminal_dag_reported_exactly_once_and_lease_released() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::restart("agent-a", Source::Manual, "r".to_owned()),
|
||||
);
|
||||
let stop = claim_one(&q);
|
||||
q.complete_node(id, stop.node_id, Ok(()));
|
||||
assert!(q.drain_terminal().is_empty(), "dag not terminal yet");
|
||||
let rec = claim_one(&q);
|
||||
q.complete_node(id, rec.node_id, Ok(()));
|
||||
let reports = q.drain_terminal();
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].dag_id, id);
|
||||
assert_eq!(reports[0].state, State::Done);
|
||||
assert!(q.drain_terminal().is_empty(), "reported exactly once");
|
||||
// Lease released: a new DAG for the agent can claim immediately.
|
||||
let next = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"stop".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, next);
|
||||
assert!(c.lease_acquired);
|
||||
}
|
||||
|
||||
/// A DAG cancelled while fully queued must still surface a terminal
|
||||
/// roll-up for the scheduler's hooks — otherwise a queued approval
|
||||
/// DAG cancelled by the operator would dangle its approval forever.
|
||||
#[test]
|
||||
fn cancelled_dag_reports_terminal_once() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::approval_deploy("agent-a", 7, "approval #7".to_owned()),
|
||||
);
|
||||
assert!(q.cancel(id));
|
||||
let reports = q.drain_terminal();
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].dag_id, id);
|
||||
assert_eq!(reports[0].state, State::Cancelled);
|
||||
assert_eq!(reports[0].approval_id, Some(7));
|
||||
// Never re-reported by later activity.
|
||||
let other = submit(&q, rebuild("agent-b", "r"));
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, other);
|
||||
q.complete_node(other, c.node_id, Err("boom".to_owned()));
|
||||
assert!(q.drain_terminal().iter().all(|t| t.dag_id != id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_children_reports_terminals() {
|
||||
let q = JobQueue::new(1);
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let _lock = claim_one(&q);
|
||||
let child = submit(
|
||||
&q,
|
||||
templates::rebuild(
|
||||
"agent-a",
|
||||
Source::MetaUpdate,
|
||||
"cascade".to_owned(),
|
||||
Some(meta),
|
||||
false,
|
||||
),
|
||||
);
|
||||
assert_eq!(q.cancel_children(meta), 1);
|
||||
let reports = q.drain_terminal();
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].dag_id, child);
|
||||
assert_eq!(reports[0].state, State::Cancelled);
|
||||
}
|
||||
|
||||
/// History trim must not evict a terminal fan-out parent while its
|
||||
/// children are still live — the dashboard groups children under it.
|
||||
#[test]
|
||||
fn trim_keeps_terminal_parent_with_live_children() {
|
||||
let q = JobQueue::new(1);
|
||||
// Pin agent-x's lease with a running stop DAG so the child below
|
||||
// stays fully queued while we churn history.
|
||||
let pin = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Stop,
|
||||
"agent-x",
|
||||
Source::Manual,
|
||||
"lease pin".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let pin_claim = claim_one(&q);
|
||||
assert_eq!(pin_claim.dag_id, pin);
|
||||
// Terminal fan-out parent + a lease-blocked child under it.
|
||||
let meta = submit(
|
||||
&q,
|
||||
templates::meta_update(vec![], Source::Manual, "bump".to_owned(), None),
|
||||
);
|
||||
let lock = claim_one(&q);
|
||||
q.complete_node(meta, lock.node_id, Ok(()));
|
||||
let mut child_spec = templates::restart("agent-x", Source::MetaUpdate, "cascade".to_owned());
|
||||
child_spec.parent_id = Some(meta);
|
||||
let child = submit(&q, child_spec);
|
||||
// Churn > MAX_HISTORY_PER_TEMPLATE terminal meta_update DAGs.
|
||||
for i in 0..7 {
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::meta_update(
|
||||
vec![format!("input-{i}")],
|
||||
Source::Manual,
|
||||
"churn".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.dag_id, id, "child is lease-blocked; churn claims freely");
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let snap = q.snapshot();
|
||||
assert!(
|
||||
snap.iter().any(|d| d.id == meta),
|
||||
"terminal parent with live child must survive trim"
|
||||
);
|
||||
assert!(snap.iter().any(|d| d.id == child));
|
||||
}
|
||||
|
||||
// ---- steps, build logs, history ----
|
||||
|
||||
#[test]
|
||||
fn set_step_only_on_running_and_signals_change() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(!q.set_step(id, 0, "too early"), "queued node refuses step");
|
||||
let c = claim_one(&q);
|
||||
assert!(q.set_step(id, c.node_id, "nix build"));
|
||||
assert!(
|
||||
!q.set_step(id, c.node_id, "nix build"),
|
||||
"same label → false"
|
||||
);
|
||||
assert!(q.set_step(id, c.node_id, "next phase"));
|
||||
assert!(q.set_step_running(id, "via running lookup"));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
|
||||
assert_eq!(node.step, None, "step cleared on completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_build_log_id_links_running_node() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
assert!(!q.set_build_log_id(id, 0, 41), "queued node refuses log id");
|
||||
let c = claim_one(&q);
|
||||
assert!(q.set_build_log_id(id, c.node_id, 42));
|
||||
assert!(q.set_build_log_id_running(id, 43));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
let snap = q.snapshot();
|
||||
let node = &snap.iter().find(|d| d.id == id).expect("dag").nodes[0];
|
||||
assert_eq!(node.build_log_id, Some(43), "log id survives completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_evicts_old_terminals_per_template() {
|
||||
let q = JobQueue::new(1);
|
||||
for i in 0..8 {
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::reconcile_only(
|
||||
Template::Start,
|
||||
&format!("agent-{i}"),
|
||||
Source::Manual,
|
||||
"start".to_owned(),
|
||||
None,
|
||||
),
|
||||
);
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
// Fresh terminals are inside the grace window: nothing evicts yet,
|
||||
// so a ~1s QueueDag poller can still observe every terminal state
|
||||
// (a broad stop/start settles many same-template DAGs at once).
|
||||
assert_eq!(
|
||||
q.snapshot().len(),
|
||||
8,
|
||||
"grace window protects fresh terminals"
|
||||
);
|
||||
// Past the grace window the per-template cap applies.
|
||||
q.trim_ignoring_grace();
|
||||
assert_eq!(q.snapshot().len(), 5, "per-template history cap");
|
||||
assert_eq!(q.live_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_is_truncated() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(&q, rebuild("agent-a", "r"));
|
||||
let c = claim_one(&q);
|
||||
q.complete_node(id, c.node_id, Err("x".repeat(5000)));
|
||||
let snap = q.snapshot();
|
||||
let err = snap.iter().find(|d| d.id == id).expect("dag").nodes[0]
|
||||
.error
|
||||
.clone()
|
||||
.expect("error stored");
|
||||
assert!(err.chars().count() <= 2001, "truncated + ellipsis");
|
||||
assert!(err.ends_with('…'));
|
||||
}
|
||||
|
||||
// ---- template shapes ----
|
||||
|
||||
#[test]
|
||||
fn graceful_stop_shape_signal_drain_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "graceful".to_owned()),
|
||||
);
|
||||
for expected in ["signal", "drain", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graceful_signal_and_drain_hold_no_build_slot() {
|
||||
// A whole-hive graceful stop overlaps every drain even at
|
||||
// buildSlots = 1 while a rebuild hogs the slot.
|
||||
let q = JobQueue::new(1);
|
||||
submit(&q, rebuild("builder", "slot hog"));
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-a", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
submit(
|
||||
&q,
|
||||
templates::graceful_stop("agent-b", Source::Manual, "g".to_owned()),
|
||||
);
|
||||
let claims = q.claim_ready();
|
||||
let kinds: Vec<&str> = claims.iter().map(|c| c.kind.as_str()).collect();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
vec!["prebuild", "signal", "signal"],
|
||||
"both agents' signals fire while the slot is held"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_shape_create_dropin_reconcile() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::spawn("newbie", 7, "approval #7 spawn".to_owned()),
|
||||
);
|
||||
for expected in ["create", "write_dropin", "reconcile"] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
assert_eq!(c.approval_id, Some(7));
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
let report_terminal = state_of(&q, id);
|
||||
assert_eq!(report_terminal, State::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perm_change_shape_prefixes_rebuild_chain() {
|
||||
let q = JobQueue::new(1);
|
||||
let id = submit(
|
||||
&q,
|
||||
templates::perm_change(
|
||||
"agent-a",
|
||||
Source::Manual,
|
||||
"perm".to_owned(),
|
||||
PermPayload::Combined {
|
||||
groups: Some(vec![]),
|
||||
caps: None,
|
||||
},
|
||||
),
|
||||
);
|
||||
for expected in [
|
||||
"write_perm_file",
|
||||
"prebuild",
|
||||
"stop_for_update",
|
||||
"swap",
|
||||
"reconcile",
|
||||
] {
|
||||
let c = claim_one(&q);
|
||||
assert_eq!(c.kind.as_str(), expected);
|
||||
q.complete_node(id, c.node_id, Ok(()));
|
||||
}
|
||||
assert_eq!(state_of(&q, id), State::Done);
|
||||
}
|
||||
|
|
@ -11,43 +11,45 @@
|
|||
//! Every module is re-exported `pub` so anything in the crate is
|
||||
//! addressable from either binary; the lib doesn't have a curated
|
||||
//! surface beyond "this is where the modules live".
|
||||
//!
|
||||
//! Cohesive clusters live in directory submodules (`stores`, `stats`,
|
||||
//! `agent_config`, `workers`); each of their children is re-exported
|
||||
//! at the crate root so pre-existing `crate::broker::…` /
|
||||
//! `hive_c0re::broker::…` paths keep compiling unchanged.
|
||||
|
||||
pub mod actions;
|
||||
pub mod agent_sockets;
|
||||
pub mod approvals;
|
||||
pub mod audit_log;
|
||||
pub mod auto_update;
|
||||
pub mod broker;
|
||||
pub mod build_logs;
|
||||
pub mod capabilities;
|
||||
pub mod agent_config;
|
||||
pub mod client;
|
||||
pub mod container_stats;
|
||||
pub mod container_view;
|
||||
pub mod coordinator;
|
||||
pub mod crash_watch;
|
||||
pub mod dashboard;
|
||||
pub mod dashboard_events;
|
||||
pub mod flake_check;
|
||||
pub mod forge;
|
||||
pub mod gateway_nginx;
|
||||
pub mod hive_stats;
|
||||
pub mod host_stats;
|
||||
pub mod knowledge;
|
||||
pub mod job_queue;
|
||||
pub mod lifecycle;
|
||||
pub mod limits;
|
||||
pub mod loose_ends;
|
||||
pub mod matrix;
|
||||
pub mod meta;
|
||||
pub mod migrate;
|
||||
pub mod operator_questions;
|
||||
pub mod paths;
|
||||
pub mod priv_client;
|
||||
pub mod questions;
|
||||
pub mod rebuild_queue;
|
||||
pub mod reminder_scheduler;
|
||||
pub mod scheduled_prompts;
|
||||
pub mod scheduled_prompts_worker;
|
||||
pub mod server;
|
||||
pub mod socket_server;
|
||||
pub mod tool_groups;
|
||||
pub mod topology;
|
||||
pub mod stats;
|
||||
pub mod stores;
|
||||
pub mod workers;
|
||||
|
||||
// Root re-exports: keep every pre-grouping `crate::<module>` /
|
||||
// `hive_c0re::<module>` path compiling without touching consumers.
|
||||
pub use agent_config::{capabilities, limits, tool_groups, topology};
|
||||
pub use stats::{container_stats, hive_stats, host_stats};
|
||||
pub use stores::{
|
||||
approvals, audit_log, broker, build_logs, db, operator_questions, power, scheduled_prompts,
|
||||
};
|
||||
pub use workers::{
|
||||
agent_sockets, auto_update, crash_watch, knowledge, reminder_scheduler,
|
||||
scheduled_prompts_worker,
|
||||
};
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
207
hive-c0re/src/lifecycle/git.rs
Normal file
207
hive-c0re/src/lifecycle/git.rs
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
//! Git shellout helpers for the per-agent proposed/applied repos: run
|
||||
//! `git` with the hive-c0re identity, resolve/plant refs and tags, and
|
||||
//! fetch proposal commits into the applied repo.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use tokio::process::Command;
|
||||
|
||||
const GIT_NAME: &str = "c0re";
|
||||
const GIT_EMAIL: &str = "c0re@hyperhive.local";
|
||||
|
||||
/// Return the SHA of the root (oldest, no-parent) commit in a repo.
|
||||
/// Used to seed the applied repo at the template baseline rather than at
|
||||
/// `main`, so the first `ApplyCommit` diff shows the manager's real changes.
|
||||
pub(super) async fn git_root_commit(dir: &Path) -> Result<String> {
|
||||
let out = git_command()
|
||||
.current_dir(dir)
|
||||
.args(["rev-list", "--max-parents=0", "HEAD"])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git rev-list --max-parents=0 HEAD in {}", dir.display()))?;
|
||||
if !out.status.success() {
|
||||
anyhow::bail!(
|
||||
"git rev-list --max-parents=0 failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> {
|
||||
git(
|
||||
dir,
|
||||
&[
|
||||
"-c",
|
||||
&format!("user.name={GIT_NAME}"),
|
||||
"-c",
|
||||
&format!("user.email={GIT_EMAIL}"),
|
||||
"commit",
|
||||
"-m",
|
||||
message,
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in
|
||||
/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise.
|
||||
#[must_use]
|
||||
pub fn git_command() -> Command {
|
||||
let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into());
|
||||
Command::new(exe)
|
||||
}
|
||||
|
||||
pub async fn git(dir: &Path, args: &[&str]) -> Result<()> {
|
||||
let out = git_command()
|
||||
.current_dir(dir)
|
||||
.args(args)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"git {} failed ({}): {}",
|
||||
args.join(" "),
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fetch the commit `sha` from the `src` git repo into `dst` and pin
|
||||
/// it as `refs/tags/<tag>`. Used at `request_apply_commit` time so
|
||||
/// hive-c0re captures an immutable handle on the manager's commit;
|
||||
/// subsequent amendments / force-pushes in `src` no longer affect
|
||||
/// what gets built. Returns the resolved full sha.
|
||||
///
|
||||
/// `sha` must be a commit sha (short or full) — the caller
|
||||
/// (`submit_apply_commit`) shape-checks it first. We resolve it
|
||||
/// LOCALLY against `src` rather than asking the remote to resolve
|
||||
/// it: `git fetch <remote> <sha>:<dst>` treats the left side as a
|
||||
/// remote *ref name*, and a bare sha is not one ("couldn't find
|
||||
/// remote ref ..."). Fetching by sha would need a full 40-hex sha
|
||||
/// plus `uploadpack.allow*SHA1InWant` on the remote, which the
|
||||
/// proposed repos don't set. hive-c0re has direct read access to
|
||||
/// `src`, so a local `rev-parse` + a branch-glob fetch sidesteps
|
||||
/// the whole sha-want negotiation.
|
||||
pub async fn git_fetch_to_tag(dst: &Path, src: &Path, sha: &str, tag: &str) -> Result<String> {
|
||||
let src_str = src.display().to_string();
|
||||
// Resolve the (short-or-full) sha to a full sha against the
|
||||
// source repo. The `^{commit}` peel + non-zero exit on a missing
|
||||
// object means a typo'd / stale sha fails loudly right here.
|
||||
let full = git_rev_parse(src, &format!("{sha}^{{commit}}"))
|
||||
.await
|
||||
.with_context(|| format!("commit '{sha}' not found in proposed repo {src_str}"))?;
|
||||
// Bring src's objects into dst. Fetching every head pulls the
|
||||
// wanted commit's history (always reachable from a branch in the
|
||||
// manager's flow) into dst's object db without sha-want.
|
||||
git(
|
||||
dst,
|
||||
&[
|
||||
"fetch",
|
||||
"--no-tags",
|
||||
&src_str,
|
||||
"+refs/heads/*:refs/remotes/proposal-src/*",
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
// Pin the exact commit as the proposal tag. The objects are now
|
||||
// local so this resolves without touching the remote.
|
||||
git(dst, &["tag", tag, &full]).await.with_context(|| {
|
||||
format!("tag {tag} at {full}: commit not reachable from any branch in proposed repo")
|
||||
})?;
|
||||
Ok(full)
|
||||
}
|
||||
|
||||
/// Resolve `refname` (a tag, branch, or sha) in `dir` to its full sha.
|
||||
pub async fn git_rev_parse(dir: &Path, refname: &str) -> Result<String> {
|
||||
let out = git_command()
|
||||
.current_dir(dir)
|
||||
.args(["rev-parse", refname])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git rev-parse {refname} in {}", dir.display()))?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"git rev-parse {refname} failed ({}): {}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
/// Plant a lightweight tag at `target`. Errors if the tag already
|
||||
/// exists — we want loud failures on id reuse, not silent
|
||||
/// overwrites.
|
||||
pub async fn git_tag(dir: &Path, name: &str, target: &str) -> Result<()> {
|
||||
git(dir, &["tag", name, target]).await
|
||||
}
|
||||
|
||||
/// Plant an annotated tag with `body` as the message. Used for
|
||||
/// `failed/<id>` (body = build error) and `denied/<id>` (body =
|
||||
/// operator note). Multi-line bodies handled via stdin so we don't
|
||||
/// have to escape anything.
|
||||
pub async fn git_tag_annotated(dir: &Path, name: &str, target: &str, body: &str) -> Result<()> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
// Annotated tags are git objects, so they need a tagger identity
|
||||
// (same constraint as a commit). Pass the hive-c0re identity
|
||||
// inline rather than relying on a global git config — applied
|
||||
// repos are hive-c0re-owned and the host's user might not have
|
||||
// user.email set.
|
||||
let mut child = git_command()
|
||||
.current_dir(dir)
|
||||
.args([
|
||||
"-c",
|
||||
&format!("user.name={GIT_NAME}"),
|
||||
"-c",
|
||||
&format!("user.email={GIT_EMAIL}"),
|
||||
"tag",
|
||||
"-a",
|
||||
name,
|
||||
target,
|
||||
"-F",
|
||||
"-",
|
||||
])
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("spawn git tag -a {name} in {}", dir.display()))?;
|
||||
if let Some(mut stdin) = child.stdin.take() {
|
||||
stdin
|
||||
.write_all(body.as_bytes())
|
||||
.await
|
||||
.context("write tag body to git stdin")?;
|
||||
// Drop closes stdin so git can finish reading.
|
||||
drop(stdin);
|
||||
}
|
||||
let out = child.wait_with_output().await.context("wait git tag -a")?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"git tag -a {name} failed ({}): {}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replace working tree + index with the tree at `target` without
|
||||
/// moving HEAD. `applied/main` stays pointing at the last known-good
|
||||
/// `deployed/*` while we let `nixos-container update` evaluate the
|
||||
/// candidate. On build failure callers reset back to HEAD; on
|
||||
/// success they fast-forward main to `target`.
|
||||
pub async fn git_read_tree_reset(dir: &Path, target: &str) -> Result<()> {
|
||||
git(dir, &["read-tree", "--reset", "-u", target]).await
|
||||
}
|
||||
|
||||
/// Hard-set a ref to `target`. Used to fast-forward `refs/heads/main`
|
||||
/// to the just-deployed proposal commit. Uses `update-ref`, not
|
||||
/// `branch -f`, so it works regardless of where HEAD currently sits.
|
||||
pub async fn git_update_ref(dir: &Path, refname: &str, target: &str) -> Result<()> {
|
||||
git(dir, &["update-ref", refname, target]).await
|
||||
}
|
||||
367
hive-c0re/src/lifecycle/host_config.rs
Normal file
367
hive-c0re/src/lifecycle/host_config.rs
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
//! Per-container host-side config: the nspawn conf rewrite (bind mounts,
|
||||
//! network isolation, forwarded credentials), the systemd resource-limits
|
||||
//! drop-in, and the `write_dropins` verb that re-applies both.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use hive_sh4re::priv_proto::{BindMount, CredentialMount};
|
||||
|
||||
use crate::coordinator::{AgentPaths, HiveEnv};
|
||||
|
||||
use super::{
|
||||
AGENT_PREFIX, CONTAINER_RUNTIME_MOUNT, CONTAINER_SHARED_MOUNT, agent_network_ip, agent_uid_gid,
|
||||
bridge_gateway_ip, container_claude_mount, container_name, validate,
|
||||
};
|
||||
|
||||
/// Re-apply the per-container host-side config: nspawn flags (bind
|
||||
/// mounts etc.), the systemd resource-limits drop-in, and a daemon
|
||||
/// reload so both take effect on the next unit (re)start. Idempotent —
|
||||
/// the job queue's `WriteDropin` node, also folded into every `Swap`
|
||||
/// (rebuild is the reconcile verb).
|
||||
pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
let container = container_name(name);
|
||||
set_nspawn_flags(
|
||||
&container,
|
||||
&paths.agent_dir,
|
||||
&paths.claude_dir,
|
||||
&paths.notes_dir,
|
||||
)
|
||||
.await?;
|
||||
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
|
||||
systemd_daemon_reload().await
|
||||
}
|
||||
|
||||
/// Write a systemd drop-in for `container@<container>.service` that applies
|
||||
/// our default resource caps. Goes under `/run/systemd/system/...` so it's
|
||||
/// ephemeral (regenerated on every spawn / rebuild).
|
||||
async fn set_resource_limits(container: &str, cpu_quota: &str, memory_max: &str) -> Result<()> {
|
||||
crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await
|
||||
}
|
||||
|
||||
async fn systemd_daemon_reload() -> Result<()> {
|
||||
crate::priv_client::daemon_reload().await
|
||||
}
|
||||
|
||||
/// Idempotently rewrite the lines in `/etc/nixos-containers/<container>.conf`
|
||||
/// that hive-c0re owns: `PRIVATE_NETWORK` (forced 0 so the agent's web UI port
|
||||
/// is reachable on the host) and `EXTRA_NSPAWN_FLAGS` (the runtime-dir bind).
|
||||
/// The start script expands `$EXTRA_NSPAWN_FLAGS` unquoted into the
|
||||
/// `systemd-nspawn` command.
|
||||
/// Where in the container's filesystem the manager sees its agents tree.
|
||||
/// Matches the `/agents` path that pre-Phase-8 hosts declared via
|
||||
/// `containers.root.bindMounts."/agents"`.
|
||||
pub const CONTAINER_MANAGER_AGENTS_MOUNT: &str = "/agents";
|
||||
|
||||
/// Where the manager sees the applied trees of every agent, read-only.
|
||||
/// Manager runs `git fetch /applied/<n>/.git refs/tags/*:refs/tags/applied/*`
|
||||
/// to learn what hive-c0re deployed (or rejected, or failed to
|
||||
/// build); the RO bind makes accidental writes impossible from
|
||||
/// inside the container.
|
||||
pub const CONTAINER_MANAGER_APPLIED_MOUNT: &str = "/applied";
|
||||
|
||||
/// The on-host root that gets bind-mounted to `/agents` inside the manager.
|
||||
/// Hard-coded to match `AGENT_STATE_ROOT` in coordinator.rs (kept duplicated
|
||||
/// here so lifecycle stays usable as a leaf module).
|
||||
pub(super) const HOST_AGENTS_ROOT: &str = "/var/lib/hyperhive/agents";
|
||||
|
||||
/// On-host applied repo root, mirrored RO into the manager. Matches
|
||||
/// `APPLIED_STATE_ROOT` in coordinator.rs.
|
||||
const HOST_APPLIED_ROOT: &str = "/var/lib/hyperhive/applied";
|
||||
|
||||
/// On-host meta repo root, mirrored RO into the manager. Matches
|
||||
/// `meta::meta_dir()` but duplicated here so lifecycle stays a leaf.
|
||||
const HOST_META_ROOT: &str = "/var/lib/hyperhive/meta";
|
||||
|
||||
/// Shared directory accessible to all agents. All agents bind-mount this RW.
|
||||
const HOST_SHARED_ROOT: &str = "/var/lib/hyperhive/shared";
|
||||
|
||||
/// Append bind flags for `child`'s state, harness, and config dirs into
|
||||
/// `binds`, all read-write. The RW on `state` is deliberate (recovery),
|
||||
/// not an oversight; see docs/persistence.md ("Parent access to child
|
||||
/// state") for the rationale. Creates missing host-side directories so
|
||||
/// nspawn doesn't refuse to start; missing dirs are non-fatal.
|
||||
fn bind_child_agent_dirs(child: &str, binds: &mut Vec<BindMount>) {
|
||||
let state_dir = format!("{HOST_AGENTS_ROOT}/{child}/state");
|
||||
let harness_dir = format!("{HOST_AGENTS_ROOT}/{child}/harness");
|
||||
let config_dir = format!("{HOST_AGENTS_ROOT}/{child}/config");
|
||||
for dir in [&state_dir, &harness_dir, &config_dir] {
|
||||
let _ = std::fs::create_dir_all(dir);
|
||||
}
|
||||
binds.push(BindMount {
|
||||
host_path: state_dir,
|
||||
container_path: format!("/agents/{child}/state"),
|
||||
read_only: false,
|
||||
});
|
||||
binds.push(BindMount {
|
||||
host_path: harness_dir,
|
||||
container_path: format!("/agents/{child}/harness"),
|
||||
read_only: false,
|
||||
});
|
||||
binds.push(BindMount {
|
||||
host_path: config_dir,
|
||||
container_path: format!("/agents/{child}/config"),
|
||||
read_only: false,
|
||||
});
|
||||
}
|
||||
|
||||
/// Hive-wide secrets forwarded into every agent container via nspawn
|
||||
/// `--load-credential=<name>:<host_path>`. Currently just the OTEL
|
||||
/// auth-header secret, when `services.hyperhive.otel.headersCredential`
|
||||
/// is set (surfaced as `HYPERHIVE_OTEL_HEADERS_CREDENTIAL` on hive-c0re's
|
||||
/// unit env — the same host option meta.rs reads to inject
|
||||
/// `hyperhive.otel.headersCredential`). The inner harness unit reads it
|
||||
/// via `LoadCredential=otel-headers` (inherit). The secret never lands in
|
||||
/// a bind mount, the nix store, or the generated config.
|
||||
///
|
||||
/// A configured-but-missing file is skipped with a warning rather than
|
||||
/// forwarded (nspawn would refuse to start the container otherwise): a
|
||||
/// host-level secret typo shouldn't take down every agent's start; OTEL
|
||||
/// just exports without the auth header until the file appears.
|
||||
fn hive_load_credentials() -> Vec<CredentialMount> {
|
||||
let mut out = Vec::new();
|
||||
let Ok(path) = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL") else {
|
||||
return out;
|
||||
};
|
||||
if path.is_empty() {
|
||||
return out;
|
||||
}
|
||||
if std::path::Path::new(&path).is_file() {
|
||||
out.push(CredentialMount {
|
||||
name: "otel-headers".to_owned(),
|
||||
host_path: path,
|
||||
});
|
||||
} else {
|
||||
tracing::warn!(
|
||||
%path,
|
||||
"HYPERHIVE_OTEL_HEADERS_CREDENTIAL is set but the file is missing; \
|
||||
skipping --load-credential (OTEL will export without the auth header)"
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::too_many_lines,
|
||||
reason = "one contiguous nspawn-flag assembly block; the length is the flag \
|
||||
surface itself, splitting it would just hide the shape"
|
||||
)]
|
||||
async fn set_nspawn_flags(
|
||||
container: &str,
|
||||
runtime_dir: &Path,
|
||||
claude_dir: &Path,
|
||||
notes_dir: &Path,
|
||||
) -> Result<()> {
|
||||
// Ensure /shared directory exists before binding. systemd-nspawn requires the bind source to exist.
|
||||
std::fs::create_dir_all(HOST_SHARED_ROOT)
|
||||
.with_context(|| format!("create {HOST_SHARED_ROOT}"))?;
|
||||
// Make /shared writable by every agent. Containers share host uids (no
|
||||
// PrivateUsers), but each agent is a distinct unix user, so a root-owned
|
||||
// 0755 dir leaves them unable to write — the documented "read/write for
|
||||
// all agents" contract was broken. A setgid group would need a
|
||||
// pinned GID declared in every container plus all agent users joined to
|
||||
// it (cross-container coordination + a rebuild cascade); instead we use
|
||||
// the /tmp model — sticky world-writable (1777). The sticky bit lets any
|
||||
// agent create files while protecting each agent's entries from deletion
|
||||
// by the others, and matches /shared's documented "free-for-all, may be
|
||||
// deleted/lost" semantics without touching any per-agent config.
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
let perms = std::fs::Permissions::from_mode(0o1777);
|
||||
std::fs::set_permissions(HOST_SHARED_ROOT, perms)
|
||||
.with_context(|| format!("chmod 1777 {HOST_SHARED_ROOT}"))?;
|
||||
}
|
||||
// Ensure /knowledge dir exists. It may be empty until forge seeds it;
|
||||
// nspawn refuses to start if the bind source is missing entirely.
|
||||
std::fs::create_dir_all(crate::knowledge::LOCAL_DIR)
|
||||
.with_context(|| format!("create {}", crate::knowledge::LOCAL_DIR))?;
|
||||
|
||||
// Logical agent name — strip the `h-` prefix.
|
||||
// For the manager: `h-ruth` → `ruth`. For sub-agents: `h-iris` → `iris`.
|
||||
let agent_name = container.strip_prefix(AGENT_PREFIX).unwrap_or(container);
|
||||
|
||||
// Claude credentials land at `/home/<agent>/.claude` so the
|
||||
// `claude` CLI (which reads `$HOME/.claude`) finds them. The
|
||||
// harness service's environment sets `HOME` to the same path
|
||||
// (`agent-base.nix` / `manager.nix`), so no `--setenv` plumbing
|
||||
// is needed here — the bind alone is enough.
|
||||
let claude_mount = container_claude_mount(agent_name);
|
||||
|
||||
// Hive-wide secrets forwarded into the container's credential store
|
||||
// (currently just the OTEL auth-header). Same for every agent.
|
||||
let load_creds = hive_load_credentials();
|
||||
|
||||
let mut binds: Vec<BindMount> = vec![
|
||||
BindMount {
|
||||
host_path: runtime_dir.to_string_lossy().into_owned(),
|
||||
container_path: CONTAINER_RUNTIME_MOUNT.to_owned(),
|
||||
read_only: false,
|
||||
},
|
||||
BindMount {
|
||||
host_path: claude_dir.to_string_lossy().into_owned(),
|
||||
container_path: claude_mount,
|
||||
read_only: false,
|
||||
},
|
||||
BindMount {
|
||||
host_path: HOST_SHARED_ROOT.to_owned(),
|
||||
container_path: CONTAINER_SHARED_MOUNT.to_owned(),
|
||||
read_only: false,
|
||||
},
|
||||
BindMount {
|
||||
host_path: crate::knowledge::LOCAL_DIR.to_owned(),
|
||||
container_path: crate::knowledge::CONTAINER_MOUNT.to_owned(),
|
||||
read_only: true,
|
||||
},
|
||||
];
|
||||
|
||||
// Own state, harness, and config dirs — same for every agent including
|
||||
// the manager. Config is RO: an agent must not edit its own config; changes
|
||||
// only ever flow through the approval queue.
|
||||
binds.push(BindMount {
|
||||
host_path: notes_dir.to_string_lossy().into_owned(),
|
||||
container_path: format!("/agents/{agent_name}/state"),
|
||||
read_only: false,
|
||||
});
|
||||
if let Some(state_parent) = notes_dir.parent() {
|
||||
let harness_dir = state_parent.join("harness");
|
||||
if !harness_dir.exists() {
|
||||
let _ = std::fs::create_dir_all(&harness_dir);
|
||||
}
|
||||
binds.push(BindMount {
|
||||
host_path: harness_dir.to_string_lossy().into_owned(),
|
||||
container_path: format!("/agents/{agent_name}/harness"),
|
||||
read_only: false,
|
||||
});
|
||||
}
|
||||
let own_config = format!("{HOST_AGENTS_ROOT}/{agent_name}/config");
|
||||
std::fs::create_dir_all(&own_config).with_context(|| format!("create {own_config}"))?;
|
||||
binds.push(BindMount {
|
||||
host_path: own_config,
|
||||
container_path: format!("/agents/{agent_name}/config"),
|
||||
read_only: true,
|
||||
});
|
||||
|
||||
// Topology-driven child mounts: every direct child of this agent gets
|
||||
// its state, harness, and config dirs bind-mounted RW (parent reads +
|
||||
// writes child state for recovery, and manages config). See
|
||||
// `bind_child_agent_dirs`.
|
||||
let direct_children = crate::topology::children_of(agent_name);
|
||||
for child in &direct_children {
|
||||
bind_child_agent_dirs(child, &mut binds);
|
||||
}
|
||||
|
||||
// `can_manage_top_level_agents` role: additionally mount every
|
||||
// parentless agent in the topology as a virtual child. Enables
|
||||
// recovery — a role holder can update those agents' configs even
|
||||
// when they are down. Also grants RO access to /applied and /meta.
|
||||
if crate::topology::has_role(
|
||||
agent_name,
|
||||
crate::topology::ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS,
|
||||
) {
|
||||
let top_level = crate::topology::top_level_agents();
|
||||
for tl in &top_level {
|
||||
if !direct_children.contains(tl) {
|
||||
bind_child_agent_dirs(tl, &mut binds);
|
||||
}
|
||||
}
|
||||
// systemd-nspawn refuses to start a container whose bind
|
||||
// source doesn't exist. The meta repo is created by the
|
||||
// startup migration, but make sure the directory is there
|
||||
// before the role holder comes up in case set_nspawn_flags
|
||||
// fires first (e.g. cold start with no agents).
|
||||
std::fs::create_dir_all(HOST_META_ROOT)
|
||||
.with_context(|| format!("create {HOST_META_ROOT}"))?;
|
||||
binds.push(BindMount {
|
||||
host_path: HOST_APPLIED_ROOT.to_owned(),
|
||||
container_path: CONTAINER_MANAGER_APPLIED_MOUNT.to_owned(),
|
||||
read_only: true,
|
||||
});
|
||||
binds.push(BindMount {
|
||||
host_path: HOST_META_ROOT.to_owned(),
|
||||
container_path: crate::meta::CONTAINER_MANAGER_META_MOUNT.to_owned(),
|
||||
read_only: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
|
||||
// container so the harness can bind `web.sock` there and the host-side
|
||||
// gateway sees it. Subdir bind (not socket file) keeps the inode
|
||||
// visible after the harness unlinks a stale socket on rebind.
|
||||
// Applies to manager and sub-agents alike.
|
||||
let socket_dir = crate::agent_sockets::agent_dir_for(agent_name);
|
||||
std::fs::create_dir_all(&socket_dir)
|
||||
.with_context(|| format!("create {}", socket_dir.display()))?;
|
||||
// Chown to the agent user so the non-root harness can bind(2) here.
|
||||
// Falls back to 0777 on first spawn when uid lookup returns None
|
||||
// (container /etc/passwd not yet rendered).
|
||||
if let Some((uid, gid)) = agent_uid_gid(agent_name) {
|
||||
if let Err(e) = crate::priv_client::chown_socket_dir(agent_name, uid, gid).await {
|
||||
tracing::warn!(%agent_name, error = ?e, "chown socket dir failed");
|
||||
}
|
||||
} else if let Err(e) = crate::priv_client::chmod_socket_dir(agent_name, 0o777).await {
|
||||
tracing::warn!(%agent_name, error = ?e, "chmod socket dir failed");
|
||||
}
|
||||
binds.push(BindMount {
|
||||
host_path: socket_dir.to_string_lossy().into_owned(),
|
||||
container_path: socket_dir.to_string_lossy().into_owned(),
|
||||
read_only: false,
|
||||
});
|
||||
|
||||
// Network isolation: when HIVE_NETWORK_ISOLATION=1 is set (by the
|
||||
// hive-network.nix module's `isolateContainers` option), flip the
|
||||
// container to a private network namespace with a veth pair attached
|
||||
// to the host bridge. Applies to all containers including the manager
|
||||
// (all hive-c0re<->agent comms go through bind-mounted UDS, not TCP).
|
||||
let isolation = {
|
||||
let isolate = std::env::var("HIVE_NETWORK_ISOLATION").ok().as_deref() == Some("1");
|
||||
let bridge = std::env::var("HIVE_NETWORK_BRIDGE").unwrap_or_default();
|
||||
let subnet = std::env::var("HIVE_NETWORK_SUBNET").unwrap_or_default();
|
||||
if isolate && !bridge.is_empty() && !subnet.is_empty() {
|
||||
let Some(agent_ip) = agent_network_ip(agent_name, &subnet) else {
|
||||
tracing::warn!(
|
||||
%agent_name, %subnet,
|
||||
"HIVE_NETWORK_SUBNET is set but could not derive a valid IP for agent \
|
||||
(bad CIDR? prefix too narrow?); skipping PRIVATE_NETWORK write to \
|
||||
avoid misconfigured isolation"
|
||||
);
|
||||
return crate::priv_client::write_nspawn_flags(
|
||||
container,
|
||||
&binds,
|
||||
None,
|
||||
&load_creds,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
let Some(gateway_ip) = bridge_gateway_ip(&subnet) else {
|
||||
tracing::warn!(
|
||||
%agent_name, %subnet,
|
||||
"HIVE_NETWORK_SUBNET is set but the bridge gateway IP is unparseable; \
|
||||
skipping PRIVATE_NETWORK write to avoid an isolated container with no \
|
||||
default route or resolver"
|
||||
);
|
||||
return crate::priv_client::write_nspawn_flags(
|
||||
container,
|
||||
&binds,
|
||||
None,
|
||||
&load_creds,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
tracing::info!(
|
||||
%agent_name, %agent_ip, %gateway_ip, %bridge,
|
||||
"network isolation: PRIVATE_NETWORK=1"
|
||||
);
|
||||
Some(hive_sh4re::priv_proto::NetworkIsolation {
|
||||
agent_ip,
|
||||
bridge,
|
||||
gateway_ip,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// Delegate the actual conf-file rewrite to hive-priv (runs as root).
|
||||
crate::priv_client::write_nspawn_flags(container, &binds, isolation, &load_creds).await
|
||||
}
|
||||
874
hive-c0re/src/lifecycle/mod.rs
Normal file
874
hive-c0re/src/lifecycle/mod.rs
Normal file
|
|
@ -0,0 +1,874 @@
|
|||
//! `nixos-container` lifecycle + per-agent config flake generation.
|
||||
|
||||
mod git;
|
||||
mod host_config;
|
||||
mod setup;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use git::{
|
||||
git, git_command, git_fetch_to_tag, git_read_tree_reset, git_rev_parse, git_tag,
|
||||
git_tag_annotated, git_update_ref,
|
||||
};
|
||||
pub use host_config::{
|
||||
CONTAINER_MANAGER_AGENTS_MOUNT, CONTAINER_MANAGER_APPLIED_MOUNT, write_dropins,
|
||||
};
|
||||
pub use setup::{
|
||||
ensure_agent_state_subvolume, ensure_claude_dir, ensure_state_dir, initial_flake_nix,
|
||||
setup_applied, setup_proposed,
|
||||
};
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use tokio::process::Command;
|
||||
|
||||
use crate::coordinator::{AgentPaths, HiveEnv};
|
||||
|
||||
/// Sub-agent container prefix. `nixos-container` caps the total container name
|
||||
/// at 11 chars (it gets encoded into network interface names), so the agent
|
||||
/// name itself can be at most `MAX_AGENT_NAME` chars.
|
||||
pub const AGENT_PREFIX: &str = "h-";
|
||||
pub const MAX_AGENT_NAME: usize = 9;
|
||||
/// Logical name of the manager agent (broker recipient, state-dir key,
|
||||
/// meta flake attribute). All persistent state lives under `ruth/`.
|
||||
pub const MANAGER_NAME: &str = "ruth";
|
||||
/// Container name of the manager. Uses the same `h-` prefix as sub-agents
|
||||
/// so `nixos-container list` output is uniform and the list filter is
|
||||
/// a single `starts_with(AGENT_PREFIX)` check. Logical name → container
|
||||
/// name: `ruth` → `h-ruth`.
|
||||
pub const MANAGER_CONTAINER: &str = "h-ruth";
|
||||
|
||||
/// Mount point of the per-agent runtime directory inside the container.
|
||||
pub const CONTAINER_RUNTIME_MOUNT: &str = "/run/hive";
|
||||
|
||||
/// Where the per-agent Claude credentials dir mounts inside the
|
||||
/// container. The harness service runs as a non-root unix user
|
||||
/// whose home is `/home/<agent>/`, so the mount path varies per
|
||||
/// agent — `container_claude_mount(name)` returns
|
||||
/// `/home/<name>/.claude` for every agent including the manager.
|
||||
/// `claude` inside the container reads
|
||||
/// `$HOME/.claude` and the service environment sets `HOME` to the
|
||||
/// same path, so the OAuth session survives container restarts.
|
||||
#[must_use]
|
||||
pub fn container_claude_mount(name: &str) -> String {
|
||||
format!("/home/{name}/.claude")
|
||||
}
|
||||
|
||||
/// Mount point of the shared directory accessible to all agents.
|
||||
/// All agents can read/write here; agents should only put things they're
|
||||
/// willing to lose (other agents may delete them).
|
||||
pub const CONTAINER_SHARED_MOUNT: &str = "/shared";
|
||||
|
||||
/// Sub-agent web UI port range. Deterministic from the agent's name (FNV-1a
|
||||
/// hash mod range size), so the dashboard can compute the same port without
|
||||
/// asking hive-c0re.
|
||||
const WEB_PORT_BASE: u16 = 8100;
|
||||
const WEB_PORT_RANGE: u16 = 900;
|
||||
|
||||
/// FNV-1a hash of a string — shared by `agent_web_port` and
|
||||
/// `agent_network_ip` so the derivation rule is identical.
|
||||
fn fnv1a(s: &str) -> u32 {
|
||||
let mut hash: u32 = 2_166_136_261;
|
||||
for b in s.bytes() {
|
||||
hash ^= u32::from(b);
|
||||
hash = hash.wrapping_mul(16_777_619);
|
||||
}
|
||||
hash
|
||||
}
|
||||
|
||||
/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) %
|
||||
/// WEB_PORT_RANGE` for every agent including the manager. The port
|
||||
/// allocation rule reads the same for every name; collisions are
|
||||
/// possible (birthday paradox at ~30 agents) and the operator
|
||||
/// resolves them by renaming an agent (different hash → different
|
||||
/// port). Stable across hosts, restarts, and dashboard renders —
|
||||
/// no state-file dance.
|
||||
#[must_use]
|
||||
pub fn agent_web_port(name: &str) -> u16 {
|
||||
// Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails.
|
||||
WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Deterministic IPv4 address for an agent inside an isolated subnet.
|
||||
///
|
||||
/// Parses `subnet_cidr` as `<network_ip>/<prefix_len>` (e.g.
|
||||
/// `"10.42.0.0/24"`), then computes:
|
||||
///
|
||||
/// ```text
|
||||
/// host_count = 2^(32 - prefix_len)
|
||||
/// usable = host_count - 3 // skip .0 (network), .1 (gateway), .255 (broadcast)
|
||||
/// offset = FNV-1a(name) % usable + 2 // .2 is the first agent slot
|
||||
/// agent_ip = network_base_u32 + offset
|
||||
/// ```
|
||||
///
|
||||
/// Returns `None` when `subnet_cidr` can't be parsed (invalid format,
|
||||
/// prefix out of range, etc.) so callers can fall back gracefully.
|
||||
/// Collisions are possible (birthday paradox) and the operator resolves
|
||||
/// them by renaming an agent, same as for port collisions.
|
||||
#[must_use]
|
||||
pub fn agent_network_ip(name: &str, subnet_cidr: &str) -> Option<String> {
|
||||
let (ip_str, prefix_str) = subnet_cidr.split_once('/')?;
|
||||
let prefix_len: u32 = prefix_str.parse().ok()?;
|
||||
if prefix_len > 30 {
|
||||
// /31 and /32 have no room for agents; /30 has 1 usable slot.
|
||||
// /0 (the other extreme) is handled further down: host_count
|
||||
// overflows checked_shl(32) → 0 → usable = 0 → None.
|
||||
return None;
|
||||
}
|
||||
// Parse dotted-decimal IPv4.
|
||||
let octets: Vec<u8> = ip_str
|
||||
.split('.')
|
||||
.map(|o| o.parse::<u8>().ok())
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
if octets.len() != 4 {
|
||||
return None;
|
||||
}
|
||||
let base_u32 = u32::from_be_bytes([octets[0], octets[1], octets[2], octets[3]]);
|
||||
// Mask off host bits to get the true network address.
|
||||
let mask = if prefix_len == 0 {
|
||||
0u32
|
||||
} else {
|
||||
!0u32 << (32 - prefix_len)
|
||||
};
|
||||
let network_base = base_u32 & mask;
|
||||
let host_count: u32 = 1u32.checked_shl(32 - prefix_len).unwrap_or(0);
|
||||
// `.0` = network, `.1` = bridge gateway, last = broadcast → 3 reserved.
|
||||
let usable = host_count.saturating_sub(3);
|
||||
if usable == 0 {
|
||||
return None;
|
||||
}
|
||||
let offset = fnv1a(name) % usable + 2; // +2: skip .0 and .1
|
||||
let ip_u32 = network_base + offset;
|
||||
let [a, b, c, d] = ip_u32.to_be_bytes();
|
||||
Some(format!("{a}.{b}.{c}.{d}"))
|
||||
}
|
||||
|
||||
/// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`.
|
||||
///
|
||||
/// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim
|
||||
/// (e.g. `10.42.0.1/24`), **not** the canonical network address — see
|
||||
/// the note in `set_nspawn_flags` + `docs/network.md`. The IP part is
|
||||
/// therefore the bridge IP itself: the host end of the bridge, the
|
||||
/// default-route target for isolated containers, and the address the
|
||||
/// hive dnsmasq resolver binds. Returns the dotted-decimal IP with the
|
||||
/// `/<prefix>` stripped, or `None` if the input isn't a valid
|
||||
/// `<ipv4>/<prefix>` pair.
|
||||
///
|
||||
/// Deliberately returns the operator-configured address verbatim rather
|
||||
/// than deriving `network + 1`: an operator who sets `bridgeIp` to a
|
||||
/// non-`.1` host address (e.g. `10.42.0.254`) runs the bridge + resolver
|
||||
/// there, so that — not `.1` — is the real gateway.
|
||||
#[must_use]
|
||||
pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option<String> {
|
||||
let (ip_str, prefix_str) = subnet_cidr.split_once('/')?;
|
||||
// Validate the prefix is a sane IPv4 CIDR length and the address is
|
||||
// dotted-decimal IPv4 — same shape `agent_network_ip` accepts — so a
|
||||
// malformed `HIVE_NETWORK_SUBNET` can't smuggle a bogus HOST_ADDRESS
|
||||
// into the nspawn conf.
|
||||
let prefix_len: u32 = prefix_str.parse().ok()?;
|
||||
if prefix_len > 32 {
|
||||
return None;
|
||||
}
|
||||
let octets: Vec<u8> = ip_str
|
||||
.split('.')
|
||||
.map(|o| o.parse::<u8>().ok())
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
if octets.len() != 4 {
|
||||
return None;
|
||||
}
|
||||
Some(ip_str.to_owned())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn container_name(name: &str) -> String {
|
||||
format!("{AGENT_PREFIX}{name}")
|
||||
}
|
||||
|
||||
/// Read the agent user's `(uid, gid)` from the container's nixos-managed
|
||||
/// `/etc/passwd`. Returns `None` when the container hasn't been built
|
||||
/// yet, the passwd file is unparseable, or the agent user is missing
|
||||
/// (e.g. legacy container that still runs as root).
|
||||
///
|
||||
/// Used by `forge` + `matrix` after writing per-agent state files so
|
||||
/// the bind-mounted host file ends up readable by the agent user
|
||||
/// without waiting for the next container activation to run the chown
|
||||
/// fixup.
|
||||
///
|
||||
/// Notes:
|
||||
/// - Reads the *container-local* passwd at
|
||||
/// `/var/lib/nixos-containers/<container>/etc/passwd`, not the host's.
|
||||
/// The container's user-namespace shares uids with the host (no
|
||||
/// `PrivateUsers`), so the uid is directly usable in host-side
|
||||
/// `chown(2)`.
|
||||
/// - Best-effort: caller treats `None` as "skip the chown".
|
||||
#[must_use]
|
||||
pub fn agent_uid_gid(agent_name: &str) -> Option<(u32, u32)> {
|
||||
let container = container_name(agent_name);
|
||||
let passwd_path = format!("/var/lib/nixos-containers/{container}/etc/passwd");
|
||||
let content = std::fs::read_to_string(&passwd_path).ok()?;
|
||||
for line in content.lines() {
|
||||
let mut parts = line.split(':');
|
||||
let user = parts.next()?;
|
||||
if user != agent_name {
|
||||
continue;
|
||||
}
|
||||
let _ = parts.next()?; // x (password placeholder)
|
||||
let uid: u32 = parts.next()?.parse().ok()?;
|
||||
let gid: u32 = parts.next()?.parse().ok()?;
|
||||
return Some((uid, gid));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Best-effort `chown(path, agent_uid, agent_gid)`. Resolves the agent's
|
||||
/// uid/gid via [`agent_uid_gid`] and shells out to `std::os::unix::fs::chown`.
|
||||
/// Silently no-ops when the container isn't built yet (`None` from
|
||||
/// [`agent_uid_gid`]) and logs at debug on chown syscall failure — the
|
||||
/// activation script in `harness-base.nix` is the steady-state safety
|
||||
/// net. Used by per-agent state writers in `forge` + `matrix` so the
|
||||
/// agent can read the file without waiting for the next container
|
||||
/// rebuild.
|
||||
pub fn chown_to_agent(name: &str, path: &Path, subsystem: &str) {
|
||||
let Some((uid, gid)) = agent_uid_gid(name) else {
|
||||
return;
|
||||
};
|
||||
if let Err(e) = std::os::unix::fs::chown(path, Some(uid), Some(gid)) {
|
||||
tracing::debug!(%name, %subsystem, path = %path.display(), error = %e, "chown to agent failed");
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(name: &str) -> Result<()> {
|
||||
if name.is_empty() {
|
||||
bail!("agent name must not be empty");
|
||||
}
|
||||
if name.len() > MAX_AGENT_NAME {
|
||||
bail!(
|
||||
"agent name '{name}' is too long ({} chars); max {MAX_AGENT_NAME}",
|
||||
name.len()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// First name (≠ `self_name`) currently running whose hashed port
|
||||
/// matches this agent's. The harness inside the colliding container
|
||||
/// would otherwise loop on `AddrInUse` forever; we surface the
|
||||
/// conflict here so spawn / rebuild fails loudly with an actionable
|
||||
/// message instead.
|
||||
async fn port_collision(self_name: &str) -> Option<String> {
|
||||
let port = agent_web_port(self_name);
|
||||
let raw = list().await.unwrap_or_default();
|
||||
for c in raw {
|
||||
let Some(other) = c.strip_prefix(AGENT_PREFIX) else {
|
||||
continue;
|
||||
};
|
||||
if other == self_name {
|
||||
continue;
|
||||
}
|
||||
if agent_web_port(other) == port && is_running(other).await {
|
||||
return Some(other.to_owned());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
create_container(name, hive, paths).await?;
|
||||
write_dropins(name, hive, paths).await?;
|
||||
priv_run("start", name).await
|
||||
}
|
||||
|
||||
/// First-spawn provisioning + `nixos-container create`, without the
|
||||
/// drop-in write or the start — the job queue's `Create` node.
|
||||
/// `spawn` composes this with `write_dropins` + start for direct
|
||||
/// callers (root-agent bootstrap).
|
||||
pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
if let Some(other) = port_collision(name).await {
|
||||
bail!(
|
||||
"port {} is already taken by '{other}' — rename one of them and retry",
|
||||
agent_web_port(name)
|
||||
);
|
||||
}
|
||||
setup_proposed(&paths.proposed_dir, name).await?;
|
||||
setup_applied(&paths.applied_dir, Some(&paths.proposed_dir), name).await?;
|
||||
ensure_agent_state_subvolume(name).await?;
|
||||
ensure_claude_dir(&paths.claude_dir)?;
|
||||
ensure_state_dir(&paths.notes_dir)?;
|
||||
// Meta flake gets the new agent's input + nixosConfiguration
|
||||
// before `nixos-container create` so the `--flake meta#<name>`
|
||||
// ref resolves.
|
||||
let agents = agents_after_spawn(name).await?;
|
||||
crate::meta::sync_agents(hive, &agents).await?;
|
||||
priv_run("create", name).await
|
||||
}
|
||||
|
||||
/// Rebuild-path preamble shared by the job queue's `Prebuild` node and
|
||||
/// `rebuild_no_meta`: fail fast on a port collision, then make sure
|
||||
/// the applied repo + state dirs exist. Container untouched.
|
||||
pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
if let Some(other) = port_collision(name).await {
|
||||
bail!(
|
||||
"port {} is already taken by '{other}' — rename one of them and retry",
|
||||
agent_web_port(name)
|
||||
);
|
||||
}
|
||||
setup_applied(&paths.applied_dir, None, name).await?;
|
||||
ensure_agent_state_subvolume(name).await?;
|
||||
ensure_claude_dir(&paths.claude_dir)?;
|
||||
ensure_state_dir(&paths.notes_dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Profile-swap for an existing, stopped container: re-apply the
|
||||
/// drop-ins, then `nixos-container update`. The job queue's `Swap`
|
||||
/// node. Requires the container stopped (the queue's `StopForUpdate`
|
||||
/// upstream); does NOT start it — the DAG's tail `Reconcile` owns
|
||||
/// bringing the agent back to its wanted power state.
|
||||
pub async fn swap_update(
|
||||
name: &str,
|
||||
hive: &HiveEnv,
|
||||
paths: &AgentPaths,
|
||||
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
|
||||
) -> Result<()> {
|
||||
write_dropins(name, hive, paths).await?;
|
||||
on_step("nixos-container update");
|
||||
priv_run_inner("update", name, Some(on_build_log_id)).await
|
||||
}
|
||||
|
||||
/// Build the `AgentSpec` list for the meta flake from `nixos-container
|
||||
/// list` + a hypothetical extra name not yet in the list (for spawn
|
||||
/// where the new agent's container doesn't exist yet). Pass empty
|
||||
/// `name_to_add` from rebuild paths where the agent is already in the
|
||||
/// container list.
|
||||
///
|
||||
/// Propagates errors from `list()` rather than swallowing them.
|
||||
/// Using `.unwrap_or_default()` here would silently produce an empty
|
||||
/// agent list when `nixos-container list` fails (priv helper down, race),
|
||||
/// which `sync_agents` would then commit to meta — dropping every agent
|
||||
/// from `flake.nix`. Callers that can tolerate failures (e.g. migration)
|
||||
/// handle the `Err` themselves with `.unwrap_or_default()`.
|
||||
async fn agents_for_meta(name_to_add: Option<&str>) -> Result<Vec<crate::meta::AgentSpec>> {
|
||||
let containers = list().await?;
|
||||
let mut out: Vec<crate::meta::AgentSpec> = containers
|
||||
.into_iter()
|
||||
.filter_map(|c| {
|
||||
let name = c.strip_prefix(AGENT_PREFIX)?.to_owned();
|
||||
Some(crate::meta::AgentSpec {
|
||||
is_manager: name == MANAGER_NAME,
|
||||
port: agent_web_port(&name),
|
||||
name,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
if let Some(extra) = name_to_add
|
||||
&& !out.iter().any(|a| a.name == extra)
|
||||
{
|
||||
out.push(crate::meta::AgentSpec {
|
||||
is_manager: extra == MANAGER_NAME,
|
||||
port: agent_web_port(extra),
|
||||
name: extra.to_owned(),
|
||||
});
|
||||
}
|
||||
out.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn agents_after_spawn(name: &str) -> Result<Vec<crate::meta::AgentSpec>> {
|
||||
agents_for_meta(Some(name)).await
|
||||
}
|
||||
|
||||
/// Like `agents_for_meta_listing` but with an extra agent added (for a
|
||||
/// container that doesn't exist yet). Used by the first-spawn path in
|
||||
/// `actions::run_apply_commit` to register the new agent in meta before
|
||||
/// `prepare_deploy` tries to update its input lock.
|
||||
pub async fn agents_for_meta_listing_with(extra: &str) -> Result<Vec<crate::meta::AgentSpec>> {
|
||||
agents_for_meta(Some(extra)).await
|
||||
}
|
||||
|
||||
/// Public enumeration of currently-existing agents (whatever
|
||||
/// `nixos-container list` says), sorted, no extras. For callers
|
||||
/// outside this module that need to reseed meta after lifecycle
|
||||
/// changes — destroy, startup reconciliation, etc.
|
||||
pub async fn agents_for_meta_listing() -> Result<Vec<crate::meta::AgentSpec>> {
|
||||
agents_for_meta(None).await
|
||||
}
|
||||
|
||||
/// True when the named container already exists (appears in
|
||||
/// `nixos-container list`). Used by the apply-commit path to decide
|
||||
/// between first-spawn (`nixos-container create`) and normal rebuild
|
||||
/// (`nixos-container update`).
|
||||
pub async fn container_exists(name: &str) -> bool {
|
||||
let container = container_name(name);
|
||||
list()
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.any(|c| c == &container)
|
||||
}
|
||||
|
||||
pub async fn kill(name: &str) -> Result<()> {
|
||||
validate(name)?;
|
||||
priv_run("stop", name).await
|
||||
}
|
||||
|
||||
pub async fn start(name: &str) -> Result<()> {
|
||||
validate(name)?;
|
||||
priv_run("start", name).await
|
||||
}
|
||||
|
||||
/// Start with the cold-start fallback: when a plain start fails (the
|
||||
/// activation-error shape), retry once via stop + kill + start before
|
||||
/// giving up. Used by the queue's fast-lane `Start` handler and the
|
||||
/// inline start-after-rebuild path.
|
||||
/// See `docs/coordinator.md::Cold-start fallback`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates the retry's start error (annotated with the original
|
||||
/// failure) when the fallback also fails.
|
||||
pub async fn start_with_fallback(name: &str) -> Result<()> {
|
||||
validate(name)?;
|
||||
if let Err(start_err) = priv_run("start", name).await {
|
||||
let container = container_name(name);
|
||||
tracing::warn!(
|
||||
container = %container,
|
||||
error = %start_err,
|
||||
"start failed (possible activation error); retrying via stop + kill + start"
|
||||
);
|
||||
priv_run("stop", name).await.unwrap_or_else(|e| {
|
||||
tracing::warn!(
|
||||
container = %container,
|
||||
error = %e,
|
||||
"stop before cold-start retry failed (ignored)"
|
||||
);
|
||||
});
|
||||
priv_run("kill", name).await.unwrap_or_else(|e| {
|
||||
tracing::warn!(
|
||||
container = %container,
|
||||
error = %e,
|
||||
"kill before cold-start retry failed (ignored)"
|
||||
);
|
||||
});
|
||||
priv_run("start", name).await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"cold-start fallback also failed: {e:#} \
|
||||
(original start error: {start_err:#})"
|
||||
)
|
||||
})
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop + start without regenerating any config. For "kick the container"
|
||||
/// without touching the flake or nspawn flags.
|
||||
pub async fn restart(name: &str) -> Result<()> {
|
||||
kill(name).await?;
|
||||
start(name).await
|
||||
}
|
||||
|
||||
/// True when the container's systemd unit is active. Used by the dashboard
|
||||
/// to gate stop/restart buttons.
|
||||
pub async fn is_running(name: &str) -> bool {
|
||||
let container = container_name(name);
|
||||
let unit = format!("container@{container}.service");
|
||||
Command::new("systemctl")
|
||||
.args(["is-active", "--quiet", &unit])
|
||||
.status()
|
||||
.await
|
||||
.is_ok_and(|s| s.success())
|
||||
}
|
||||
|
||||
/// Fully tear down a sub-agent's container: stop + remove via `nixos-container
|
||||
/// destroy`, then clean our own systemd drop-in. Leaves it to the caller to
|
||||
/// wipe `/var/lib/hyperhive/...` state and the per-agent runtime dir.
|
||||
pub async fn destroy(name: &str) -> Result<()> {
|
||||
validate(name)?;
|
||||
let container = container_name(name);
|
||||
// nixos-container destroy handles stop + removal of /var/lib/nixos-containers/<C>
|
||||
// and /etc/nixos-containers/<C>.conf. Tolerate "no such container".
|
||||
if let Err(e) = priv_run("destroy", name).await {
|
||||
tracing::warn!(error = ?e, "nixos-container destroy returned an error; continuing cleanup");
|
||||
}
|
||||
// Remove the systemd resource-limits drop-in via hive-priv.
|
||||
if let Err(e) = crate::priv_client::remove_service_dropin(&container).await {
|
||||
tracing::warn!(error = ?e, "remove service drop-in failed (non-fatal)");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Container-level rebuild without touching the meta repo. The one
|
||||
/// remaining fused stop/update/start pipeline: the approval deploy
|
||||
/// (`actions::deploy_applied_target`) drives meta through the
|
||||
/// two-phase prepare/finalize/abort flow itself and needs the inline
|
||||
/// start to verify the agent comes back up before finalizing. Every
|
||||
/// other rebuild is a job-queue DAG (`Prebuild → StopForUpdate → Swap
|
||||
/// → Reconcile`) whose `Prebuild` executor owns the meta sync +
|
||||
/// relock this path's deleted `rebuild` wrapper used to do.
|
||||
///
|
||||
/// `on_step` is called at each phase boundary with a short human-readable
|
||||
/// label so callers can surface progress (e.g. update the rebuild-queue
|
||||
/// step shown in the dashboard). Pass `&|_| ()` when progress reporting
|
||||
/// is not needed.
|
||||
///
|
||||
/// `on_build_log_id` is called with the build-log row id immediately after
|
||||
/// the `nixos-container update` log row opens, before the actual update
|
||||
/// command starts. Callers can use this to link the queue entry to the log
|
||||
/// for live streaming. Pass `&|_| ()` when not needed.
|
||||
///
|
||||
/// `defer_start` skips the start-after-update for a previously-running
|
||||
/// container and returns `true` instead, so a queue-side caller can hand
|
||||
/// the (potentially slow) container boot to the fast lane rather than
|
||||
/// holding the serialized build lane through it. With `defer_start =
|
||||
/// false` the start (with cold-start fallback) runs inline as before and
|
||||
/// the return value is always `false`. The spawn path always starts
|
||||
/// inline — a freshly-created container boots as part of provisioning.
|
||||
pub async fn rebuild_no_meta(
|
||||
name: &str,
|
||||
hive: &HiveEnv,
|
||||
paths: &AgentPaths,
|
||||
defer_start: bool,
|
||||
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
|
||||
) -> Result<bool> {
|
||||
prepare_rebuild_dirs(name, paths).await?;
|
||||
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
||||
if container_exists(name).await {
|
||||
// Rebuild strategy: stop-before-update + pre-build.
|
||||
// See `docs/coordinator.md::Container lifecycle`.
|
||||
let was_running = is_running(name).await;
|
||||
write_dropins(name, hive, paths).await?;
|
||||
if was_running {
|
||||
on_step("nix build");
|
||||
prebuild_toplevel(name, &flake_ref, &|_| ()).await?;
|
||||
on_step("nixos-container stop");
|
||||
priv_run("stop", name).await?;
|
||||
}
|
||||
on_step("nixos-container update");
|
||||
let update_result = priv_run_inner("update", name, Some(on_build_log_id)).await;
|
||||
if let Err(ref update_err) = update_result {
|
||||
// The update failed (e.g. nix build error). If the agent was
|
||||
// running before we stopped it, try to bring it back up on the
|
||||
// previous successful configuration so it doesn't stay dead.
|
||||
// The start failure is logged but not promoted to an error —
|
||||
// we always propagate the original update error (below).
|
||||
if was_running {
|
||||
tracing::warn!(
|
||||
%name,
|
||||
error = %update_err,
|
||||
"nixos-container update failed; attempting restart on old config"
|
||||
);
|
||||
on_step("nixos-container start (recovery)");
|
||||
if let Err(e) = priv_run("start", name).await {
|
||||
tracing::warn!(%name, error = %e, "recovery start after failed update also failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
update_result?;
|
||||
if was_running {
|
||||
if defer_start {
|
||||
// The caller re-queues the start on the fast lane so the
|
||||
// build lane is freed for the next entry instead of
|
||||
// waiting out the container boot here.
|
||||
return Ok(true);
|
||||
}
|
||||
on_step("nixos-container start");
|
||||
start_with_fallback(name).await?;
|
||||
}
|
||||
Ok(false)
|
||||
} else {
|
||||
// Spawn path: create is atomic, no prebuild needed.
|
||||
// See `docs/coordinator.md::Spawn path`.
|
||||
on_step("nixos-container create");
|
||||
priv_run("create", name).await?;
|
||||
write_dropins(name, hive, paths).await?;
|
||||
on_step("nixos-container start");
|
||||
priv_run("start", name).await?;
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Pre-build `system.build.toplevel` against `meta#<name>` so the
|
||||
/// subsequent `nixos-container update` finds the result cached and
|
||||
/// skips straight to the profile-swap. Store-warming only — container
|
||||
/// is untouched. See `docs/coordinator.md::Rebuild path` for why
|
||||
/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild
|
||||
/// attr path` for why the explicit nixosConfigurations attr is required.
|
||||
///
|
||||
/// `on_build_log_id` fires with the `build_logs` row id as soon as the
|
||||
/// row opens, so queue-side callers can link their node to the live
|
||||
/// stream. Pass `&|_| ()` when not needed.
|
||||
pub async fn prebuild_toplevel(
|
||||
name: &str,
|
||||
flake_ref: &str,
|
||||
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
|
||||
) -> Result<()> {
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
// Split `<root>#<name>` so we can re-emit with the explicit
|
||||
// `nixosConfigurations.<name>` segment. The flake_ref shape is
|
||||
// constructed by `rebuild_no_meta` and always contains exactly one
|
||||
// `#`; `split_once` returning None here would be a programmer
|
||||
// error we'd want to surface loudly rather than paper over.
|
||||
let (flake_root, fragment) = flake_ref
|
||||
.split_once('#')
|
||||
.with_context(|| format!("flake_ref {flake_ref:?} missing '#<name>' fragment"))?;
|
||||
// Sanity-check the fragment matches the agent name we were
|
||||
// passed — guards against future calls that pass a divergent
|
||||
// pair (no current callsite does, but the pair is redundant
|
||||
// and worth checking once).
|
||||
if fragment != name {
|
||||
anyhow::bail!("prebuild_toplevel: flake_ref fragment '{fragment}' ≠ agent name '{name}'");
|
||||
}
|
||||
let attr = format!("{flake_root}#nixosConfigurations.{name}.config.system.build.toplevel");
|
||||
let args = vec![
|
||||
"--extra-experimental-features",
|
||||
"nix-command flakes",
|
||||
"build",
|
||||
"--no-link",
|
||||
"--print-out-paths",
|
||||
&attr,
|
||||
];
|
||||
let cmdline = format!("nix {}", args.join(" "));
|
||||
tracing::info!(%name, %cmdline, "prebuild: warming system toplevel");
|
||||
|
||||
// Open a build_logs row for this attempt (best-effort — None when
|
||||
// the global handle hasn't been installed, e.g. early startup
|
||||
// or standalone tests). Lines pumped from stdout/stderr append
|
||||
// into the row; `finish` lands the terminal status before we bail.
|
||||
let logs = crate::build_logs::global();
|
||||
let log_id = logs.as_ref().and_then(|h| {
|
||||
h.start(name, "prebuild", &cmdline)
|
||||
.map_err(|e| {
|
||||
tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)");
|
||||
})
|
||||
.ok()
|
||||
});
|
||||
if let Some(id) = log_id {
|
||||
on_build_log_id(id);
|
||||
}
|
||||
|
||||
let mut child = Command::new("nix")
|
||||
.args(&args)
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("spawn {cmdline}"))?;
|
||||
|
||||
let stdout = child.stdout.take().expect("piped stdout");
|
||||
let stderr = child.stderr.take().expect("piped stderr");
|
||||
|
||||
let stdout_cmdline = cmdline.clone();
|
||||
let stdout_logs = logs.clone();
|
||||
let pump_stdout = tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(stdout).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
tracing::info!(target: "nix-prebuild", cmdline = %stdout_cmdline, "{line}");
|
||||
if let (Some(h), Some(id)) = (&stdout_logs, log_id) {
|
||||
h.append_stdout(id, &line);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let stderr_cmdline = cmdline.clone();
|
||||
let stderr_logs = logs.clone();
|
||||
let pump_stderr = tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(stderr).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}");
|
||||
if let (Some(h), Some(id)) = (&stderr_logs, log_id) {
|
||||
h.append_stderr(id, &line);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let status = child
|
||||
.wait()
|
||||
.await
|
||||
.with_context(|| format!("wait {cmdline}"))?;
|
||||
let _ = pump_stdout.await;
|
||||
let _ = pump_stderr.await;
|
||||
|
||||
let ok = status.success();
|
||||
if let (Some(h), Some(id)) = (&logs, log_id) {
|
||||
h.finish(
|
||||
id,
|
||||
if ok {
|
||||
crate::build_logs::BuildStatus::Ok
|
||||
} else {
|
||||
crate::build_logs::BuildStatus::Fail
|
||||
},
|
||||
);
|
||||
}
|
||||
if !ok {
|
||||
match log_id {
|
||||
Some(id) => bail!("prebuild {cmdline} failed ({status}); see build log #{id}"),
|
||||
None => bail!("prebuild {cmdline} failed ({status})"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn list() -> Result<Vec<String>> {
|
||||
let stdout = crate::priv_client::list_containers().await?;
|
||||
Ok(stdout
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| line.starts_with(AGENT_PREFIX))
|
||||
.map(str::to_owned)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Build the per-line callback for `create_container_streaming` /
|
||||
/// `update_container_streaming`. Both ops share identical dispatch logic
|
||||
/// (stdout → info + `append_stdout`, stderr → warn + `append_stderr`); this
|
||||
/// helper avoids duplicating that match body across the two call sites.
|
||||
fn make_log_callback(
|
||||
logs: Option<std::sync::Arc<crate::build_logs::BuildLogs>>,
|
||||
log_id: Option<i64>,
|
||||
cmdline: String,
|
||||
) -> impl FnMut(hive_sh4re::priv_proto::PrivStream, &str) {
|
||||
use hive_sh4re::priv_proto::PrivStream;
|
||||
move |stream, line| match stream {
|
||||
PrivStream::Stdout => {
|
||||
tracing::info!(target: "nixos-container", cmdline = %cmdline, "{line}");
|
||||
if let (Some(h), Some(id)) = (&logs, log_id) {
|
||||
h.append_stdout(id, line);
|
||||
}
|
||||
}
|
||||
PrivStream::Stderr => {
|
||||
tracing::warn!(target: "nixos-container", cmdline = %cmdline, "{line}");
|
||||
if let (Some(h), Some(id)) = (&logs, log_id) {
|
||||
h.append_stderr(id, line);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute a container operation via hive-priv and integrate with
|
||||
/// `build_logs.sqlite`. hive-priv runs as root and forwards output lines
|
||||
/// to hive-c0re in real time via the streaming priv protocol. Each line
|
||||
/// is appended to the build-log row as it arrives, so the dashboard
|
||||
/// shows live progress during long `nixos-container create` / `update` runs.
|
||||
async fn priv_run(kind: &str, name: &str) -> Result<()> {
|
||||
priv_run_inner(kind, name, None).await
|
||||
}
|
||||
|
||||
/// Like `priv_run` but calls `on_log_id(log_id)` immediately after the
|
||||
/// build-log row is opened — before the actual container op starts.
|
||||
/// This lets callers surface the row id for live streaming (e.g. the
|
||||
/// rebuild-queue worker sets `build_log_id` on the queue entry so the
|
||||
/// dashboard can link to `/api/build-logs/id/{id}/stream`).
|
||||
///
|
||||
/// The callback fires only when a build-log row is successfully opened
|
||||
/// (i.e. the global `BuildLogs` handle is installed AND `h.start()`
|
||||
/// succeeds). No-op when `on_log_id` is `None` — that's the path for
|
||||
/// all callers that don't need the id.
|
||||
async fn priv_run_inner(
|
||||
kind: &str,
|
||||
name: &str,
|
||||
on_log_id: Option<&(dyn Fn(i64) + Send + Sync)>,
|
||||
) -> Result<()> {
|
||||
let container = container_name(name);
|
||||
let cmdline = format!("nixos-container {kind} {container}");
|
||||
|
||||
let logs = crate::build_logs::global();
|
||||
let log_id = logs.as_ref().and_then(|h| {
|
||||
h.start(name, kind, &cmdline)
|
||||
.map_err(|e| {
|
||||
tracing::warn!(error = ?e, "build_logs: start failed (priv_run log dropped)");
|
||||
})
|
||||
.ok()
|
||||
});
|
||||
// Notify the caller as soon as the log row exists so it can surface
|
||||
// the id for live streaming before the container op even starts.
|
||||
if let (Some(id), Some(cb)) = (log_id, on_log_id) {
|
||||
cb(id);
|
||||
}
|
||||
|
||||
// For long-running ops use the streaming protocol so build_logs
|
||||
// receives lines in real time rather than as a batch at completion.
|
||||
let result: Result<()> = match kind {
|
||||
"create" => {
|
||||
crate::priv_client::create_container_streaming(
|
||||
name,
|
||||
make_log_callback(logs.clone(), log_id, cmdline.clone()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
"update" => {
|
||||
crate::priv_client::update_container_streaming(
|
||||
name,
|
||||
make_log_callback(logs.clone(), log_id, cmdline.clone()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
"start" => crate::priv_client::start_container(name).await,
|
||||
"stop" => crate::priv_client::stop_container(name).await,
|
||||
"kill" => crate::priv_client::kill_container(name).await,
|
||||
"destroy" => crate::priv_client::destroy_container(name).await,
|
||||
other => Err(anyhow::anyhow!("unknown container op: {other}")),
|
||||
};
|
||||
|
||||
let succeeded = result.is_ok();
|
||||
if let (Some(h), Some(id)) = (&logs, log_id) {
|
||||
h.finish(
|
||||
id,
|
||||
if succeeded {
|
||||
crate::build_logs::BuildStatus::Ok
|
||||
} else {
|
||||
crate::build_logs::BuildStatus::Fail
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) => {
|
||||
let journal = if kind == "update" {
|
||||
container_journal_tail(&container).await
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
match log_id {
|
||||
Some(id) => bail!("{e:#}; see build log #{id}{journal}"),
|
||||
None => bail!("{e:#}{journal}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// On a failed `nixos-container update`, the stderr nixos-container
|
||||
/// itself prints is often terse ("failed to reload container") — the
|
||||
/// real reason (which unit failed `switch-to-configuration` during
|
||||
/// the reload phase) lands in the *container's* own journal, not on
|
||||
/// the host. Fetch the tail of it so a failed rebuild self-documents
|
||||
/// the failing unit in the error string, no second round-trip.
|
||||
///
|
||||
/// Scoped to `update`: that's the reload-phase case, and the
|
||||
/// container is still up (running the old generation) so
|
||||
/// `journalctl -M` works. Best-effort — returns "" for other verbs
|
||||
/// or when the journal can't be read (machine gone, journalctl
|
||||
/// missing); it never produces an error of its own.
|
||||
async fn container_journal_tail(container: &str) -> String {
|
||||
// `-M` enters the container namespace and needs root, so the read
|
||||
// is delegated to hive-priv (hive-c0re itself runs unprivileged).
|
||||
let res = crate::priv_client::read_container_journal(
|
||||
container,
|
||||
hive_sh4re::priv_proto::JournalQuery {
|
||||
lines: 40,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match res {
|
||||
Ok((stdout, _)) if !stdout.is_empty() => format!(
|
||||
"\n--- last 40 journal lines from container '{container}' ---\n{}",
|
||||
stdout.trim_end()
|
||||
),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
251
hive-c0re/src/lifecycle/setup.rs
Normal file
251
hive-c0re/src/lifecycle/setup.rs
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
//! First-spawn provisioning: seed the manager-editable proposed repo and
|
||||
//! the hive-c0re-owned applied repo, and ensure the per-agent state /
|
||||
//! claude-credentials dirs (btrfs subvolume when available) exist.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
|
||||
use super::git::{
|
||||
git, git_command, git_commit, git_read_tree_reset, git_rev_parse, git_root_commit, git_tag,
|
||||
};
|
||||
use super::host_config::HOST_AGENTS_ROOT;
|
||||
|
||||
/// Initialize the manager-editable proposed repo. Seeds two tracked
|
||||
/// files: `agent.nix` (the module the manager edits) and `flake.nix`
|
||||
/// (the boilerplate that lets the meta flake import this repo as an
|
||||
/// input — meta locks at a specific sha and reads
|
||||
/// `nixosModules.default`, so `flake.nix` must be in the commit). The
|
||||
/// manager shouldn't edit `flake.nix` (the prompt says so) but it's
|
||||
/// visible so they can introspect.
|
||||
///
|
||||
/// Touched by hive-c0re only on first spawn — never again — so the
|
||||
/// manager can't be surprised by hive-c0re commits or working-tree
|
||||
/// resets.
|
||||
pub async fn setup_proposed(proposed_dir: &Path, name: &str) -> Result<()> {
|
||||
let fresh = !proposed_dir.join(".git").exists();
|
||||
if fresh {
|
||||
std::fs::create_dir_all(proposed_dir)
|
||||
.with_context(|| format!("create {}", proposed_dir.display()))?;
|
||||
let agent_path = proposed_dir.join("agent.nix");
|
||||
if !agent_path.exists() {
|
||||
std::fs::write(&agent_path, initial_agent_nix(name))
|
||||
.with_context(|| format!("write {}", agent_path.display()))?;
|
||||
}
|
||||
let flake_path = proposed_dir.join("flake.nix");
|
||||
if !flake_path.exists() {
|
||||
std::fs::write(&flake_path, initial_flake_nix())
|
||||
.with_context(|| format!("write {}", flake_path.display()))?;
|
||||
}
|
||||
git(proposed_dir, &["init", "--initial-branch=main"]).await?;
|
||||
git(proposed_dir, &["add", "agent.nix", "flake.nix"]).await?;
|
||||
git_commit(proposed_dir, "hive-c0re init").await?;
|
||||
}
|
||||
// Idempotently wire the `applied` remote — purely for the
|
||||
// manager's ergonomics. The URL is the path inside the manager
|
||||
// container (`/applied/<n>/.git`), where the RO bind in
|
||||
// `set_nspawn_flags` makes it real. hive-c0re itself never
|
||||
// dereferences this remote; the host-side fetch in
|
||||
// `request_apply_commit` uses absolute host paths.
|
||||
ensure_applied_remote(proposed_dir, name).await
|
||||
}
|
||||
|
||||
async fn ensure_applied_remote(proposed_dir: &Path, name: &str) -> Result<()> {
|
||||
let want = format!("/applied/{name}/.git");
|
||||
let existing = git_command()
|
||||
.current_dir(proposed_dir)
|
||||
.args(["remote", "get-url", "applied"])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git remote get-url applied in {}", proposed_dir.display()))?;
|
||||
if existing.status.success() {
|
||||
let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned();
|
||||
if current == want {
|
||||
return Ok(());
|
||||
}
|
||||
// URL drifted (path scheme changed, etc.) — re-point it.
|
||||
return git(proposed_dir, &["remote", "set-url", "applied", &want]).await;
|
||||
}
|
||||
git(proposed_dir, &["remote", "add", "applied", &want]).await
|
||||
}
|
||||
|
||||
/// Set up the applied repo. First-spawn only: init the repo, pull
|
||||
/// proposed's initial commit in via `git fetch`, tag it `deployed/0`.
|
||||
/// This is the *only* time hive-c0re reads from `proposed` for an
|
||||
/// agent — subsequent proposals are fetched at `request_apply_commit`
|
||||
/// time and tagged `proposal/<id>` (see `actions::approve` for the
|
||||
/// tag state machine).
|
||||
///
|
||||
/// `proposed_dir` is `None` on rebuild paths where the repo already
|
||||
/// exists — we just verify it's the right shape and bail otherwise.
|
||||
/// Unlike the pre-overhaul code path, `flake.nix` is no longer
|
||||
/// regenerated at the host level: it's tracked in proposed (seeded by
|
||||
/// `setup_proposed`) and rides along on every fetch.
|
||||
pub async fn setup_applied(
|
||||
applied_dir: &Path,
|
||||
proposed_dir: Option<&Path>,
|
||||
name: &str,
|
||||
) -> Result<()> {
|
||||
std::fs::create_dir_all(applied_dir)
|
||||
.with_context(|| format!("create {}", applied_dir.display()))?;
|
||||
|
||||
if !applied_dir.join(".git").exists() {
|
||||
let Some(proposed) = proposed_dir else {
|
||||
bail!(
|
||||
"applied repo at {} is missing its .git directory; \
|
||||
cannot rebuild without a proposed source to seed from. \
|
||||
destroy --purge and re-spawn this agent.",
|
||||
applied_dir.display()
|
||||
);
|
||||
};
|
||||
git(applied_dir, &["init", "--initial-branch=main"]).await?;
|
||||
let proposed_str = proposed.display().to_string();
|
||||
// Seed the applied repo at the root (template) commit of proposed,
|
||||
// not at `main`. This ensures `deployed/0` is the template baseline
|
||||
// so the first ApplyCommit diff shows the manager's real changes
|
||||
// rather than an empty diff (which happens when the manager has
|
||||
// already committed their config and proposed/main == proposal/<id>).
|
||||
let root_sha = git_root_commit(proposed).await?;
|
||||
git(
|
||||
applied_dir,
|
||||
// --update-head-ok lets us fetch into refs/heads/main while
|
||||
// HEAD still points there. git's default safeguard refuses
|
||||
// to avoid index/working-tree desync, but the working tree
|
||||
// is empty (we just `init`'d) and we read-tree-reset right
|
||||
// after, so the safeguard is moot here.
|
||||
&[
|
||||
"fetch",
|
||||
"--no-tags",
|
||||
"--update-head-ok",
|
||||
&proposed_str,
|
||||
&format!("{root_sha}:refs/heads/main"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
git_read_tree_reset(applied_dir, "refs/heads/main").await?;
|
||||
git_tag(applied_dir, "deployed/0", "refs/heads/main").await?;
|
||||
} else if git_rev_parse(applied_dir, "refs/tags/deployed/0")
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
// Pre-overhaul applied repo — no deployed/* tag scheme,
|
||||
// flake.nix may be untracked, agent.nix possibly authored by
|
||||
// hive-c0re directly. The startup auto-migration fixes this
|
||||
// in place; if it didn't run (or got skipped), surface a
|
||||
// clear error.
|
||||
bail!(
|
||||
"applied repo at {} predates the meta-flake layout. \
|
||||
Restart hive-c0re to let the auto-migration run, or \
|
||||
destroy --purge {name} and re-spawn.",
|
||||
applied_dir.display()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create the per-agent Claude credentials dir if missing. Mode 0755 — hive-core
|
||||
/// needs read+execute to list the directory so `claude_has_session` can detect a
|
||||
/// valid session; credential files inside (`.credentials.json` etc.) are 0600 so
|
||||
/// secrets stay private regardless of the directory mode. Idempotent: existing
|
||||
/// dirs are left untouched (an agent's OAuth tokens survive `destroy`/recreate).
|
||||
/// Public for the `InitConfig` approval path in `actions.rs` which seeds
|
||||
/// dirs without calling the full `spawn`.
|
||||
pub fn ensure_claude_dir(claude_dir: &Path) -> Result<()> {
|
||||
use std::io;
|
||||
if !claude_dir.exists() {
|
||||
std::fs::create_dir_all(claude_dir)
|
||||
.with_context(|| format!("create {}", claude_dir.display()))?;
|
||||
}
|
||||
// 0755: hive-core (different user from the agent) needs read+execute to
|
||||
// list the directory so `claude_has_session` can detect a valid session.
|
||||
// The credential files inside (`.credentials.json` etc.) are 0600 so the
|
||||
// secrets themselves stay private regardless of the directory mode.
|
||||
//
|
||||
// Best-effort: on the first container boot, `hive-agent-user-migrate`
|
||||
// chowns this dir to the agent user. After that, hive-core (a different
|
||||
// user) cannot chmod it (EPERM) — that's fine because the mode set during
|
||||
// initial creation (0755) is preserved through the chown. Any other error
|
||||
// (ENOENT, I/O error) is unexpected and propagated.
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
match std::fs::set_permissions(claude_dir, std::fs::Permissions::from_mode(0o755)) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == io::ErrorKind::PermissionDenied => {
|
||||
tracing::debug!(
|
||||
path = %claude_dir.display(),
|
||||
"ensure_claude_dir: chmod 755 skipped (dir likely owned by agent user after migration)"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e).with_context(|| format!("chmod 755 {}", claude_dir.display()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Public for the `InitConfig` approval path in `actions.rs` which seeds
|
||||
/// dirs without calling the full `spawn`. Also creates the sibling `harness/`
|
||||
/// dir so the first harness startup can write its sqlite files immediately.
|
||||
pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
||||
if !notes_dir.exists() {
|
||||
std::fs::create_dir_all(notes_dir)
|
||||
.with_context(|| format!("create {}", notes_dir.display()))?;
|
||||
}
|
||||
// Harness dir is a sibling of the agent-visible state dir.
|
||||
if let Some(parent) = notes_dir.parent() {
|
||||
let harness_dir = parent.join("harness");
|
||||
if !harness_dir.exists() {
|
||||
std::fs::create_dir_all(&harness_dir)
|
||||
.with_context(|| format!("create {}", harness_dir.display()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure agent `name`'s persistent state root
|
||||
/// (`/var/lib/hyperhive/agents/<name>`) is a btrfs subvolume — when the host
|
||||
/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`,
|
||||
/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`.
|
||||
///
|
||||
/// Progressive enhancement: if the root already exists
|
||||
/// (any agent provisioned before this landed, plain dir or subvol) it's left
|
||||
/// exactly as-is — no auto-migration — and the priv round-trip is skipped. On
|
||||
/// a non-btrfs host the priv op no-ops and the root is later created as a
|
||||
/// plain dir by `ensure_*_dir`, identical to the old behaviour. Only a
|
||||
/// brand-new agent on a btrfs host gets a real subvolume. Subvolume creation
|
||||
/// is privileged, so it's delegated to hive-priv.
|
||||
pub async fn ensure_agent_state_subvolume(name: &str) -> Result<()> {
|
||||
let root = Path::new(HOST_AGENTS_ROOT).join(name);
|
||||
if root.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
crate::priv_client::ensure_agent_subvolume(name)
|
||||
.await
|
||||
.with_context(|| format!("ensure btrfs subvolume for agent {name}"))
|
||||
}
|
||||
|
||||
fn initial_agent_nix(name: &str) -> String {
|
||||
format!(
|
||||
"{{ config, pkgs, lib, ... }}:\n{{\n # Per-agent overrides for {name}. This is a regular NixOS module\n # — add packages, services, modules, imports as needed.\n #\n # imports = [ ./extra-module.nix ];\n # environment.systemPackages = with pkgs; [ ];\n}}\n",
|
||||
)
|
||||
}
|
||||
|
||||
/// Module-only flake exposed by every agent's repo. Consumed by the
|
||||
/// hive-c0re-owned meta flake at `/var/lib/hyperhive/meta/` as a flake
|
||||
/// input. The wrapper is intentionally permissive:
|
||||
///
|
||||
/// - Manager edits `inputs.* = …` to add other flakes (e.g. an MCP
|
||||
/// server's own flake) — the lock for those lands in the agent's
|
||||
/// own `flake.lock` and rolls up into meta's lock transitively.
|
||||
/// - The outputs block forwards every input (minus `self`) into
|
||||
/// `agent.nix` as the `flakeInputs` module argument, so the
|
||||
/// manager just references `flakeInputs.<name>.packages.${pkgs.system}.default`
|
||||
/// without further plumbing.
|
||||
///
|
||||
/// Identity injection (`HIVE_PORT` / `HIVE_LABEL` / dashboard port /
|
||||
/// git committer) still lives in the meta flake's wrapper.
|
||||
pub fn initial_flake_nix() -> &'static str {
|
||||
"{\n description = \"hyperhive agent\";\n inputs = { };\n outputs =\n { self, ... }@inputs:\n {\n nixosModules.default = {\n imports = [ ./agent.nix ];\n _module.args.flakeInputs = builtins.removeAttrs inputs [ \"self\" ];\n };\n };\n}\n"
|
||||
}
|
||||
155
hive-c0re/src/lifecycle/tests.rs
Normal file
155
hive-c0re/src/lifecycle/tests.rs
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
//! Unit tests for the lifecycle module (moved verbatim from the old
|
||||
//! single-file `lifecycle.rs` `#[cfg(test)]` block).
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Regression test: `setup_proposed` must seed both agent.nix and flake.nix
|
||||
/// in the initial commit. Before commit 5b5a93e flake.nix was missing from
|
||||
/// the scaffold, requiring manual creation (seen with the damocles agent).
|
||||
#[tokio::test]
|
||||
async fn setup_proposed_seeds_flake_nix() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let proposed = dir.path().join("proposed");
|
||||
setup_proposed(&proposed, "test-agent")
|
||||
.await
|
||||
.expect("setup_proposed");
|
||||
|
||||
// Both files must exist on disk.
|
||||
assert!(proposed.join("agent.nix").exists(), "agent.nix missing");
|
||||
assert!(proposed.join("flake.nix").exists(), "flake.nix missing");
|
||||
|
||||
// flake.nix must export nixosModules.default (the meta-flake contract).
|
||||
let flake = std::fs::read_to_string(proposed.join("flake.nix")).unwrap();
|
||||
assert!(
|
||||
flake.contains("nixosModules.default"),
|
||||
"flake.nix does not export nixosModules.default"
|
||||
);
|
||||
|
||||
// Both files must be tracked in the initial git commit.
|
||||
let out = git_command()
|
||||
.current_dir(&proposed)
|
||||
.args(["show", "--name-only", "--format=", "HEAD"])
|
||||
.output()
|
||||
.await
|
||||
.expect("git show");
|
||||
let tracked = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(tracked.contains("agent.nix"), "agent.nix not committed");
|
||||
assert!(tracked.contains("flake.nix"), "flake.nix not committed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_is_in_subnet() {
|
||||
// Default subnet 10.42.0.0/24 — agents get .2 to .254.
|
||||
let ip = agent_network_ip("alice", "10.42.0.0/24").expect("should produce an IP");
|
||||
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
|
||||
assert_eq!(&octets[..3], &[10, 42, 0], "wrong /24 prefix");
|
||||
assert!(
|
||||
octets[3] >= 2 && octets[3] <= 254,
|
||||
"host byte {}",
|
||||
octets[3]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_stable() {
|
||||
// Same name + subnet must always produce the same IP.
|
||||
let a = agent_network_ip("damocles", "10.42.0.0/24");
|
||||
let b = agent_network_ip("damocles", "10.42.0.0/24");
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_different_agents() {
|
||||
// Different agent names very likely produce different IPs (not guaranteed,
|
||||
// but for these two names the hashes don't collide).
|
||||
let alice = agent_network_ip("alice", "10.42.0.0/24").unwrap();
|
||||
let bob = agent_network_ip("bob", "10.42.0.0/24").unwrap();
|
||||
assert_ne!(alice, bob, "alice and bob collide — rename one");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_different_subnet() {
|
||||
let ip = agent_network_ip("alice", "192.168.5.0/24").expect("should produce an IP");
|
||||
let octets: Vec<u8> = ip.split('.').map(|o| o.parse().unwrap()).collect();
|
||||
assert_eq!(&octets[..3], &[192, 168, 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bridge_gateway_ip_extracts_verbatim_address() {
|
||||
// HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the
|
||||
// canonical network — the gateway is the address before the `/`.
|
||||
assert_eq!(
|
||||
bridge_gateway_ip("10.42.0.1/24").as_deref(),
|
||||
Some("10.42.0.1")
|
||||
);
|
||||
// Non-`.1` operator override: the gateway is wherever the bridge is.
|
||||
assert_eq!(
|
||||
bridge_gateway_ip("10.42.0.254/24").as_deref(),
|
||||
Some("10.42.0.254")
|
||||
);
|
||||
assert_eq!(
|
||||
bridge_gateway_ip("172.30.0.1/16").as_deref(),
|
||||
Some("172.30.0.1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bridge_gateway_ip_rejects_bad_input() {
|
||||
assert!(bridge_gateway_ip("notanip/24").is_none());
|
||||
assert!(bridge_gateway_ip("10.42.0.1").is_none()); // no prefix
|
||||
assert!(bridge_gateway_ip("10.42.0.1/33").is_none()); // prefix > 32
|
||||
assert!(bridge_gateway_ip("10.42.0.999/24").is_none()); // octet > 255
|
||||
assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_rejects_bad_input() {
|
||||
assert!(agent_network_ip("alice", "notanip/24").is_none());
|
||||
assert!(agent_network_ip("alice", "10.0.0.0/33").is_none()); // prefix > 32
|
||||
assert!(agent_network_ip("alice", "10.0.0.0/31").is_none()); // too small
|
||||
assert!(agent_network_ip("alice", "10.0.0.0").is_none()); // no prefix
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_network_ip_normalizes_bridge_ip_subnet() {
|
||||
// HIVE_NETWORK_SUBNET carries the bridge IP (10.42.0.1/24), not
|
||||
// canonical network (10.42.0.0/24). Both must produce the same result
|
||||
// after host-bit masking.
|
||||
let from_bridge = agent_network_ip("alice", "10.42.0.1/24");
|
||||
let from_canonical = agent_network_ip("alice", "10.42.0.0/24");
|
||||
assert_eq!(
|
||||
from_bridge, from_canonical,
|
||||
"bridge-IP and canonical-network form should normalize to the same result"
|
||||
);
|
||||
// Result must still be in .2-.254.
|
||||
let ip = from_bridge.unwrap();
|
||||
let last: u8 = ip.rsplit('.').next().unwrap().parse().unwrap();
|
||||
assert!((2..=254).contains(&last), "host byte {last}");
|
||||
}
|
||||
|
||||
/// `setup_proposed` is idempotent: calling it on an existing repo is a
|
||||
/// no-op (the fresh guard skips all writes).
|
||||
#[tokio::test]
|
||||
async fn setup_proposed_idempotent() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let proposed = dir.path().join("proposed");
|
||||
setup_proposed(&proposed, "test-agent")
|
||||
.await
|
||||
.expect("first call");
|
||||
// Second call must not error even though .git already exists.
|
||||
setup_proposed(&proposed, "test-agent")
|
||||
.await
|
||||
.expect("second call");
|
||||
// Still one commit.
|
||||
let out = git_command()
|
||||
.current_dir(&proposed)
|
||||
.args(["rev-list", "--count", "HEAD"])
|
||||
.output()
|
||||
.await
|
||||
.expect("git rev-list");
|
||||
let count = String::from_utf8_lossy(&out.stdout).trim().to_owned();
|
||||
assert_eq!(
|
||||
count, "1",
|
||||
"expected exactly one commit after idempotent call"
|
||||
);
|
||||
}
|
||||
|
|
@ -13,12 +13,11 @@
|
|||
//! the dashboard uses, so the bottleneck would be json
|
||||
//! (de)serialisation, not the read.
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::Result;
|
||||
use hive_sh4re::LooseEnd;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
|
||||
/// Open threads pending against `agent`:
|
||||
/// - undelivered inbox messages this agent still owes itself a `recv`
|
||||
|
|
@ -143,14 +142,6 @@ fn saturating_age(now: i64, then: i64) -> u64 {
|
|||
u64::try_from(delta).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ use hive_sh4re::{HostRequest, HostResponse};
|
|||
use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig};
|
||||
use hive_c0re::{
|
||||
agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge,
|
||||
knowledge, matrix, migrate, rebuild_queue, reminder_scheduler, scheduled_prompts_worker,
|
||||
server, socket_server,
|
||||
job_queue, knowledge, matrix, migrate, reminder_scheduler, scheduled_prompts_worker, server,
|
||||
socket_server,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -85,6 +85,11 @@ enum Cmd {
|
|||
/// option.
|
||||
#[arg(long)]
|
||||
model_prices: Option<String>,
|
||||
/// Override: number of concurrent nix-heavy job-queue nodes
|
||||
/// (prebuild / profile-swap / create / meta lock). Set via the
|
||||
/// `services.hyperhive.c0re.buildSlots` NixOS option.
|
||||
#[arg(long)]
|
||||
build_slots: Option<usize>,
|
||||
},
|
||||
/// Spawn a new agent container directly (`hive-agent-<name>`). Bypasses
|
||||
/// the approval queue — use only as an operator on the host. For
|
||||
|
|
@ -156,6 +161,7 @@ async fn main() -> Result<()> {
|
|||
agent_cpu_quota,
|
||||
agent_memory_max,
|
||||
model_prices,
|
||||
build_slots,
|
||||
} => {
|
||||
// Base config from the --config file (or the built-in
|
||||
// defaults), then apply any per-flag overrides — config
|
||||
|
|
@ -195,7 +201,10 @@ async fn main() -> Result<()> {
|
|||
sc.model_prices =
|
||||
serde_json::from_str(&v).context("--model-prices: invalid JSON")?;
|
||||
}
|
||||
cmd_serve(sc.env, sc.model_prices, db, &cli.socket).await
|
||||
if let Some(v) = build_slots {
|
||||
sc.build_slots = v;
|
||||
}
|
||||
cmd_serve(sc.env, sc.model_prices, sc.build_slots, db, &cli.socket).await
|
||||
}
|
||||
Cmd::Spawn { name } => {
|
||||
render(client::request(&cli.socket, HostRequest::Spawn { name }).await?)
|
||||
|
|
@ -244,6 +253,7 @@ async fn main() -> Result<()> {
|
|||
async fn cmd_serve(
|
||||
env: HiveEnv,
|
||||
model_prices: hive_c0re::hive_stats::PriceTable,
|
||||
build_slots: usize,
|
||||
db: std::path::PathBuf,
|
||||
socket: &std::path::Path,
|
||||
) -> Result<()> {
|
||||
|
|
@ -255,7 +265,7 @@ async fn cmd_serve(
|
|||
// `dashboard_port` is consumed into the Coordinator below; capture the
|
||||
// Copy value first for the dashboard + knowledge-webhook tasks.
|
||||
let dashboard_port = env.dashboard_port;
|
||||
let coord = Arc::new(Coordinator::open(&db, env, model_prices)?);
|
||||
let coord = Arc::new(Coordinator::open(&db, env, model_prices, build_slots)?);
|
||||
socket_server::start_manager(coord.clone())?;
|
||||
// Idempotent pre-flight: rewrite pre-meta-layout applied
|
||||
// repos, ensure proposed repos carry the `applied`
|
||||
|
|
@ -413,25 +423,17 @@ async fn cmd_serve(
|
|||
// and fans the body out to each active target's inbox. See
|
||||
// scheduled_prompts_worker.rs.
|
||||
scheduled_prompts_worker::spawn(coord.clone());
|
||||
// Rebuild-queue worker: drains the global rebuild/meta-update/
|
||||
// spawn queue FIFO so hive-c0re never runs two heavyweight
|
||||
// container ops concurrently. Existing rebuild call sites
|
||||
// (auto_update, dashboard, manager, approval handler) enqueue
|
||||
// here instead of awaiting `rebuild_agent` inline. See
|
||||
// `rebuild_queue.rs`.
|
||||
// Job-queue scheduler: drives the global DAG queue (rebuild /
|
||||
// meta-update / spawn / power ops). Concurrency comes from the
|
||||
// build-slot count + per-agent leases inside the queue, not from
|
||||
// multiple workers — cheap nodes (graceful signals, drains,
|
||||
// reconciles) overlap nix-heavy ones structurally. Call sites
|
||||
// (auto_update, dashboard, manager, approval handler) submit DAGs
|
||||
// instead of awaiting lifecycle work inline. See `job_queue/`.
|
||||
{
|
||||
let q_coord = coord.clone();
|
||||
tokio::spawn(async move {
|
||||
rebuild_queue::run_worker(q_coord).await;
|
||||
});
|
||||
// Fast lane: a second serial worker for hard Start/Stop, running
|
||||
// concurrently with the build worker above so a stop/start never
|
||||
// waits behind a slow build for another container. Per-agent
|
||||
// ordering vs that agent's own build is enforced in the queue's
|
||||
// claim logic (a fast op defers behind its agent's running build).
|
||||
let fast_coord = coord.clone();
|
||||
tokio::spawn(async move {
|
||||
rebuild_queue::run_fast_worker(fast_coord).await;
|
||||
job_queue::scheduler::run_worker(q_coord).await;
|
||||
});
|
||||
}
|
||||
// Forward every broker event onto the unified dashboard
|
||||
|
|
|
|||
|
|
@ -29,6 +29,25 @@ const GIT_EMAIL: &str = "c0re@hyperhive.local";
|
|||
/// take turns instead of colliding.
|
||||
static META_LOCK: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
/// Coarse exclusivity for meta-repo *windows* that span multiple
|
||||
/// `META_LOCK` acquisitions — above all the two-phase deploy
|
||||
/// (`prepare_deploy` stages `flake.lock` uncommitted for the whole
|
||||
/// container build; `finalize_deploy` / `abort_deploy` resolve it).
|
||||
/// `META_LOCK` serializes individual git ops but cannot keep another
|
||||
/// op out of that staged window: a perm-file or lock-bump commit
|
||||
/// landing mid-window would sweep the staged deploy lock into its own
|
||||
/// commit and neuter `abort_deploy`. Job-queue executors that mutate
|
||||
/// the meta repo hold this gate for their mutation span; the opaque
|
||||
/// approval-deploy node holds it across its whole prepare→finalize
|
||||
/// span. Never acquired inside this module's functions (they run
|
||||
/// *under* a caller's window — nesting would deadlock).
|
||||
static DEPLOY_GATE: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
/// Acquire the deploy/meta-mutation window gate. See [`DEPLOY_GATE`].
|
||||
pub async fn exclusive() -> tokio::sync::MutexGuard<'static, ()> {
|
||||
DEPLOY_GATE.lock().await
|
||||
}
|
||||
|
||||
/// Where the manager sees this directory inside its container (RO bind).
|
||||
pub const CONTAINER_MANAGER_META_MOUNT: &str = "/meta";
|
||||
|
||||
|
|
@ -239,11 +258,16 @@ pub async fn prepare_deploy(name: &str) -> Result<()> {
|
|||
pub async fn finalize_deploy(name: &str, sha: &str, tag: &str) -> Result<()> {
|
||||
let _guard = META_LOCK.lock().await;
|
||||
let dir = meta_dir();
|
||||
if !has_staged_changes(&dir).await? {
|
||||
if !paths_dirty(&dir, &["flake.lock"]).await? {
|
||||
return Ok(());
|
||||
}
|
||||
let short = &sha[..sha.len().min(12)];
|
||||
git_commit(&dir, &format!("deploy {name} {tag} {short}")).await
|
||||
git_commit_paths(
|
||||
&dir,
|
||||
&format!("deploy {name} {tag} {short}"),
|
||||
&["flake.lock"],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Phase 2-failure. Unstage + restore the lock so meta returns to
|
||||
|
|
@ -256,21 +280,6 @@ pub async fn abort_deploy() -> Result<()> {
|
|||
git(&dir, &["restore", "flake.lock"]).await
|
||||
}
|
||||
|
||||
async fn has_staged_changes(dir: &Path) -> Result<bool> {
|
||||
let st = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(["diff", "--cached", "--quiet"])
|
||||
.status()
|
||||
.await
|
||||
.with_context(|| format!("git diff --cached in {}", dir.display()))?;
|
||||
// exit 1 = differences present, 0 = no diff, other = error
|
||||
match st.code() {
|
||||
Some(0) => Ok(false),
|
||||
Some(1) => Ok(true),
|
||||
_ => bail!("git diff --cached exited unexpectedly"),
|
||||
}
|
||||
}
|
||||
|
||||
/// One-shot used by the manual-rebuild path: relock just one
|
||||
/// agent's input and commit the lock change if any. Single-phase
|
||||
/// (no separate finalize) because rebuild has no failure-revert
|
||||
|
|
@ -280,11 +289,16 @@ pub async fn lock_update_for_rebuild(name: &str) -> Result<()> {
|
|||
let dir = meta_dir();
|
||||
let input = format!("agent-{name}");
|
||||
nix(&dir, &["flake", "update", &input]).await?;
|
||||
if git_is_clean(&dir).await? {
|
||||
if !paths_dirty(&dir, &["flake.lock"]).await? {
|
||||
return Ok(());
|
||||
}
|
||||
git(&dir, &["add", "flake.lock"]).await?;
|
||||
git_commit(&dir, &format!("rebuild {name}: lock update")).await
|
||||
git_commit_paths(
|
||||
&dir,
|
||||
&format!("rebuild {name}: lock update"),
|
||||
&["flake.lock"],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Build the `--override-input` value pinning an agent's config repo to
|
||||
|
|
@ -349,7 +363,7 @@ pub async fn lock_update(inputs: &[String]) -> Result<()> {
|
|||
args.push(i.as_str());
|
||||
}
|
||||
nix(&dir, &args).await?;
|
||||
if git_is_clean(&dir).await? {
|
||||
if !paths_dirty(&dir, &["flake.lock"]).await? {
|
||||
return Ok(());
|
||||
}
|
||||
git(&dir, &["add", "flake.lock"]).await?;
|
||||
|
|
@ -360,7 +374,7 @@ pub async fn lock_update(inputs: &[String]) -> Result<()> {
|
|||
} else {
|
||||
format!("lock update: {}", inputs.join(", "))
|
||||
};
|
||||
git_commit(&dir, &msg).await
|
||||
git_commit_paths(&dir, &msg, &["flake.lock"]).await
|
||||
}
|
||||
|
||||
/// One-shot used by the auto-update path: pin the latest hyperhive
|
||||
|
|
@ -370,11 +384,11 @@ pub async fn lock_update_hyperhive() -> Result<()> {
|
|||
let _guard = META_LOCK.lock().await;
|
||||
let dir = meta_dir();
|
||||
nix(&dir, &["flake", "update", "hyperhive"]).await?;
|
||||
if git_is_clean(&dir).await? {
|
||||
if !paths_dirty(&dir, &["flake.lock"]).await? {
|
||||
return Ok(());
|
||||
}
|
||||
git(&dir, &["add", "flake.lock"]).await?;
|
||||
git_commit(&dir, "bump hyperhive").await
|
||||
git_commit_paths(&dir, "bump hyperhive", &["flake.lock"]).await
|
||||
}
|
||||
|
||||
/// Write the tool-groups file for `agent` and commit it atomically
|
||||
|
|
@ -388,8 +402,13 @@ pub async fn commit_tool_groups(agent: &str, groups: &[String]) -> Result<()> {
|
|||
if crate::tool_groups::tool_groups_path().exists() {
|
||||
git(&dir, &["add", "tool-groups.json"]).await?;
|
||||
}
|
||||
if has_staged_changes(&dir).await? {
|
||||
git_commit(&dir, &format!("set tool-groups for {agent}")).await?;
|
||||
if paths_dirty(&dir, &["tool-groups.json"]).await? {
|
||||
git_commit_paths(
|
||||
&dir,
|
||||
&format!("set tool-groups for {agent}"),
|
||||
&["tool-groups.json"],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -404,8 +423,13 @@ pub async fn commit_capabilities(agent: &str, caps: &[String]) -> Result<()> {
|
|||
if crate::capabilities::capabilities_path().exists() {
|
||||
git(&dir, &["add", "capabilities.json"]).await?;
|
||||
}
|
||||
if has_staged_changes(&dir).await? {
|
||||
git_commit(&dir, &format!("set capabilities for {agent}")).await?;
|
||||
if paths_dirty(&dir, &["capabilities.json"]).await? {
|
||||
git_commit_paths(
|
||||
&dir,
|
||||
&format!("set capabilities for {agent}"),
|
||||
&["capabilities.json"],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -445,8 +469,14 @@ pub async fn commit_perms(
|
|||
}
|
||||
parts.push("capabilities");
|
||||
}
|
||||
if has_staged_changes(&dir).await? {
|
||||
git_commit(&dir, &format!("set {} for {agent}", parts.join(" + "))).await?;
|
||||
let paths = ["tool-groups.json", "capabilities.json"];
|
||||
if paths_dirty(&dir, &paths).await? {
|
||||
git_commit_paths(
|
||||
&dir,
|
||||
&format!("set {} for {agent}", parts.join(" + ")),
|
||||
&paths,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -467,10 +497,11 @@ pub async fn commit_topology(
|
|||
let dir = meta_dir();
|
||||
let stage = async {
|
||||
git(&dir, &["add", "topology.json"]).await?;
|
||||
if has_staged_changes(&dir).await? {
|
||||
git_commit(
|
||||
if paths_dirty(&dir, &["topology.json"]).await? {
|
||||
git_commit_paths(
|
||||
&dir,
|
||||
&format!("topology: {} → {}", child, new_parent.unwrap_or("<root>")),
|
||||
&["topology.json"],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
|
@ -541,8 +572,8 @@ pub async fn bulk_commit_topology(
|
|||
};
|
||||
let stage = async {
|
||||
git(&dir, &["add", "topology.json"]).await?;
|
||||
if has_staged_changes(&dir).await? {
|
||||
git_commit(&dir, &commit_msg).await?;
|
||||
if paths_dirty(&dir, &["topology.json"]).await? {
|
||||
git_commit_paths(&dir, &commit_msg, &["topology.json"]).await?;
|
||||
}
|
||||
Ok::<_, anyhow::Error>(())
|
||||
};
|
||||
|
|
@ -1158,16 +1189,6 @@ where
|
|||
out
|
||||
}
|
||||
|
||||
async fn git_is_clean(dir: &Path) -> Result<bool> {
|
||||
let out = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(["status", "--porcelain"])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git status in {}", dir.display()))?;
|
||||
Ok(out.stdout.iter().all(u8::is_ascii_whitespace))
|
||||
}
|
||||
|
||||
/// Return the list of file names that are currently staged (index differs
|
||||
/// from HEAD). On the initial commit (`HEAD` doesn't exist yet) falls back
|
||||
/// to `git diff --cached --name-only HEAD` failing gracefully by using
|
||||
|
|
@ -1248,6 +1269,43 @@ async fn git_commit(dir: &Path, message: &str) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Path-limited commit: commits ONLY the given paths, so unrelated
|
||||
/// staged content — above all a `prepare_deploy`-staged `flake.lock`
|
||||
/// — can never be swept into someone else's commit. Every targeted
|
||||
/// meta commit (perm files, topology, lock bumps) goes through this;
|
||||
/// only `sync_agents` uses the bare [`git_commit`], because its
|
||||
/// staged set *is* its intentional commit set.
|
||||
async fn git_commit_paths(dir: &Path, message: &str, paths: &[&str]) -> Result<()> {
|
||||
let name = format!("user.name={GIT_NAME}");
|
||||
let email = format!("user.email={GIT_EMAIL}");
|
||||
let mut args = vec!["-c", &name, "-c", &email, "commit", "-m", message, "--"];
|
||||
args.extend_from_slice(paths);
|
||||
git(dir, &args).await?;
|
||||
if let Err(e) = crate::forge::push_meta(dir).await {
|
||||
tracing::warn!(error = ?e, "forge: meta push after commit failed (non-fatal)");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// True when any of `paths` differs between HEAD and the index or
|
||||
/// working tree — the path-scoped replacement for whole-tree
|
||||
/// `git_is_clean` / `has_staged_changes` guards, which a concurrently
|
||||
/// staged deploy lock would otherwise trip.
|
||||
async fn paths_dirty(dir: &Path, paths: &[&str]) -> Result<bool> {
|
||||
let mut args = vec!["diff", "--quiet", "HEAD", "--"];
|
||||
args.extend_from_slice(paths);
|
||||
let out = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(&args)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git diff --quiet in {}", dir.display()))?;
|
||||
// Exit 0 = no differences; 1 = differences; anything else (e.g.
|
||||
// no HEAD yet on a fresh repo) → treat as dirty so the commit
|
||||
// path runs and surfaces real errors loudly.
|
||||
Ok(!out.status.success())
|
||||
}
|
||||
|
||||
async fn nix(dir: &Path, args: &[&str]) -> Result<()> {
|
||||
// `--extra-experimental-features` belt-and-suspenders for hosts
|
||||
// that haven't set this in nix.conf. The hyperhive module's
|
||||
|
|
@ -1276,6 +1334,43 @@ async fn nix(dir: &Path, args: &[&str]) -> Result<()> {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The regression the deploy-window bug review surfaced: a
|
||||
/// path-limited commit must leave an unrelated staged file (the
|
||||
/// prepare_deploy-staged `flake.lock`) untouched, so a later
|
||||
/// `abort_deploy` still has something to restore.
|
||||
#[tokio::test]
|
||||
async fn path_limited_commit_leaves_unrelated_staged_file_alone() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let dir = tmp.path();
|
||||
git(dir, &["init", "--initial-branch=main"])
|
||||
.await
|
||||
.expect("git init");
|
||||
std::fs::write(dir.join("tool-groups.json"), "{}").expect("write");
|
||||
std::fs::write(dir.join("flake.lock"), "v1").expect("write");
|
||||
git(dir, &["add", "-A"]).await.expect("add");
|
||||
git_commit(dir, "seed").await.expect("seed commit");
|
||||
// A deploy stages a new lock (uncommitted)…
|
||||
std::fs::write(dir.join("flake.lock"), "v2-staged-by-deploy").expect("write");
|
||||
git(dir, &["add", "flake.lock"]).await.expect("stage lock");
|
||||
// …and a perm change commits, path-limited.
|
||||
std::fs::write(dir.join("tool-groups.json"), r#"{"alice":[]}"#).expect("write");
|
||||
git(dir, &["add", "tool-groups.json"]).await.expect("add");
|
||||
git_commit_paths(dir, "set tool-groups for alice", &["tool-groups.json"])
|
||||
.await
|
||||
.expect("path-limited commit");
|
||||
// The perm file is committed; the deploy's staged lock is not.
|
||||
assert!(
|
||||
!paths_dirty(dir, &["tool-groups.json"])
|
||||
.await
|
||||
.expect("check"),
|
||||
"perm file must be committed"
|
||||
);
|
||||
assert!(
|
||||
paths_dirty(dir, &["flake.lock"]).await.expect("check"),
|
||||
"staged deploy lock must survive the perm commit"
|
||||
);
|
||||
}
|
||||
|
||||
fn sample_spec(name: &str, is_manager: bool, port: u16) -> AgentSpec {
|
||||
AgentSpec {
|
||||
name: name.to_owned(),
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::approvals::kind_to_str;
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::limits;
|
||||
use crate::socket_server::spawn_question_watchdog;
|
||||
|
|
@ -229,7 +228,7 @@ pub fn handle_cancel_loose_end(
|
|||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id: approval.id,
|
||||
agent: &approval.agent,
|
||||
approval_kind: kind_to_str(approval.kind),
|
||||
approval_kind: approval.kind.as_str(),
|
||||
sha_short,
|
||||
status: "cancelled",
|
||||
note: approval.note,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -90,13 +90,9 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
|||
tracing::info!(%id, %name, "spawn approval queued");
|
||||
HostResponse::success()
|
||||
}
|
||||
HostRequest::Kill { name } => handle_kill(&coord, name).await?,
|
||||
HostRequest::Restart { name } => {
|
||||
tracing::info!(%name, "restart");
|
||||
lifecycle::restart(name).await?;
|
||||
HostResponse::success()
|
||||
}
|
||||
HostRequest::RestartAll => handle_restart_all().await?,
|
||||
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill),
|
||||
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart),
|
||||
HostRequest::RestartAll => handle_restart_all(&coord).await?,
|
||||
HostRequest::Stop { scope, graceful } => {
|
||||
// Resolve the scope to explicit container names at the entry
|
||||
// point, then operate on names — never pass the bare "all
|
||||
|
|
@ -131,13 +127,23 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
|||
agents.retain(|a| prev.contains(a));
|
||||
}
|
||||
let infra = scoped_infra(scope);
|
||||
handle_start(&agents, &infra).await?
|
||||
handle_start(&coord, &agents, &infra).await?
|
||||
}
|
||||
HostRequest::Destroy { name, purge } => {
|
||||
actions::destroy(&coord, name, *purge).await?;
|
||||
HostResponse::success()
|
||||
}
|
||||
HostRequest::Rebuild { name } => handle_rebuild(&coord, name).await?,
|
||||
HostRequest::Rebuild { name } => submit_single(&coord, name, Verb::Rebuild),
|
||||
HostRequest::QueueDag { id } => {
|
||||
// The polled DAG first, then its live fan-out children.
|
||||
let dags = coord
|
||||
.job_queue
|
||||
.snapshot()
|
||||
.into_iter()
|
||||
.filter(|d| d.id == *id || d.parent_id == Some(*id))
|
||||
.collect();
|
||||
HostResponse::dags(dags)
|
||||
}
|
||||
HostRequest::List => HostResponse::list(lifecycle::list().await?),
|
||||
HostRequest::AgentStatus => {
|
||||
let rows = crate::container_view::build_all(&coord)
|
||||
|
|
@ -201,6 +207,9 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
|
|||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
match lifecycle::spawn(name, &hive, &paths).await {
|
||||
Ok(()) => {
|
||||
if let Err(e) = coord.power.set(name, crate::power::Wanted::Up) {
|
||||
tracing::warn!(%name, error = ?e, "agent_power: set wanted=up failed");
|
||||
}
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Spawned {
|
||||
agent: name.to_owned(),
|
||||
ok: true,
|
||||
|
|
@ -223,44 +232,81 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
|
|||
Ok(HostResponse::success())
|
||||
}
|
||||
|
||||
/// Kill `name`'s container, unregister its socket, notify the manager.
|
||||
async fn handle_kill(coord: &Arc<Coordinator>, name: &str) -> Result<HostResponse> {
|
||||
tracing::info!(%name, "kill");
|
||||
lifecycle::kill(name).await?;
|
||||
coord.unregister_agent(name);
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
|
||||
agent: name.to_owned(),
|
||||
});
|
||||
Ok(HostResponse::success())
|
||||
/// Single-agent queue verbs the admin socket exposes. Each submits the
|
||||
/// matching DAG (persisting the `wanted` intent, serializing on the
|
||||
/// agent's lease, with the transient/crash-watch suppression the old
|
||||
/// direct lifecycle calls lacked) and returns the DAG id for the
|
||||
/// client's wait loop.
|
||||
#[derive(Clone, Copy)]
|
||||
enum Verb {
|
||||
/// Stop DAG (`wanted = Offline`; Reconcile kills + unregisters +
|
||||
/// fires `Killed`).
|
||||
Kill,
|
||||
/// Restart DAG (`wanted = Up`; mechanical stop + reconcile-start).
|
||||
Restart,
|
||||
/// Rebuild DAG — the Swap tail owns the manager `Rebuilt` events +
|
||||
/// kick, so the CLI path can't drift from the dashboard's.
|
||||
Rebuild,
|
||||
}
|
||||
|
||||
/// Restart every container, aggregating per-agent failures into one
|
||||
/// response rather than aborting on the first error.
|
||||
async fn handle_restart_all() -> Result<HostResponse> {
|
||||
fn submit_single(coord: &Arc<Coordinator>, name: &str, verb: Verb) -> HostResponse {
|
||||
use crate::job_queue::{Source, submit};
|
||||
let id = match verb {
|
||||
Verb::Kill => {
|
||||
tracing::info!(%name, "kill");
|
||||
submit::stop(
|
||||
coord,
|
||||
name,
|
||||
Source::Manual,
|
||||
"manual kill via hivectl".to_owned(),
|
||||
)
|
||||
}
|
||||
Verb::Restart => {
|
||||
tracing::info!(%name, "restart");
|
||||
submit::restart(
|
||||
coord,
|
||||
name,
|
||||
Source::Manual,
|
||||
"manual restart via hivectl".to_owned(),
|
||||
)
|
||||
}
|
||||
Verb::Rebuild => {
|
||||
tracing::info!(%name, "rebuild");
|
||||
submit::rebuild(
|
||||
coord,
|
||||
name,
|
||||
Source::Manual,
|
||||
"manual rebuild via hivectl".to_owned(),
|
||||
)
|
||||
}
|
||||
};
|
||||
HostResponse::queued(vec![id])
|
||||
}
|
||||
|
||||
/// Restart every container by submitting one restart DAG per agent —
|
||||
/// each serializes on its own lease, so unrelated agents' restarts
|
||||
/// overlap while nothing races an in-flight rebuild. Returns once all
|
||||
/// are queued; per-agent results surface on the queue.
|
||||
async fn handle_restart_all(coord: &Arc<Coordinator>) -> Result<HostResponse> {
|
||||
tracing::info!("restart-all");
|
||||
let agents = lifecycle::list().await?;
|
||||
let mut ok_agents: Vec<String> = Vec::new();
|
||||
let mut errors: Vec<String> = Vec::new();
|
||||
let mut queued: Vec<u64> = Vec::new();
|
||||
for agent in &agents {
|
||||
if let Err(e) = lifecycle::restart(agent).await {
|
||||
tracing::warn!(%agent, error = ?e, "restart-all: failed to restart agent");
|
||||
errors.push(format!("{agent}: {e:#}"));
|
||||
} else {
|
||||
ok_agents.push(agent.clone());
|
||||
}
|
||||
}
|
||||
if errors.is_empty() {
|
||||
Ok(HostResponse::list(ok_agents))
|
||||
} else {
|
||||
Ok(HostResponse {
|
||||
ok: false,
|
||||
error: Some(errors.join("; ")),
|
||||
agents: Some(ok_agents),
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
})
|
||||
let Some(logical) = agent.strip_prefix(lifecycle::AGENT_PREFIX) else {
|
||||
continue;
|
||||
};
|
||||
queued.push(crate::job_queue::submit::restart(
|
||||
coord,
|
||||
logical,
|
||||
crate::job_queue::Source::Manual,
|
||||
"manual restart via hivectl restart-all".to_owned(),
|
||||
));
|
||||
ok_agents.push(logical.to_owned());
|
||||
}
|
||||
let mut resp = HostResponse::list(ok_agents);
|
||||
resp.queued_dags = Some(queued);
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Stop the given `agents` (resolved logical names) then `infra` containers
|
||||
|
|
@ -270,11 +316,13 @@ async fn handle_restart_all() -> Result<HostResponse> {
|
|||
/// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these
|
||||
/// explicit name lists up front — this never sees the "all" flag.
|
||||
///
|
||||
/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the
|
||||
/// harness, run one stop-checkpoint turn, drain, then container stop, with a
|
||||
/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1`
|
||||
/// path. `graceful` applies to agents only - infra containers have no harness
|
||||
/// turn loop, so they're always hard-stopped.
|
||||
/// Every agent rides the job queue: a `graceful` stop submits the
|
||||
/// quiesce DAG (signal → drain → reconcile-stop; all drains overlap),
|
||||
/// a hard stop a plain stop DAG — both persist `wanted = Offline` and
|
||||
/// serialize on the agent's lease so nothing races an in-flight
|
||||
/// rebuild. The response carries the DAG ids so `hivectl` can wait
|
||||
/// with per-node progress. Infra containers have no harness / lease
|
||||
/// and stay direct + synchronous.
|
||||
async fn handle_stop(
|
||||
coord: &Arc<Coordinator>,
|
||||
agents: &[String],
|
||||
|
|
@ -284,37 +332,41 @@ async fn handle_stop(
|
|||
tracing::info!(?agents, ?infra, graceful, "stop");
|
||||
let mut ok_items: Vec<String> = Vec::new();
|
||||
let mut errors: Vec<String> = Vec::new();
|
||||
let mut enqueued_graceful = false;
|
||||
let mut queued: Vec<u64> = Vec::new();
|
||||
|
||||
for agent in agents {
|
||||
if graceful {
|
||||
// Graceful stop: enqueue the quiesce orchestration rather than a
|
||||
// hard kill. Serialised through the rebuild queue so it can't race
|
||||
// an in-flight rebuild for the same agent, and its per-step
|
||||
// progress surfaces on the queue snapshot + build log.
|
||||
coord.rebuild_queue.enqueue(
|
||||
crate::rebuild_queue::QueueKind::GracefulStop,
|
||||
agent.clone(),
|
||||
crate::rebuild_queue::QueueSource::Manual,
|
||||
"manual via hivectl graceful stop".to_owned(),
|
||||
None,
|
||||
);
|
||||
ok_items.push(agent.clone());
|
||||
enqueued_graceful = true;
|
||||
continue;
|
||||
}
|
||||
match lifecycle::kill(agent).await {
|
||||
Ok(()) => ok_items.push(agent.clone()),
|
||||
Err(e) => {
|
||||
tracing::warn!(%agent, error = ?e, "stop: agent kill failed");
|
||||
errors.push(format!("{agent}: {e:#}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if enqueued_graceful {
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
let reason = if graceful {
|
||||
"manual via hivectl graceful stop"
|
||||
} else {
|
||||
"manual via hivectl stop"
|
||||
};
|
||||
let id = if graceful {
|
||||
crate::job_queue::submit::graceful_stop(
|
||||
coord,
|
||||
agent,
|
||||
crate::job_queue::Source::Manual,
|
||||
reason.to_owned(),
|
||||
)
|
||||
} else {
|
||||
crate::job_queue::submit::stop(
|
||||
coord,
|
||||
agent,
|
||||
crate::job_queue::Source::Manual,
|
||||
reason.to_owned(),
|
||||
)
|
||||
};
|
||||
queued.push(id);
|
||||
ok_items.push(agent.clone());
|
||||
}
|
||||
|
||||
// Agents go down before infra so they're not mid-request against a
|
||||
// forge/matrix that's already gone. Hard stops are quick kills —
|
||||
// await their DAGs (bounded) before touching infra. Graceful stops
|
||||
// keep the immediate return (drains take minutes and the
|
||||
// agents-then-infra race pre-existed there).
|
||||
if !graceful && !infra.is_empty() {
|
||||
await_dags(coord, &queued, std::time::Duration::from_mins(2)).await;
|
||||
}
|
||||
for &container in infra {
|
||||
let name = container.unit_name();
|
||||
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {
|
||||
|
|
@ -326,14 +378,42 @@ async fn handle_stop(
|
|||
}
|
||||
}
|
||||
|
||||
Ok(finish_lifecycle(ok_items, &errors))
|
||||
let mut resp = finish_lifecycle(ok_items, &errors);
|
||||
resp.queued_dags = Some(queued);
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Best-effort server-side wait for a set of DAGs to settle terminal,
|
||||
/// bounded by `timeout` — used to preserve ordering invariants inside
|
||||
/// one request (agent stops before infra stops) without trusting the
|
||||
/// client to wait.
|
||||
async fn await_dags(coord: &Arc<Coordinator>, ids: &[u64], timeout: std::time::Duration) {
|
||||
let deadline = std::time::Instant::now() + timeout;
|
||||
loop {
|
||||
let snap = coord.job_queue.snapshot();
|
||||
let pending = ids
|
||||
.iter()
|
||||
.any(|id| snap.iter().any(|d| d.id == *id && !d.state.is_terminal()));
|
||||
if !pending {
|
||||
return;
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
tracing::warn!(?ids, "await_dags: timed out; proceeding");
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the given `infra` containers then `agents` (`hivectl start`) — the
|
||||
/// inverse of [`handle_stop`]. Infra comes up before agents so the agents
|
||||
/// find forge/matrix/gateway ready. Per-target failures aggregated. Callers
|
||||
/// resolve the [`LifecycleScope`] to these explicit name lists up front.
|
||||
async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result<HostResponse> {
|
||||
async fn handle_start(
|
||||
coord: &Arc<Coordinator>,
|
||||
agents: &[String],
|
||||
infra: &[InfraContainer],
|
||||
) -> Result<HostResponse> {
|
||||
tracing::info!(?agents, ?infra, "start");
|
||||
let mut ok_items: Vec<String> = Vec::new();
|
||||
let mut errors: Vec<String> = Vec::new();
|
||||
|
|
@ -349,17 +429,23 @@ async fn handle_start(agents: &[String], infra: &[InfraContainer]) -> Result<Hos
|
|||
}
|
||||
}
|
||||
|
||||
let mut queued: Vec<u64> = Vec::new();
|
||||
for agent in agents {
|
||||
match lifecycle::start(agent).await {
|
||||
Ok(()) => ok_items.push(agent.clone()),
|
||||
Err(e) => {
|
||||
tracing::warn!(%agent, error = ?e, "start: agent start failed");
|
||||
errors.push(format!("{agent}: {e:#}"));
|
||||
}
|
||||
}
|
||||
// Through the queue: persists `wanted = Up`, upgrades a
|
||||
// stale-rev start to a full rebuild, and serializes on the
|
||||
// agent's lease. Ids ride back for hivectl's wait loop.
|
||||
queued.push(crate::job_queue::submit::start(
|
||||
coord,
|
||||
agent,
|
||||
crate::job_queue::Source::Manual,
|
||||
"manual via hivectl start".to_owned(),
|
||||
));
|
||||
ok_items.push(agent.clone());
|
||||
}
|
||||
|
||||
Ok(finish_lifecycle(ok_items, &errors))
|
||||
let mut resp = finish_lifecycle(ok_items, &errors);
|
||||
resp.queued_dags = Some(queued);
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Resolve which sub-agent logical names a scope targets: every live
|
||||
|
|
@ -444,48 +530,7 @@ fn finish_lifecycle(ok_items: Vec<String>, errors: &[String]) -> HostResponse {
|
|||
ok: false,
|
||||
error: Some(errors.join("; ")),
|
||||
agents: Some(ok_items),
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..HostResponse::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild `name`'s container, notifying the manager of the outcome
|
||||
/// (success or failure) and kicking the agent's next turn on success.
|
||||
async fn handle_rebuild(coord: &Arc<Coordinator>, name: &str) -> Result<HostResponse> {
|
||||
tracing::info!(%name, "rebuild");
|
||||
let agent_dir = coord.ensure_runtime(name)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||
let result = lifecycle::rebuild(name, &hive, &paths, true, false, &|_| (), &|_| ()).await;
|
||||
// Mirror auto_update::rebuild_agent — the manager wants to know
|
||||
// about every rebuild attempt regardless of which surface triggered
|
||||
// it, especially failures (build error → manager can adjust the
|
||||
// agent's agent.nix). Without this the admin-socket CLI was a
|
||||
// notify-gap.
|
||||
match &result {
|
||||
Ok(_) => {
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: name.to_owned(),
|
||||
ok: true,
|
||||
note: None,
|
||||
sha: None,
|
||||
tag: None,
|
||||
});
|
||||
// Wake the agent's next turn with the "you were rebuilt"
|
||||
// hint. Same pattern as auto_update::rebuild_agent and the
|
||||
// dashboard rebuild path — this is the CLI's equivalent.
|
||||
coord.kick_agent(name, "container rebuilt");
|
||||
}
|
||||
Err(e) => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
||||
agent: name.to_owned(),
|
||||
ok: false,
|
||||
note: Some(format!("{e:#}")),
|
||||
sha: None,
|
||||
tag: None,
|
||||
}),
|
||||
}
|
||||
result?;
|
||||
Ok(HostResponse::success())
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
343
hive-c0re/src/socket_server/config_approvals.rs
Normal file
343
hive-c0re/src/socket_server/config_approvals.rs
Normal file
|
|
@ -0,0 +1,343 @@
|
|||
//! Config-approval request handlers: `RequestInitConfig` /
|
||||
//! `RequestApplyCommit` / `RequestUpdateMetaInputs`, plus the shared
|
||||
//! submit helpers (`submit_init_config` / `submit_apply_commit`) and the
|
||||
//! commit-sha shape check (`validate_commit_ref`).
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use hive_sh4re::AgentResponse;
|
||||
|
||||
use super::require_new_child;
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
/// `RequestInitConfig` — queue an `InitConfig` approval for an agent. The
|
||||
/// `name` must be brand-new (absent from the topology) or already in the
|
||||
/// caller's subtree; the requester is recorded as the new agent's parent (the
|
||||
/// root requesting a new agent → a top-level agent, matching reconcile's
|
||||
/// default).
|
||||
pub(super) fn handle_request_init_config(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
name: &str,
|
||||
description: Option<String>,
|
||||
) -> AgentResponse {
|
||||
if let Some(err) = require_new_child(agent, name, "request_init_config for") {
|
||||
return err;
|
||||
}
|
||||
tracing::info!(%agent, %name, "request_init_config");
|
||||
match submit_init_config(coord, name, Some(agent), description) {
|
||||
Ok(_id) => AgentResponse::Ok,
|
||||
Err(e) => AgentResponse::Err {
|
||||
message: format!("{e:#}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// `RequestApplyCommit` — queue an apply-commit approval for an agent. The
|
||||
/// target must be in the caller's subtree (the root covers every agent).
|
||||
pub(super) async fn handle_request_apply_commit(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
target_agent: &str,
|
||||
commit_ref: &str,
|
||||
description: Option<&str>,
|
||||
) -> AgentResponse {
|
||||
if let Some(err) = require_new_child(agent, target_agent, "request_apply_commit for") {
|
||||
return err;
|
||||
}
|
||||
tracing::info!(%agent, %target_agent, %commit_ref, "request_apply_commit");
|
||||
match submit_apply_commit(coord, target_agent, commit_ref, description, agent).await {
|
||||
Ok((id, sha)) => {
|
||||
tracing::info!(%id, %target_agent, %sha, "apply_commit approval queued");
|
||||
AgentResponse::Ok
|
||||
}
|
||||
Err(e) => AgentResponse::Err {
|
||||
message: format!("{e:#}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// `RequestUpdateMetaInputs` — queue an `UpdateMetaInputs` approval
|
||||
/// carrying the JSON-encoded input list in `commit_ref` (no git commit
|
||||
/// is involved; the field is the payload the approval handler decodes).
|
||||
pub(super) fn handle_request_update_meta_inputs(
|
||||
coord: &Arc<Coordinator>,
|
||||
requester: &str,
|
||||
inputs: &[String],
|
||||
description: Option<&str>,
|
||||
) -> AgentResponse {
|
||||
let label = if inputs.is_empty() {
|
||||
"all inputs".to_string()
|
||||
} else {
|
||||
inputs.join(", ")
|
||||
};
|
||||
tracing::info!(%requester, %label, "request_update_meta_inputs");
|
||||
let commit_ref = serde_json::to_string(inputs).unwrap_or_default();
|
||||
let id = match coord
|
||||
.approvals
|
||||
.submit_kind(
|
||||
requester,
|
||||
hive_sh4re::ApprovalKind::UpdateMetaInputs,
|
||||
&commit_ref,
|
||||
description,
|
||||
requester,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("{e:#}"))
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("queue update_meta_inputs approval: {e:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
tracing::info!(%id, %label, "update_meta_inputs approval queued");
|
||||
coord.emit_approval_added(crate::coordinator::ApprovalAdded {
|
||||
id,
|
||||
agent: requester,
|
||||
approval_kind: "update_meta_inputs",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description: description.map(str::to_owned),
|
||||
pr_number: None,
|
||||
});
|
||||
AgentResponse::Ok
|
||||
}
|
||||
|
||||
/// `request_apply_commit` takes a commit SHA only — not a branch or
|
||||
/// tag name. A branch is mutable; pinning the proposal to a concrete
|
||||
/// sha keeps "what the manager asked to deploy" unambiguous and means
|
||||
/// the `proposal/<id>` tag is a faithful record of the request.
|
||||
/// Accepts a 7..=40 char hex string (short or full sha); the exact
|
||||
/// commit is resolved + existence-checked against the proposed repo
|
||||
/// later in `lifecycle::git_fetch_to_tag`.
|
||||
pub(crate) fn validate_commit_ref(commit_ref: &str) -> Result<()> {
|
||||
let n = commit_ref.len();
|
||||
let hex = commit_ref.chars().all(|c| c.is_ascii_hexdigit());
|
||||
if !(7..=40).contains(&n) || !hex {
|
||||
anyhow::bail!(
|
||||
"commit_ref '{commit_ref}' is not a commit sha — request_apply_commit \
|
||||
takes a 7-40 char hex sha, not a branch or tag name"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Queue an `InitConfig` approval for a brand-new agent whose config repo
|
||||
/// does not yet exist. Shared between the manager and agent sockets.
|
||||
///
|
||||
/// `parent`, when `Some`, is the agent that will own the new child once
|
||||
/// the operator approves: it is stashed in the approval's `commit_ref`
|
||||
/// field (unused for `InitConfig` otherwise — same pattern
|
||||
/// `UpdateMetaInputs` uses to carry its inputs JSON) and consumed in
|
||||
/// `run_approval_init_config` to write the `child -> parent` topology
|
||||
/// edge. Callers pass the requesting agent, so the requester becomes the
|
||||
/// new agent's parent (the root requesting a new agent → a top-level agent,
|
||||
/// matching `topology::reconcile`'s default). `None` writes no explicit
|
||||
/// edge (reconcile-default placement) — retained for that fallback.
|
||||
pub(crate) fn submit_init_config(
|
||||
coord: &Arc<Coordinator>,
|
||||
name: &str,
|
||||
parent: Option<&str>,
|
||||
description: Option<String>,
|
||||
) -> anyhow::Result<i64> {
|
||||
let proposed_dir = crate::coordinator::Coordinator::agent_proposed_dir(name);
|
||||
if proposed_dir.join(".git").exists() {
|
||||
anyhow::bail!(
|
||||
"proposed config repo for '{name}' already exists at {} - \
|
||||
use request_apply_commit to update an existing agent's config",
|
||||
proposed_dir.display()
|
||||
);
|
||||
}
|
||||
let id = coord
|
||||
.approvals
|
||||
.submit_kind(
|
||||
name,
|
||||
hive_sh4re::ApprovalKind::InitConfig,
|
||||
parent.unwrap_or(""),
|
||||
description.as_deref(),
|
||||
// `parent` is the requesting agent (becomes the new child's
|
||||
// parent); it's also the submitter the approval events route
|
||||
// back to. No declared parent = operator-initiated path.
|
||||
parent.unwrap_or("operator"),
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("queue approval row: {e:#}"))?;
|
||||
tracing::info!(%id, %name, "init_config approval queued");
|
||||
coord.emit_approval_added(crate::coordinator::ApprovalAdded {
|
||||
id,
|
||||
agent: name,
|
||||
approval_kind: "init_config",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description,
|
||||
pr_number: None,
|
||||
});
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Submit-time half of the apply flow: queue the approval row, then
|
||||
/// fetch the manager's commit from the proposed repo into applied and
|
||||
/// pin it as `refs/tags/proposal/<id>`. From this point on the manager
|
||||
/// repo is irrelevant for this approval — even if the manager amends
|
||||
/// or force-pushes, the canonical sha hive-c0re will eventually
|
||||
/// approve/deny lives in applied's object DB.
|
||||
///
|
||||
/// If anything fails after the row is inserted (sha missing in
|
||||
/// proposed, fs error, git plumbing crash) we mark the row failed and
|
||||
/// surface the error to the manager. We don't try to roll the row
|
||||
/// back — the failure is part of the audit trail.
|
||||
pub(crate) async fn submit_apply_commit(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
commit_ref: &str,
|
||||
description: Option<&str>,
|
||||
submitter: &str,
|
||||
) -> anyhow::Result<(i64, String)> {
|
||||
validate_commit_ref(commit_ref)?;
|
||||
let proposed_dir = crate::coordinator::Coordinator::agent_proposed_dir(agent);
|
||||
let applied_dir = crate::coordinator::Coordinator::agent_applied_dir(agent);
|
||||
if !proposed_dir.exists() {
|
||||
anyhow::bail!(
|
||||
"proposed repo missing for agent '{agent}' (expected at {})",
|
||||
proposed_dir.display()
|
||||
);
|
||||
}
|
||||
if !applied_dir.join(".git").exists() {
|
||||
// First deploy: seed the applied repo from proposed so we can plant
|
||||
// the proposal/<id> tag below. setup_applied seeds at the root
|
||||
// (template) commit of proposed, not at main, so deployed/0 is the
|
||||
// template baseline. This makes the diff mara sees on approval
|
||||
// show the manager's actual changes rather than an empty diff.
|
||||
crate::lifecycle::setup_applied(&applied_dir, Some(&proposed_dir), agent)
|
||||
.await
|
||||
.context("seed applied repo for first spawn")?;
|
||||
}
|
||||
let id = coord
|
||||
.approvals
|
||||
.submit_kind(
|
||||
agent,
|
||||
hive_sh4re::ApprovalKind::ApplyCommit,
|
||||
commit_ref,
|
||||
description,
|
||||
submitter,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("queue approval row: {e:#}"))?;
|
||||
let tag = format!("proposal/{id}");
|
||||
let sha =
|
||||
match crate::lifecycle::git_fetch_to_tag(&applied_dir, &proposed_dir, commit_ref, &tag)
|
||||
.await
|
||||
{
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
// Surface the failure on the approval row so the
|
||||
// dashboard reflects it instead of leaving a phantom
|
||||
// pending entry. The note doubles as the operator-visible
|
||||
// explanation of why the approval can't be approved.
|
||||
let note = format!("{e:#}");
|
||||
let _ = coord.approvals.mark_failed(id, ¬e);
|
||||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id,
|
||||
agent,
|
||||
approval_kind: "apply_commit",
|
||||
sha_short: None,
|
||||
status: "failed",
|
||||
note: Some(note),
|
||||
description: description.map(str::to_owned),
|
||||
});
|
||||
return Err(anyhow::anyhow!("git_fetch_to_tag: {e:#}"));
|
||||
}
|
||||
};
|
||||
coord
|
||||
.approvals
|
||||
.set_fetched_sha(id, &sha)
|
||||
.map_err(|e| anyhow::anyhow!("persist fetched_sha: {e:#}"))?;
|
||||
// Pre-flight gates: both reject the apply before approval if
|
||||
// the agent's flake state would inflate meta's lock with duplicates
|
||||
// or lie about what nix will fetch. Both checks independently read
|
||||
// `<tag>:flake.lock` via git — they don't share state. Order matters
|
||||
// only for early-exit + messaging: sync first means a stale lock
|
||||
// bails with the actionable "run `nix flake lock`" hint rather than
|
||||
// a dedup pass on a lock nix would never produce.
|
||||
//
|
||||
// Runs after `set_fetched_sha` so the failed row carries the sha
|
||||
// that broke. Both failure paths mark + emit, then bail.
|
||||
let sha_short = sha[..sha.len().min(12)].to_owned();
|
||||
if let Err(e) = crate::flake_check::check_lock_in_sync(&applied_dir, &tag, id).await {
|
||||
let note = format!("{e:#}");
|
||||
let _ = coord.approvals.mark_failed(id, ¬e);
|
||||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id,
|
||||
agent,
|
||||
approval_kind: "apply_commit",
|
||||
sha_short: Some(sha_short.clone()),
|
||||
status: "failed",
|
||||
note: Some(note),
|
||||
description: description.map(str::to_owned),
|
||||
});
|
||||
return Err(anyhow::anyhow!("flake lock-sync check: {e:#}"));
|
||||
}
|
||||
if let Err(e) = crate::flake_check::check_no_duplicate_inputs(&applied_dir, &tag).await {
|
||||
let note = format!("{e:#}");
|
||||
let _ = coord.approvals.mark_failed(id, ¬e);
|
||||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id,
|
||||
agent,
|
||||
approval_kind: "apply_commit",
|
||||
sha_short: Some(sha_short.clone()),
|
||||
status: "failed",
|
||||
note: Some(note),
|
||||
description: description.map(str::to_owned),
|
||||
});
|
||||
return Err(anyhow::anyhow!("flake dedup check: {e:#}"));
|
||||
}
|
||||
// Mirror the freshly-planted proposal/<id> tag to the forge.
|
||||
if let Err(e) = crate::forge::push_config(agent).await {
|
||||
tracing::warn!(%agent, %id, error = ?e, "forge: push_config after submit failed");
|
||||
}
|
||||
// Phase 5b: surface the new pending approval on the dashboard
|
||||
// event channel. Compute the diff once here so live subscribers
|
||||
// get a fully-formed row without a snapshot refetch. `sha_short`
|
||||
// is reused from the dedup gate above.
|
||||
let diff = crate::dashboard::approval_diff(agent, id).await;
|
||||
coord.emit_approval_added(crate::coordinator::ApprovalAdded {
|
||||
id,
|
||||
agent,
|
||||
approval_kind: "apply_commit",
|
||||
sha_short: Some(sha_short),
|
||||
diff: Some(diff),
|
||||
description: description.map(str::to_owned),
|
||||
pr_number: None,
|
||||
});
|
||||
Ok((id, sha))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn accepts_short_and_full_sha() {
|
||||
assert!(validate_commit_ref("e194f78").is_ok());
|
||||
assert!(validate_commit_ref("e194f7812ab").is_ok());
|
||||
assert!(validate_commit_ref(&"a".repeat(40)).is_ok());
|
||||
// Uppercase hex resolves fine through `git rev-parse`.
|
||||
assert!(validate_commit_ref("E194F78").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_branch_and_tag_names() {
|
||||
// The exact bug class this guard exists for.
|
||||
assert!(validate_commit_ref("main").is_err());
|
||||
assert!(validate_commit_ref("HEAD").is_err());
|
||||
assert!(validate_commit_ref("deployed/0").is_err());
|
||||
assert!(validate_commit_ref("feature-branch").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_too_short_too_long_and_empty() {
|
||||
assert!(validate_commit_ref("").is_err());
|
||||
assert!(validate_commit_ref("abc123").is_err()); // 6 chars
|
||||
assert!(validate_commit_ref(&"a".repeat(41)).is_err());
|
||||
}
|
||||
}
|
||||
201
hive-c0re/src/socket_server/lifecycle_handlers.rs
Normal file
201
hive-c0re/src/socket_server/lifecycle_handlers.rs
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
//! Container-lifecycle request handlers (`Start` / `Restart` / `Kill` /
|
||||
//! `Update` / `ListDescendants`), including the capability-gated
|
||||
//! infra-container restart path. All are topology-guarded via
|
||||
//! `super::require_descendant`.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use hive_sh4re::AgentResponse;
|
||||
|
||||
use super::require_descendant;
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
/// `Start` — start a container, kicking its next turn. The caller must be an
|
||||
/// ancestor of `name` in the topology (the root covers every agent).
|
||||
pub(super) fn handle_start(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResponse {
|
||||
if let Some(err) = require_descendant(agent, name, "start") {
|
||||
return err;
|
||||
}
|
||||
tracing::info!(%agent, %name, "start container");
|
||||
// Persist `wanted = Up` and submit the Start DAG; the submit layer
|
||||
// upgrades a stale-rev start to a full rebuild so the container
|
||||
// runs current nix derivations before it starts.
|
||||
crate::job_queue::submit::start(
|
||||
coord,
|
||||
name,
|
||||
crate::job_queue::Source::Manual,
|
||||
format!("agent `{agent}` start tool"),
|
||||
);
|
||||
AgentResponse::Ok
|
||||
}
|
||||
|
||||
/// `Restart` — enqueue a restart for a container. The caller must be an
|
||||
/// ancestor of `name` in the topology. The infra-container branch is
|
||||
/// orthogonal: it is gated on the `infra_admin` capability and audited, so it
|
||||
/// stays ahead of the topology guard.
|
||||
pub(super) async fn handle_restart(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
name: &str,
|
||||
) -> AgentResponse {
|
||||
// Infra-container restart: an agent holding the `infra_admin`
|
||||
// capability can restart a hive infrastructure container (hive-ci /
|
||||
// hive-gateway / hive-forge / hive-matrix) by passing its name to the
|
||||
// same restart tool. The `InfraContainer` enum parse both recognises
|
||||
// these (never agent children, so disjoint from the child path below)
|
||||
// and yields the typed value the restart path needs.
|
||||
if let Ok(container) = name.parse::<hive_sh4re::priv_proto::InfraContainer>() {
|
||||
return handle_restart_infra(coord, agent, container).await;
|
||||
}
|
||||
if let Some(err) = require_descendant(agent, name, "restart") {
|
||||
return err;
|
||||
}
|
||||
tracing::info!(%agent, %name, "submit restart");
|
||||
crate::job_queue::submit::restart(
|
||||
coord,
|
||||
name,
|
||||
crate::job_queue::Source::Manual,
|
||||
format!("agent `{agent}` restart tool"),
|
||||
);
|
||||
AgentResponse::Ok
|
||||
}
|
||||
|
||||
/// Restart a hive infrastructure container on behalf of an agent that
|
||||
/// holds the `infra_admin` capability. The `container` is already a valid
|
||||
/// [`InfraContainer`] (the caller parsed it); this gates on the capability
|
||||
/// and routes the systemctl restart through hive-priv. Direct, not
|
||||
/// approval-gated.
|
||||
async fn handle_restart_infra(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
container: hive_sh4re::priv_proto::InfraContainer,
|
||||
) -> AgentResponse {
|
||||
let name = container.unit_name();
|
||||
// Record the attempt in the operator-visible privileged-action audit
|
||||
// trail, then emit a live `AuditEntryAdded` so the dashboard audit view
|
||||
// appends it off `/dashboard/stream`. Best-effort: `record` returns the
|
||||
// canonical row (or `None` on a sqlite blip), and we stream exactly that
|
||||
// row so the stored + streamed views can't drift. `action` is stable so
|
||||
// the dashboard can group/filter.
|
||||
let audit = |outcome: crate::audit_log::AuditOutcome, detail: Option<&str>| {
|
||||
if let Some(entry) = coord
|
||||
.audit_log
|
||||
.record(agent, "restart_infra", name, outcome, detail)
|
||||
{
|
||||
coord.emit_audit_entry(entry);
|
||||
}
|
||||
};
|
||||
if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::InfraAdmin) {
|
||||
tracing::warn!(%agent, %name, "agent: infra restart denied (no infra_admin capability)");
|
||||
audit(
|
||||
crate::audit_log::AuditOutcome::Err,
|
||||
Some("denied: missing infra_admin capability"),
|
||||
);
|
||||
return AgentResponse::Err {
|
||||
message: format!(
|
||||
"restarting infra container `{name}` requires the `infra_admin` capability"
|
||||
),
|
||||
};
|
||||
}
|
||||
tracing::info!(%agent, %name, "agent: restart infra container");
|
||||
match crate::priv_client::restart_infra_container(container).await {
|
||||
Ok(()) => {
|
||||
audit(crate::audit_log::AuditOutcome::Ok, None);
|
||||
AgentResponse::Ok
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("{e:#}");
|
||||
audit(crate::audit_log::AuditOutcome::Err, Some(&msg));
|
||||
AgentResponse::Err { message: msg }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `Kill` — kill a container, unregister it, notify the manager. The caller
|
||||
/// must be an ancestor of `name` in the topology.
|
||||
pub(super) async fn handle_kill(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
name: &str,
|
||||
) -> AgentResponse {
|
||||
if let Some(err) = require_descendant(agent, name, "kill") {
|
||||
return err;
|
||||
}
|
||||
tracing::info!(%agent, %name, "kill container");
|
||||
// Persist the intent even if the kill fails — otherwise the next
|
||||
// reconcile would restart the container.
|
||||
if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) {
|
||||
tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed");
|
||||
}
|
||||
let result: anyhow::Result<()> = async {
|
||||
crate::lifecycle::kill(name).await?;
|
||||
coord.unregister_agent(name);
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(()) => {
|
||||
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
|
||||
agent: name.to_owned(),
|
||||
});
|
||||
AgentResponse::Ok
|
||||
}
|
||||
Err(e) => AgentResponse::Err {
|
||||
message: format!("{e:#}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// `Update` — enqueue a rebuild for a container. The caller must be an
|
||||
/// ancestor of `name` in the topology.
|
||||
pub(super) fn handle_update(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResponse {
|
||||
if let Some(err) = require_descendant(agent, name, "rebuild") {
|
||||
return err;
|
||||
}
|
||||
tracing::info!(%agent, %name, "submit rebuild");
|
||||
crate::job_queue::submit::rebuild(
|
||||
coord,
|
||||
name,
|
||||
crate::job_queue::Source::Manual,
|
||||
format!("agent `{agent}` update tool"),
|
||||
);
|
||||
AgentResponse::Ok
|
||||
}
|
||||
|
||||
/// `ListDescendants` — every topological descendant of `agent` with
|
||||
/// its running/stopped state, parents before children.
|
||||
pub(super) async fn handle_list_descendants(agent: &str) -> AgentResponse {
|
||||
tracing::debug!(%agent, "agent: list descendants");
|
||||
// All containers known to nixos-container (running only).
|
||||
let running_set: std::collections::HashSet<String> = match crate::lifecycle::list().await {
|
||||
Ok(names) => names
|
||||
.into_iter()
|
||||
.filter_map(|c| {
|
||||
c.strip_prefix(crate::lifecycle::AGENT_PREFIX)
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("list containers failed: {e:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
// Walk the full topology and collect every descendant.
|
||||
let topo = crate::topology::read();
|
||||
let mut names: Vec<String> = topo
|
||||
.keys()
|
||||
.filter(|name| crate::topology::is_descendant_of(name, agent))
|
||||
.cloned()
|
||||
.collect();
|
||||
// Parents before children, then alpha within each tier.
|
||||
crate::auto_update::topology_sort(&mut names, &topo);
|
||||
let containers = names
|
||||
.into_iter()
|
||||
.map(|name| {
|
||||
let running = running_set.contains(&name);
|
||||
hive_sh4re::ContainerInfo { name, running }
|
||||
})
|
||||
.collect();
|
||||
AgentResponse::Containers { containers }
|
||||
}
|
||||
1092
hive-c0re/src/socket_server/mod.rs
Normal file
1092
hive-c0re/src/socket_server/mod.rs
Normal file
File diff suppressed because it is too large
Load diff
229
hive-c0re/src/socket_server/reminders.rs
Normal file
229
hive-c0re/src/socket_server/reminders.rs
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
//! Reminder request handling: the `Remind` handler, the shared
|
||||
//! `store_remind` storage path with its pending-cap and large-body
|
||||
//! auto-save dance, timing resolution, and the agent-state target
|
||||
//! resolution shared by the loose-ends / reminder query handlers.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use hive_sh4re::AgentResponse;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
pub(super) fn handle_remind(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
message: &str,
|
||||
timing: &hive_sh4re::ReminderTiming,
|
||||
file_path: Option<&str>,
|
||||
) -> AgentResponse {
|
||||
match store_remind(coord, agent, message, timing, file_path) {
|
||||
Ok(()) => AgentResponse::Ok,
|
||||
Err(message) => AgentResponse::Err { message },
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared remind-storage path used by both the agent and the manager
|
||||
/// dispatchers. Validates timing, applies the auto-file overflow
|
||||
/// dance (see [`prepare_remind_storage`]), and writes the reminder
|
||||
/// row. Returns `Ok(())` on success, or a caller-ready error string
|
||||
/// the dispatcher wraps in `*Response::Err`.
|
||||
/// Maximum pending (un-delivered) reminders per agent. Exceeding this
|
||||
/// causes `store_remind` to return an error so the agent knows to back
|
||||
/// off instead of silently dropping. Override via
|
||||
/// `HIVE_REMIND_MAX_PENDING_PER_AGENT`; set to `0` to disable the cap
|
||||
/// (not recommended — a runaway agent can still flood the scheduler).
|
||||
const DEFAULT_REMIND_MAX_PENDING: u64 = 50;
|
||||
|
||||
fn remind_max_pending() -> u64 {
|
||||
std::env::var("HIVE_REMIND_MAX_PENDING_PER_AGENT")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_REMIND_MAX_PENDING)
|
||||
}
|
||||
|
||||
pub(crate) fn store_remind(
|
||||
coord: &Arc<Coordinator>,
|
||||
agent: &str,
|
||||
message: &str,
|
||||
timing: &hive_sh4re::ReminderTiming,
|
||||
file_path: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
let max = remind_max_pending();
|
||||
if max > 0 {
|
||||
let pending = coord.broker.count_pending_reminders_for(agent).unwrap_or(0);
|
||||
if pending >= max {
|
||||
return Err(format!(
|
||||
"reminder rejected: agent `{agent}` already has {pending} pending \
|
||||
reminders (cap {max}). Cancel some via `cancel_loose_end` or wait \
|
||||
for them to fire before scheduling more. Override the cap with \
|
||||
`HIVE_REMIND_MAX_PENDING_PER_AGENT`."
|
||||
));
|
||||
}
|
||||
}
|
||||
let due_at = resolve_due_at(timing).map_err(|e| format!("invalid reminder timing: {e:#}"))?;
|
||||
let (stored_message, stored_path) = prepare_remind_storage(agent, message, file_path)?;
|
||||
let id = coord
|
||||
.broker
|
||||
.store_reminder(agent, &stored_message, stored_path.as_deref(), due_at)
|
||||
.map_err(|e| format!("failed to store reminder: {e:#}"))?;
|
||||
tracing::info!(%id, %agent, %due_at, "reminder scheduled");
|
||||
coord.emit_reminders_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decide what we actually store in the reminders row, applying the
|
||||
/// same byte cap as the rest of the wire protocol
|
||||
/// ([`crate::limits::MESSAGE_MAX_BYTES`]). Three outcomes:
|
||||
///
|
||||
/// 1. Body within the cap → stored verbatim, with whatever `file_path`
|
||||
/// the caller passed (None or Some). The scheduler honours
|
||||
/// `file_path` at delivery time as before.
|
||||
/// 2. Body over the cap, no caller `file_path` → auto-generate a path
|
||||
/// under `/agents/<agent>/state/reminders/auto-<ts>.md`, write the
|
||||
/// body to disk now, store a short pointer hint as the message and
|
||||
/// clear `file_path` (so the scheduler doesn't re-write at
|
||||
/// delivery and overwrite the body with the hint).
|
||||
/// 3. Body over the cap, caller provided `file_path` → honour the
|
||||
/// caller's path: write the body to it now, store the same hint
|
||||
/// and clear `file_path` for the same reason as (2).
|
||||
///
|
||||
/// Returns `(stored_message, stored_file_path)` on success, or a
|
||||
/// caller-ready error string on auto-save failure (which is the only
|
||||
/// way a Remind request can be refused for size — the agent never has
|
||||
/// to think about the cap).
|
||||
fn prepare_remind_storage(
|
||||
agent: &str,
|
||||
message: &str,
|
||||
file_path: Option<&str>,
|
||||
) -> Result<(String, Option<String>), String> {
|
||||
if message.len() <= crate::limits::MESSAGE_MAX_BYTES {
|
||||
return Ok((message.to_owned(), file_path.map(str::to_owned)));
|
||||
}
|
||||
let req_path = match file_path {
|
||||
Some(p) => p.to_owned(),
|
||||
None => auto_reminder_path(agent),
|
||||
};
|
||||
let host_path = crate::reminder_scheduler::resolve_host_path(agent, &req_path)
|
||||
.map_err(|reason| format!("auto-save path `{req_path}` rejected: {reason}"))?;
|
||||
crate::reminder_scheduler::write_payload(agent, &host_path, message).map_err(|reason| {
|
||||
format!("auto-save of large reminder body to `{req_path}` failed: {reason}")
|
||||
})?;
|
||||
let hint = format!(
|
||||
"[reminder body of {} bytes auto-saved to `{req_path}`; read with your filesystem tools]",
|
||||
message.len()
|
||||
);
|
||||
Ok((hint, None))
|
||||
}
|
||||
|
||||
/// Generate a per-agent path for an auto-saved reminder body. Uses
|
||||
/// `unix_nanos` plus the agent name to keep collisions infinitesimal
|
||||
/// across the agent's own state subtree (we're not stamping a hostname
|
||||
/// since hive-c0re is single-host).
|
||||
fn auto_reminder_path(agent: &str) -> String {
|
||||
let ts_ns = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_nanos());
|
||||
format!("/agents/{agent}/state/reminders/auto-{ts_ns}.md")
|
||||
}
|
||||
|
||||
/// Resolve the target agent name for a *named* `GetLooseEnds` /
|
||||
/// `CountPendingReminders` / `ReminderRollup` query. Rules:
|
||||
///
|
||||
/// - `None` (or `Some(caller)`) → the caller's own threads (always allowed).
|
||||
/// - `Some(descendant)` in the caller's subtree (the root's subtree is the whole hive) → allowed, no extra capability.
|
||||
/// - `Some(other)` outside the subtree → requires the `query_agent_state` capability; error otherwise.
|
||||
/// - `Some("*")` → rejected here; the hive-wide sweep is handled by `handle_get_loose_ends` under the same `query_agent_state` gate.
|
||||
pub(super) fn resolve_agent_state_target<'a>(
|
||||
caller: &'a str,
|
||||
target: Option<&'a str>,
|
||||
) -> Result<&'a str, String> {
|
||||
match target {
|
||||
None => Ok(caller),
|
||||
Some("*") => Err(
|
||||
"hive-wide query (agent=\"*\") is only valid for loose-ends; \
|
||||
not available for this query"
|
||||
.to_owned(),
|
||||
),
|
||||
Some(name) => {
|
||||
// Own subtree (the root covers all) is visible without extra
|
||||
// capability; `is_descendant_of` returns true for `name == caller`.
|
||||
if crate::topology::is_descendant_of(name, caller) {
|
||||
return Ok(name);
|
||||
}
|
||||
if crate::capabilities::has_cap(caller, hive_sh4re::Capability::QueryAgentState) {
|
||||
Ok(name)
|
||||
} else {
|
||||
Err(format!(
|
||||
"agent `{caller}` cannot query `{name}`: not in its subtree and \
|
||||
`query_agent_state` capability is not granted"
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the `due_at` unix timestamp for a Remind request. Returns
|
||||
/// distinct error messages for each failure mode (overflow on
|
||||
/// `InSeconds`, pre-epoch clock, `i64` cast wrap) so the caller can tell
|
||||
/// what went wrong without inspecting the chain.
|
||||
fn resolve_due_at(timing: &hive_sh4re::ReminderTiming) -> anyhow::Result<i64> {
|
||||
use hive_sh4re::ReminderTiming;
|
||||
match timing {
|
||||
ReminderTiming::InSeconds { seconds } => {
|
||||
let now = std::time::SystemTime::now();
|
||||
let future = now
|
||||
.checked_add(std::time::Duration::from_secs(*seconds))
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!("InSeconds overflow: {seconds}s exceeds system time range")
|
||||
})?;
|
||||
let duration = future
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_err(|e| anyhow::anyhow!("system time before UNIX_EPOCH: {e}"))?;
|
||||
i64::try_from(duration.as_secs())
|
||||
.map_err(|e| anyhow::anyhow!("unix timestamp exceeds i64 range: {e}"))
|
||||
}
|
||||
ReminderTiming::At { unix_timestamp } => Ok(*unix_timestamp),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn auto_reminder_path_format() {
|
||||
let p = auto_reminder_path("damocles");
|
||||
assert!(p.starts_with("/agents/damocles/state/reminders/auto-"));
|
||||
assert!(
|
||||
std::path::Path::new(&p)
|
||||
.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_remind_storage_passthrough_under_cap() {
|
||||
let (msg, fp) = prepare_remind_storage("foo", "small body", None).unwrap();
|
||||
assert_eq!(msg, "small body");
|
||||
assert_eq!(fp, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_remind_storage_passthrough_with_caller_file_path() {
|
||||
let (msg, fp) =
|
||||
prepare_remind_storage("foo", "small", Some("/agents/foo/state/x.md")).unwrap();
|
||||
assert_eq!(msg, "small");
|
||||
assert_eq!(fp.as_deref(), Some("/agents/foo/state/x.md"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_agent_state_target_self_and_default_are_free() {
|
||||
// No topology/capability state needed for these: `None` and the
|
||||
// caller's own name resolve to the caller (`is_descendant_of` short-
|
||||
// circuits to true when candidate == ancestor); `"*"` is rejected
|
||||
// (the hive-wide sweep is handled by the loose-ends caller instead).
|
||||
assert_eq!(resolve_agent_state_target("iris", None), Ok("iris"));
|
||||
assert_eq!(resolve_agent_state_target("iris", Some("iris")), Ok("iris"));
|
||||
assert!(resolve_agent_state_target("iris", Some("*")).is_err());
|
||||
}
|
||||
}
|
||||
404
hive-c0re/src/socket_server/schedules.rs
Normal file
404
hive-c0re/src/socket_server/schedules.rs
Normal file
|
|
@ -0,0 +1,404 @@
|
|||
//! Scheduled-prompt request handlers (`ListSchedules` /
|
||||
//! `RequestSchedulePrompt` / `CancelSchedule` / `EditSchedule` /
|
||||
//! `FireScheduleNow`), their shared ownership check, and the
|
||||
//! schedule-to-wire mapping reused by the dashboard
|
||||
//! (`schedule_to_wire_public` / `filter_ghost_schedule_targets`).
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use hive_sh4re::AgentResponse;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
/// `ListSchedules` — snapshot every scheduled prompt onto the wire.
|
||||
pub(super) fn handle_list_schedules(coord: &Arc<Coordinator>) -> AgentResponse {
|
||||
match coord.scheduled_prompts.list() {
|
||||
Ok(schedules) => AgentResponse::Schedules {
|
||||
schedules: schedules.into_iter().map(schedule_to_wire).collect(),
|
||||
},
|
||||
Err(e) => AgentResponse::Err {
|
||||
message: format!("list scheduled prompts: {e:#}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit a `RequestSchedulePrompt` payload as an `ApprovalKind::SchedulePrompt`
|
||||
/// row. Encodes the payload into the approval's `commit_ref` so the
|
||||
/// approve handler can re-parse it without a side table. Validates
|
||||
/// inputs (non-empty targets, non-empty body, sane interval) at
|
||||
/// submit time — the operator should never see a malformed schedule
|
||||
/// pending approval.
|
||||
pub(super) fn handle_request_schedule_prompt(
|
||||
coord: &Arc<Coordinator>,
|
||||
requester: &str,
|
||||
payload: &hive_sh4re::SchedulePromptPayload,
|
||||
) -> AgentResponse {
|
||||
if payload.targets.is_empty() {
|
||||
return AgentResponse::Err {
|
||||
message: "schedule must have at least one target".into(),
|
||||
};
|
||||
}
|
||||
if payload.body.trim().is_empty() {
|
||||
return AgentResponse::Err {
|
||||
message: "schedule body must be non-empty".into(),
|
||||
};
|
||||
}
|
||||
if let Some(0) = payload.interval_seconds {
|
||||
return AgentResponse::Err {
|
||||
message: "interval_seconds must be > 0 (use None for one-shot)".into(),
|
||||
};
|
||||
}
|
||||
let commit_ref = match serde_json::to_string(payload) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("encode SchedulePromptPayload: {e:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
let id = match coord.approvals.submit_kind(
|
||||
requester,
|
||||
hive_sh4re::ApprovalKind::SchedulePrompt,
|
||||
&commit_ref,
|
||||
payload.description.as_deref(),
|
||||
requester,
|
||||
) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("queue schedule_prompt approval: {e:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
%id,
|
||||
requester,
|
||||
targets = ?payload.targets,
|
||||
first_fire_at = payload.first_fire_at_unix,
|
||||
interval = ?payload.interval_seconds,
|
||||
"schedule_prompt approval queued"
|
||||
);
|
||||
coord.emit_approval_added(crate::coordinator::ApprovalAdded {
|
||||
id,
|
||||
agent: requester,
|
||||
approval_kind: "schedule_prompt",
|
||||
sha_short: None,
|
||||
diff: None,
|
||||
description: payload.description.clone(),
|
||||
pr_number: None,
|
||||
});
|
||||
AgentResponse::Ok
|
||||
}
|
||||
|
||||
/// Cancel a schedule (whole or per-target). Manager-surface
|
||||
/// authorization: a manager can cancel its own schedules + any
|
||||
/// schedule whose owner is one of its sub-agents (topology-walked).
|
||||
/// The operator surface bypasses this and can cancel anything;
|
||||
/// agents reaching this path through the manager get the
|
||||
/// topology-scoped check.
|
||||
pub(super) fn handle_cancel_schedule(
|
||||
coord: &Arc<Coordinator>,
|
||||
requester: &str,
|
||||
schedule_id: i64,
|
||||
targets: Option<&[String]>,
|
||||
) -> AgentResponse {
|
||||
let schedule = match coord.scheduled_prompts.get(schedule_id) {
|
||||
Ok(Some(s)) => s,
|
||||
Ok(None) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("schedule {schedule_id} not found"),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("read schedule {schedule_id}: {e:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
if !cancel_authorized(requester, &schedule.owner) {
|
||||
return AgentResponse::Err {
|
||||
message: format!(
|
||||
"not authorized: {requester} cannot cancel schedule owned by {owner}",
|
||||
owner = schedule.owner
|
||||
),
|
||||
};
|
||||
}
|
||||
let result = match targets {
|
||||
Some(list) if !list.is_empty() => coord
|
||||
.scheduled_prompts
|
||||
.cancel_targets(schedule_id, list)
|
||||
.map_err(|e| format!("cancel targets: {e:#}")),
|
||||
_ => coord
|
||||
.scheduled_prompts
|
||||
.cancel_all(schedule_id)
|
||||
.map_err(|e| format!("cancel all: {e:#}")),
|
||||
};
|
||||
match result {
|
||||
Ok(()) => {
|
||||
coord.emit_schedules_snapshot();
|
||||
AgentResponse::Ok
|
||||
}
|
||||
Err(message) => AgentResponse::Err { message },
|
||||
}
|
||||
}
|
||||
|
||||
/// Authorize + dispatch a `FireScheduleNow` request from the
|
||||
/// manager surface. Same ownership rules as `CancelSchedule`:
|
||||
/// requester can fire its own schedules + any owned by an agent
|
||||
/// in its subtree. The actual fan-out lives in
|
||||
/// `scheduled_prompts_worker::fire_now`.
|
||||
pub(super) async fn handle_fire_schedule_now(
|
||||
coord: &Arc<Coordinator>,
|
||||
requester: &str,
|
||||
schedule_id: i64,
|
||||
) -> AgentResponse {
|
||||
let schedule = match coord.scheduled_prompts.get(schedule_id) {
|
||||
Ok(Some(s)) => s,
|
||||
Ok(None) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("schedule {schedule_id} not found"),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("read schedule {schedule_id}: {e:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
if !cancel_authorized(requester, &schedule.owner) {
|
||||
return AgentResponse::Err {
|
||||
message: format!(
|
||||
"not authorized: {requester} cannot fire schedule owned by {owner}",
|
||||
owner = schedule.owner
|
||||
),
|
||||
};
|
||||
}
|
||||
// MCP fire_schedule_now stays no-reset (cadence intact); the
|
||||
// reset-timer option is a dashboard-dialog affordance.
|
||||
match crate::scheduled_prompts_worker::fire_now(coord, schedule_id, false).await {
|
||||
Ok(_report) => {
|
||||
coord.emit_schedules_snapshot();
|
||||
AgentResponse::Ok
|
||||
}
|
||||
Err(e) => AgentResponse::Err {
|
||||
message: format!("fire schedule {schedule_id} now: {e:#}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Field-named PATCH payload for [`handle_edit_schedule`]. Every
|
||||
/// field is "leave alone" when `None`; the double-`Option` fields
|
||||
/// additionally distinguish clear (`Some(None)`) from set
|
||||
/// (`Some(Some(v))`).
|
||||
#[allow(
|
||||
clippy::option_option,
|
||||
reason = "double-Option carries three-state PATCH semantics: outer None = \
|
||||
leave alone, Some(None) = clear, Some(Some(v)) = set"
|
||||
)]
|
||||
pub(super) struct EditSchedulePatch {
|
||||
pub(super) body: Option<String>,
|
||||
pub(super) description: Option<Option<String>>,
|
||||
pub(super) interval_seconds: Option<Option<u64>>,
|
||||
pub(super) next_fire_at_unix: Option<i64>,
|
||||
pub(super) targets_add: Option<Vec<String>>,
|
||||
pub(super) targets_remove: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
/// Authorize + dispatch a `EditSchedule` patch. Same ownership
|
||||
/// rules as `CancelSchedule` — the manager can edit
|
||||
/// schedules it owns + any owned by an agent in its subtree.
|
||||
/// Forwards the partial payload to
|
||||
/// `ScheduledPrompts::update` which enforces the cancelled-row /
|
||||
/// zero-interval validation. Returns `Ok` on a clean update;
|
||||
/// `Err` with the underlying message on any auth / validation
|
||||
/// failure so the dashboard can surface it verbatim.
|
||||
pub(super) fn handle_edit_schedule(
|
||||
coord: &Arc<Coordinator>,
|
||||
requester: &str,
|
||||
schedule_id: i64,
|
||||
patch: EditSchedulePatch,
|
||||
) -> AgentResponse {
|
||||
let EditSchedulePatch {
|
||||
body,
|
||||
description,
|
||||
interval_seconds,
|
||||
next_fire_at_unix,
|
||||
targets_add,
|
||||
targets_remove,
|
||||
} = patch;
|
||||
let schedule = match coord.scheduled_prompts.get(schedule_id) {
|
||||
Ok(Some(s)) => s,
|
||||
Ok(None) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("schedule {schedule_id} not found"),
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
return AgentResponse::Err {
|
||||
message: format!("read schedule {schedule_id}: {e:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
if !cancel_authorized(requester, &schedule.owner) {
|
||||
return AgentResponse::Err {
|
||||
message: format!(
|
||||
"not authorized: {requester} cannot edit schedule owned by {owner}",
|
||||
owner = schedule.owner
|
||||
),
|
||||
};
|
||||
}
|
||||
let patch = crate::scheduled_prompts::UpdateSchedule {
|
||||
body,
|
||||
description,
|
||||
interval_seconds,
|
||||
next_fire_at_unix,
|
||||
targets_add,
|
||||
targets_remove,
|
||||
};
|
||||
match coord.scheduled_prompts.update(schedule_id, patch) {
|
||||
Ok(()) => {
|
||||
coord.emit_schedules_snapshot();
|
||||
AgentResponse::Ok
|
||||
}
|
||||
Err(e) => AgentResponse::Err {
|
||||
message: format!("edit schedule {schedule_id}: {e:#}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Permission check for `CancelSchedule` on the manager surface.
|
||||
/// `requester` (always `ruth` here) can cancel its own schedules.
|
||||
/// Sub-agent ownership is delegated to topology — see
|
||||
/// `crate::topology::is_descendant_of`. Also reused by
|
||||
/// `handle_fire_schedule_now` — fire-auth follows the same shape.
|
||||
fn cancel_authorized(requester: &str, owner: &str) -> bool {
|
||||
if requester == owner {
|
||||
return true;
|
||||
}
|
||||
if requester == hive_sh4re::OPERATOR_RECIPIENT {
|
||||
return true;
|
||||
}
|
||||
// Manager can cancel anything owned by an agent in its subtree.
|
||||
// For the current single-manager topology that covers everything,
|
||||
// but the check stays correct as the tree grows.
|
||||
crate::topology::is_descendant_of(owner, requester)
|
||||
}
|
||||
|
||||
/// Map a `scheduled_prompts::Schedule` to its public wire shape.
|
||||
/// Field-by-field copy — the two types are intentionally identical;
|
||||
/// the separation keeps hive-sh4re free of hive-c0re-internal types.
|
||||
/// Public alias `schedule_to_wire_public` re-exports for
|
||||
/// `dashboard.rs::api_schedules` without crossing the module
|
||||
/// boundary into the socket-server file.
|
||||
pub fn schedule_to_wire_public(s: crate::scheduled_prompts::Schedule) -> hive_sh4re::WireSchedule {
|
||||
schedule_to_wire(s)
|
||||
}
|
||||
|
||||
/// Drop schedule targets that point at agents which no longer exist, so
|
||||
/// the dashboard's schedule table doesn't render ghost columns for
|
||||
/// destroyed agents. `live` is the set of logical agent names from the
|
||||
/// last `nixos-container list` scan (stopped agents included, destroyed
|
||||
/// ones absent); the `operator` pseudo-target is always retained since
|
||||
/// it isn't a container. Applied only to the dashboard wire paths
|
||||
/// (`api_schedules` + the `SchedulesChanged` SSE emit) — the
|
||||
/// manager-facing `list_schedules` stays unfiltered so agents can still
|
||||
/// see and cancel stale targets. This is a view filter: the underlying
|
||||
/// schedule rows keep every target, so a re-spawned agent's targets
|
||||
/// reappear on their own.
|
||||
pub(crate) fn filter_ghost_schedule_targets(
|
||||
schedules: &mut [hive_sh4re::WireSchedule],
|
||||
live: &std::collections::HashSet<String>,
|
||||
) {
|
||||
for s in schedules.iter_mut() {
|
||||
s.targets
|
||||
.retain(|t| t.target == hive_sh4re::OPERATOR_RECIPIENT || live.contains(&t.target));
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule_to_wire(s: crate::scheduled_prompts::Schedule) -> hive_sh4re::WireSchedule {
|
||||
hive_sh4re::WireSchedule {
|
||||
id: s.id,
|
||||
owner: s.owner,
|
||||
body: s.body,
|
||||
interval_seconds: s.interval_seconds,
|
||||
next_fire_at_unix: hive_sh4re::wire_time::from_secs(s.next_fire_at_unix),
|
||||
created_at_unix: hive_sh4re::wire_time::from_secs(s.created_at_unix),
|
||||
source: match s.source {
|
||||
crate::scheduled_prompts::ScheduleSource::Operator => {
|
||||
hive_sh4re::WireScheduleSource::Operator
|
||||
}
|
||||
crate::scheduled_prompts::ScheduleSource::Approval { id } => {
|
||||
hive_sh4re::WireScheduleSource::Approval { id }
|
||||
}
|
||||
},
|
||||
cancelled_at_unix: s.cancelled_at_unix.map(hive_sh4re::wire_time::from_secs),
|
||||
paused_at_unix: s.paused_at_unix.map(hive_sh4re::wire_time::from_secs),
|
||||
description: s.description,
|
||||
targets: s
|
||||
.targets
|
||||
.into_iter()
|
||||
.map(|t| hive_sh4re::WireScheduleTarget {
|
||||
target: t.target,
|
||||
cancelled_at_unix: t.cancelled_at_unix.map(hive_sh4re::wire_time::from_secs),
|
||||
last_fired_at_unix: t.last_fired_at_unix.map(hive_sh4re::wire_time::from_secs),
|
||||
last_result: t.last_result,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn target(name: &str) -> hive_sh4re::WireScheduleTarget {
|
||||
hive_sh4re::WireScheduleTarget {
|
||||
target: name.to_owned(),
|
||||
cancelled_at_unix: None,
|
||||
last_fired_at_unix: None,
|
||||
last_result: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule(targets: &[&str]) -> hive_sh4re::WireSchedule {
|
||||
hive_sh4re::WireSchedule {
|
||||
id: 1,
|
||||
owner: "operator".to_owned(),
|
||||
body: "ping".to_owned(),
|
||||
interval_seconds: None,
|
||||
next_fire_at_unix: hive_sh4re::wire_time::from_secs(0),
|
||||
created_at_unix: hive_sh4re::wire_time::from_secs(0),
|
||||
source: hive_sh4re::WireScheduleSource::Operator,
|
||||
cancelled_at_unix: None,
|
||||
paused_at_unix: None,
|
||||
description: None,
|
||||
targets: targets.iter().map(|t| target(t)).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghost_filter_drops_dead_agents_keeps_live_and_operator() {
|
||||
let live: std::collections::HashSet<String> = ["iris".to_owned(), "damocles".to_owned()]
|
||||
.into_iter()
|
||||
.collect();
|
||||
let mut schedules = vec![schedule(&["iris", "ghost", "operator", "damocles"])];
|
||||
filter_ghost_schedule_targets(&mut schedules, &live);
|
||||
let kept: Vec<&str> = schedules[0]
|
||||
.targets
|
||||
.iter()
|
||||
.map(|t| t.target.as_str())
|
||||
.collect();
|
||||
// `ghost` (destroyed) dropped; live agents + operator pseudo-target kept.
|
||||
assert_eq!(kept, vec!["iris", "operator", "damocles"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghost_filter_can_empty_targets_when_all_dead() {
|
||||
let live: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
let mut schedules = vec![schedule(&["gone1", "gone2"])];
|
||||
filter_ghost_schedule_targets(&mut schedules, &live);
|
||||
// operator is never in the live set but is always retained; here
|
||||
// there's no operator target, so everything drops.
|
||||
assert!(schedules[0].targets.is_empty());
|
||||
}
|
||||
}
|
||||
|
|
@ -18,12 +18,13 @@
|
|||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::Duration;
|
||||
|
||||
use rusqlite::{Connection, OpenFlags};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
|
||||
/// Window accepted by `/api/stats-hive?window=`. Maps to a lookback
|
||||
/// span; the hive view is a flat rollup (no per-bucket trend — the
|
||||
|
|
@ -225,12 +226,6 @@ struct AgentAgg {
|
|||
bash: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
fn now_secs() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::cast_sign_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
|
|
@ -322,7 +317,7 @@ fn read_bash_heads(conn: &Connection, from: i64) -> HashMap<String, u64> {
|
|||
/// per-agent db is skipped (logged), never fatal.
|
||||
#[must_use]
|
||||
pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
// Fixed windows look back a constant span; `all` aggregates every
|
||||
// recorded turn (`from == 0`). The hive rollup isn't time-bucketed, so
|
||||
// unlike the per-agent snapshot it needs no adaptive bucket sizing.
|
||||
|
|
@ -426,7 +421,7 @@ mod tests {
|
|||
let conn = Connection::open_in_memory().unwrap();
|
||||
conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
|
||||
.unwrap();
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
for (ts, head) in [
|
||||
(now - 100, "cargo"),
|
||||
(now - 200, "cargo"),
|
||||
8
hive-c0re/src/stats/mod.rs
Normal file
8
hive-c0re/src/stats/mod.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
//! Metrics aggregation for the dashboard: hive-wide turn-stats
|
||||
//! rollups, host-system probes / server warnings, and live
|
||||
//! per-container cgroup load. Each submodule is re-exported at the
|
||||
//! crate root, so `crate::hive_stats::…` etc. keep working unchanged.
|
||||
|
||||
pub mod container_stats;
|
||||
pub mod hive_stats;
|
||||
pub mod host_stats;
|
||||
|
|
@ -4,9 +4,9 @@
|
|||
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use hive_sh4re::{Approval, ApprovalKind, ApprovalStatus};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
|
|
@ -24,66 +24,26 @@ CREATE INDEX IF NOT EXISTS idx_approvals_pending
|
|||
ON approvals (id) WHERE status = 'pending';
|
||||
";
|
||||
|
||||
/// Add the `description` column to pre-description databases. Manager-supplied
|
||||
/// note shown on the dashboard approval card at submission time (distinct from
|
||||
/// `note` which is set on denial/failure).
|
||||
fn ensure_description_column(conn: &Connection) -> Result<()> {
|
||||
let has: bool = conn
|
||||
.prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'description'")?
|
||||
.exists([])?;
|
||||
if !has {
|
||||
conn.execute_batch("ALTER TABLE approvals ADD COLUMN description TEXT;")
|
||||
.context("add approvals.description column")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add the `kind` column to pre-Phase-8 databases. ALTER TABLE ADD COLUMN is
|
||||
/// idempotent here only via a column-existence check (sqlite doesn't support
|
||||
/// IF NOT EXISTS on ADD COLUMN). Defaults legacy rows to `apply_commit`,
|
||||
/// which matches their actual semantics.
|
||||
fn ensure_kind_column(conn: &Connection) -> Result<()> {
|
||||
let has_kind: bool = conn
|
||||
.prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'kind'")?
|
||||
.exists([])?;
|
||||
if !has_kind {
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE approvals ADD COLUMN kind TEXT NOT NULL DEFAULT 'apply_commit';",
|
||||
)
|
||||
.context("add approvals.kind column")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Same shape as `ensure_kind_column` but for `fetched_sha` — the
|
||||
/// canonical sha hive-c0re vouched for at `request_apply_commit` time.
|
||||
/// Distinct from `commit_ref` (manager-supplied, may not even resolve
|
||||
/// in proposed by the time we approve).
|
||||
fn ensure_fetched_sha_column(conn: &Connection) -> Result<()> {
|
||||
let has: bool = conn
|
||||
.prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'fetched_sha'")?
|
||||
.exists([])?;
|
||||
if !has {
|
||||
conn.execute_batch("ALTER TABLE approvals ADD COLUMN fetched_sha TEXT;")
|
||||
.context("add approvals.fetched_sha column")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Same shape as `ensure_fetched_sha_column` but for `submitter` — the
|
||||
/// agent that submitted the approval (the authenticated socket caller).
|
||||
/// Approval-scoped helper events route to this agent. Legacy rows have
|
||||
/// NULL; callers fall back to the root agent for those.
|
||||
fn ensure_submitter_column(conn: &Connection) -> Result<()> {
|
||||
let has: bool = conn
|
||||
.prepare("SELECT 1 FROM pragma_table_info('approvals') WHERE name = 'submitter'")?
|
||||
.exists([])?;
|
||||
if !has {
|
||||
conn.execute_batch("ALTER TABLE approvals ADD COLUMN submitter TEXT;")
|
||||
.context("add approvals.submitter column")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Additive column migrations for pre-existing databases, applied via
|
||||
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
|
||||
const MIGRATIONS: &[&str] = &[
|
||||
// `kind` (pre-Phase-8 dbs): legacy rows default to `apply_commit`,
|
||||
// which matches their actual semantics.
|
||||
"ALTER TABLE approvals ADD COLUMN kind TEXT NOT NULL DEFAULT 'apply_commit'",
|
||||
// `description`: manager-supplied note shown on the dashboard
|
||||
// approval card at submission time (distinct from `note`, set on
|
||||
// denial/failure).
|
||||
"ALTER TABLE approvals ADD COLUMN description TEXT",
|
||||
// `fetched_sha`: the canonical sha hive-c0re vouched for at
|
||||
// `request_apply_commit` time. Distinct from `commit_ref`
|
||||
// (manager-supplied, may not even resolve by approve time).
|
||||
"ALTER TABLE approvals ADD COLUMN fetched_sha TEXT",
|
||||
// `submitter`: the agent that submitted the approval (the
|
||||
// authenticated socket caller); approval-scoped helper events
|
||||
// route to it. Legacy rows are NULL → callers fall back to the
|
||||
// root agent.
|
||||
"ALTER TABLE approvals ADD COLUMN submitter TEXT",
|
||||
];
|
||||
|
||||
pub struct Approvals {
|
||||
conn: Mutex<Connection>,
|
||||
|
|
@ -91,18 +51,10 @@ pub struct Approvals {
|
|||
|
||||
impl Approvals {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("create approvals db parent {}", parent.display()))?;
|
||||
}
|
||||
let conn = Connection::open(path)
|
||||
.with_context(|| format!("open approvals db {}", path.display()))?;
|
||||
let conn = crate::db::open(path, "approvals")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply approvals schema")?;
|
||||
ensure_kind_column(&conn).context("migrate approvals.kind")?;
|
||||
ensure_fetched_sha_column(&conn).context("migrate approvals.fetched_sha")?;
|
||||
ensure_description_column(&conn).context("migrate approvals.description")?;
|
||||
ensure_submitter_column(&conn).context("migrate approvals.submitter")?;
|
||||
crate::db::apply_migrations(&conn, "approvals", MIGRATIONS)?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
|
|
@ -123,7 +75,7 @@ impl Approvals {
|
|||
VALUES (?1, ?2, ?3, ?4, 'pending', ?5, ?6)",
|
||||
params![
|
||||
agent,
|
||||
kind_to_str(kind),
|
||||
kind.as_str(),
|
||||
commit_ref,
|
||||
now_unix(),
|
||||
description,
|
||||
|
|
@ -415,21 +367,6 @@ fn row_to_approval(row: &rusqlite::Row<'_>) -> rusqlite::Result<Approval> {
|
|||
})
|
||||
}
|
||||
|
||||
/// Stable kind→str mapping used wherever we emit `ApprovalResolved`
|
||||
/// or persist a kind to sqlite. `pub(crate)` so callers like
|
||||
/// `questions::handle_cancel_loose_end` don't have to duplicate the
|
||||
/// match; bumping a kind here is the single source of truth.
|
||||
pub(crate) fn kind_to_str(kind: ApprovalKind) -> &'static str {
|
||||
match kind {
|
||||
ApprovalKind::ApplyCommit => "apply_commit",
|
||||
ApprovalKind::Spawn => "spawn",
|
||||
ApprovalKind::InitConfig => "init_config",
|
||||
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
||||
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
||||
ApprovalKind::MergeConfigPr => "merge_config_pr",
|
||||
}
|
||||
}
|
||||
|
||||
fn kind_from_str(s: &str) -> Result<ApprovalKind> {
|
||||
Ok(match s {
|
||||
"apply_commit" => ApprovalKind::ApplyCommit,
|
||||
|
|
@ -442,14 +379,6 @@ fn kind_from_str(s: &str) -> Result<ApprovalKind> {
|
|||
})
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -21,11 +21,11 @@
|
|||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::Serialize;
|
||||
|
||||
|
|
@ -119,11 +119,8 @@ impl AuditLog {
|
|||
/// Returns an error if the directory can't be created, the sqlite
|
||||
/// file can't be opened, or applying the schema fails.
|
||||
pub fn open(db_dir: &Path) -> Result<Self> {
|
||||
std::fs::create_dir_all(db_dir)
|
||||
.with_context(|| format!("create audit_log db parent {}", db_dir.display()))?;
|
||||
let path = db_dir.join("audit_log.sqlite");
|
||||
let conn = Connection::open(&path)
|
||||
.with_context(|| format!("open audit_log db {}", path.display()))?;
|
||||
let conn = crate::db::open(&path, "audit_log")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply audit_log schema")?;
|
||||
Ok(Self {
|
||||
|
|
@ -151,7 +148,7 @@ impl AuditLog {
|
|||
outcome: AuditOutcome,
|
||||
detail: Option<&str>,
|
||||
) -> Option<AuditEntry> {
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
match conn.execute(
|
||||
"INSERT INTO audit_log (ts_unix, agent, action, target, outcome, detail)
|
||||
|
|
@ -218,7 +215,7 @@ impl AuditLog {
|
|||
/// # Errors
|
||||
/// Returns an error if the `DELETE` query fails.
|
||||
pub fn vacuum(&self) -> Result<u64> {
|
||||
let cutoff = now_secs() - KEEP_SECS;
|
||||
let cutoff = now_unix() - KEEP_SECS;
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let removed = conn.execute("DELETE FROM audit_log WHERE ts_unix < ?1", params![cutoff])?;
|
||||
Ok(u64::try_from(removed).unwrap_or(0))
|
||||
|
|
@ -262,14 +259,6 @@ fn row_to_entry(r: &rusqlite::Row) -> rusqlite::Result<AuditEntry> {
|
|||
})
|
||||
}
|
||||
|
||||
fn now_secs() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -329,7 +318,7 @@ mod tests {
|
|||
let conn = db.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"UPDATE audit_log SET ts_unix = ?1",
|
||||
params![now_secs() - KEEP_SECS - 60],
|
||||
params![now_unix() - KEEP_SECS - 60],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
|
@ -4,11 +4,11 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use hive_sh4re::{InboxRow, Message};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
|
|
@ -157,12 +157,7 @@ pub struct Broker {
|
|||
|
||||
impl Broker {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("create db parent {}", parent.display()))?;
|
||||
}
|
||||
let conn =
|
||||
Connection::open(path).with_context(|| format!("open broker db {}", path.display()))?;
|
||||
let conn = crate::db::open(path, "broker")?;
|
||||
conn.execute_batch(SCHEMA).context("apply broker schema")?;
|
||||
ensure_message_columns(&conn).context("migrate messages columns")?;
|
||||
ensure_reminder_columns(&conn).context("migrate reminders columns")?;
|
||||
|
|
@ -1126,41 +1121,19 @@ fn ensure_message_columns(conn: &Connection) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Idempotent reminder-table migrations. `ALTER TABLE ADD COLUMN`
|
||||
/// has no `IF NOT EXISTS` form in sqlite, so we probe
|
||||
/// `pragma_table_info` per column. New deploys (table created by
|
||||
/// SCHEMA in this commit cycle) skip the ALTER; pre-existing
|
||||
/// broker.sqlite files get the columns added on next boot.
|
||||
/// Idempotent reminder-table migrations — plain additive columns, via
|
||||
/// `db::apply_migrations`. (The messages-table migration above stays
|
||||
/// bespoke: its backfill must run only when `acked_at` was just
|
||||
/// created, which try-and-ignore can't express.)
|
||||
fn ensure_reminder_columns(conn: &Connection) -> Result<()> {
|
||||
for (name, sql) in [
|
||||
(
|
||||
"attempt_count",
|
||||
"ALTER TABLE reminders ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0;",
|
||||
),
|
||||
(
|
||||
"last_error",
|
||||
"ALTER TABLE reminders ADD COLUMN last_error TEXT;",
|
||||
),
|
||||
] {
|
||||
let has: bool = conn
|
||||
.prepare(&format!(
|
||||
"SELECT 1 FROM pragma_table_info('reminders') WHERE name = '{name}'"
|
||||
))?
|
||||
.exists([])?;
|
||||
if !has {
|
||||
conn.execute_batch(sql)
|
||||
.with_context(|| format!("add reminders.{name} column"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
crate::db::apply_migrations(
|
||||
conn,
|
||||
"broker reminders",
|
||||
&[
|
||||
"ALTER TABLE reminders ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0",
|
||||
"ALTER TABLE reminders ADD COLUMN last_error TEXT",
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -5,9 +5,9 @@
|
|||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
use tokio::sync::broadcast;
|
||||
|
|
@ -147,11 +147,8 @@ pub struct BuildLogs {
|
|||
|
||||
impl BuildLogs {
|
||||
pub fn open(db_dir: &Path) -> Result<Self> {
|
||||
std::fs::create_dir_all(db_dir)
|
||||
.with_context(|| format!("create build_logs db parent {}", db_dir.display()))?;
|
||||
let path = db_dir.join("build_logs.sqlite");
|
||||
let conn = Connection::open(&path)
|
||||
.with_context(|| format!("open build_logs db {}", path.display()))?;
|
||||
let conn = crate::db::open(&path, "build_logs")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply build_logs schema")?;
|
||||
let (notify_tx, _) = broadcast::channel(NOTIFY_CAP);
|
||||
|
|
@ -172,7 +169,7 @@ impl BuildLogs {
|
|||
/// — the caller threads it through `append_stdout` / `append_stderr`
|
||||
/// while the child runs and into `finish` once it exits.
|
||||
pub fn start(&self, agent: &str, kind: &str, cmdline: &str) -> Result<i64> {
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO build_logs (agent, kind, cmdline, started_at) VALUES (?1, ?2, ?3, ?4)",
|
||||
|
|
@ -222,7 +219,7 @@ impl BuildLogs {
|
|||
/// Finalize a build attempt. Sets `finished_at` to now and
|
||||
/// `status` to the terminal state. Best-effort.
|
||||
pub fn finish(&self, id: i64, status: BuildStatus) {
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
if let Err(e) = conn.execute(
|
||||
"UPDATE build_logs SET finished_at = ?1, status = ?2 WHERE id = ?3",
|
||||
|
|
@ -377,7 +374,7 @@ impl BuildLogs {
|
|||
/// a long-running build shouldn't disappear from its own log
|
||||
/// viewer mid-stream.
|
||||
pub fn vacuum(&self) -> Result<u64> {
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let fail_cutoff = now - KEEP_FAIL_SECS;
|
||||
let ok_cutoff = now - KEEP_OK_SECS;
|
||||
|
|
@ -441,14 +438,6 @@ fn row_to_header(r: &rusqlite::Row) -> rusqlite::Result<BuildLogHeader> {
|
|||
})
|
||||
}
|
||||
|
||||
fn now_secs() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -536,7 +525,7 @@ mod tests {
|
|||
// stays within KEEP_FAIL_SECS so it survives; old_fail goes
|
||||
// beyond; old_ok goes past KEEP_OK_SECS but inside
|
||||
// KEEP_FAIL_SECS — proves the per-status rule.
|
||||
let now = now_secs();
|
||||
let now = now_unix();
|
||||
{
|
||||
let conn = db.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
58
hive-c0re/src/stores/db.rs
Normal file
58
hive-c0re/src/stores/db.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
//! Shared sqlite connection setup for hive-c0re's host-side stores.
|
||||
//!
|
||||
//! Several modules keep their own tables — and their own
|
||||
//! `Mutex<Connection>` — in the coordinator DB
|
||||
//! (`db/broker.sqlite`: broker, approvals, operator questions,
|
||||
//! scheduled prompts, agent power) or in a sibling file under the same
|
||||
//! `db/` dir (`build_logs.sqlite`, `audit_log.sqlite`). The open dance
|
||||
//! is identical everywhere: ensure the parent dir exists, open the
|
||||
//! connection, set a busy timeout so concurrent same-process writers
|
||||
//! wait each other out instead of surfacing `SQLITE_BUSY`. This helper
|
||||
//! owns that dance; schema creation + column migrations stay with each
|
||||
//! store (they're per-table concerns).
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use rusqlite::Connection;
|
||||
|
||||
/// How long a write waits on another connection's lock before erroring.
|
||||
/// Generous relative to the stores' tiny transactions — a timeout here
|
||||
/// means something is genuinely wedged, not ordinary contention.
|
||||
const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Apply additive, idempotent migrations: run each statement and
|
||||
/// ignore `duplicate column name` errors (sqlite has no
|
||||
/// `ADD COLUMN IF NOT EXISTS`; try-and-ignore is the portable path —
|
||||
/// same pattern as hive-ag3nt's `turn_stats`). Any other error
|
||||
/// propagates. Fits plain `ALTER TABLE ADD COLUMN` (new columns must
|
||||
/// carry a default or tolerate NULL) and `IF NOT EXISTS` index DDL;
|
||||
/// migrations with creation-conditional backfills (broker's
|
||||
/// `acked_at`) stay bespoke in their store.
|
||||
pub fn apply_migrations(conn: &Connection, subsystem: &str, statements: &[&str]) -> Result<()> {
|
||||
for stmt in statements {
|
||||
if let Err(e) = conn.execute_batch(stmt) {
|
||||
if e.to_string().contains("duplicate column name") {
|
||||
continue;
|
||||
}
|
||||
return Err(e).with_context(|| format!("{subsystem} migration failed: {stmt}"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open a connection to the sqlite file at `path`, creating the parent
|
||||
/// directory if needed. `subsystem` labels error contexts (`"broker"`,
|
||||
/// `"approvals"`, …).
|
||||
pub fn open(path: &Path, subsystem: &str) -> Result<Connection> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("create {subsystem} db parent {}", parent.display()))?;
|
||||
}
|
||||
let conn = Connection::open(path)
|
||||
.with_context(|| format!("open {subsystem} db {}", path.display()))?;
|
||||
conn.busy_timeout(BUSY_TIMEOUT)
|
||||
.with_context(|| format!("set {subsystem} busy_timeout"))?;
|
||||
Ok(conn)
|
||||
}
|
||||
14
hive-c0re/src/stores/mod.rs
Normal file
14
hive-c0re/src/stores/mod.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
//! Sqlite-backed host-side stores (broker, approval / question /
|
||||
//! schedule queues, build logs, audit trail, power intent) plus the
|
||||
//! shared connection open/migration helper (`db`). Each submodule is
|
||||
//! re-exported at the crate root, so `crate::broker::…` etc. keep
|
||||
//! working unchanged.
|
||||
|
||||
pub mod approvals;
|
||||
pub mod audit_log;
|
||||
pub mod broker;
|
||||
pub mod build_logs;
|
||||
pub mod db;
|
||||
pub mod operator_questions;
|
||||
pub mod power;
|
||||
pub mod scheduled_prompts;
|
||||
|
|
@ -11,11 +11,11 @@
|
|||
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
|
||||
|
|
@ -33,41 +33,18 @@ CREATE INDEX IF NOT EXISTS idx_operator_questions_pending
|
|||
ON operator_questions (id) WHERE answered_at IS NULL;
|
||||
";
|
||||
|
||||
/// Add late-added columns to pre-existing databases. `ALTER TABLE
|
||||
/// ADD COLUMN` has no `IF NOT EXISTS` form in sqlite, so we check
|
||||
/// `pragma_table_info` first per column.
|
||||
fn ensure_columns(conn: &Connection) -> Result<()> {
|
||||
for (name, sql) in [
|
||||
(
|
||||
"multi",
|
||||
"ALTER TABLE operator_questions ADD COLUMN multi INTEGER NOT NULL DEFAULT 0;",
|
||||
),
|
||||
(
|
||||
"deadline_at",
|
||||
"ALTER TABLE operator_questions ADD COLUMN deadline_at INTEGER;",
|
||||
),
|
||||
// `target` = recipient of the question. NULL = operator
|
||||
// (back-compat default for rows written before agent-to-agent
|
||||
// questions existed); a non-null agent name = peer-to-peer
|
||||
// question. Dashboard's `pending()` filters on `target IS NULL`
|
||||
// so peer questions never leak into the operator's queue.
|
||||
(
|
||||
"target",
|
||||
"ALTER TABLE operator_questions ADD COLUMN target TEXT;",
|
||||
),
|
||||
] {
|
||||
let has: bool = conn
|
||||
.prepare(&format!(
|
||||
"SELECT 1 FROM pragma_table_info('operator_questions') WHERE name = '{name}'"
|
||||
))?
|
||||
.exists([])?;
|
||||
if !has {
|
||||
conn.execute_batch(sql)
|
||||
.with_context(|| format!("add operator_questions.{name} column"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Additive column migrations for pre-existing databases, applied via
|
||||
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
|
||||
const MIGRATIONS: &[&str] = &[
|
||||
"ALTER TABLE operator_questions ADD COLUMN multi INTEGER NOT NULL DEFAULT 0",
|
||||
"ALTER TABLE operator_questions ADD COLUMN deadline_at INTEGER",
|
||||
// `target` = recipient of the question. NULL = operator
|
||||
// (back-compat default for rows written before agent-to-agent
|
||||
// questions existed); a non-null agent name = peer-to-peer
|
||||
// question. Dashboard's `pending()` filters on `target IS NULL`
|
||||
// so peer questions never leak into the operator's queue.
|
||||
"ALTER TABLE operator_questions ADD COLUMN target TEXT",
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OpQuestion {
|
||||
|
|
@ -97,16 +74,10 @@ pub struct OperatorQuestions {
|
|||
|
||||
impl OperatorQuestions {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).with_context(|| {
|
||||
format!("create operator_questions db parent {}", parent.display())
|
||||
})?;
|
||||
}
|
||||
let conn = Connection::open(path)
|
||||
.with_context(|| format!("open operator_questions db {}", path.display()))?;
|
||||
let conn = crate::db::open(path, "operator_questions")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply operator_questions schema")?;
|
||||
ensure_columns(&conn).context("migrate operator_questions columns")?;
|
||||
crate::db::apply_migrations(&conn, "operator_questions", MIGRATIONS)?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
|
|
@ -300,11 +271,3 @@ fn row_to_question(row: &rusqlite::Row<'_>) -> rusqlite::Result<OpQuestion> {
|
|||
target: row.get(9)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
198
hive-c0re/src/stores/power.rs
Normal file
198
hive-c0re/src/stores/power.rs
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
//! Durable per-agent power *intent* (`wanted: Up | Offline`) — the
|
||||
//! spec half of spec-vs-status desired-state reconciliation.
|
||||
//! `container_view` remains the observed *status*; the job queue's
|
||||
//! `Reconcile` nodes are the mechanism that converges the two.
|
||||
//!
|
||||
//! Stored as the `agent_power` table in the coordinator DB
|
||||
//! (`/var/lib/hyperhive/db/broker.sqlite`, one tiny row per agent) —
|
||||
//! same one-file-many-modules pattern as `approvals` /
|
||||
//! `operator_questions` / `scheduled_prompts`, each with its own
|
||||
//! connection. Intent persists across hive-c0re restarts; in-flight
|
||||
//! queue work deliberately does not. Setting `wanted` is never a
|
||||
//! queued node: operator/intent actions update the row synchronously
|
||||
//! at request time, then submit the DAG whose terminal `Reconcile`
|
||||
//! reads the fresh value — rapid toggles are last-writer-wins and the
|
||||
//! reconciles converge. Power toggles never commit to the meta repo.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS agent_power (
|
||||
agent TEXT PRIMARY KEY,
|
||||
wanted TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
";
|
||||
|
||||
/// Per-agent power intent.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Wanted {
|
||||
Up,
|
||||
Offline,
|
||||
}
|
||||
|
||||
impl Wanted {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Wanted::Up => "up",
|
||||
Wanted::Offline => "offline",
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"up" => Some(Wanted::Up),
|
||||
"offline" => Some(Wanted::Offline),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed value from an observed running state (first boot after
|
||||
/// this store lands, or an agent spawned outside the normal path).
|
||||
pub fn from_running(running: bool) -> Self {
|
||||
if running { Wanted::Up } else { Wanted::Offline }
|
||||
}
|
||||
}
|
||||
|
||||
/// What a `Reconcile` should do given intent + observation. Pure so
|
||||
/// the `{Up,Offline} × {up,down}` matrix is unit-testable.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReconcileAction {
|
||||
Start,
|
||||
Stop,
|
||||
Noop,
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn reconcile_action(wanted: Wanted, running: bool) -> ReconcileAction {
|
||||
match (wanted, running) {
|
||||
(Wanted::Up, false) => ReconcileAction::Start,
|
||||
(Wanted::Offline, true) => ReconcileAction::Stop,
|
||||
(Wanted::Up, true) | (Wanted::Offline, false) => ReconcileAction::Noop,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sqlite-backed store. `Arc`-friendly: all methods take `&self`, the
|
||||
/// internal `Mutex<Connection>` serializes access.
|
||||
pub struct PowerStore {
|
||||
conn: Mutex<Connection>,
|
||||
}
|
||||
|
||||
impl PowerStore {
|
||||
/// Open (a connection to) the shared coordinator DB and ensure the
|
||||
/// `agent_power` table exists. `db_path` is the same sqlite file
|
||||
/// the broker / approvals / questions stores open.
|
||||
pub fn open(db_path: &Path) -> Result<Self> {
|
||||
let conn = crate::db::open(db_path, "agent_power")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply agent_power schema")?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
}
|
||||
|
||||
/// In-memory store for tests.
|
||||
#[cfg(test)]
|
||||
pub fn open_in_memory() -> Result<Self> {
|
||||
let conn = Connection::open_in_memory().context("open in-memory agent_power db")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply agent_power schema")?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
}
|
||||
|
||||
/// Read an agent's intent. `None` when the agent has no row yet
|
||||
/// (callers seed from observed state via [`Self::get_or_seed`]).
|
||||
pub fn get(&self, agent: &str) -> Result<Option<Wanted>> {
|
||||
let conn = self.conn.lock().expect("agent_power mutex poisoned");
|
||||
let row: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT wanted FROM agent_power WHERE agent = ?1",
|
||||
params![agent],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()
|
||||
.context("select agent_power")?;
|
||||
Ok(row.and_then(|s| Wanted::parse(&s)))
|
||||
}
|
||||
|
||||
/// Write an agent's intent (last-writer-wins, synchronous at
|
||||
/// request time).
|
||||
pub fn set(&self, agent: &str, wanted: Wanted) -> Result<()> {
|
||||
let conn = self.conn.lock().expect("agent_power mutex poisoned");
|
||||
conn.execute(
|
||||
"INSERT INTO agent_power (agent, wanted, updated_at) VALUES (?1, ?2, ?3)
|
||||
ON CONFLICT(agent) DO UPDATE SET wanted = ?2, updated_at = ?3",
|
||||
params![agent, wanted.as_str(), now_unix()],
|
||||
)
|
||||
.context("upsert agent_power")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read an agent's intent, seeding the row from the observed
|
||||
/// running state when absent — the migration rule for agents that
|
||||
/// predate this store (running ⇒ `Up`, stopped ⇒ `Offline`), after
|
||||
/// which the DB is authoritative.
|
||||
pub fn get_or_seed(&self, agent: &str, running: bool) -> Result<Wanted> {
|
||||
if let Some(w) = self.get(agent)? {
|
||||
return Ok(w);
|
||||
}
|
||||
let seeded = Wanted::from_running(running);
|
||||
self.set(agent, seeded)?;
|
||||
tracing::info!(%agent, wanted = seeded.as_str(), "agent_power: seeded from observed state");
|
||||
Ok(seeded)
|
||||
}
|
||||
|
||||
/// Drop an agent's row (container destroyed).
|
||||
pub fn remove(&self, agent: &str) -> Result<()> {
|
||||
let conn = self.conn.lock().expect("agent_power mutex poisoned");
|
||||
conn.execute("DELETE FROM agent_power WHERE agent = ?1", params![agent])
|
||||
.context("delete agent_power")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The full `{Up,Offline} × {up,down}` reconcile matrix:
|
||||
/// start / stop / noop / noop.
|
||||
#[test]
|
||||
fn reconcile_matrix() {
|
||||
assert_eq!(reconcile_action(Wanted::Up, false), ReconcileAction::Start);
|
||||
assert_eq!(
|
||||
reconcile_action(Wanted::Offline, true),
|
||||
ReconcileAction::Stop
|
||||
);
|
||||
assert_eq!(reconcile_action(Wanted::Up, true), ReconcileAction::Noop);
|
||||
assert_eq!(
|
||||
reconcile_action(Wanted::Offline, false),
|
||||
ReconcileAction::Noop
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_set_roundtrip_and_seed() {
|
||||
let store = PowerStore::open_in_memory().expect("open");
|
||||
assert_eq!(store.get("alice").expect("get"), None);
|
||||
// Seed from observed running state, once.
|
||||
assert_eq!(store.get_or_seed("alice", true).expect("seed"), Wanted::Up);
|
||||
// Thereafter the DB is authoritative — observed state no longer
|
||||
// overrides.
|
||||
assert_eq!(
|
||||
store.get_or_seed("alice", false).expect("seeded"),
|
||||
Wanted::Up
|
||||
);
|
||||
store.set("alice", Wanted::Offline).expect("set");
|
||||
assert_eq!(store.get("alice").expect("get"), Some(Wanted::Offline));
|
||||
store.remove("alice").expect("remove");
|
||||
assert_eq!(store.get("alice").expect("get"), None);
|
||||
}
|
||||
}
|
||||
|
|
@ -16,6 +16,7 @@ use std::path::Path;
|
|||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
|
|
@ -178,13 +179,7 @@ pub struct ScheduledPrompts {
|
|||
|
||||
impl ScheduledPrompts {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).with_context(|| {
|
||||
format!("create scheduled_prompts db parent {}", parent.display())
|
||||
})?;
|
||||
}
|
||||
let conn = Connection::open(path)
|
||||
.with_context(|| format!("open scheduled_prompts db {}", path.display()))?;
|
||||
let conn = crate::db::open(path, "scheduled_prompts")?;
|
||||
// Required for ON DELETE CASCADE to actually fire — sqlite
|
||||
// ships with FKs disabled per connection by default.
|
||||
conn.execute_batch("PRAGMA foreign_keys = ON;")
|
||||
|
|
@ -192,12 +187,11 @@ impl ScheduledPrompts {
|
|||
conn.execute_batch(SCHEMA)
|
||||
.context("apply scheduled_prompts schema")?;
|
||||
// Migration: add paused_at_unix to existing databases.
|
||||
// Silently ignores "duplicate column name" errors so this is
|
||||
// idempotent across daemon restarts on already-migrated DBs.
|
||||
let _ = conn.execute(
|
||||
"ALTER TABLE scheduled_prompts ADD COLUMN paused_at_unix INTEGER",
|
||||
[],
|
||||
);
|
||||
crate::db::apply_migrations(
|
||||
&conn,
|
||||
"scheduled_prompts",
|
||||
&["ALTER TABLE scheduled_prompts ADD COLUMN paused_at_unix INTEGER"],
|
||||
)?;
|
||||
// Migration: recreate the due-rows index to also exclude paused
|
||||
// rows. `CREATE INDEX IF NOT EXISTS` won't update an existing
|
||||
// index's WHERE clause, so we drop + recreate on every open.
|
||||
|
|
@ -681,14 +675,6 @@ fn load_targets(conn: &Connection, schedule_id: i64) -> Result<Vec<ScheduleTarge
|
|||
Ok(out)
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
320
hive-c0re/src/workers/auto_update.rs
Normal file
320
hive-c0re/src/workers/auto_update.rs
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path
|
||||
//! for agents whose per-agent rev marker is stale — a `StartupSweep`
|
||||
//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for
|
||||
//! the stale agents whose `wanted` power intent is `Up` — and (b)
|
||||
//! converge every other drifted agent to its persisted `wanted` via
|
||||
//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal:
|
||||
//!
|
||||
//! 1. **Stale but wanted-offline agents** get no rebuild — their
|
||||
//! rebuild happens the first time they're started (the start
|
||||
//! submit path upgrades a stale start to rebuild+start). The sweep
|
||||
//! parent still runs whenever *any* marker is stale so the meta
|
||||
//! hyperhive lock is bumped for those later start-upgrades.
|
||||
//! 2. **Agents whose rev marker matches** the current hyperhive flake
|
||||
//! path are skipped — nothing changed, no nix work to do.
|
||||
//!
|
||||
//! Booting with no config change performs no meta commit — only
|
||||
//! reconciles. See `docs/coordinator.md::Boot reconcile`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
|
||||
|
||||
/// Marker file recording the hyperhive rev a sub-agent's container was last
|
||||
/// built against. Sibling of `applied/<name>/` (rather than inside it) to
|
||||
/// keep it out of the applied repo's git history. Uses a leading dot so a
|
||||
/// glob over `applied/*` doesn't include it.
|
||||
pub fn rev_marker_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev"))
|
||||
}
|
||||
|
||||
/// Resolve the current rev of `hyperhive_flake`. For a path on disk we
|
||||
/// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/...
|
||||
/// update yields a different string. For anything else we return None.
|
||||
#[must_use]
|
||||
pub fn current_flake_rev(hyperhive_flake: &str) -> Option<String> {
|
||||
let path = Path::new(hyperhive_flake);
|
||||
if !path.exists() {
|
||||
return None;
|
||||
}
|
||||
std::fs::canonicalize(path)
|
||||
.ok()
|
||||
.map(|p| p.display().to_string())
|
||||
}
|
||||
|
||||
/// Returns true when the applied repo has commits that have not yet been
|
||||
/// deployed (i.e. the applied HEAD differs from the sha currently locked in
|
||||
/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip
|
||||
/// conveys: "there is a config change ready to apply via rebuild."
|
||||
///
|
||||
/// Async on purpose: this runs per agent inside `container_view::build_all`,
|
||||
/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and
|
||||
/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous
|
||||
/// `git` fork here blocks a tokio worker for the whole exec — under
|
||||
/// nix-build disk saturation that's long enough that concurrent sweeps
|
||||
/// starved the runtime and stalled the per-agent sockets.
|
||||
pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
|
||||
let applied_head = tokio::process::Command::new("git")
|
||||
.args([
|
||||
"-C",
|
||||
&format!("/var/lib/hyperhive/applied/{name}"),
|
||||
"rev-parse",
|
||||
"HEAD",
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.ok()
|
||||
.filter(|o| o.status.success())
|
||||
.and_then(|o| String::from_utf8(o.stdout).ok())
|
||||
.map(|s| s.trim().to_owned());
|
||||
|
||||
match (applied_head.as_deref(), deployed_sha) {
|
||||
(Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this hive is "ruthless" — running with no root/manager agent at
|
||||
/// all (no ruth). When true, hive-c0re skips the root-agent create/start
|
||||
/// sweep entirely. Controlled by the host option
|
||||
/// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS`
|
||||
/// env var. Defaults to `false` when the var is unset (back-compat: the
|
||||
/// root agent was always auto-managed before this opt-out existed); only
|
||||
/// an explicit `true` / `1` / `yes` enables ruthless mode.
|
||||
fn ruthless() -> bool {
|
||||
match std::env::var("HYPERHIVE_RUTHLESS") {
|
||||
Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Auto-create the manager container on startup if it isn't already there.
|
||||
/// hive-c0re manages the manager end-to-end: operators no longer declare
|
||||
/// `containers.h-ruth` in their host NixOS config. Bypasses the approval
|
||||
/// queue — the root/manager is auto-managed by default. Operators who
|
||||
/// don't want a root agent at all set `services.hyperhive.ruthless = true`,
|
||||
/// which short-circuits this whole function. Idempotent.
|
||||
pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
|
||||
if ruthless() {
|
||||
tracing::info!(
|
||||
"ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let existing = lifecycle::list().await.unwrap_or_default();
|
||||
let current_rev = current_flake_rev(&coord.hyperhive_flake);
|
||||
if existing
|
||||
.iter()
|
||||
.any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME))
|
||||
{
|
||||
// Container exists already. If it predates the unified lifecycle
|
||||
// (no applied flake on disk) we must rebuild — otherwise it's
|
||||
// running whatever the host-declarative config was at create
|
||||
// time, with a wrong systemd unit and port.
|
||||
let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
|
||||
if !applied_flake.exists() && current_rev.is_some() {
|
||||
tracing::warn!(
|
||||
"manager container exists but no applied flake — forcing rebuild to migrate"
|
||||
);
|
||||
if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild(
|
||||
MANAGER_NAME,
|
||||
crate::job_queue::Source::AutoUpdate,
|
||||
"manager migration: no applied flake".to_owned(),
|
||||
None,
|
||||
true,
|
||||
)) {
|
||||
tracing::warn!(error = ?e, "manager migration rebuild submit failed");
|
||||
}
|
||||
} else {
|
||||
tracing::debug!("manager container already present");
|
||||
}
|
||||
// hive-c0re auto-manages the root/manager by default, so a
|
||||
// present-but-stopped root (e.g. a first-start failure on a fresh
|
||||
// install) is brought back up here: the startup sweep's rebuild only
|
||||
// restarts a container that was already running, so without this it
|
||||
// stays down until a manual `nixos-container start`. The sub-agent
|
||||
// `was_running` guard is intentionally left untouched. (Operators
|
||||
// opt out of this whole auto-management with
|
||||
// `services.hyperhive.ruthless = true`, gated at the top of
|
||||
// this function.)
|
||||
if !lifecycle::is_running(MANAGER_NAME).await {
|
||||
tracing::info!("manager container present but not running — starting");
|
||||
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
|
||||
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
|
||||
}
|
||||
if let Err(e) = lifecycle::start(MANAGER_NAME).await {
|
||||
tracing::warn!(error = ?e, "manager start failed");
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!("manager container missing — spawning");
|
||||
let runtime = coord.ensure_runtime(MANAGER_NAME)?;
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
|
||||
lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
|
||||
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
|
||||
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
|
||||
}
|
||||
if let Some(rev) = current_rev {
|
||||
let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sort `names` in-place so parents precede their children in the topology.
|
||||
/// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last,
|
||||
/// alphabetically within their tier. Stable within each depth tier.
|
||||
pub fn topology_sort(
|
||||
names: &mut [String],
|
||||
topo: &std::collections::BTreeMap<String, Option<String>>,
|
||||
) {
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
// Build depth map using owned clones so the borrow on `names` is released
|
||||
// before the sort_by mutable borrow.
|
||||
let name_set: Vec<String> = names.to_vec();
|
||||
let mut depth: HashMap<String, usize> = HashMap::new();
|
||||
let mut queue: VecDeque<String> = VecDeque::new();
|
||||
// Seed roots: entries with no parent, or names not present in topo at all.
|
||||
for name in &name_set {
|
||||
if topo.get(name).is_none_or(Option::is_none) {
|
||||
depth.insert(name.clone(), 0);
|
||||
queue.push_back(name.clone());
|
||||
}
|
||||
}
|
||||
// BFS to assign depths to children.
|
||||
while let Some(parent) = queue.pop_front() {
|
||||
let d = depth[&parent] + 1;
|
||||
for name in &name_set {
|
||||
let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str());
|
||||
if is_child && !depth.contains_key(name) {
|
||||
depth.insert(name.clone(), d);
|
||||
queue.push_back(name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
names.sort_by(|a, b| {
|
||||
let da = depth.get(a).copied().unwrap_or(usize::MAX);
|
||||
let db = depth.get(b).copied().unwrap_or(usize::MAX);
|
||||
da.cmp(&db).then(a.cmp(b))
|
||||
});
|
||||
}
|
||||
|
||||
/// Boot reconcile (see the module doc): classify every agent by rev
|
||||
/// freshness + persisted `wanted` intent, submit one `StartupSweep`
|
||||
/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up
|
||||
/// agents) when anything is stale, and `Reconcile` DAGs for agents
|
||||
/// whose observed power state drifted from `wanted`. Returns Ok even
|
||||
/// if some submissions failed.
|
||||
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
||||
let containers = match lifecycle::list().await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let current_rev = current_flake_rev(&coord.hyperhive_flake);
|
||||
|
||||
// Resolve container names to logical agent names, then sort by
|
||||
// topology depth so parents are always rebuilt before their
|
||||
// children. Root agents (depth 0) go first; agents absent from
|
||||
// the topology file sort last (stable, alphabetical within tier).
|
||||
let mut logical_names: Vec<String> = containers
|
||||
.iter()
|
||||
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
|
||||
.collect();
|
||||
let topo = crate::topology::read();
|
||||
topology_sort(&mut logical_names, &topo);
|
||||
|
||||
// Classify. `get_or_seed` doubles as the one-time migration: an
|
||||
// agent without an `agent_power` row is seeded from its observed
|
||||
// state (running ⇒ Up), after which the DB is authoritative.
|
||||
let mut any_stale = false;
|
||||
let mut fanout: Vec<String> = Vec::new(); // stale ∧ wanted=Up → sweep rebuild
|
||||
let mut drifted: Vec<String> = Vec::new(); // fresh ∧ wanted≠observed → reconcile
|
||||
let mut n_deferred = 0usize;
|
||||
let mut n_skipped = 0usize;
|
||||
for name in &logical_names {
|
||||
let running = lifecycle::is_running(name).await;
|
||||
let wanted = match coord.power.get_or_seed(name, running) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed");
|
||||
crate::power::Wanted::from_running(running)
|
||||
}
|
||||
};
|
||||
let fresh = current_rev.as_ref().is_some_and(|rev| {
|
||||
std::fs::read_to_string(rev_marker_path(name))
|
||||
.is_ok_and(|stored| stored == rev.as_str())
|
||||
});
|
||||
if fresh {
|
||||
n_skipped += 1;
|
||||
} else {
|
||||
any_stale = true;
|
||||
if wanted == crate::power::Wanted::Up {
|
||||
// Rebuild against the post-bump lock; the DAG's tail
|
||||
// Reconcile brings the agent (back) up — covering both
|
||||
// the running-stale and stopped-but-wanted-up cases.
|
||||
fanout.push(name.clone());
|
||||
continue;
|
||||
}
|
||||
// Stale but wanted offline: no boot-time nix work — the
|
||||
// start submit path upgrades a stale start to a rebuild.
|
||||
n_deferred += 1;
|
||||
tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start");
|
||||
}
|
||||
if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop {
|
||||
drifted.push(name.clone());
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
total = containers.len(),
|
||||
rebuilds = fanout.len(),
|
||||
reconciles = drifted.len(),
|
||||
deferred = n_deferred,
|
||||
up_to_date = n_skipped,
|
||||
"boot reconcile"
|
||||
);
|
||||
|
||||
// Sweep parent whenever ANY marker is stale — even when every
|
||||
// stale agent is wanted-offline: the hyperhive lock bump must land
|
||||
// now so their later start-upgrade rebuilds build against it.
|
||||
// No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
|
||||
if any_stale {
|
||||
let reason = format!(
|
||||
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
|
||||
fanout.len(),
|
||||
n_deferred,
|
||||
n_skipped,
|
||||
);
|
||||
if let Err(e) = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::startup_sweep(reason, fanout))
|
||||
{
|
||||
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
|
||||
}
|
||||
}
|
||||
for name in drifted {
|
||||
if let Err(e) = coord
|
||||
.job_queue
|
||||
.submit(crate::job_queue::templates::reconcile_only(
|
||||
crate::job_queue::Template::Reconcile,
|
||||
&name,
|
||||
crate::job_queue::Source::AutoUpdate,
|
||||
"boot reconcile".to_owned(),
|
||||
None,
|
||||
))
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
|
||||
}
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
12
hive-c0re/src/workers/mod.rs
Normal file
12
hive-c0re/src/workers/mod.rs
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
//! Background tasks and periodic sweeps: crash/login watcher, the
|
||||
//! reminder and scheduled-prompt delivery loops, boot-time auto-update
|
||||
//! reconcile, the agent-sockets.json writer loop, and knowledge-repo
|
||||
//! sync. Each submodule is re-exported at the crate root, so
|
||||
//! `crate::crash_watch::…` etc. keep working unchanged.
|
||||
|
||||
pub mod agent_sockets;
|
||||
pub mod auto_update;
|
||||
pub mod crash_watch;
|
||||
pub mod knowledge;
|
||||
pub mod reminder_scheduler;
|
||||
pub mod scheduled_prompts_worker;
|
||||
|
|
@ -10,6 +10,7 @@ use hive_sh4re::Message;
|
|||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::scheduled_prompts::Schedule;
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
|
||||
/// Per-tick cap. Each schedule fires once per tick at most;
|
||||
/// 100/tick × 5s tick = sustained throughput cap of ~20/sec,
|
||||
|
|
@ -236,14 +237,6 @@ fn notify_operator_missing_target(coord: &Coordinator, schedule: &Schedule, targ
|
|||
}
|
||||
}
|
||||
|
||||
fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Per-target outcome counts for one `fire_now` invocation.
|
||||
/// Returned to the operator so the dashboard can render
|
||||
/// "fired to N (M failed, K missing)" without a follow-up GET.
|
||||
191
hive-sh4re/src/jobs.rs
Normal file
191
hive-sh4re/src/jobs.rs
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
//! Wire shapes of hive-c0re's job-DAG queue: what a queued job looks
|
||||
//! like on the dashboard SSE channel (`rebuild_queue_changed`), the
|
||||
//! `/api/state.rebuild_queue` snapshot, and the host admin socket's
|
||||
//! `QueueDag` polling surface (`hivectl`'s wait/progress loop). The
|
||||
//! queue *internals* — node kinds, dependency edges, scheduling state —
|
||||
//! live in `hive-c0re::job_queue`; these are the serialized views it
|
||||
//! produces. Semantics: `docs/coordinator.md::Job queue`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What a DAG *means* — the request-level shape. Wire strings match
|
||||
/// the pre-DAG queue's `kind` values so dashboards key off the same
|
||||
/// tags.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Template {
|
||||
/// Rebuild one agent's container (prebuild → stop → profile-swap →
|
||||
/// reconcile).
|
||||
Rebuild,
|
||||
/// Bump meta flake locks; child `Rebuild` DAGs fan out on
|
||||
/// completion for every affected agent.
|
||||
MetaUpdate,
|
||||
/// First-deploy spawn (approval-driven).
|
||||
Spawn,
|
||||
/// Reserved for a future destroy integration.
|
||||
Destroy,
|
||||
/// Boot-time config sweep (hyperhive lock bump + stale-agent
|
||||
/// rebuild fan-out).
|
||||
StartupSweep,
|
||||
/// Mechanical stop + converge to `wanted = Up` (a restart).
|
||||
Restart,
|
||||
/// Perm-file commit followed by the rebuild subgraph.
|
||||
PermChange,
|
||||
/// Quiesce the harness, drain, then stop (`wanted = Offline`).
|
||||
GracefulStop,
|
||||
/// Converge to `wanted = Up`.
|
||||
Start,
|
||||
/// Converge to `wanted = Offline`.
|
||||
Stop,
|
||||
/// Bare converge of observed power state to the persisted intent
|
||||
/// (boot reconcile).
|
||||
Reconcile,
|
||||
}
|
||||
|
||||
impl Template {
|
||||
#[must_use]
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Template::Rebuild => "rebuild",
|
||||
Template::MetaUpdate => "meta_update",
|
||||
Template::Spawn => "spawn",
|
||||
Template::Destroy => "destroy",
|
||||
Template::StartupSweep => "startup_sweep",
|
||||
Template::Restart => "restart",
|
||||
Template::PermChange => "perm_change",
|
||||
Template::GracefulStop => "graceful_stop",
|
||||
Template::Start => "start",
|
||||
Template::Stop => "stop",
|
||||
Template::Reconcile => "reconcile",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the submit request originated — drives the "why" chip on the
|
||||
/// dashboard.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// Operator action (dashboard button, CLI, manager tool).
|
||||
Manual,
|
||||
/// Cascade child of a `MetaUpdate` DAG's fan-out.
|
||||
MetaUpdate,
|
||||
/// Boot-time submission (sweep parent, boot reconciles).
|
||||
AutoUpdate,
|
||||
/// Cascade child of a `StartupSweep` DAG's fan-out.
|
||||
StartupSweep,
|
||||
/// Crash recovery path (future use).
|
||||
CrashRecover,
|
||||
/// Operator approved a pending `Approval` row; `approval_id` on
|
||||
/// the DAG points back at the source row.
|
||||
Approval,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
#[must_use]
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Source::Manual => "manual",
|
||||
Source::MetaUpdate => "meta_update",
|
||||
Source::AutoUpdate => "auto_update",
|
||||
Source::StartupSweep => "startup_sweep",
|
||||
Source::CrashRecover => "crash_recover",
|
||||
Source::Approval => "approval",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lifecycle state of a node — and, rolled up, of a DAG.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum State {
|
||||
Queued,
|
||||
Running,
|
||||
Done,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl State {
|
||||
#[must_use]
|
||||
pub fn is_terminal(self) -> bool {
|
||||
matches!(self, State::Done | State::Failed | State::Cancelled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Kind-specific payload for `Template::PermChange` DAGs.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum PermPayload {
|
||||
/// Set the tool groups for one agent (`tool-groups.json`).
|
||||
ToolGroups { groups: Vec<String> },
|
||||
/// Set the capabilities for one agent (`capabilities.json`).
|
||||
Capabilities { caps: Vec<String> },
|
||||
/// Set both perm-types in one entry — the batch
|
||||
/// `POST /api/permissions` path. `None` leaves that file untouched;
|
||||
/// present fields commit together and rebuild once.
|
||||
Combined {
|
||||
groups: Option<Vec<String>>,
|
||||
caps: Option<Vec<String>>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Node id, unique within its DAG.
|
||||
pub type NodeId = u32;
|
||||
|
||||
/// One node of a queued DAG, as serialized. Step labels, build-log
|
||||
/// links, errors, and timestamps are per-node; the DAG-level `state`
|
||||
/// is a roll-up.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NodeView {
|
||||
pub id: NodeId,
|
||||
/// Node primitive tag: `"prebuild"`, `"stop_for_update"`,
|
||||
/// `"swap"`, `"create"`, `"meta_lock"`, `"reconcile"`, `"signal"`,
|
||||
/// `"drain"`, `"write_dropin"`, `"write_perm_file"`,
|
||||
/// `"approval_deploy"`.
|
||||
pub kind: String,
|
||||
/// Ids of the nodes this one waits for.
|
||||
#[serde(default)]
|
||||
pub deps: Vec<NodeId>,
|
||||
pub state: State,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub step: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub build_log_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// A queued/running/recent DAG. DAG-level fields mirror the pre-DAG
|
||||
/// `QueueEntry` names (`kind` = template string, roll-up `state`);
|
||||
/// everything per-node appears exactly once, inside `nodes`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DagView {
|
||||
pub id: u64,
|
||||
pub agent: String,
|
||||
/// Template wire string — same values the old `kind` field used.
|
||||
pub kind: Template,
|
||||
/// Roll-up: `failed` if any node failed, else `running` /
|
||||
/// `queued` / `cancelled` / `done`.
|
||||
pub state: State,
|
||||
pub source: Source,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub parent_id: Option<u64>,
|
||||
pub reason: String,
|
||||
pub enqueued_at: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub inputs: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub approval_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
pub nodes: Vec<NodeView>,
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ use chrono::{DateTime, Utc};
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub mod assets;
|
||||
pub mod jobs;
|
||||
pub mod paths;
|
||||
pub mod priv_proto;
|
||||
pub mod wire_time;
|
||||
|
|
@ -69,6 +70,10 @@ pub enum HostRequest {
|
|||
/// matrix GUI disabled). Backs `hivectl open` + the federation
|
||||
/// peer-config block (which reads the bare `domain`).
|
||||
Urls,
|
||||
/// Fetch one job-queue DAG (plus its live fan-out children, linked
|
||||
/// via `parent_id`) by id — the polling surface behind `hivectl`'s
|
||||
/// wait/progress loop. Result: [`HostResponse::dags`].
|
||||
QueueDag { id: u64 },
|
||||
/// List pending approval requests.
|
||||
Pending,
|
||||
/// Approve a pending request by id; the action runs immediately.
|
||||
|
|
@ -170,7 +175,7 @@ pub struct HiveUrls {
|
|||
pub matrix: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct HostResponse {
|
||||
pub ok: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
|
|
@ -188,6 +193,16 @@ pub struct HostResponse {
|
|||
/// request kind.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub agent_statuses: Option<Vec<AgentStatusRow>>,
|
||||
/// Ids of the job-queue DAGs this request submitted (rebuild /
|
||||
/// restart / power ops). Clients poll them via
|
||||
/// [`HostRequest::QueueDag`]; `None` for non-submitting requests.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub queued_dags: Option<Vec<u64>>,
|
||||
/// `QueueDag` result — the requested DAG followed by its live
|
||||
/// fan-out children ([`jobs::DagView`]). Empty when the DAG has
|
||||
/// been evicted from the queue's history tail.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub dags: Option<Vec<jobs::DagView>>,
|
||||
}
|
||||
|
||||
/// One row in the approval queue. `commit_ref` is overloaded per
|
||||
|
|
@ -250,6 +265,24 @@ pub enum ApprovalKind {
|
|||
MergeConfigPr,
|
||||
}
|
||||
|
||||
impl ApprovalKind {
|
||||
/// Wire/UI string — the same value serde's `snake_case` rename
|
||||
/// produces. The single source of truth for every place that needs
|
||||
/// the kind as a `&'static str` (sqlite storage, dashboard events),
|
||||
/// so adding a variant can't silently miss a hand-rolled match.
|
||||
#[must_use]
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
ApprovalKind::ApplyCommit => "apply_commit",
|
||||
ApprovalKind::Spawn => "spawn",
|
||||
ApprovalKind::InitConfig => "init_config",
|
||||
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
||||
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
||||
ApprovalKind::MergeConfigPr => "merge_config_pr",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ApprovalStatus {
|
||||
|
|
@ -280,11 +313,7 @@ impl HostResponse {
|
|||
pub fn success() -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -293,10 +322,7 @@ impl HostResponse {
|
|||
Self {
|
||||
ok: false,
|
||||
error: Some(message.into()),
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -304,11 +330,8 @@ impl HostResponse {
|
|||
pub fn list(agents: Vec<String>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: Some(agents),
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -316,11 +339,8 @@ impl HostResponse {
|
|||
pub fn pending(approvals: Vec<Approval>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: Some(approvals),
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -329,11 +349,8 @@ impl HostResponse {
|
|||
pub fn urls(urls: HiveUrls) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: Some(urls),
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -342,11 +359,29 @@ impl HostResponse {
|
|||
pub fn agent_statuses(rows: Vec<AgentStatusRow>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: Some(rows),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// A request that submitted job-queue DAGs — carries their ids for
|
||||
/// the client's wait/progress loop.
|
||||
#[must_use]
|
||||
pub fn queued(ids: Vec<u64>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
queued_dags: Some(ids),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// `QueueDag` result — the polled DAG + its live children.
|
||||
#[must_use]
|
||||
pub fn dags(dags: Vec<jobs::DagView>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
dags: Some(dags),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,19 @@ pub fn from_secs(secs: i64) -> DateTime<Utc> {
|
|||
DateTime::<Utc>::from_timestamp(secs, 0).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Current unix timestamp in seconds — the single definition behind
|
||||
/// every store's `created_at` / `sent_at` / … stamp (this module owns
|
||||
/// the epoch-seconds convention; a dozen local copies of this fn used
|
||||
/// to float around both binaries). Clamps to 0 on a pre-epoch clock.
|
||||
#[must_use]
|
||||
pub fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::{DateTime, Utc};
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ let
|
|||
agent_cpu_quota = cfg.agentCpuQuota;
|
||||
agent_memory_max = cfg.agentMemoryMax;
|
||||
model_prices = cfg.modelPrices;
|
||||
build_slots = cfg.buildSlots;
|
||||
};
|
||||
|
||||
# Stylix theme integration (zero-op auto-detect). When the operator's
|
||||
|
|
@ -771,6 +772,22 @@ in
|
|||
`"2G"`.
|
||||
'';
|
||||
};
|
||||
|
||||
buildSlots = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 1;
|
||||
example = 2;
|
||||
description = ''
|
||||
Number of nix-heavy job-queue nodes (container prebuilds,
|
||||
profile swaps, first-spawn creates, meta lock bumps) hive-c0re
|
||||
runs concurrently. The default of 1 serializes all heavy nix
|
||||
work like the pre-DAG rebuild queue did; raise it on hosts with
|
||||
the cores/RAM to build several agent toplevels at once.
|
||||
Per-agent correctness is independent of this count — each
|
||||
agent's container-affecting operations are serialized by its
|
||||
lifecycle lease regardless.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
|
|
|
|||
Loading…
Reference in a new issue