diff --git a/docs/coordinator.md b/docs/coordinator.md index 1ac89697..acd1f84b 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -22,7 +22,7 @@ The **DAG** is the unit of dedup / cancel / approval-resolution and the dashboard group; the **node** is the unit of scheduling / execution / build-log / step label. Deps are intra-DAG edges only (`AfterOk` by default: the dep must succeed, a failed/cancelled dep cancels the dependent — -cancel-downstream). Cross-DAG ordering comes from the per-agent lease + dedup, +cancel-downstream). Cross-DAG ordering comes from the per-agent lease, never from edges between DAGs. Submit-time validation (petgraph `toposort`) rejects cyclic specs outright, fixing the old queue's "circular dep silently deadlocks" caveat. @@ -139,11 +139,13 @@ resources are free. Resources: 1. **Build slots** — `services.hyperhive.c0re.buildSlots` permits (default 1), held by nix-heavy nodes for the node's duration. -2. **Per-agent lifecycle lease** — DAG-scoped: acquired at the DAG's first - container-affecting node (`StopForUpdate`, `Swap`, `Signal`, `Drain`, - `Reconcile`, `WriteDropin`, `Create`, `ApprovalDeploy`), held until the DAG - is terminal, so two lifecycle DAGs for one agent never interleave their - container ops. **Lease-exempt**: `Prebuild`, `MetaLock`, `WritePermFile` — +2. **Per-agent lifecycle lease** — keyed on the **node's** agent (agent is + per-node; a DAG can span agents) and globally exclusive per agent across + all DAGs: acquired at a container-affecting node (`StopForUpdate`, `Swap`, + `Signal`, `Drain`, `Reconcile`, `WriteDropin`, `Create`, `ApprovalDeploy`), + held by the owning DAG until it's terminal, so two DAGs never interleave + container ops on the same agent. A DAG touching several agents holds one + lease per agent. **Lease-exempt**: `Prebuild`, `MetaLock`, `WritePermFile` — they touch the store / meta, not the running container, which is exactly why a stop can land while another DAG's prebuild is still building. @@ -156,14 +158,11 @@ The queue is in-memory only and lost on hive-c0re restart — deliberate: desired state is re-derived at boot from the DB + rev markers (see _Boot reconcile_), so there is no durable-recovery machinery to go wrong. -### Dedup, cancel, history +### Cancel, history -Dedup at **DAG granularity**: a repeat submit against a DAG whose roll-up is -still `Queued` with the same `(template, agent, parent_id, approval_id)` — -plus `inputs` for meta-updates and the perm-type discriminant for perm -changes — returns the existing id and appends an "also requested by …" line. -`parent_id` in the key keeps a cascade child from collapsing into a -standalone or sweep rebuild. Running/terminal DAGs never dedup. +Submit-time dedup was removed with the agent-per-node move (a multi-agent DAG +has no single agent to key a dedup on), so every submit enqueues a fresh DAG; +whether any dedup needs reintroducing is tracked as a follow-up. Cancel only applies to still-fully-queued DAGs (an in-flight nix build isn't interruptible); `cancel_children` cancels a parent's still-queued child DAGs. @@ -186,12 +185,14 @@ of dangling it). ### Wire shape `RebuildQueueChanged { seq, queue: [DagView…] }` (event name kept). Each -`DagView` carries the old entry-level fields (`id`, `kind` = template string, -roll-up `state`, `agent`, `source`, `parent_id`, `reason`, timestamps, -`inputs`, `approval_id`) plus `nodes: [NodeView…]` — per-node `kind`, `deps`, -`state`, `step`, `build_log_id`, timestamps, `error`. Step labels and build -logs are **per-node**; the dashboard renders the node chain on each queue -card and keys the live-log panel off the running node. +`DagView` carries the entry-level fields (`id`, `kind` = template string, +roll-up `state`, `source`, `parent_id`, `reason`, timestamps, `inputs`, +`approval_id`) plus `nodes: [NodeView…]` — per-node `agent`, `kind`, `deps`, +`state`, `step`, `build_log_id`, timestamps, `error`. There is **no +DAG-level `agent`** (agent is per-node, so a DAG can span agents); consumers +derive a DAG's agent(s) from its nodes. Step labels and build logs are +**per-node**; the dashboard renders the node chain on each queue card and +keys the live-log panel off the running node. --- diff --git a/frontend/packages/dashboard/src/builds.js b/frontend/packages/dashboard/src/builds.js index e56d5060..ac01cca2 100644 --- a/frontend/packages/dashboard/src/builds.js +++ b/frontend/packages/dashboard/src/builds.js @@ -165,11 +165,22 @@ function firstFailedNode(entry) { return (entry.nodes || []).find((n) => n.state === 'failed') || null; } +// Distinct agents across a DAG's nodes, comma-joined for display. Agent is +// per-node now (a DAG can span agents, e.g. a hive-wide restart), so there's +// no DAG-level `agent` field — derive it from the nodes. +function entryAgents(entry) { + const seen = []; + for (const n of entry.nodes || []) { + if (n.agent && !seen.includes(n.agent)) seen.push(n.agent); + } + return seen.join(','); +} + function rebuildQueueEntryFingerprint(entry, isChild) { return JSON.stringify({ state: entry.state, kind: entry.kind, - agent: entry.agent, + agent: entryAgents(entry), source: entry.source, started_at: entry.started_at, enqueued_at: entry.enqueued_at, @@ -265,7 +276,7 @@ function renderQueueEntry(entry, _byId, isChild) { el('span', { class: 'rqe-kind', title: entry.kind }, (QUEUE_KIND_GLYPH[entry.kind] || '?') + ' ' + entry.kind), ' ', - el('code', { class: 'rqe-agent' }, entry.agent), + el('code', { class: 'rqe-agent' }, entryAgents(entry)), ); li.append(' ', el('span', { class: 'rqe-source rqe-source-' + entry.source }, entry.source)); if (entry.state === 'queued') { @@ -331,14 +342,14 @@ function renderQueueEntry(entry, _byId, isChild) { class: 'inline rqe-cancel', 'data-async': '', 'data-confirm': - `cancel ${entry.kind} for \`${entry.agent}\` (queue id ${entry.id})? ` + + `cancel ${entry.kind} for \`${entryAgents(entry)}\` (queue id ${entry.id})? ` + `the row drops from the queue and never runs. running / done / failed entries can't be cancelled this way.`, }); cancelForm.append(el('button', { type: 'submit', class: 'rqe-cancel-btn', title: 'cancel this queued ' + entry.kind, - 'aria-label': 'cancel queued ' + entry.kind + ' for ' + entry.agent, + 'aria-label': 'cancel queued ' + entry.kind + ' for ' + entryAgents(entry), }, '✗')); li.append(cancelForm); } @@ -414,7 +425,7 @@ function renderRebuildLiveLog(queue) { const header = el('div', { class: 'rebuild-live-log-header' }, toggle, ' ', el('span', { class: 'rebuild-live-log-title' }, 'live build log — '), - el('code', { class: 'rqe-agent' }, running.agent), + el('code', { class: 'rqe-agent' }, entryAgents(running)), ' ', el('span', { class: 'rqe-kind' }, (running.kind || 'rebuild') + ' · ' + (NODE_KIND_LABEL[liveNode.kind] || liveNode.kind)), ' ', badge, ' ', diff --git a/hive-c0re/src/bin/hivectl/dag_progress.rs b/hive-c0re/src/bin/hivectl/dag_progress.rs index a50b7007..0875b814 100644 --- a/hive-c0re/src/bin/hivectl/dag_progress.rs +++ b/hive-c0re/src/bin/hivectl/dag_progress.rs @@ -68,7 +68,7 @@ async fn wait_for_dags_plain(socket: &Path, ids: Vec) -> Result<()> { // sees whether the agent came back. if d.nodes.iter().all(|n| n.state.is_terminal()) { if d.state == hive_sh4re::jobs::State::Failed { - failed.push(format!("{} {}", d.kind.as_str(), d.agent)); + failed.push(format!("{} {}", d.kind.as_str(), dag_agents(d))); } } else { all_terminal = false; @@ -138,7 +138,7 @@ async fn wait_for_dags_animated(socket: &Path, ids: Vec) -> Result<()> { "{} {} {} · {}", state_glyph(d.state), d.kind.as_str(), - d.agent, + dag_agents(d), fmt_dur(dag_elapsed(d, now)), )); for n in &d.nodes { @@ -166,7 +166,7 @@ async fn wait_for_dags_animated(socket: &Path, ids: Vec) -> Result<()> { } if d.nodes.iter().all(|n| n.state.is_terminal()) { if d.state == hive_sh4re::jobs::State::Failed { - failed.push(format!("{} {}", d.kind.as_str(), d.agent)); + failed.push(format!("{} {}", d.kind.as_str(), dag_agents(d))); } } else { all_terminal = false; @@ -204,6 +204,19 @@ fn finish_wait(mut failed: Vec) -> Result<()> { } } +/// Distinct agents across a DAG's nodes, comma-joined for display — the +/// per-node replacement for the old DAG-level `agent` field. Single-agent +/// DAGs render one name; a hive-wide DAG lists each. +fn dag_agents(d: &hive_sh4re::jobs::DagView) -> String { + let mut seen: Vec<&str> = Vec::new(); + for n in &d.nodes { + if !seen.contains(&n.agent.as_str()) { + seen.push(&n.agent); + } + } + seen.join(",") +} + /// Current unix time in seconds (0 on the impossible pre-epoch error). fn now_unix() -> i64 { std::time::SystemTime::now() @@ -290,7 +303,7 @@ fn render_dag_line(d: &hive_sh4re::jobs::DagView) -> String { "{} {} {:<12}", state_glyph(d.state), d.kind.as_str(), - d.agent + dag_agents(d) ); for (i, n) in d.nodes.iter().enumerate() { let sep = if i == 0 { " " } else { " → " }; @@ -314,9 +327,10 @@ mod tests { use super::render_dag_line; - fn node(id: u32, kind: &str, state: State, step: Option<&str>) -> NodeView { + fn node(id: u32, agent: &str, kind: &str, state: State, step: Option<&str>) -> NodeView { NodeView { id, + agent: agent.to_owned(), kind: kind.to_owned(), deps: if id == 0 { vec![] } else { vec![id - 1] }, state, @@ -332,7 +346,6 @@ mod tests { fn render_dag_line_shows_chain_and_running_step() { let dag = DagView { id: 7, - agent: "alice".to_owned(), kind: Template::Rebuild, state: State::Running, source: Source::Manual, @@ -345,10 +358,16 @@ mod tests { approval_id: None, perm_payload: None, nodes: vec![ - node(0, "prebuild", State::Done, None), - node(1, "stop_for_update", State::Done, None), - node(2, "swap", State::Running, Some("nixos-container update")), - node(3, "reconcile", State::Queued, None), + node(0, "alice", "prebuild", State::Done, None), + node(1, "alice", "stop_for_update", State::Done, None), + node( + 2, + "alice", + "swap", + State::Running, + Some("nixos-container update"), + ), + node(3, "alice", "reconcile", State::Queued, None), ], }; let line = render_dag_line(&dag); @@ -363,11 +382,10 @@ mod tests { #[test] fn render_dag_line_surfaces_first_node_error() { - let mut failed = node(0, "prebuild", State::Failed, None); + let mut failed = node(0, "bob", "prebuild", State::Failed, None); failed.error = Some("nix build exploded".to_owned()); let dag = DagView { id: 8, - agent: "bob".to_owned(), kind: Template::Rebuild, state: State::Failed, source: Source::Manual,