refactor(#2441): update queue consumers + docs for agent-per-node
DagView no longer has a DAG-level agent, so consumers derive it from the per-node agents: - hivectl dag_progress.rs: dag_agents(d) helper (distinct node agents, comma-joined) in place of d.agent. - dashboard builds.js: entryAgents(entry) helper likewise for the rebuild-queue card + live-log header + cancel confirms. - docs/coordinator.md: lease prose (node-agent-keyed, global per agent), wire shape (NodeView.agent, no DagView.agent), and dropped the removed dedup section.
This commit is contained in:
atlas
mara
parent
2a59f2f5fc
commit
e2b48d2014
3 changed files with 66 additions and 36 deletions
|
|
@ -22,7 +22,7 @@ The **DAG** is the unit of dedup / cancel / approval-resolution and the
|
|||
dashboard group; the **node** is the unit of scheduling / execution /
|
||||
build-log / step label. Deps are intra-DAG edges only (`AfterOk` by default:
|
||||
the dep must succeed, a failed/cancelled dep cancels the dependent —
|
||||
cancel-downstream). Cross-DAG ordering comes from the per-agent lease + dedup,
|
||||
cancel-downstream). Cross-DAG ordering comes from the per-agent lease,
|
||||
never from edges between DAGs. Submit-time validation (petgraph `toposort`)
|
||||
rejects cyclic specs outright, fixing the old queue's "circular dep silently
|
||||
deadlocks" caveat.
|
||||
|
|
@ -139,11 +139,13 @@ resources are free. Resources:
|
|||
|
||||
1. **Build slots** — `services.hyperhive.c0re.buildSlots` permits (default 1),
|
||||
held by nix-heavy nodes for the node's duration.
|
||||
2. **Per-agent lifecycle lease** — DAG-scoped: acquired at the DAG's first
|
||||
container-affecting node (`StopForUpdate`, `Swap`, `Signal`, `Drain`,
|
||||
`Reconcile`, `WriteDropin`, `Create`, `ApprovalDeploy`), held until the DAG
|
||||
is terminal, so two lifecycle DAGs for one agent never interleave their
|
||||
container ops. **Lease-exempt**: `Prebuild`, `MetaLock`, `WritePermFile` —
|
||||
2. **Per-agent lifecycle lease** — keyed on the **node's** agent (agent is
|
||||
per-node; a DAG can span agents) and globally exclusive per agent across
|
||||
all DAGs: acquired at a container-affecting node (`StopForUpdate`, `Swap`,
|
||||
`Signal`, `Drain`, `Reconcile`, `WriteDropin`, `Create`, `ApprovalDeploy`),
|
||||
held by the owning DAG until it's terminal, so two DAGs never interleave
|
||||
container ops on the same agent. A DAG touching several agents holds one
|
||||
lease per agent. **Lease-exempt**: `Prebuild`, `MetaLock`, `WritePermFile` —
|
||||
they touch the store / meta, not the running container, which is exactly
|
||||
why a stop can land while another DAG's prebuild is still building.
|
||||
|
||||
|
|
@ -156,14 +158,11 @@ The queue is in-memory only and lost on hive-c0re restart — deliberate:
|
|||
desired state is re-derived at boot from the DB + rev markers (see _Boot
|
||||
reconcile_), so there is no durable-recovery machinery to go wrong.
|
||||
|
||||
### Dedup, cancel, history
|
||||
### Cancel, history
|
||||
|
||||
Dedup at **DAG granularity**: a repeat submit against a DAG whose roll-up is
|
||||
still `Queued` with the same `(template, agent, parent_id, approval_id)` —
|
||||
plus `inputs` for meta-updates and the perm-type discriminant for perm
|
||||
changes — returns the existing id and appends an "also requested by …" line.
|
||||
`parent_id` in the key keeps a cascade child from collapsing into a
|
||||
standalone or sweep rebuild. Running/terminal DAGs never dedup.
|
||||
Submit-time dedup was removed with the agent-per-node move (a multi-agent DAG
|
||||
has no single agent to key a dedup on), so every submit enqueues a fresh DAG;
|
||||
whether any dedup needs reintroducing is tracked as a follow-up.
|
||||
|
||||
Cancel only applies to still-fully-queued DAGs (an in-flight nix build isn't
|
||||
interruptible); `cancel_children` cancels a parent's still-queued child DAGs.
|
||||
|
|
@ -186,12 +185,14 @@ of dangling it).
|
|||
### Wire shape
|
||||
|
||||
`RebuildQueueChanged { seq, queue: [DagView…] }` (event name kept). Each
|
||||
`DagView` carries the old entry-level fields (`id`, `kind` = template string,
|
||||
roll-up `state`, `agent`, `source`, `parent_id`, `reason`, timestamps,
|
||||
`inputs`, `approval_id`) plus `nodes: [NodeView…]` — per-node `kind`, `deps`,
|
||||
`state`, `step`, `build_log_id`, timestamps, `error`. Step labels and build
|
||||
logs are **per-node**; the dashboard renders the node chain on each queue
|
||||
card and keys the live-log panel off the running node.
|
||||
`DagView` carries the entry-level fields (`id`, `kind` = template string,
|
||||
roll-up `state`, `source`, `parent_id`, `reason`, timestamps, `inputs`,
|
||||
`approval_id`) plus `nodes: [NodeView…]` — per-node `agent`, `kind`, `deps`,
|
||||
`state`, `step`, `build_log_id`, timestamps, `error`. There is **no
|
||||
DAG-level `agent`** (agent is per-node, so a DAG can span agents); consumers
|
||||
derive a DAG's agent(s) from its nodes. Step labels and build logs are
|
||||
**per-node**; the dashboard renders the node chain on each queue card and
|
||||
keys the live-log panel off the running node.
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -165,11 +165,22 @@ function firstFailedNode(entry) {
|
|||
return (entry.nodes || []).find((n) => n.state === 'failed') || null;
|
||||
}
|
||||
|
||||
// Distinct agents across a DAG's nodes, comma-joined for display. Agent is
|
||||
// per-node now (a DAG can span agents, e.g. a hive-wide restart), so there's
|
||||
// no DAG-level `agent` field — derive it from the nodes.
|
||||
function entryAgents(entry) {
|
||||
const seen = [];
|
||||
for (const n of entry.nodes || []) {
|
||||
if (n.agent && !seen.includes(n.agent)) seen.push(n.agent);
|
||||
}
|
||||
return seen.join(',');
|
||||
}
|
||||
|
||||
function rebuildQueueEntryFingerprint(entry, isChild) {
|
||||
return JSON.stringify({
|
||||
state: entry.state,
|
||||
kind: entry.kind,
|
||||
agent: entry.agent,
|
||||
agent: entryAgents(entry),
|
||||
source: entry.source,
|
||||
started_at: entry.started_at,
|
||||
enqueued_at: entry.enqueued_at,
|
||||
|
|
@ -265,7 +276,7 @@ function renderQueueEntry(entry, _byId, isChild) {
|
|||
el('span', { class: 'rqe-kind', title: entry.kind },
|
||||
(QUEUE_KIND_GLYPH[entry.kind] || '?') + ' ' + entry.kind),
|
||||
' ',
|
||||
el('code', { class: 'rqe-agent' }, entry.agent),
|
||||
el('code', { class: 'rqe-agent' }, entryAgents(entry)),
|
||||
);
|
||||
li.append(' ', el('span', { class: 'rqe-source rqe-source-' + entry.source }, entry.source));
|
||||
if (entry.state === 'queued') {
|
||||
|
|
@ -331,14 +342,14 @@ function renderQueueEntry(entry, _byId, isChild) {
|
|||
class: 'inline rqe-cancel',
|
||||
'data-async': '',
|
||||
'data-confirm':
|
||||
`cancel ${entry.kind} for \`${entry.agent}\` (queue id ${entry.id})? ` +
|
||||
`cancel ${entry.kind} for \`${entryAgents(entry)}\` (queue id ${entry.id})? ` +
|
||||
`the row drops from the queue and never runs. running / done / failed entries can't be cancelled this way.`,
|
||||
});
|
||||
cancelForm.append(el('button', {
|
||||
type: 'submit',
|
||||
class: 'rqe-cancel-btn',
|
||||
title: 'cancel this queued ' + entry.kind,
|
||||
'aria-label': 'cancel queued ' + entry.kind + ' for ' + entry.agent,
|
||||
'aria-label': 'cancel queued ' + entry.kind + ' for ' + entryAgents(entry),
|
||||
}, '✗'));
|
||||
li.append(cancelForm);
|
||||
}
|
||||
|
|
@ -414,7 +425,7 @@ function renderRebuildLiveLog(queue) {
|
|||
const header = el('div', { class: 'rebuild-live-log-header' },
|
||||
toggle, ' ',
|
||||
el('span', { class: 'rebuild-live-log-title' }, 'live build log — '),
|
||||
el('code', { class: 'rqe-agent' }, running.agent),
|
||||
el('code', { class: 'rqe-agent' }, entryAgents(running)),
|
||||
' ', el('span', { class: 'rqe-kind' },
|
||||
(running.kind || 'rebuild') + ' · ' + (NODE_KIND_LABEL[liveNode.kind] || liveNode.kind)),
|
||||
' ', badge, ' ',
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ async fn wait_for_dags_plain(socket: &Path, ids: Vec<u64>) -> Result<()> {
|
|||
// sees whether the agent came back.
|
||||
if d.nodes.iter().all(|n| n.state.is_terminal()) {
|
||||
if d.state == hive_sh4re::jobs::State::Failed {
|
||||
failed.push(format!("{} {}", d.kind.as_str(), d.agent));
|
||||
failed.push(format!("{} {}", d.kind.as_str(), dag_agents(d)));
|
||||
}
|
||||
} else {
|
||||
all_terminal = false;
|
||||
|
|
@ -138,7 +138,7 @@ async fn wait_for_dags_animated(socket: &Path, ids: Vec<u64>) -> Result<()> {
|
|||
"{} {} {} · {}",
|
||||
state_glyph(d.state),
|
||||
d.kind.as_str(),
|
||||
d.agent,
|
||||
dag_agents(d),
|
||||
fmt_dur(dag_elapsed(d, now)),
|
||||
));
|
||||
for n in &d.nodes {
|
||||
|
|
@ -166,7 +166,7 @@ async fn wait_for_dags_animated(socket: &Path, ids: Vec<u64>) -> Result<()> {
|
|||
}
|
||||
if d.nodes.iter().all(|n| n.state.is_terminal()) {
|
||||
if d.state == hive_sh4re::jobs::State::Failed {
|
||||
failed.push(format!("{} {}", d.kind.as_str(), d.agent));
|
||||
failed.push(format!("{} {}", d.kind.as_str(), dag_agents(d)));
|
||||
}
|
||||
} else {
|
||||
all_terminal = false;
|
||||
|
|
@ -204,6 +204,19 @@ fn finish_wait(mut failed: Vec<String>) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Distinct agents across a DAG's nodes, comma-joined for display — the
|
||||
/// per-node replacement for the old DAG-level `agent` field. Single-agent
|
||||
/// DAGs render one name; a hive-wide DAG lists each.
|
||||
fn dag_agents(d: &hive_sh4re::jobs::DagView) -> String {
|
||||
let mut seen: Vec<&str> = Vec::new();
|
||||
for n in &d.nodes {
|
||||
if !seen.contains(&n.agent.as_str()) {
|
||||
seen.push(&n.agent);
|
||||
}
|
||||
}
|
||||
seen.join(",")
|
||||
}
|
||||
|
||||
/// Current unix time in seconds (0 on the impossible pre-epoch error).
|
||||
fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
|
|
@ -290,7 +303,7 @@ fn render_dag_line(d: &hive_sh4re::jobs::DagView) -> String {
|
|||
"{} {} {:<12}",
|
||||
state_glyph(d.state),
|
||||
d.kind.as_str(),
|
||||
d.agent
|
||||
dag_agents(d)
|
||||
);
|
||||
for (i, n) in d.nodes.iter().enumerate() {
|
||||
let sep = if i == 0 { " " } else { " → " };
|
||||
|
|
@ -314,9 +327,10 @@ mod tests {
|
|||
|
||||
use super::render_dag_line;
|
||||
|
||||
fn node(id: u32, kind: &str, state: State, step: Option<&str>) -> NodeView {
|
||||
fn node(id: u32, agent: &str, kind: &str, state: State, step: Option<&str>) -> NodeView {
|
||||
NodeView {
|
||||
id,
|
||||
agent: agent.to_owned(),
|
||||
kind: kind.to_owned(),
|
||||
deps: if id == 0 { vec![] } else { vec![id - 1] },
|
||||
state,
|
||||
|
|
@ -332,7 +346,6 @@ mod tests {
|
|||
fn render_dag_line_shows_chain_and_running_step() {
|
||||
let dag = DagView {
|
||||
id: 7,
|
||||
agent: "alice".to_owned(),
|
||||
kind: Template::Rebuild,
|
||||
state: State::Running,
|
||||
source: Source::Manual,
|
||||
|
|
@ -345,10 +358,16 @@ mod tests {
|
|||
approval_id: None,
|
||||
perm_payload: None,
|
||||
nodes: vec![
|
||||
node(0, "prebuild", State::Done, None),
|
||||
node(1, "stop_for_update", State::Done, None),
|
||||
node(2, "swap", State::Running, Some("nixos-container update")),
|
||||
node(3, "reconcile", State::Queued, None),
|
||||
node(0, "alice", "prebuild", State::Done, None),
|
||||
node(1, "alice", "stop_for_update", State::Done, None),
|
||||
node(
|
||||
2,
|
||||
"alice",
|
||||
"swap",
|
||||
State::Running,
|
||||
Some("nixos-container update"),
|
||||
),
|
||||
node(3, "alice", "reconcile", State::Queued, None),
|
||||
],
|
||||
};
|
||||
let line = render_dag_line(&dag);
|
||||
|
|
@ -363,11 +382,10 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn render_dag_line_surfaces_first_node_error() {
|
||||
let mut failed = node(0, "prebuild", State::Failed, None);
|
||||
let mut failed = node(0, "bob", "prebuild", State::Failed, None);
|
||||
failed.error = Some("nix build exploded".to_owned());
|
||||
let dag = DagView {
|
||||
id: 8,
|
||||
agent: "bob".to_owned(),
|
||||
kind: Template::Rebuild,
|
||||
state: State::Failed,
|
||||
source: Source::Manual,
|
||||
|
|
|
|||
Loading…
Reference in a new issue