hyperhive/hive-c0re/src/job_queue/tests.rs

//! Queue-core unit tests: what c0re's templates **declare** — node kinds,
//! parent nesting, dep edges with the outcomes that satisfy them, and the
//! resources each construction site states it holds — plus the read layer over
//! that graph (wire projection, history retention, error truncation).
//!
//! **Nothing here runs a node.** Everything a template declares is in the graph
//! the moment `insert` returns, so the assertions read it there. Whether the
//! scheduler then honours those declarations — cascade, roll-up, grant
//! borrow/release, fairness, the `Finishing` gate — is `hive_jobq`'s property
//! and is tested in `hive_jobq`, against its own primitives rather than through
//! this module's templates.
//!
//! That split is why this file can't claim or complete: those are not part of
//! c0re's surface. A test helper that reached for them was reaching across the
//! boundary the two crates exist to draw.

use super::model::NodeKind;
use super::*;

/// Insert a declared job, naming nothing — the shape assertions read the whole
/// graph. A test that needs a handle calls `q.insert` directly and names the
/// node it cares about.
fn insert(q: &JobQueue, declare: impl FnOnce(&JobBuilder)) {
    q.insert_job(|b| {
        declare(b);
        Vec::new()
    })
    .expect("valid shape");
}

/// Insert a declared job and hand back the ids of the nodes it **named**, in
/// the order it named them.
///
/// This is the handle that replaced the DAG id: there is no container to point
/// at any more, so a test that needs to cancel a job or read its state names
/// the roots it cares about — exactly what production does with the ids
/// [`JobQueue::insert_job`] returns.
/// Handed back as raw `u64`, the same form production passes to `cancel` and
/// `node_subtrees` — a `NodeId` cannot be fabricated, so the read surface takes
/// raw ids and searches for them.
fn insert_named(
    q: &JobQueue,
    declare: impl FnOnce(&JobBuilder) -> Vec<hive_jobq::NodeGuid>,
) -> Vec<u64> {
    q.insert_job(declare)
        .expect("valid shape")
        .into_iter()
        .map(NodeId::get)
        .collect()
}

fn ident(s: &str) -> hive_types::Ident {
    hive_types::Ident::parse(s).expect("valid test ident")
}

fn rebuild(builder: &JobBuilder, agent: &str) -> Vec<hive_jobq::NodeGuid> {
    templates::rebuild(builder, agent, true)
}

/// Restart shape with every agent treated as **running** — the online
/// shape (`[Signal→Drain→] StopForUpdate → Reconcile`, no `SetWanted` head)
/// most queue-mechanics tests assume. Mirrors the pre-dynamic
/// `templates::restart` (which is now the state-aware `power::restart_nodes`).
fn restart_online(
    builder: &JobBuilder,
    agents: &[&str],
    graceful: bool,
) -> Vec<hive_jobq::NodeGuid> {
    let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
    power::restart_nodes(builder, &targets, graceful)
}

/// Stop shape with every agent treated as **running** — the online shape
/// (`SetWanted → [Signal→Drain→](graceful) Reconcile`).
fn stop_online(builder: &JobBuilder, agents: &[&str], graceful: bool) -> Vec<hive_jobq::NodeGuid> {
    let targets: Vec<(String, bool)> = agents.iter().map(|a| ((*a).to_owned(), true)).collect();
    power::stop_nodes(builder, &targets, graceful)
}

// `Claimed` / `ClaimReady` / `CompleteNode` lived here: a claim snapshot type
// and two extension traits that let this module start and finish nodes by
// hand. Nothing in this file drives the scheduler any more, so they are gone
// — which is the point. Production completes a node **inside** the future
// `hive_jobq::scheduler::Scheduler::claim_next` hands back, so "run the node,
// then remember to complete it" is not an expressible sequence there. A test
// helper that re-expressed it was a hole in exactly the seam it was testing.

/// One node's **declared** shape: what it is, what it hangs under, and what it
/// waits for — all by kind, since ids are not stable across runs.
#[derive(Debug, PartialEq, Eq)]
struct Declared {
    kind: &'static str,
    /// Parent kind, or `None` when the node is a group root — which is now a
    /// genuine `parent = None`, not "hangs under the container".
    parent: Option<&'static str>,
    /// Kinds this node declared a node-dep on, in declaration order, each with
    /// the outcome set that satisfies it.
    ///
    /// The outcome set is **not** decoration: a template emits its tails as a
    /// pair edged on the same upstream nodes, and the *only* thing telling the
    /// ok-tail from the fail-tail is which outcomes each accepts. Without it
    /// two structurally different nodes read as identical.
    after: Vec<(&'static str, String)>,
}

/// Render a dep's outcome set as the outcomes it actually accepts.
///
/// ⚠️ Spelled out rather than bucketed into `ok` / `any` / other. The first
/// version of this did bucket, and a template's three `ResolveApproval` tails —
/// which differ ONLY in their accepted outcomes — all rendered as `"other"`.
/// A helper that prints two structurally different nodes identically turns an
/// assertion into a tautology.
fn when_tag(when: hive_jobq::DepWhen) -> String {
    [
        (TerminalState::Done, "done"),
        (TerminalState::Failed, "failed"),
        (TerminalState::Cancelled, "cancelled"),
        (TerminalState::Skipped, "skipped"),
    ]
    .into_iter()
    .filter(|(outcome, _)| when.accepts(*outcome))
    .map(|(_, name)| name)
    .collect::<Vec<_>>()
    .join("|")
}

/// Every work node in the graph, in insertion order, as its declared shape.
///
/// **This is what the template tests are actually about.** A template's output
/// is fully determined the moment `insert` returns: the kinds, the parent
/// nesting and the dep edges are all sitting in the graph. Reading them here
/// keeps the assertion on c0re's own product. Whether the scheduler then
/// *honours* those edges — runs a chain serially, holds a grant across a
/// subtree — is `hive_jobq`'s property and is tested in `hive_jobq`.
fn declared_shape(q: &JobQueue) -> Vec<Declared> {
    declared_shape_filtered(q, &|_| true)
}

/// Every node in the graph, since each test inserts into a fresh [`JobQueue`].
///
/// This used to take a DAG id and filter by `root_of(n) == Some(container)`.
/// With no container node there is no per-DAG root to filter on — and nothing
/// to exclude either, because every node in the graph is now real work. Tests
/// that insert more than one job name a node per job and assert on the ids
/// [`JobQueue::insert`] hands back.
fn declared_shape_filtered(q: &JobQueue, keep: &dyn Fn(&NodeKind) -> bool) -> Vec<Declared> {
    let sched = q.sched().lock().expect("job_queue mutex poisoned");
    let graph = sched.graph();
    let kind_of = |id: NodeId| graph.node(id).map(|n| n.payload.as_str());
    graph
        .nodes()
        .filter(|n| keep(&n.payload))
        .map(|n| Declared {
            kind: n.payload.as_str(),
            parent: n.parent.and_then(kind_of),
            after: n
                .deps
                .iter()
                .filter_map(|dep| match dep {
                    hive_jobq::Dep::Node { id, when } => kind_of(*id).map(|k| (k, when_tag(*when))),
                    hive_jobq::Dep::Resource { .. } => None,
                })
                .collect(),
        })
        .collect()
}

/// The id of the one node of `kind`, for the resource assertions.
///
/// Panics unless there is exactly one — every caller is about a shape where the
/// kind is unique, so two would mean the assertion had quietly stopped being
/// about the node the test names.
fn node_of(q: &JobQueue, kind: &str) -> hive_jobq::NodeId {
    let sched = q.sched().lock().expect("job_queue mutex poisoned");
    let graph = sched.graph();
    let mut found: Vec<_> = graph
        .nodes()
        .filter(|n| n.payload.as_str() == kind)
        .map(|n| n.id)
        .collect();
    assert_eq!(
        found.len(),
        1,
        "expected exactly one {kind} node in the graph"
    );
    found.pop().expect("checked above")
}

/// The payload of the one node of `kind`, for assertions about what a node
/// *carries* rather than how it is wired.
fn payload_of(q: &JobQueue, kind: &str) -> NodeKind {
    let id = node_of(q, kind);
    let sched = q.sched().lock().expect("job_queue mutex poisoned");
    sched.graph().node(id).expect("node exists").payload.clone()
}

/// Kinds of every node still `Pending` — the nodes that could yet run.
/// Stronger than asking the scheduler what is *ready right now*: a node
/// blocked on a dep is not ready but is very much still alive.
fn pending_kinds(q: &JobQueue) -> Vec<&'static str> {
    pending_kinds_filtered(q, &|_| true)
}

/// [`pending_kinds`] restricted to the nodes whose payload names `agent`.
fn pending_kinds_for(q: &JobQueue, agent: &str) -> Vec<&'static str> {
    pending_kinds_filtered(q, &|kind: &NodeKind| kind.agent() == agent)
}

/// The payloads of every node still `Pending`, for the cases where *which* of a
/// family of same-kind nodes survived is the assertion — a template emits one
/// tail per outcome and they differ only in what they carry.
fn pending_payloads(q: &JobQueue) -> Vec<NodeKind> {
    let sched = q.sched().lock().expect("job_queue mutex poisoned");
    let graph = sched.graph();
    graph
        .nodes()
        .filter(|n| n.state == State::Pending)
        .map(|n| n.payload.clone())
        .collect()
}

fn pending_kinds_filtered(q: &JobQueue, keep: &dyn Fn(&NodeKind) -> bool) -> Vec<&'static str> {
    let sched = q.sched().lock().expect("job_queue mutex poisoned");
    let graph = sched.graph();
    graph
        .nodes()
        .filter(|n| n.state == State::Pending && keep(&n.payload))
        .map(|n| n.payload.as_str())
        .collect()
}

/// Shorthand for one expected row, so the tables below read as a shape.
fn row(
    kind: &'static str,
    parent: Option<&'static str>,
    after: &[(&'static str, &str)],
) -> Declared {
    Declared {
        kind,
        parent,
        after: after.iter().map(|(k, w)| (*k, (*w).to_owned())).collect(),
    }
}

/// The resources a node **declared**, read off its graph edges.
///
/// The declaration is the thing under test now that construction sites state
/// their own holdings: asking the `NodeKind` what it "should" need would just
/// re-run the derivation this module removed, and would pass even if the
/// construction site declared nothing.
fn declared_resources(q: &JobQueue, node_id: hive_jobq::NodeId) -> Vec<Resource> {
    let inner = q.lock();
    inner
        .graph()
        .node(node_id)
        .expect("node exists")
        .deps
        .iter()
        .filter_map(|dep| match dep {
            hive_jobq::Dep::Resource { name, .. } => Some(name.clone()),
            hive_jobq::Dep::Node { .. } => None,
        })
        .collect()
}

/// The resources declared by **every** node of `kind`, one row per
/// node, sorted so the rows read as a set rather than an insertion order.
///
/// The per-agent templates emit several nodes of one kind — one per agent — and
/// what makes them concurrent is that each holds only its *own* agent's lease.
/// That is a statement about the whole family, so it needs all the rows, not
/// [`declared_resources`]'s single node.
fn declared_resources_of_kind(q: &JobQueue, kind: &str) -> Vec<Vec<Resource>> {
    let sched = q.sched().lock().expect("job_queue mutex poisoned");
    let graph = sched.graph();
    let mut rows: Vec<Vec<Resource>> = graph
        .nodes()
        .filter(|n| n.payload.as_str() == kind)
        .map(|n| {
            n.deps
                .iter()
                .filter_map(|dep| match dep {
                    hive_jobq::Dep::Resource { name, .. } => Some(name.clone()),
                    hive_jobq::Dep::Node { .. } => None,
                })
                .collect()
        })
        .collect();
    rows.sort_by_key(|r| format!("{r:?}"));
    rows
}

/// [`declared_shape`] restricted to the nodes whose payload names `agent`.
///
/// A hive-wide DAG interleaves one subgraph per agent, and the kinds alone
/// cannot tell them apart — two `set_wanted` rows look identical. Slicing by
/// agent is what makes "the fresh agent goes straight to reconcile while the
/// stale one rebuilds first" expressible as a declared shape.
fn declared_shape_for(q: &JobQueue, agent: &str) -> Vec<Declared> {
    declared_shape_filtered(q, &|kind: &NodeKind| kind.agent() == agent)
}

fn state_of(q: &JobQueue, dag_id: u64) -> State {
    // A group root's own state *is* its subtree's roll-up — that is the
    // scheduler's contract (`Finishing` until children settle, then the
    // rolled-up outcome), so there is nothing to derive here any more.
    // A root aged out of the retained history reads `Done`: it settled, or
    // it would still be live.
    q.graph_snapshot(None)
        .iter()
        .find(|n| n.id == dag_id)
        .map_or(State::Done, |n| n.state)
}

/// How many groups the queue is showing — roots, not nodes.
///
/// `graph_snapshot` is flat (every node under every visible root), so a test
/// asking "how many DAGs" counts the parentless ones. Counting rows would
/// count steps, which is a different number: one rebuild is ~7 nodes.
fn dag_count(q: &JobQueue) -> usize {
    q.graph_snapshot(None)
        .iter()
        .filter(|n| n.parent.is_none())
        .count()
}

// ---- insert (dedup removed — every insert is a fresh job) ----
//
// `submit_assigns_distinct_ids` lived here and is gone. Its whole body was
// "two inserts get different container ids" — an assertion about the id
// allocation of a node type this issue deleted, and in any case `hive_jobq`'s
// property rather than c0re's. What the tests below keep is the part that was
// about c0re: **no dedup**, now read off the group count instead of off an id.

/// Insert-time dedup was removed with the agent-per-node refactor (a
/// multi-agent job has no single agent to key a dedup on), so an identical
/// re-insert — same template + agent, still queued — now enqueues a distinct
/// group instead of collapsing into the pending one. Whether any dedup needs
/// reintroducing is tracked as a follow-up.
#[test]
fn identical_resubmit_is_a_distinct_dag() {
    let q = JobQueue::new(1);
    let first = insert_named(&q, |builder| rebuild(builder, "agent-a"));
    let after_first = dag_count(&q);
    let resubmit = insert_named(&q, |builder| rebuild(builder, "agent-a"));
    assert_ne!(
        first, resubmit,
        "no dedup: an identical re-insert declares its own nodes"
    );
    // Counted as a doubling rather than against a literal: one rebuild is
    // several group roots now, and pinning the number here would make this
    // test fail on any shape change while saying nothing about dedup.
    assert_eq!(
        dag_count(&q),
        after_first * 2,
        "the re-insert added its own roots instead of collapsing into the pending ones"
    );
}

#[test]
fn distinct_submits_never_collapse() {
    let q = JobQueue::new(1);
    let rebuild_a = insert_named(&q, |builder| rebuild(builder, "agent-a"));
    let one_rebuild = dag_count(&q);
    let rebuild_b = insert_named(&q, |builder| rebuild(builder, "agent-b"));
    let two_rebuilds = dag_count(&q);
    let restart_a = insert_named(&q, |builder| restart_online(builder, &["agent-a"], false));
    assert_ne!(rebuild_a, rebuild_b);
    assert_ne!(rebuild_a, restart_a);
    assert_eq!(
        two_rebuilds,
        one_rebuild * 2,
        "two rebuilds, nothing merged"
    );
    assert_eq!(
        dag_count(&q),
        two_rebuilds + restart_a.len(),
        "a restart of an agent that already has a queued rebuild is still its own group"
    );
}

// `resubmit_while_running_is_new_dag` lived here: the same two inserts as
// above, kept under a second name so a reader looking for "a config bump
// mid-build must not be swallowed" would find it. It asserted nothing the
// test above doesn't — `insert_job` never consults the state of an existing
// node, so "while running" could not change the outcome and was never staged.
// The scenario is named in that test's doc instead; a duplicate test is a
// second place for the same fact to rot.

// ---- malformed specs: no longer expressible ----
//
// Three tests lived here — a dependency cycle, a dependency on a node that
// does not exist, and an out-of-range parent index — each asserting that
// `submit` refused the spec. All three built their spec by hand out of
// positional indices, which is exactly the representation that made those
// shapes possible: an index can name a node that isn't there, or one that
// comes later.
//
// A job is now declared against handles that only exist for nodes already
// declared, so there is no index to put out of range, and every edge points
// backwards — a cycle needs a forward edge. The guard those tests covered was
// deleted along with the failure mode. What remains — a handle used against a
// builder that never issued it — is `hive_jobq`'s to reject, and its builder
// tests cover it (`a_forward_edge_is_rejected_by_name`,
// `a_forward_parent_is_rejected_by_name`, `graph_rejection_surfaces_as_is`).

// ---- dependency order within a DAG ----

#[test]
fn rebuild_chain_is_declared_serial() {
    // Was `rebuild_chain_claims_in_dep_order`, which drove the whole DAG to
    // observe an order that is fully declared the moment `insert` returns.
    //
    // ⚠️ The old name was also wrong about the mechanism, and reading it rather
    // than the graph is how you'd stay wrong: **only part of this chain is dep
    // edges.** `swap` declares no deps at all — it is ordered by *parent
    // nesting* ("a node's sub-nodes run after its own logic"). Both axes are
    // asserted below because a template can break either one independently.
    //
    // ⚠️ "Serial" is now about the *declared* order, not about concurrency:
    // `prebuild` and the graceful quiesce chain are siblings under the brace and
    // run **in parallel** (see `graceful_rebuild_chain_drains_before_stopping`).
    // The non-graceful shape asserted here has no quiesce chain, so nothing here
    // is concurrent — but the name would mislead about the graceful one.
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        rebuild(builder, "agent-a");
    });
    assert_eq!(
        declared_shape(&q),
        vec![
            row("meta_sync", None, &[]),
            // The brace: holds the lease + slot for everything nested below it.
            row("agent_window", None, &[("meta_sync", "done")]),
            // No dep: ordered by hanging under the brace.
            row("prebuild", Some("agent_window"), &[]),
            // A real dep, not nesting: the container must not go down until the
            // build that replaces it has succeeded.
            row(
                "stop_for_update",
                Some("agent_window"),
                &[("prebuild", "done")]
            ),
            row("swap", Some("stop_for_update"), &[]),
            row(
                "rebuild_bookkeeping",
                Some("stop_for_update"),
                &[("swap", "done")]
            ),
            row(
                "reconcile",
                None,
                &[("agent_window", "done|failed|skipped")]
            ),
            // The tail pair. The ok tail needs every root to succeed; the !ok
            // tail hangs off the ok tail's *elimination* (`skipped`), which is
            // what makes exactly one of them run.
            row(
                "emit_rebuilt",
                None,
                &[
                    ("meta_sync", "done"),
                    ("agent_window", "done"),
                    ("reconcile", "done"),
                ],
            ),
            row(
                "emit_rebuilt",
                None,
                &[
                    ("emit_rebuilt", "skipped"),
                    ("meta_sync", "done|failed|skipped"),
                    ("agent_window", "done|failed|skipped"),
                    ("reconcile", "done|failed|skipped"),
                ],
            ),
        ]
    );
}

/// The boot sweep's graceful shape: the agent gets `Signal` → `Drain` to finish
/// its turn before `StopForUpdate` takes the container down — **concurrently
/// with the nix build**, which is the whole reason the brace exists. The drain
/// window costs nothing it doesn't already cost; nesting it under `Prebuild`
/// used to hide the entire `GRACEFUL_STOP_TIMEOUT` behind the build.
///
/// Lease continuity across the bounce is preserved by `AgentWindow` holding the
/// lease above all of them, which is what makes them safe as siblings — the
/// older shape had to nest `Signal` over the rest of the stop for exactly that
/// reason, since otherwise each would acquire the lease separately and leave a
/// window for another DAG to claim the agent mid-stop.
#[test]
fn graceful_rebuild_chain_drains_before_stopping() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::graceful_rebuild_nodes(builder, "agent-a", true, None);
    });
    // Asserted as full rows, not just kinds: the kind list is identical whether
    // the quiesce chain runs beside the build or nested under it, so a
    // kind-only assertion cannot see the bug this shape exists to fix.
    assert_eq!(
        declared_shape(&q),
        vec![
            row("meta_sync", None, &[]),
            row("agent_window", None, &[("meta_sync", "done")]),
            row("prebuild", Some("agent_window"), &[]),
            // 🎯 **The fix, and the thing to guard.** `signal` hangs off the
            // *brace*, not off `prebuild`, and declares no dep on it — so the
            // drain window runs concurrently with the nix build instead of
            // behind it. Both halves matter: re-parenting it under `prebuild`
            // OR adding an `AfterOk(prebuild)` edge would each silently restore
            // the original defect (up to GRACEFUL_STOP_TIMEOUT hidden behind
            // every agent's build, on every boot sweep).
            row("signal", Some("agent_window"), &[]),
            // Siblings under the brace, dep-ordered — not nested. Nesting is
            // only needed where `Signal` itself holds the lease.
            row("drain", Some("agent_window"), &[("signal", "done")]),
            // The container still goes down only when *both* are ready: the
            // build succeeded and the agent has checkpointed. Running the drain
            // early is the win; stopping early would just be downtime.
            row(
                "stop_for_update",
                Some("agent_window"),
                &[("prebuild", "done"), ("drain", "done")]
            ),
            row("swap", Some("stop_for_update"), &[]),
            row(
                "rebuild_bookkeeping",
                Some("stop_for_update"),
                &[("swap", "done")]
            ),
            row(
                "reconcile",
                None,
                &[("agent_window", "done|failed|skipped")]
            ),
        ],
        "graceful runs signal + drain beside the build, and stops only after both"
    );
}

/// The non-graceful shape is the default everywhere except the boot sweep:
/// a manual rebuild, a meta-update cascade child and a deploy must NOT spend a
/// drain window, so `StopForUpdate` waits only on `Prebuild`.
#[test]
fn non_graceful_rebuild_has_no_signal_or_drain() {
    // Read the shape off the queue rather than out of a node list: a declared
    // job keeps its nodes to itself and inserts them, so what it built is
    // observable where it matters — in what the scheduler runs.
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::rebuild_nodes(builder, "agent-a", true, None);
    });
    assert_eq!(
        declared_shape(&q)
            .iter()
            .map(|d| d.kind)
            .collect::<Vec<_>>(),
        vec![
            "meta_sync",
            "agent_window",
            "prebuild",
            "stop_for_update",
            "swap",
            "rebuild_bookkeeping",
            "reconcile"
        ],
        "exactly seven nodes, and none of them is signal or drain"
    );
}

// ---- build slots ----

#[test]
fn rebuild_chain_declares_its_resources_on_the_brace() {
    // Was `rebuild_chain_declares_the_slot_where_the_nix_work_is` (and before
    // that `fifo_fairness_for_the_slot`).
    //
    // ⚠️ **The rename is a reversal, not tidying.** The old name asserted "the
    // slot follows the nix work and the lease follows the container" — each node
    // declaring what it personally needed. The brace inverts that for a
    // coordinated subtree; see `docs/scheduler/coordinator.md`, _Braces_.
    //
    // (`hive_jobq` owns slot *fairness*, pinned there by
    // `a_contended_resource_goes_to_the_oldest_waiter`. What is c0re's is
    // *which* nodes contend in the first place — a declaration, asserted here.)
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        rebuild(builder, "agent-a");
    });
    let res = |kind: &str| declared_resources(&q, node_of(&q, kind));

    let agent = || Resource::Agent("agent-a".to_owned());
    assert_eq!(
        [
            res("meta_sync"),
            res("agent_window"),
            res("prebuild"),
            res("stop_for_update"),
            res("swap"),
            res("reconcile"),
        ],
        [
            // The meta preamble takes the global window and *nothing else* —
            // no slot (it does no nix work) and no lease. It stays a sibling
            // root so that hive-global window is not held across any build.
            vec![Resource::MetaWindow],
            // The brace takes both, atomically, and holds them for its whole
            // subtree. Hoisting the slot here is not a widening: it already
            // spanned the entire rebuild when `Prebuild` held it, because a unit
            // is held until the acquirer's subtree settles and everything below
            // was inside `Prebuild`.
            vec![Resource::BuildSlot, agent()],
            // The coordinated children declare nothing and re-enter the brace's
            // grants. An empty vec here is the *point* of the shape, not an
            // omission — if any of these regains a declaration it will silently
            // stop running in parallel with its siblings.
            vec![],
            vec![],
            vec![],
            // `Reconcile` is the exception that stays: a top-level root outside
            // the brace, so it takes a genuinely fresh lease after the window
            // has released.
            vec![agent()],
        ],
        "the brace declares for the subtree; coordinated children declare nothing"
    );
}

// ---- per-agent lease ----

#[test]
fn multi_agent_restart_declares_concurrent_per_agent_subgraphs() {
    let q = JobQueue::new(4);
    let roots = insert_named(&q, |builder| {
        restart_online(builder, &["agent-a", "agent-b"], false)
    });
    // Was "a hive-wide restart is ONE DAG": one container over both agents.
    // With the container gone it is one *insert* over N independent groups —
    // which is the same claim about the operator's action and a better one
    // about the graph, since independence is what lets them run at once.
    // Asserted against the named count rather than a literal: the point is
    // that every root the job named is top-level, with nothing above it.
    assert_eq!(dag_count(&q), roots.len(), "every named root is top-level");
    // Each agent's subgraph head (StopForUpdate, since both are running) is a
    // group root with no deps, so nothing orders them against each other; and
    // each declares only its OWN agent's lease, so nothing makes them contend.
    // Those two declared facts are what "they run concurrently" *means* here —
    // that a scheduler then does run independent, resource-disjoint roots at
    // once is hive_jobq's property, tested there.
    let heads: Vec<_> = declared_shape(&q)
        .into_iter()
        .filter(|d| d.kind == "stop_for_update")
        .collect();
    assert_eq!(
        heads,
        vec![
            row("stop_for_update", None, &[]),
            row("stop_for_update", None, &[]),
        ],
        "both per-agent heads are independent group roots"
    );
    assert_eq!(
        declared_resources_of_kind(&q, "stop_for_update"),
        vec![
            vec![Resource::Agent("agent-a".to_owned())],
            vec![Resource::Agent("agent-b".to_owned())],
        ],
        "each head declares only its own agent's lease — disjoint, so no contention"
    );
}

#[test]
fn multi_agent_stop_declares_concurrent_per_agent_subgraphs() {
    let q = JobQueue::new(4);
    let roots = insert_named(&q, |builder| {
        stop_online(builder, &["agent-a", "agent-b"], false)
    });
    // See the restart case above for why this is a named-root count now.
    assert_eq!(dag_count(&q), roots.len(), "every named root is top-level");
    // Same declared story as the restart case above: each agent's subgraph head
    // is a group root with no node-deps, holding only its own agent's lease.
    // Independent roots on disjoint resources is what "concurrently" means at
    // this layer — the running of them is hive_jobq's.
    let heads: Vec<_> = declared_shape(&q)
        .into_iter()
        .filter(|d| d.kind == "set_wanted")
        .collect();
    assert_eq!(
        heads,
        vec![row("set_wanted", None, &[]), row("set_wanted", None, &[])],
        "both per-agent stop subgraph heads are independent group roots"
    );
    assert_eq!(
        declared_resources_of_kind(&q, "set_wanted"),
        vec![
            vec![Resource::Agent("agent-a".to_owned())],
            vec![Resource::Agent("agent-b".to_owned())],
        ],
        "each head declares only its own agent's lease"
    );
}

#[test]
fn multi_agent_start_folds_per_agent_stale_rebuild() {
    let q = JobQueue::new(4);
    // fresh: offline + not stale → SetWanted → Reconcile.
    // stale: offline + stale → SetWanted → «rebuild subgraph».
    let roots = insert_named(&q, |builder| {
        power::start_nodes(
            builder,
            &[
                ("fresh".to_owned(), false, false),
                ("stale".to_owned(), false, true),
            ],
        )
    });
    // One insert spanning both agents — and an *uneven* number of roots, which
    // is the shape this test is about: the fresh agent names one, the stale one
    // names four (its rebuild chains behind `SetWanted` rather than nesting
    // under it, so the head alone would report the start done mid-rebuild).
    assert_eq!(dag_count(&q), roots.len(), "every named root is top-level");
    // The fold is a *declared* difference, readable the moment submit returns:
    // both agents get a `SetWanted(Up)` group root, but the fresh agent's
    // subgraph ends at the Reconcile behind it while the stale agent's carries
    // the whole rebuild chain in between.
    assert_eq!(
        declared_shape_for(&q, "fresh"),
        vec![
            row("set_wanted", None, &[]),
            row("reconcile", Some("set_wanted"), &[]),
        ],
        "a fresh agent is intent + convergence, nothing in between"
    );
    assert_eq!(
        declared_shape_for(&q, "stale"),
        vec![
            row("set_wanted", None, &[]),
            row("meta_sync", None, &[("set_wanted", "done")]),
            row("agent_window", None, &[("meta_sync", "done")]),
            row("prebuild", Some("agent_window"), &[]),
            row(
                "stop_for_update",
                Some("agent_window"),
                &[("prebuild", "done")]
            ),
            row("swap", Some("stop_for_update"), &[]),
            row(
                "rebuild_bookkeeping",
                Some("stop_for_update"),
                &[("swap", "done")]
            ),
            row(
                "reconcile",
                None,
                &[("agent_window", "done|failed|skipped")]
            ),
        ],
        "a stale agent gets the whole rebuild chain wedged between intent and \
         convergence — same DAG, same head kind, more in the middle"
    );
}

#[test]
fn offline_agents_skip_mechanical_nodes_but_keep_reconcile() {
    // The dynamic build skips Signal/Drain/StopForUpdate for a down agent
    // (nothing to quiesce/stop) but ALWAYS keeps the Reconcile tail — the
    // convergence guarantee that catches a race-up between the is_running
    // read and node exec.
    let q = JobQueue::new(4);
    // Offline graceful stop → SetWanted(Off) → Reconcile (no Signal/Drain).
    let stop = insert_named(&q, |builder| {
        power::stop_nodes(builder, &[("down".to_owned(), false)], true)
    });
    // Offline restart → a lone Reconcile (no SetWanted, no StopForUpdate):
    // nothing to bounce, and restart never rewrites intent, so the tail
    // Reconcile converges the down agent to its existing `wanted`.
    let restart = insert_named(&q, |builder| {
        power::restart_nodes(builder, &[("down2".to_owned(), false)], true)
    });
    // The whole group, **root included** — the root is a work node now
    // (`SetWanted` for the stop, the lone `Reconcile` for the restart), not a
    // container to be filtered out. One id per agent, which is what the chain
    // named.
    let shape = |roots: &[u64]| -> Vec<String> {
        q.node_subtrees(roots)
            .iter()
            .map(|n| n.payload.label.clone())
            .collect()
    };
    assert_eq!(
        shape(&stop),
        vec!["set_wanted".to_owned(), "reconcile".to_owned()],
        "offline graceful stop skips the signal/drain quiesce, keeps Reconcile"
    );
    assert_eq!(
        shape(&restart),
        vec!["reconcile".to_owned()],
        "offline restart is a lone Reconcile (no SetWanted head, nothing to stop)"
    );
}

#[test]
fn boot_sweep_nodes_declare_their_own_resources() {
    // Regression, and the reason it needs its own test: `workers::auto_update`
    // is the only place job nodes are constructed *outside* `job_queue/`, so
    // nothing in this module covered it. When resource derivation moved to the
    // construction sites, this path was missed and both kinds silently declared
    // nothing — dropping the agent lease a boot `Reconcile` needs to not race
    // another DAG's container ops, and letting the sweep `MetaLock` land its
    // meta commit inside another node's staged deploy window. Nothing failed to
    // compile; only an exhaustive caller list would have caught it.
    let q = JobQueue::new(4);
    insert(&q, |builder| {
        crate::workers::auto_update::boot_nodes(
            builder,
            true,
            vec!["stale-agent".to_owned()],
            vec!["drifted-agent".to_owned()],
        );
    });

    let mut lock = declared_resources(&q, node_of(&q, "meta_lock"));
    lock.sort_by_key(|r| format!("{r:?}"));
    assert_eq!(
        lock,
        vec![Resource::BuildSlot, Resource::MetaWindow],
        "the sweep MetaLock runs a nix lock bump and commits to meta"
    );

    assert_eq!(
        declared_resources(&q, node_of(&q, "reconcile")),
        vec![Resource::Agent("drifted-agent".to_owned())],
        "a boot Reconcile touches the container, so it holds that agent's lease"
    );
}

/// Crash-watch suppression for a cascade rebuild, which the deleted half of
/// `meta_update_grows_cascade_in_dag` used to assert via a DAG-level
/// `transient` field on the submit-time spec (both the field and the spec type
/// are gone).
///
/// The property is unchanged — a container going down under a rebuild must not
/// read as a crash — but it is no longer a DAG-level declaration: each node
/// answers for itself, so the assertion moves to the nodes a cascade actually
/// runs. Kept as its own test rather than dropped, because it is the *property*
/// that mattered, not the field that used to carry it.
#[test]
fn rebuild_chain_nodes_suppress_crash_watch() {
    for kind in [
        NodeKind::StopForUpdate {
            agent: "a".to_owned(),
        },
        NodeKind::Swap {
            agent: "a".to_owned(),
        },
        NodeKind::Drain {
            agent: "a".to_owned(),
        },
    ] {
        assert!(
            kind.takes_container_down(),
            "{} must suppress crash-watch — a rebuild takes the container down \
             on purpose",
            kind.as_str()
        );
    }
    // The counter-case, and the reason this can't be "any node in a rebuild":
    // the tail brings the container back up, so a container that dies there
    // really did crash.
    assert!(
        !NodeKind::Start {
            agent: "a".to_owned()
        }
        .takes_container_down()
    );
}

// ---- failure: cancel-downstream + AfterAny ----
//
// `failed_node_cancels_downstream_but_afterany_reconcile_runs` lived here. It
// drove a rebuild to a failed `Prebuild` and then asserted three unrelated
// things at once, which is why it needed a running scheduler at all:
//
//   1. the cascade — a failed node cancels its `AfterOk` dependants while the
//      `AfterAny` reconcile still runs. That is hive_jobq's rule, and it owns
//      the test: `failed_after_ok_dep_cancels_dependents_but_after_any_runs`.
//      c0re's declaration of *which* edge is which is asserted in
//      `rebuild_chain_is_declared_serial` — the reconcile's accepted outcome
//      set is right there in the shape table.
//   2. the wire projection — which nodes ride and which are filtered out.
//      **That decision no longer exists:** the generic view serves every node
//      under a visible root, terminal ones included, so there is no predicate
//      left to test. The `Done`-off/`Skipped`-on filter died with `DagView`.
//   3. the roll-up — a group with a failed node reads `Failed`. That is the
//      root node's own `state`, stamped by `hive_jobq`'s settle loop and
//      tested there; nothing re-derives it host-side any more.
//
// Reconstructing all three from one arranged run made none of them
// individually legible, and the arrangement was the only reason this module
// needed to claim and complete nodes.

/// The swap-success path: `Swap` ok → the `AfterOk` `RebuildBookkeeping` (bookkeeping
/// tail) runs, and only then does `Reconcile` fire — serialized behind
/// `RebuildBookkeeping` (not racing it) because `Reconcile` deps `AfterAny(RebuildBookkeeping)`.
#[test]
fn rebuild_reconcile_waits_for_the_whole_build_subtree() {
    // Replaces `swap_ok_runs_post_swap_before_reconcile` and
    // `swap_failure_still_runs_reconcile`, which walked the same DAG with the
    // swap succeeding in one and failing in the other.
    //
    // The interesting claim was "Reconcile must wait for RebuildBookkeeping, not race
    // it" — and it does *not* come from an edge between them. `reconcile` deps
    // `AfterAny(agent_window)`, while `rebuild_bookkeeping` sits inside the brace's
    // subtree (rebuild_bookkeeping → stop_for_update → agent_window). A parent is not
    // terminal until its subtree is, so the brace cannot satisfy that edge while
    // rebuild_bookkeeping is outstanding. **The ordering is the parent chain, not a
    // dependency.**
    //
    // Both facts are asserted in `rebuild_chain_is_declared_serial`; this test
    // states the derived property explicitly because the indirection is the
    // easy thing to break — someone flattening the chain would keep every edge
    // and still lose the guarantee. That warning earned itself: `AgentWindow`
    // **did** flatten this chain, and the guarantee survives only because the
    // roll-up point moved with it.
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        rebuild(builder, "agent-a");
    });
    let shape = declared_shape(&q);
    let parent_of = |kind: &str| {
        shape
            .iter()
            .find(|d| d.kind == kind)
            .unwrap_or_else(|| panic!("{kind} node"))
            .parent
    };
    assert_eq!(parent_of("rebuild_bookkeeping"), Some("stop_for_update"));
    assert_eq!(parent_of("stop_for_update"), Some("agent_window"));
    assert_eq!(
        shape
            .iter()
            .find(|d| d.kind == "reconcile")
            .expect("reconcile node")
            .after,
        vec![("agent_window", "done|failed|skipped".to_owned())],
        "reconcile gates on the brace's roll-up, which covers the whole \
         subtree — including rebuild_bookkeeping — and runs on failure too"
    );
}

// `swap_failure_still_runs_reconcile` lived here.
//
// It asserted that a failed swap leaves `rebuild_bookkeeping` `Skipped` and `swap`
// `Failed`, and that reconcile still runs. All three are hive_jobq's cascade
// (`failed_after_ok_dep_cancels_dependents_but_after_any_still_runs`), and the
// "says so on the wire" half turned out to be nothing: `snapshot` fills
// `NodeView { state: node.state, .. }`, a straight copy of the same `State`
// type, so there is no c0re-side mapping to get wrong.
//
// `failed_reconcile_marks_dag_failed` lived here too — a one-node DAG whose
// node fails, asserting the DAG reads `Failed`. That is `failed_child_rolls_
// parent_up_to_failed` in hive_jobq, restated through a c0re template.

// `multi_agent_lease_frees_per_subgraph_not_whole_dag` and
// `dag_settles_terminal_and_releases_lease_after_work` lived here.
//
// Both drove a DAG to completion to watch an agent lease free up — one when a
// single agent's subgraph settled inside a still-running multi-agent DAG, the
// other when a whole power op finished. Releasing a grant once its owner's
// subtree is terminal is hive_jobq's (`owner_holds_grant_for_its_whole_subtree`,
// `child_borrows_ancestor_grant_released_when_subtree_done`,
// `leaf_owner_goes_done_directly_and_releases`).
//
// The c0re halves are declared and asserted elsewhere: that each agent's
// subgraph is an independent root holding only its own lease is in
// `multi_agent_restart_is_one_dag_with_concurrent_per_agent_subgraphs`, and
// that a power op emits no tail node is in
// `cancelled_power_op_runs_no_compensating_node`, which checks the DAG has no
// pending nodes left at all.

/// `Start` / `Stop` were lease-exempt *as kinds*, which was only safe because
/// every construction site fans them out from inside a lease-holding ancestor.
/// They declare the lease themselves now, and this pins that they do.
///
/// Was `a_fanned_out_start_declares_the_lease_and_re_enters_its_reconciles_
/// grant`, which submitted a `Reconcile`, claimed it, and then **re-declared
/// the fan-out inline** — *"same two calls the scheduler makes"*. That is a
/// copy of production in a test: had `exec.rs` stopped declaring the lease, it
/// would have kept passing. The declaration now lives in `templates::
/// fanned_out_mechanical`, so this calls the real thing.
///
/// The other half of the old test — that a descendant *re-enters* its
/// ancestor's grant rather than taking a second unit of a cap-1 lease — is
/// `hive_jobq`'s, and is tested there by
/// `child_borrows_ancestor_grant_released_when_subtree_done` and
/// `nested_borrowers_never_deadlock`.
#[test]
fn a_fanned_out_mechanical_node_declares_its_agent_lease() {
    let q = JobQueue::new(4);
    insert(&q, |builder| {
        templates::fanned_out_mechanical(
            builder,
            NodeKind::Start {
                agent: "agent-a".to_owned(),
            },
        );
    });
    assert_eq!(declared_shape(&q), vec![row("start", None, &[])]);
    assert_eq!(
        declared_resources(&q, node_of(&q, "start")),
        vec![Resource::Agent("agent-a".to_owned())],
        "the fanned-out node carries the lease itself, rather than relying on \
         whoever happened to fan it out"
    );
}

/// A running `MetaLock` grows one rebuild subgraph per agent into **its own
/// DAG**, rooted on itself — not as child DAGs. That is what keeps a boot sweep
/// (or a meta-update cascade) one unit of work, with every rebuild building
/// against the lock the emitter just bumped.
///
/// Replaces `grown_subgraph_roots_on_emitter_and_rebases_local_deps` and
/// `meta_update_grows_cascade_in_dag`, which differed only in which rebuild
/// flavour they grew and each minted a builder by hand to simulate the graft.
/// What they were checking is the `grown_*_rebuilds` templates, so this calls
/// one — the boot sweep's, since that is the caller that grows a graceful one.
///
/// That the grafted work lands under the emitter, and that the emitter parks in
/// `Finishing` until it settles, is `hive_jobq`'s
/// (`a_completing_node_grows_the_work_it_declared`).
#[test]
fn a_meta_lock_grows_one_rebuild_subgraph_per_agent() {
    let q = JobQueue::new(4);
    let agents = vec!["alice".to_owned(), "bob".to_owned()];
    insert(&q, |builder| {
        templates::grown_graceful_rebuilds(builder, &agents, true);
    });

    // One chain per agent, each an independent group root — so the two rebuild
    // concurrently, each on its own lease.
    let shape = declared_shape(&q);
    let heads: Vec<_> = shape
        .iter()
        .filter(|d| d.kind == "meta_sync")
        .map(|d| d.parent)
        .collect();
    assert_eq!(heads, vec![None, None], "one root chain per agent");
    assert_eq!(
        shape.iter().filter(|d| d.kind == "prebuild").count(),
        2,
        "both agents get their own build"
    );
    // `graceful: true` is the sweep's distinguishing knob — agents mid-turn
    // when the host came up get their drain window rather than being cut off.
    assert_eq!(
        shape.iter().filter(|d| d.kind == "drain").count(),
        2,
        "a boot sweep is graceful, so each agent gets a drain"
    );
}

// ---- cancel ----

#[test]
fn cancel_clears_queued_dag() {
    let q = JobQueue::new(1);
    let roots = insert_named(&q, |builder| rebuild(builder, "agent-a"));
    let [head, brace, tail] = roots.as_slice() else {
        panic!("a rebuild names three roots, got {roots:?}")
    };
    assert!(q.cancel(*head), "fully-queued dag cancels");
    // The operator sees `Cancelled` the moment the cancel returns — a group must
    // not read `Queued` back to the operator who just cancelled it (the
    // dashboard renders this roll-up from the snapshot
    // `post_rebuild_queue_cancel` emits synchronously).
    assert_eq!(state_of(&q, *head), State::Cancelled, "no stale Queued gap");
    // Both `EmitRebuilt` tails go with it — the ok one is `AFTER_OK`, the
    // failure one keys on elimination — so a rebuild that never ran emits
    // nothing.
    //
    // 🎯 **But `Reconcile` survives, and that is the finding.** Its edge onto
    // the brace accepts `done|failed|skipped`, and a cancel-cascade *skips* the
    // brace rather than cancelling it — so the edge is satisfied and the tail
    // stays claimable. With a container above them all, one cancel took the
    // whole group; a job is its roots now, and dropping it means dropping every
    // id the insert returned. That is what the ids are for.
    assert_eq!(
        pending_kinds(&q),
        vec!["reconcile"],
        "the convergence tail outlives its head's cancel"
    );
    assert!(
        !q.cancel(*brace),
        "the brace was eliminated with the head — there is nothing left to cancel"
    );
    assert!(q.cancel(*tail), "the surviving tail cancels on its own id");
    assert!(
        pending_kinds(&q).is_empty(),
        "cancelling every named root leaves nothing alive, got {:?}",
        pending_kinds(&q)
    );
}

/// `cancel` takes a **node** id, not a DAG id — so an interior node can be
/// dropped without touching the rest of the group.
///
/// This is the capability the DAG-scoped version couldn't express, and the
/// reason it reads naturally: a DAG id *is* its root node's id, so the
/// whole-group cancel every other test does is just this called on a root.
/// Here a hive-wide restart drops **one agent's** subgraph and the other agent
/// still runs.
#[test]
fn cancel_drops_one_agents_branch_leaving_the_rest() {
    let q = JobQueue::new(2);
    let roots = insert_named(&q, |builder| {
        restart_online(builder, &["agent-a", "agent-b"], false)
    });
    // One root per agent, **in the order the chain named them** — that ordering
    // is `insert_job`'s contract, and it is what replaced digging the right
    // subgraph out of a snapshot by matching on its payload's agent field.
    let [a_root, _b_root] = roots.as_slice() else {
        panic!("a two-agent restart names one root per agent, got {roots:?}")
    };

    assert!(q.cancel(*a_root), "an interior/group root cancels alone");

    // agent-a's subgraph is gone; agent-b's is untouched and still alive.
    assert!(
        pending_kinds_for(&q, "agent-a").is_empty(),
        "agent-a's branch was dropped whole, got {:?}",
        pending_kinds_for(&q, "agent-a")
    );
    assert!(
        !pending_kinds_for(&q, "agent-b").is_empty(),
        "agent-b's branch survives its sibling's cancel"
    );
}

/// A cancelled power op must run **no** compensating node — not even one that
/// carries a `SetWanted` head.
///
/// Now structural rather than a property of a hook enum: a power op emits no
/// tail node at all, so once its work nodes cancel there is simply nothing left
/// to claim. `cancel` also refuses unless every work node is still `Pending`
/// (`hive_jobq`'s `cancel_node_refuses_a_group_with_anything_running`), so a
/// `Cancelled` DAG provably never executed a node: its `SetWanted` never ran
/// and the agent's intent still reads whatever
/// the operator last set. A "revert" instead writes the agent's *observed*
/// state, which for a down-but-`wanted = Up` agent (crashed, or caught
/// mid-bounce) flips the intent to `Offline` and leaves it
/// deliberately-stopped as far as reconcile and crash-watch are concerned.
#[test]
fn cancelled_power_op_runs_no_compensating_node() {
    /// Insert-cancel-assert for one power op. Taking the roots the job already
    /// named is what removes the need to put three differently-typed recipes in
    /// one array: each caller inserts its own, so no closure type has to be
    /// erased to a boxed one.
    fn assert_cancels_clean(q: &JobQueue, roots: &[u64], writes_intent: bool, case: &str) {
        // Read the intent head off the inserted nodes rather than out of a
        // spec: a declared job holds its own nodes and inserts them. The queue
        // is fresh per case, so the whole graph is this one op.
        let has_intent = declared_shape(q).iter().any(|d| d.kind == "set_wanted");
        assert_eq!(has_intent, writes_intent, "{case}: intent head");
        for root in roots {
            assert!(q.cancel(*root), "{case}: cancelled while queued");
            assert_eq!(state_of(q, *root), State::Cancelled);
        }
        // Nothing is left that *could* run. Asserting on the pending set rather
        // than on "what is ready this instant" also covers a node that is alive
        // but blocked — which is exactly what a leftover compensating node
        // would look like.
        assert_eq!(
            pending_kinds(q),
            Vec::<&str>::new(),
            "{case}: a power op emits no tail node, so a cancelled one leaves nothing"
        );
    }

    for graceful in [false, true] {
        for running in [false, true] {
            let targets = vec![("agent-a".to_owned(), running)];
            let case = format!("graceful={graceful} running={running}");

            let q = JobQueue::new(1);
            let roots = insert_named(&q, |builder| {
                power::restart_nodes(builder, &targets, graceful)
            });
            assert_cancels_clean(&q, &roots, false, &format!("restart {case}"));

            let q = JobQueue::new(1);
            let roots = insert_named(&q, |builder| power::stop_nodes(builder, &targets, graceful));
            assert_cancels_clean(&q, &roots, true, &format!("stop {case}"));

            let q = JobQueue::new(1);
            let roots = insert_named(&q, |builder| {
                power::start_nodes(builder, &[("agent-a".to_owned(), running, false)])
            });
            assert_cancels_clean(&q, &roots, true, &format!("start {case}"));
        }
    }
}

// ---- terminal reporting + lease release ----

/// A DAG cancelled while fully queued must still **run its tail**, or a queued
/// approval DAG cancelled by the operator would dangle its approval forever.
///
/// This is the load-bearing case for sparing tails in [`JobQueue::cancel`]: the
/// work nodes all cancel, but `ResolveApproval` is weak-edged, so a `Cancelled`
/// dep satisfies its edge and it becomes claimable instead of being cancelled
/// along with everything else. It reads `Cancelled` off its own deps and resolves
/// the approval as "cancelled before completion".
#[test]
fn cancelled_dag_still_runs_its_approval_tail() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::approval_deploy(builder, "agent-a", 7);
    });
    // Found by kind, not returned: `approval_deploy` deliberately names
    // nothing, because nothing polls it — the approval row is how an operator
    // follows a deploy, so the template is fire-and-forget in production and a
    // test must not make it return an id it would otherwise have no use for.
    let window = node_of(&q, "deploy_window").get();
    assert!(q.cancel(window), "fully-queued dag cancels");
    // The `Cancelled` tail is the only node whose edge accepts a dropped
    // dependency, so it is the only one `cancel` spares — and *which* tail
    // survives is the whole assertion: the template emits one per outcome and
    // the spared one names how the approval row is about to be resolved.
    // Nothing computes it, so reading the survivor is reading the answer.
    let spared = pending_payloads(&q);
    assert!(
        matches!(
            spared.as_slice(),
            [NodeKind::ResolveApproval {
                approval_id: 7,
                outcome: TerminalState::Cancelled
            }]
        ),
        "only the cancelled-outcome tail is spared, got {spared:?}"
    );
    // ⚠️ `Cancelled`, and it reads terminal **while the spared tail is still
    // pending** — the one place this differs from the container era, where the
    // cancel landed on a node *above* the window and the window rolled up
    // `Finishing`. Here the operator cancels the window itself, so its own
    // state is `Cancelled` however its subtree is doing. Deliberately asserted
    // rather than routed around: the group's card goes terminal while a
    // bookkeeping node runs on. That is acceptable for the tail this test
    // protects (it resolves the approval row and nothing waits on it), and it
    // would not be for work an operator expects to still be watching.
    assert_eq!(state_of(&q, window), State::Cancelled);
    // An unrelated group landing in the same graph doesn't disturb this one's
    // state — a root rolls up its own subtree, not the graph.
    let _other = insert_named(&q, |builder| rebuild(builder, "agent-b"));
    assert_eq!(state_of(&q, window), State::Cancelled);
}

// ---- approval deploy subtree ----

/// The config-PR deploy is a subtree, not one opaque node. The
/// resource-holding root completes immediately (its `Finishing` state is the
/// parent gate that releases the children), then the phases run strictly in
/// order — and the `AfterAny` tail still runs when the irreversible half fails,
/// because it's the node that compensates for it.
#[test]
fn deploy_dag_runs_phases_in_order_and_tails_a_failed_apply() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::approval_deploy(builder, "agent-a", 7);
    });

    assert_eq!(
        declared_shape(&q),
        vec![
            // The window is the group root and holds the meta window for the
            // whole subtree; the three phases are its sub-nodes.
            row("deploy_window", None, &[]),
            row("merge_verify", Some("deploy_window"), &[]),
            // Apply only on a clean verify — a failed verify cancel-cascades
            // it, which is what leaves the forge and the applied repo untouched.
            row(
                "deploy_apply",
                Some("deploy_window"),
                &[("merge_verify", "done")]
            ),
            // The compensation tail accepts every terminal outcome of apply,
            // *including `skipped`* — which is the state apply lands in when
            // verify failed and it never ran. That one edge is the entire
            // "still tails a failed apply / a failed verify" behaviour, and it
            // is why two separate DAG-driving tests collapsed into this table.
            row(
                "deploy_tail",
                Some("deploy_window"),
                &[("deploy_apply", "done|failed|skipped")]
            ),
            // One approval tail per outcome, gated on the window's roll-up.
            row("resolve_approval", None, &[("deploy_window", "done")]),
            row("resolve_approval", None, &[("deploy_window", "failed")]),
            row("resolve_approval", None, &[("deploy_window", "cancelled")]),
        ]
    );
}

/// The deploy's happy path: `DeployApply` does not build. It grows the ordinary
/// rebuild chain into the live DAG under itself, and `FinalizeDeploy` — gated on
/// that graft finishing — plants the deploy tag last.
///
/// The queue is built with **one** build slot on purpose. `DeployWindow` already
/// holds that slot (and the meta window) for the whole subtree, so the grafted
/// `Prebuild` can only ever claim by *re-entering* its ancestor's hold. If the
/// graft were rooted anywhere outside `DeployWindow`'s subtree it would block on
/// a resource its own DAG owns and deadlock — this test is what pins that down.
#[test]
fn deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it() {
    // What `DeployApply` grows is `deploy_rebuild_nodes`' output, and that is a
    // pure declaration — so it is declared here directly rather than by running
    // a deploy far enough to graft it. **Reproducing the runtime path is not
    // needed to test what the runtime path declares.**
    //
    // The grafting mechanism itself is hive_jobq's and tested there: the work
    // lands under the emitter *before* it settles, and the emitter parks in
    // `Finishing` so a downstream `AfterAny` gate stays shut while the new
    // children run (`a_completing_node_grows_the_work_it_declared`,
    // `parent_parks_in_finishing_until_children_roll_up`).
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::deploy_rebuild_nodes(builder, "agent-a", 11);
    });

    assert_eq!(
        declared_shape(&q),
        vec![
            row("meta_sync", None, &[]),
            row("agent_window", None, &[("meta_sync", "done")]),
            row("prebuild", Some("agent_window"), &[]),
            row(
                "stop_for_update",
                Some("agent_window"),
                &[("prebuild", "done")]
            ),
            row("swap", Some("stop_for_update"), &[]),
            row(
                "rebuild_bookkeeping",
                Some("stop_for_update"),
                &[("swap", "done")]
            ),
            row(
                "reconcile",
                None,
                &[("agent_window", "done|failed|skipped")]
            ),
            // The deploy tag is planted only after the rebuild came up clean:
            // `AfterOk` on **both** roots, so either one failing skips it. That
            // pair of edges is the whole "skips finalize on a failed graft"
            // behaviour — no run needed to see it.
            row(
                "finalize_deploy",
                None,
                &[("agent_window", "done"), ("reconcile", "done")]
            ),
        ]
    );
}

// `deploy_dag_skips_finalize_but_still_tails_a_failed_graft` lived here.
//
// A failure *inside* the grafted rebuild is the failure mode subgraph growth
// introduces: the deploy is already merged and the container half-swapped, so
// `FinalizeDeploy` must be cancel-cascaded (no `deployed/<id>` tag planted)
// while the tail still runs to compensate, and `Reconcile` is deliberately
// still reached — it boots the container back up.
//
// Every declared half of that is asserted by
// `deploy_apply_grows_rebuild_subgraph_and_finalizes_after_it`, which reads
// `deploy_rebuild_nodes`' shape directly:
//
//   - "a failed swap still reaches Reconcile" is the `reconcile` row's
//     `AfterAny` edge on `prebuild` (`done|failed|skipped`);
//   - "finalize is cancel-cascaded" is `finalize_deploy`'s `AfterOk` pair —
//     either root failing skips it;
//   - "the tail still runs" is `deploy_tail`'s own `done|failed|skipped` edge
//     on apply, asserted in the `approval_deploy` table above.
//
// The runtime halves are hive_jobq's: cascade on failure, roll-up, and
// `first_error` digging past a group root that rolled up `Failed` while
// carrying no error of its own (`first_error_skips_a_rolled_up_failure_
// carrying_no_error`). That last one is why the DAG reports "profile swap
// failed" rather than nothing — the mechanism, not this shape.

// `deploy_dag_skips_apply_but_still_runs_tail_when_verify_fails` lived here.
//
// A pre-merge rejection (drift gate, eval failure) cancel-cascades the
// irreversible half via its `AfterOk` edge, while the tail is still reached —
// it owns the forge mirror, not just compensation. That test and
// `deploy_dag_runs_phases_in_order_and_tails_a_failed_apply` differed only in
// *where* they injected the failure, and each drove the whole DAG to watch the
// tail run anyway.
//
// Both outcomes follow from one declared edge, which the surviving test now
// asserts directly: the tail accepts `done|failed|skipped` on apply, and
// `skipped` is exactly the state apply lands in when verify failed and it never
// ran. The runtime halves are hive_jobq's and tested there —
// `failed_after_ok_dep_cancels_dependents_but_after_any_still_runs` and
// `failed_child_rolls_parent_up_to_failed` (an Ok tail cannot launder a failed
// deploy into a success).

// ---- history ----
//
// The node → build-log link is no longer queue state: the log row carries
// `node_id` and the lookup lives in `stores::build_logs` (see
// `node_link_survives_completion_and_newest_wins` there). Nothing in the queue
// needs testing for it any more, which is the point of that move.

/// History retention is a **flat** newest-first cap over all terminal DAGs
/// (`MAX_HISTORY_DAGS`), not a per-template bucket behind a grace window.
/// The dashboard renders one recent-builds list, so one number bounds it —
/// and with no bucketing there's nothing for a burst of same-shaped DAGs to
/// evict early, which is what the grace window used to paper over.
///
/// `retain_history` is a sort-and-truncate over `(handle, finished_at,
/// tiebreak)`. This used to submit `MAX_HISTORY_DAGS + 8` DAGs, claim and fail
/// each one's node, then read the ids back out of a snapshot — the scheduler,
/// the roll-up and the wire projection all in the path of a policy that reads
/// none of them.
///
/// Calling it directly also reaches the half the round-trip never could: every
/// DAG in that loop settled inside the same wall-clock second, so `finished_at`
/// tied on all of them and **only** the tiebreak was ever exercised. Eviction
/// by time — the actual policy — went untested.
#[test]
fn history_retains_live_dags_and_the_newest_terminals() {
    const CAP: usize = 3;
    // Live DAGs are kept whole, regardless of the cap.
    assert_eq!(
        retain_history(vec!["live-a", "live-b"], vec![], CAP),
        vec!["live-a", "live-b"],
    );
    // Terminals: newest `finished_at` first, oldest evicted past the cap.
    assert_eq!(
        retain_history(
            vec![],
            vec![
                ("oldest", 100, 1),
                ("newest", 400, 2),
                ("middle", 200, 3),
                ("later", 300, 4),
            ],
            CAP,
        ),
        vec!["newest", "later", "middle"],
        "the oldest terminal falls off"
    );
    // Same second → the tiebreak decides, descending, so the DAG inserted
    // last wins. This is the *common* case: a burst settles together.
    assert_eq!(
        retain_history(
            vec![],
            vec![("a", 100, 1), ("b", 100, 2), ("c", 100, 3), ("d", 100, 4)],
            CAP,
        ),
        vec!["d", "c", "b"],
    );
    // A live DAG never competes with history for the cap.
    assert_eq!(
        retain_history(
            vec!["live"],
            vec![("a", 100, 1), ("b", 200, 2), ("c", 300, 3), ("d", 400, 4)],
            CAP,
        ),
        vec!["live", "d", "c", "b"],
    );
}

/// `graph_snapshot`'s `states` ask, exercised through the real scheduler:
/// a queued (never-run) DAG cancels as one unit, so every node in it shares
/// one state and this only proves root-level inclusion/exclusion — the
/// per-node case (a live group holding a mix of finished and unfinished
/// steps) isn't expressible without driving the scheduler, which this test
/// module deliberately can't do (see the module doc comment). See
/// `filter_nodes_by_state_keeps_matching_nodes_from_a_mixed_state_tree`
/// below for that half, exercised directly against hand-built `GraphNode`s.
/// `None` is the identity filter (every visible node, current default
/// behaviour).
#[test]
fn graph_snapshot_states_filters_by_node_state() {
    let q = JobQueue::new(2);
    let roots_a = insert_named(&q, |builder| restart_online(builder, &["agent-a"], false));
    let roots_b = insert_named(&q, |builder| restart_online(builder, &["agent-b"], false));
    let [head_a] = roots_a.as_slice() else {
        panic!("a one-agent restart names one root, got {roots_a:?}")
    };
    let [head_b] = roots_b.as_slice() else {
        panic!("a one-agent restart names one root, got {roots_b:?}")
    };

    assert!(q.cancel(*head_a), "queued dag cancels");
    assert_eq!(state_of(&q, *head_a), State::Cancelled);
    assert_eq!(state_of(&q, *head_b), State::Pending, "untouched sibling");

    let roots_of = |states: Option<&[State]>| -> Vec<u64> {
        q.graph_snapshot(states)
            .into_iter()
            .filter(|n| n.parent.is_none())
            .map(|n| n.id)
            .collect()
    };

    assert_eq!(
        roots_of(Some(&[State::Cancelled])),
        vec![*head_a],
        "only the cancelled group's root rides"
    );

    assert_eq!(
        roots_of(Some(&[State::Pending])),
        vec![*head_b],
        "only the pending group's root rides"
    );

    let mut both = roots_of(None);
    both.sort_unstable();
    let mut expected = vec![*head_a, *head_b];
    expected.sort_unstable();
    assert_eq!(
        both, expected,
        "no filter shows every visible root, as before"
    );
}

#[test]
fn error_truncation_cuts_on_a_char_boundary() {
    // `truncate_error` is a pure `&str -> String`. This used to submit a DAG,
    // claim its head, fail it with a long string and read the error back out of
    // a snapshot — four moving parts to observe one transform, and the DAG
    // round-trip is covered by its own tests either way.
    //
    // Testing it directly also reaches the case the round-trip never did: the
    // cap is a **byte** length, so a multibyte char straddling it would panic
    // the slice. That boundary scan is the only non-obvious line in the fn and
    // it had no coverage at all.
    assert_eq!(
        truncate_error("short"),
        "short",
        "under the cap is untouched"
    );

    let ascii = truncate_error(&"x".repeat(5000));
    assert!(ascii.ends_with('…'));
    assert!(ascii.len() <= MAX_ERROR_LEN + '…'.len_utf8());

    // 'é' is 2 bytes, so the 2000-byte cap lands mid-char.
    let multibyte = truncate_error(&"é".repeat(5000));
    assert!(multibyte.ends_with('…'));
    assert!(multibyte.len() <= MAX_ERROR_LEN + '…'.len_utf8());
}

// ---- template shapes ----

#[test]
fn graceful_stop_shape_signal_drain_reconcile() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        stop_online(builder, &["agent-a"], true);
    });
    assert_eq!(
        declared_shape(&q),
        vec![
            // The whole stop hangs under `set_wanted`: the durable intent is
            // written first, and the mechanical steps are its sub-nodes.
            row("set_wanted", None, &[]),
            row("signal", Some("set_wanted"), &[]),
            row("drain", Some("set_wanted"), &[("signal", "done")]),
            row("reconcile", Some("set_wanted"), &[("drain", "done")]),
        ]
    );
    // The quiesce pair declares **nothing**: `set_wanted` is the brace and holds
    // the lease for the whole subtree, so both borrow that grant. Same shape,
    // same helper (`templates::quiesce`) as the rebuild's window.
    //
    // ⚠️ In particular neither takes a build slot — which is what lets a
    // whole-hive graceful *stop* overlap every agent's drain even at
    // `buildSlots = 1`: the ceiling is one `GRACEFUL_STOP_TIMEOUT` in total,
    // not one per agent. That holds here because nothing in a stop chain is
    // slot-needing. It does **not** hold for the boot *sweep*, whose rebuilds
    // do hold the slot across their drains — see the note on the sweep in
    // `exec.rs`.
    assert_eq!(
        [
            declared_resources(&q, node_of(&q, "signal")),
            declared_resources(&q, node_of(&q, "drain")),
        ],
        [vec![], vec![]],
        "the quiesce pair borrows the brace's lease and declares nothing"
    );
}

#[test]
fn pause_shape_signal_drain() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        power::pause_nodes(builder, &["agent-a".to_owned()]);
    });
    assert_eq!(
        declared_shape(&q),
        vec![
            // Unlike the stop quiesce pair (which hangs under an existing
            // `set_wanted` head), pausing has no natural parent to reuse, so
            // this shape declares its own `AgentWindow` brace — the pair are
            // plain siblings under it, not a signal-holds-the-lease-itself
            // shape (that was tried first and rejected at insert: a child
            // can't `after_ok` its own parent — see `pause_quiesce`'s doc
            // comment for the exact error).
            row("agent_window", None, &[]),
            row("pause_signal", Some("agent_window"), &[]),
            row(
                "pause_drain",
                Some("agent_window"),
                &[("pause_signal", "done")]
            ),
        ]
    );
    assert_eq!(
        declared_resources(&q, node_of(&q, "agent_window")),
        vec![Resource::Agent("agent-a".to_owned())],
        "the brace holds the lease for the whole pair"
    );
    assert_eq!(
        [
            declared_resources(&q, node_of(&q, "pause_signal")),
            declared_resources(&q, node_of(&q, "pause_drain")),
        ],
        [vec![], vec![]],
        "the pause pair borrows the brace's lease and declares nothing itself \
         — same shape the stop quiesce pair uses"
    );
}

#[test]
fn spawn_shape_provision_create_dropin_reconcile() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::spawn(builder, "newbie", 7);
    });
    assert_eq!(
        declared_shape(&q),
        vec![
            row("provision", None, &[]),
            row("create", Some("provision"), &[]),
            row("write_dropin", Some("create"), &[]),
            row("reconcile", Some("create"), &[("write_dropin", "done")]),
            // One tail per outcome, each edged to accept only that one — so
            // *which* tail the graph lets run already is the answer, and
            // nothing branches at runtime. The three differ **only** in their
            // accepted outcome, which is why `declared_shape` spells the
            // outcome set out instead of bucketing it.
            row("resolve_approval", None, &[("provision", "done")]),
            row("resolve_approval", None, &[("provision", "failed")]),
            row("resolve_approval", None, &[("provision", "cancelled")]),
        ]
    );
}

/// The destroy chain, and the reason it is a chain: `Stop` is reused so the
/// crash-watch answer comes from the node that actually stops the container.
///
/// Asserting the *edges* is the point. `destroy_container` runs `after_ok` a
/// `stop`, which is what makes "the container is already down here" a
/// structural fact rather than a convention — see the companion test below for
/// why that matters.
#[test]
fn destroy_shape_stop_then_destroy_then_bookkeeping() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::destroy(builder, "doomed", false);
    });
    assert_eq!(
        declared_shape(&q),
        vec![
            row("stop", None, &[]),
            // No explicit edge to `stop`: `part_of` already gates the child on
            // its parent reaching `Finishing`, and declaring a dep on your own
            // parent is rejected outright (it would deadlock). The precondition
            // is the group membership.
            row("destroy_container", Some("stop"), &[]),
            row(
                "destroy_bookkeeping",
                Some("stop"),
                &[("destroy_container", "done")]
            ),
        ]
    );
}

/// `purge` inserts the irreversible delete as its own node, between the destroy
/// and the bookkeeping tail — so the meta sync that stops referencing the agent
/// runs *after* its trees are actually gone, and a purge is visibly distinct
/// from a plain destroy on the graph instead of being a hidden boolean.
#[test]
fn destroy_shape_purge_inserts_purge_state_before_the_tail() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::destroy(builder, "doomed", true);
    });
    assert_eq!(
        declared_shape(&q),
        vec![
            row("stop", None, &[]),
            row("destroy_container", Some("stop"), &[]),
            row(
                "purge_state",
                Some("stop"),
                &[("destroy_container", "done")]
            ),
            row(
                "destroy_bookkeeping",
                Some("stop"),
                &[("purge_state", "done")]
            ),
        ]
    );
}

/// The counter-case to `rebuild_chain_nodes_suppress_crash_watch`, and the one
/// assertion in this file that exists to stop a *plausible* edit rather than a
/// wrong one.
///
/// `destroy_container` is the most obvious candidate for `takes_container_down`
/// on the whole list and must stay `false`. It is edged downstream of a `Stop`
/// that already carries the flag, so the intentional stop is already accounted
/// for; a container still alive when this node claims is a genuine bug. Since a
/// wrong `true` **silently swallows a real crash** while a wrong `false` only
/// costs a spurious event, this is the asymmetry that has to be pinned.
#[test]
fn destroy_container_must_not_suppress_crash_watch() {
    assert!(
        !NodeKind::DestroyContainer {
            agent: "a".to_owned()
        }
        .takes_container_down(),
        "destroy_container runs after a Stop that already declared the \
         container is going down; claiming it again would suppress the alert \
         for a container found unexpectedly alive"
    );
    // The upstream node is where the `true` lives — assert it here too, so the
    // pair reads as one property and moving the flag breaks this test.
    assert!(
        NodeKind::Stop {
            agent: "a".to_owned()
        }
        .takes_container_down()
    );
}

#[test]
fn perm_change_shape_prefixes_rebuild_chain() {
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::perm_change(
            builder,
            "agent-a",
            PermPayload::Combined {
                groups: Some(vec![]),
                caps: None,
            },
        );
    });
    assert_eq!(
        declared_shape(&q)
            .iter()
            .map(|d| d.kind)
            .collect::<Vec<_>>(),
        vec![
            "write_perm_file",
            "meta_sync",
            "agent_window",
            "prebuild",
            "stop_for_update",
            "swap",
            "rebuild_bookkeeping",
            "reconcile",
            // the ok / !ok tail pair
            "emit_rebuilt",
            "emit_rebuilt",
        ],
        "the perm write prefixes an otherwise ordinary rebuild chain"
    );
}

#[test]
fn reparent_shape_is_a_lone_agentless_meta_window_node() {
    // Single-move `set-parent` shape: one node, no rebuild subgraph (no
    // container rebuild needed for a parent move), agentless like
    // `MetaLock`, and it must declare the meta window — a topology commit
    // must not land inside another node's staged deploy window.
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::reparent(builder, vec![(ident("alice"), Some(ident("bob")))]);
    });
    assert_eq!(
        declared_shape(&q),
        vec![row("reparent", None, &[])],
        "one node, no rebuild subgraph"
    );
    let node = node_of(&q, "reparent");
    assert_eq!(
        declared_resources(&q, node),
        vec![Resource::MetaWindow],
        "a topology commit must declare the same MetaWindow as WritePermFile, \
         and nothing else — no lease (agentless), no build slot (no nix work)"
    );
}

#[test]
fn reparent_bulk_shape_carries_every_move_on_one_node() {
    // `set-parent-bulk`: still ONE node (one git commit, `moves.len() > 1`),
    // not one node per move — bulk atomicity across every move in the
    // request is the reason a single node was chosen in the first place.
    let moves = vec![(ident("alice"), Some(ident("bob"))), (ident("carol"), None)];
    let q = JobQueue::new(1);
    insert(&q, |builder| {
        templates::reparent(builder, moves.clone());
    });
    assert_eq!(
        declared_shape(&q),
        vec![row("reparent", None, &[])],
        "one node for the whole request, not one per move"
    );
    let NodeKind::Reparent { moves: got } = payload_of(&q, "reparent") else {
        panic!("expected a Reparent node");
    };
    assert_eq!(got, moves, "every move rides the single node");
}