feat(#2591): hive-jobq Node lifecycle — started/finished timestamps + failure reason

Node gains started_at/finished_at (chrono DateTime<Utc>, serialized
RFC 3339 on the wire per hive_sh4re::wire_time) plus error (String).
Graph::set_state self-stamps started_at on the first Running transition
and finished_at on the first terminal one, via an internal now_utc()
clock (keeps settle/complete signatures stable). Outcome::Failed(String)
carries the failure reason, set on the terminal transition.

hive-c0re complete_node builds Outcome::Failed(msg); its node_rt
side-table stays i64 for now (double-write) until #2637 reads the Node.

Toward #2637: the jobq graph becomes the source of truth for per-node
lifecycle so the queue can be sent to the client as-is.
This commit is contained in:
atlas 2026-07-22 23:02:24 +02:00 committed by mara
commit 03eb64cb5c
5 changed files with 115 additions and 9 deletions

View file

@ -38,12 +38,14 @@ use crate::{Dep, DepWhen, Graph, GraphError, NodeId, State};
///
/// `Cancelled` is not an outcome a runner reports — it is scheduler-driven (an
/// `AfterOk` dependency failed), so a runner only ever says `Done` or `Failed`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
/// The node's work succeeded.
Done,
/// The node's work failed.
Failed,
/// The node's work failed, carrying the failure reason — recorded on
/// [`crate::Node::error`] for the failed node itself (a node that rolls up
/// `Failed` from a child, or is cancelled, carries no error of its own).
Failed(String),
}
/// Drives a [`Graph`] over an owned resource pool: claim runnable nodes, record
@ -211,7 +213,10 @@ impl<N, R: Clone + Eq + Hash> Scheduler<N, R> {
/// to start newly-unblocked work.
pub fn complete(&mut self, id: NodeId, outcome: Outcome) {
match outcome {
Outcome::Failed => {
Outcome::Failed(error) => {
// Record the reason before the terminal transition so it's set
// by the time `set_state` stamps `finished_at`.
self.graph.set_error(id, error);
self.graph.set_state(id, State::Failed);
self.cascade_cancel(id);
}
@ -454,6 +459,45 @@ mod tests {
assert_eq!(avail(&s, "build-slot"), 1);
}
#[test]
fn lifecycle_timestamps_and_error_are_stamped() {
let mut s = scheduler_with_slots(2);
let ok = s.append("ok", vec![], None).expect("insert");
let bad = s.append("bad", vec![], None).expect("insert");
let downstream = s.append("down", vec![after_ok(bad)], None).expect("insert");
// Before running: no timestamps.
assert!(s.graph().node(ok).unwrap().started_at.is_none());
assert!(s.graph().node(ok).unwrap().finished_at.is_none());
let started = s.settle();
assert!(started.contains(&ok) && started.contains(&bad));
// Running → started_at stamped, finished_at still none.
assert!(s.graph().node(ok).unwrap().started_at.is_some());
assert!(s.graph().node(ok).unwrap().finished_at.is_none());
// Done → finished_at stamped, no error.
s.complete(ok, Outcome::Done);
let n = s.graph().node(ok).unwrap();
assert!(n.finished_at.is_some());
assert_eq!(n.error, None);
// Failed → the reason rides `Outcome::Failed`, finished_at stamped.
s.complete(bad, Outcome::Failed("boom".to_owned()));
let n = s.graph().node(bad).unwrap();
assert_eq!(n.state, State::Failed);
assert_eq!(n.error.as_deref(), Some("boom"));
assert!(n.finished_at.is_some());
// The `AfterOk`-cancelled node: cascade-cancelled, so finished_at is set,
// but it never ran (no started_at) and carries no error of its own.
let n = s.graph().node(downstream).unwrap();
assert_eq!(n.state, State::Cancelled);
assert!(n.started_at.is_none());
assert!(n.finished_at.is_some());
assert_eq!(n.error, None);
}
#[test]
fn build_slot_cap_limits_concurrency_and_release_unblocks() {
let mut s = scheduler_with_slots(2);
@ -514,7 +558,7 @@ mod tests {
assert_eq!(s.settle(), vec![root]);
s.complete(root, Outcome::Done);
assert_eq!(s.settle(), vec![child]);
s.complete(child, Outcome::Failed);
s.complete(child, Outcome::Failed(String::new()));
assert_eq!(
s.graph().node(root).unwrap().state,
State::Failed,
@ -689,7 +733,7 @@ mod tests {
)
.expect("weak");
assert_eq!(s.settle(), vec![root]);
s.complete(root, Outcome::Failed);
s.complete(root, Outcome::Failed(String::new()));
assert_eq!(s.graph().node(strong1).unwrap().state, State::Cancelled);
assert_eq!(s.graph().node(strong2).unwrap().state, State::Cancelled);
assert_eq!(s.settle(), vec![weak]);
@ -704,7 +748,7 @@ mod tests {
let child = s.append("child", vec![], Some(root)).expect("child");
let grandchild = s.append("gc", vec![], Some(child)).expect("gc");
assert_eq!(s.settle(), vec![root]);
s.complete(root, Outcome::Failed);
s.complete(root, Outcome::Failed(String::new()));
assert_eq!(s.graph().node(child).unwrap().state, State::Cancelled);
assert_eq!(s.graph().node(grandchild).unwrap().state, State::Cancelled);
}