feat(#2591): hive-jobq Node lifecycle — started/finished timestamps + failure reason
Node gains started_at/finished_at (chrono DateTime<Utc>, serialized RFC 3339 on the wire per hive_sh4re::wire_time) plus error (String). Graph::set_state self-stamps started_at on the first Running transition and finished_at on the first terminal one, via an internal now_utc() clock (keeps settle/complete signatures stable). Outcome::Failed(String) carries the failure reason, set on the terminal transition. hive-c0re complete_node builds Outcome::Failed(msg); its node_rt side-table stays i64 for now (double-write) until #2637 reads the Node. Toward #2637: the jobq graph becomes the source of truth for per-node lifecycle so the queue can be sent to the client as-is.
This commit is contained in:
parent
30a2a2e9de
commit
03eb64cb5c
5 changed files with 115 additions and 9 deletions
|
|
@ -38,12 +38,14 @@ use crate::{Dep, DepWhen, Graph, GraphError, NodeId, State};
|
|||
///
|
||||
/// `Cancelled` is not an outcome a runner reports — it is scheduler-driven (an
|
||||
/// `AfterOk` dependency failed), so a runner only ever says `Done` or `Failed`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Outcome {
|
||||
/// The node's work succeeded.
|
||||
Done,
|
||||
/// The node's work failed.
|
||||
Failed,
|
||||
/// The node's work failed, carrying the failure reason — recorded on
|
||||
/// [`crate::Node::error`] for the failed node itself (a node that rolls up
|
||||
/// `Failed` from a child, or is cancelled, carries no error of its own).
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
/// Drives a [`Graph`] over an owned resource pool: claim runnable nodes, record
|
||||
|
|
@ -211,7 +213,10 @@ impl<N, R: Clone + Eq + Hash> Scheduler<N, R> {
|
|||
/// to start newly-unblocked work.
|
||||
pub fn complete(&mut self, id: NodeId, outcome: Outcome) {
|
||||
match outcome {
|
||||
Outcome::Failed => {
|
||||
Outcome::Failed(error) => {
|
||||
// Record the reason before the terminal transition so it's set
|
||||
// by the time `set_state` stamps `finished_at`.
|
||||
self.graph.set_error(id, error);
|
||||
self.graph.set_state(id, State::Failed);
|
||||
self.cascade_cancel(id);
|
||||
}
|
||||
|
|
@ -454,6 +459,45 @@ mod tests {
|
|||
assert_eq!(avail(&s, "build-slot"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_timestamps_and_error_are_stamped() {
|
||||
let mut s = scheduler_with_slots(2);
|
||||
let ok = s.append("ok", vec![], None).expect("insert");
|
||||
let bad = s.append("bad", vec![], None).expect("insert");
|
||||
let downstream = s.append("down", vec![after_ok(bad)], None).expect("insert");
|
||||
|
||||
// Before running: no timestamps.
|
||||
assert!(s.graph().node(ok).unwrap().started_at.is_none());
|
||||
assert!(s.graph().node(ok).unwrap().finished_at.is_none());
|
||||
|
||||
let started = s.settle();
|
||||
assert!(started.contains(&ok) && started.contains(&bad));
|
||||
// Running → started_at stamped, finished_at still none.
|
||||
assert!(s.graph().node(ok).unwrap().started_at.is_some());
|
||||
assert!(s.graph().node(ok).unwrap().finished_at.is_none());
|
||||
|
||||
// Done → finished_at stamped, no error.
|
||||
s.complete(ok, Outcome::Done);
|
||||
let n = s.graph().node(ok).unwrap();
|
||||
assert!(n.finished_at.is_some());
|
||||
assert_eq!(n.error, None);
|
||||
|
||||
// Failed → the reason rides `Outcome::Failed`, finished_at stamped.
|
||||
s.complete(bad, Outcome::Failed("boom".to_owned()));
|
||||
let n = s.graph().node(bad).unwrap();
|
||||
assert_eq!(n.state, State::Failed);
|
||||
assert_eq!(n.error.as_deref(), Some("boom"));
|
||||
assert!(n.finished_at.is_some());
|
||||
|
||||
// The `AfterOk`-cancelled node: cascade-cancelled, so finished_at is set,
|
||||
// but it never ran (no started_at) and carries no error of its own.
|
||||
let n = s.graph().node(downstream).unwrap();
|
||||
assert_eq!(n.state, State::Cancelled);
|
||||
assert!(n.started_at.is_none());
|
||||
assert!(n.finished_at.is_some());
|
||||
assert_eq!(n.error, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_slot_cap_limits_concurrency_and_release_unblocks() {
|
||||
let mut s = scheduler_with_slots(2);
|
||||
|
|
@ -514,7 +558,7 @@ mod tests {
|
|||
assert_eq!(s.settle(), vec![root]);
|
||||
s.complete(root, Outcome::Done);
|
||||
assert_eq!(s.settle(), vec![child]);
|
||||
s.complete(child, Outcome::Failed);
|
||||
s.complete(child, Outcome::Failed(String::new()));
|
||||
assert_eq!(
|
||||
s.graph().node(root).unwrap().state,
|
||||
State::Failed,
|
||||
|
|
@ -689,7 +733,7 @@ mod tests {
|
|||
)
|
||||
.expect("weak");
|
||||
assert_eq!(s.settle(), vec![root]);
|
||||
s.complete(root, Outcome::Failed);
|
||||
s.complete(root, Outcome::Failed(String::new()));
|
||||
assert_eq!(s.graph().node(strong1).unwrap().state, State::Cancelled);
|
||||
assert_eq!(s.graph().node(strong2).unwrap().state, State::Cancelled);
|
||||
assert_eq!(s.settle(), vec![weak]);
|
||||
|
|
@ -704,7 +748,7 @@ mod tests {
|
|||
let child = s.append("child", vec![], Some(root)).expect("child");
|
||||
let grandchild = s.append("gc", vec![], Some(child)).expect("gc");
|
||||
assert_eq!(s.settle(), vec![root]);
|
||||
s.complete(root, Outcome::Failed);
|
||||
s.complete(root, Outcome::Failed(String::new()));
|
||||
assert_eq!(s.graph().node(child).unwrap().state, State::Cancelled);
|
||||
assert_eq!(s.graph().node(grandchild).unwrap().state, State::Cancelled);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue