feat(#2591): hive-jobq Node lifecycle — started/finished timestamps + failure reason

Node gains started_at/finished_at (chrono DateTime<Utc>, serialized
RFC 3339 on the wire per hive_sh4re::wire_time) plus error (String).
Graph::set_state self-stamps started_at on the first Running transition
and finished_at on the first terminal one, via an internal now_utc()
clock (keeps settle/complete signatures stable). Outcome::Failed(String)
carries the failure reason, set on the terminal transition.

hive-c0re complete_node builds Outcome::Failed(msg); its node_rt
side-table stays i64 for now (double-write) until #2637 reads the Node.

Toward #2637: the jobq graph becomes the source of truth for per-node
lifecycle so the queue can be sent to the client as-is.
This commit is contained in:
atlas 2026-07-22 23:02:24 +02:00 committed by mara
commit 03eb64cb5c
5 changed files with 115 additions and 9 deletions

View file

@ -29,6 +29,8 @@
pub mod resources;
pub mod scheduler;
use chrono::{DateTime, Utc};
/// Opaque, stable, monotonic node identifier.
///
/// Assigned by the [`Graph`] on insert and persisted, so it is stable across
@ -141,6 +143,19 @@ impl State {
}
}
/// Wall-clock UTC now — the source for node lifecycle timestamps
/// ([`Node::started_at`] / [`Node::finished_at`]). The graph stamps its own
/// timestamps rather than threading a clock through every call, so a node's
/// timing is self-contained. Derived from `SystemTime` (the workspace `chrono`
/// carries no `clock` feature, matching `hive_sh4re::wire_time`), truncated to
/// whole seconds; a pre-epoch or out-of-range clock clamps to the epoch.
fn now_utc() -> DateTime<Utc> {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX));
DateTime::<Utc>::from_timestamp(secs, 0).unwrap_or_default()
}
/// A node in the graph, carrying a caller-defined payload `N`.
///
/// The library schedules over `Node`s and resources without interpreting the
@ -165,6 +180,17 @@ pub struct Node<N, R> {
pub deps: Vec<Dep<R>>,
/// Lifecycle state.
pub state: State,
/// UTC instant the node entered [`State::Running`] (`None` until it starts;
/// a cancelled node never ran, so it stays `None`). Stamped by the graph.
pub started_at: Option<DateTime<Utc>>,
/// UTC instant the node reached a terminal state (`Done` / `Failed` /
/// `Cancelled`). `None` while non-terminal. Stamped by the graph.
pub finished_at: Option<DateTime<Utc>>,
/// Failure reason for a `Failed` node, supplied by the runner via
/// [`scheduler::Outcome::Failed`]. `None` unless this node's own logic
/// failed (a node that rolled up `Failed` from a child, or was cancelled,
/// carries no error of its own).
pub error: Option<String>,
}
/// An error from inserting into or loading a [`Graph`] with a dangling id.
@ -308,6 +334,9 @@ impl<N, R> Graph<N, R> {
payload,
deps,
state: State::Pending,
started_at: None,
finished_at: None,
error: None,
});
Ok(id)
}
@ -353,16 +382,35 @@ impl<N, R> Graph<N, R> {
/// Set a node's lifecycle state, returning `false` for an unknown id. The
/// scheduler drives every state transition — nothing else mutates state,
/// which is what keeps the resource guards + terminality in sync.
/// which is what keeps the resource guards + terminality in sync. This is
/// also where the node's lifecycle timestamps are stamped: `started_at` on
/// the first transition to [`State::Running`], `finished_at` on the first
/// transition to a terminal state (`Done` / `Failed` / `Cancelled`).
pub(crate) fn set_state(&mut self, id: NodeId, state: State) -> bool {
if let Some(node) = self.nodes.iter_mut().find(|n| n.id == id) {
node.state = state;
if state == State::Running {
if node.started_at.is_none() {
node.started_at = Some(now_utc());
}
} else if state.is_terminal() && node.finished_at.is_none() {
node.finished_at = Some(now_utc());
}
true
} else {
false
}
}
/// Record a node's failure reason ([`Node::error`]). No-op for an unknown
/// id. Called by the scheduler on an [`scheduler::Outcome::Failed`] before
/// the terminal state transition.
pub(crate) fn set_error(&mut self, id: NodeId, error: String) {
if let Some(node) = self.nodes.iter_mut().find(|n| n.id == id) {
node.error = Some(error);
}
}
/// Check that every id the graph holds resolves: every [`Dep::Node`] id
/// names a node present in the graph, and `next_id` is past the largest
/// existing id. Deserialization runs this, so a loaded graph is internally
@ -568,6 +616,9 @@ mod tests {
when: DepWhen::AfterOk,
}],
state: State::Pending,
started_at: None,
finished_at: None,
error: None,
}],
next_id: 1,
};
@ -585,6 +636,9 @@ mod tests {
payload: "x",
deps: vec![],
state: State::Pending,
started_at: None,
finished_at: None,
error: None,
}],
next_id: 3,
};