hive-c0re: close build-log rows a dead process left in flight
A build_logs row is opened with finished_at NULL and closed only by the process that runs the build. If hive-c0re stopped in between, the row stayed open for good: the dashboard listed it with no status, its log stream never sent `done`, and the retention sweep, which skips open rows, never deleted it. Coordinator::open now closes every open row as `fail` and appends a line saying hive-c0re stopped. It runs before this process starts any build, and only there: BuildLogs::open itself leaves rows alone so a second opener can't close another process's live rows.
This commit is contained in:
parent
0472ed3ab5
commit
466e32b69f
3 changed files with 98 additions and 0 deletions
|
|
@ -462,6 +462,15 @@ impl Coordinator {
|
|||
let build_logs = Arc::new(
|
||||
crate::build_logs::BuildLogs::open(build_logs_dir).context("open build_logs")?,
|
||||
);
|
||||
// This process is the database's only writer and has started no
|
||||
// build yet, so a row still in-flight belongs to a dead process.
|
||||
let reaped = build_logs.reap_orphans()?;
|
||||
if reaped > 0 {
|
||||
tracing::warn!(
|
||||
reaped,
|
||||
"build_logs: closed rows left in-flight by a previous run"
|
||||
);
|
||||
}
|
||||
// Install the process-wide handle so `lifecycle::run` /
|
||||
// `lifecycle::prebuild_toplevel` can write without us having
|
||||
// to thread an `Arc<BuildLogs>` through every public entry
|
||||
|
|
@ -1376,3 +1385,35 @@ mod agent_paths_tests {
|
|||
assert_eq!(paths.agent, std::path::PathBuf::from("/run/x"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod open_tests {
|
||||
use super::{Coordinator, HiveEnv};
|
||||
|
||||
fn open(db: &std::path::Path) -> Coordinator {
|
||||
Coordinator::open(
|
||||
db,
|
||||
HiveEnv::default(),
|
||||
crate::hive_stats::PriceTable::new(),
|
||||
1,
|
||||
)
|
||||
.expect("open coordinator")
|
||||
}
|
||||
|
||||
/// A build log row the previous process left open is closed as failed
|
||||
/// when the next one opens the coordinator.
|
||||
#[tokio::test]
|
||||
async fn open_closes_build_logs_left_in_flight() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = dir.path().join("broker.sqlite");
|
||||
let id = open(&db)
|
||||
.build_logs
|
||||
.start("alice", "run", "cmd", None)
|
||||
.expect("start");
|
||||
|
||||
let coord = open(&db);
|
||||
let row = coord.build_logs.get_full(id).expect("get").expect("Some");
|
||||
assert_eq!(row.header.status.as_deref(), Some("fail"));
|
||||
assert!(row.header.finished_at.is_some());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue