hive-c0re: fail on a malformed agent name and on an unreadable container list
An agent name that is not a valid Ident made `Coordinator::agent_paths` panic. Job payloads carry names as plain strings (the swarm's published wanted state is one source), and a panic inside a job-queue node never reaches `complete_growing`, so the node's resources (the deploy window included) were held until hive-c0re restarted. `agent_paths` now returns an error; the job-queue nodes, the admin-socket spawn and set-limits paths, the root-agent spawn and the dashboard set-limits handler propagate it. `lifecycle::list().await.unwrap_or_default()` turned a failed container list into "no agents": - meta-update cascade: the lock bump committed and zero rebuilds fanned out, reported as success. The cascade is now resolved before the lock bump and a list failure fails the node. - dashboard update-all: queued nothing and returned 200 "ok". Now 500 with the error. - container rescan: every row was emitted as removed and the cache emptied. Now the last snapshot stands; `hivectl status` gets an error. - dashboard journal: answered 404 "no managed container". Now 500. - spawn/rebuild port-collision check: silently skipped. Now fails. - startup migration: the per-agent phases ran over nothing, and phase 3 handed an empty agent list to `meta::sync_agents`, which renders the meta flake with exactly the agents it is given. Both now log the list failure and skip. The hive-jobq scheduler still leaks a node's resources on any executor panic; that root is not addressed here. Refs #4723
This commit is contained in:
parent
ee25b7de20
commit
7ac6819652
10 changed files with 153 additions and 67 deletions
|
|
@ -63,7 +63,13 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
|
|||
Err(e) => tracing::warn!(error = ?e, "clear stale meta lock failed"),
|
||||
}
|
||||
}
|
||||
let names = enumerate_agents().await;
|
||||
let names = match enumerate_agents().await {
|
||||
Ok(names) => names,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "migration: container list failed; skipping per-agent phases");
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
tracing::info!(count = names.len(), "migration: scanning");
|
||||
|
||||
// Phase 0: move harness-owned files out of state/ into harness/.
|
||||
|
|
@ -94,9 +100,15 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
|
|||
|
||||
// Phase 3: meta repo.
|
||||
tracing::debug!("migration: phase 3 (meta sync_agents)");
|
||||
let agents = lifecycle::agents_for_meta_listing()
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
// `sync_agents` renders exactly the agents it is given, so an empty
|
||||
// stand-in for an unreadable list would drop every agent from the flake.
|
||||
let agents = match lifecycle::agents_for_meta_listing().await {
|
||||
Ok(agents) => agents,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "migration: container list failed; skipping meta sync_agents");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await {
|
||||
Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"),
|
||||
Err(_) => {
|
||||
|
|
@ -139,9 +151,9 @@ fn migrate_harness_files(name: &hive_types::Ident) {
|
|||
}
|
||||
}
|
||||
|
||||
async fn enumerate_agents() -> Vec<hive_types::Ident> {
|
||||
let containers = lifecycle::list().await.unwrap_or_default();
|
||||
containers
|
||||
async fn enumerate_agents() -> Result<Vec<hive_types::Ident>> {
|
||||
let containers = lifecycle::list().await?;
|
||||
Ok(containers
|
||||
.into_iter()
|
||||
.filter_map(|c| {
|
||||
let name = if c == MANAGER_CONTAINER {
|
||||
|
|
@ -151,7 +163,7 @@ async fn enumerate_agents() -> Vec<hive_types::Ident> {
|
|||
};
|
||||
hive_types::Ident::parse(name).ok()
|
||||
})
|
||||
.collect()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn migrate_applied_repo(name: &str) -> Result<()> {
|
||||
|
|
|
|||
Loading…
Reference in a new issue