hive-c0re: fail on a malformed agent name and on an unreadable container list
An agent name that is not a valid Ident made `Coordinator::agent_paths` panic. Job payloads carry names as plain strings (the swarm's published wanted state is one source), and a panic inside a job-queue node never reaches `complete_growing`, so the node's resources (the deploy window included) were held until hive-c0re restarted. `agent_paths` now returns an error; the job-queue nodes, the admin-socket spawn and set-limits paths, the root-agent spawn and the dashboard set-limits handler propagate it. `lifecycle::list().await.unwrap_or_default()` turned a failed container list into "no agents": - meta-update cascade: the lock bump committed and zero rebuilds fanned out, reported as success. The cascade is now resolved before the lock bump and a list failure fails the node. - dashboard update-all: queued nothing and returned 200 "ok". Now 500 with the error. - container rescan: every row was emitted as removed and the cache emptied. Now the last snapshot stands; `hivectl status` gets an error. - dashboard journal: answered 404 "no managed container". Now 500. - spawn/rebuild port-collision check: silently skipped. Now fails. - startup migration: the per-agent phases ran over nothing, and phase 3 handed an empty agent list to `meta::sync_agents`, which renders the meta flake with exactly the agents it is given. Both now log the list failure and skip. The hive-jobq scheduler still leaks a node's resources on any executor panic; that root is not addressed here. Refs #4723
This commit is contained in:
parent
ee25b7de20
commit
7ac6819652
10 changed files with 153 additions and 67 deletions
|
|
@ -269,9 +269,11 @@ fn validate(name: &str) -> Result<()> {
|
|||
/// would otherwise loop on `AddrInUse` forever; we surface the
|
||||
/// conflict here so spawn / rebuild fails loudly with an actionable
|
||||
/// message instead.
|
||||
async fn port_collision(self_name: &str) -> Option<String> {
|
||||
async fn port_collision(self_name: &str) -> Result<Option<String>> {
|
||||
let port = agent_web_port(self_name);
|
||||
let raw = list().await.unwrap_or_default();
|
||||
let raw = list()
|
||||
.await
|
||||
.context("listing containers for the port-collision check")?;
|
||||
for c in raw {
|
||||
let Some(other) = c.strip_prefix(AGENT_PREFIX) else {
|
||||
continue;
|
||||
|
|
@ -280,10 +282,10 @@ async fn port_collision(self_name: &str) -> Option<String> {
|
|||
continue;
|
||||
}
|
||||
if agent_web_port(other) == port && is_running(other).await {
|
||||
return Some(other.to_owned());
|
||||
return Ok(Some(other.to_owned()));
|
||||
}
|
||||
}
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
|
|
@ -303,7 +305,7 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
|
|||
/// create the container — that's `create_only` / the `Create` node.
|
||||
pub async fn provision_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
if let Some(other) = port_collision(name).await {
|
||||
if let Some(other) = port_collision(name).await? {
|
||||
bail!(
|
||||
"port {} is already taken by '{other}' — rename one of them and retry",
|
||||
agent_web_port(name)
|
||||
|
|
@ -356,7 +358,7 @@ pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) ->
|
|||
/// than the clear bail this replaces.
|
||||
pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
if let Some(other) = port_collision(name).await {
|
||||
if let Some(other) = port_collision(name).await? {
|
||||
bail!(
|
||||
"port {} is already taken by '{other}' — rename one of them and retry",
|
||||
agent_web_port(name)
|
||||
|
|
@ -888,6 +890,16 @@ pub async fn list() -> Result<Vec<String>> {
|
|||
.collect())
|
||||
}
|
||||
|
||||
/// Logical agent names from a [`list`] result. An unreadable list stays an
|
||||
/// error rather than becoming "no agents": a caller fanning work out over
|
||||
/// every agent would otherwise report success after doing nothing.
|
||||
pub fn agent_names(listed: Result<Vec<String>>) -> Result<Vec<String>> {
|
||||
Ok(listed?
|
||||
.into_iter()
|
||||
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Sync `/etc/tmpfiles.d/hyperhive-agents.conf` with the currently-known
|
||||
/// agent set (from `nixos-container list`). Strips the `h-` prefix to get
|
||||
/// logical names. Best-effort: errors are logged but never propagated — a
|
||||
|
|
|
|||
|
|
@ -377,3 +377,18 @@ fn failed_destroy_with_an_unreadable_list_fails() {
|
|||
.expect_err("an unreadable list must not pass for an absent container");
|
||||
assert!(format!("{err:#}").contains("cannot confirm"), "{err:#}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_names_strips_the_container_prefix() {
|
||||
let names = agent_names(Ok(vec![format!("{AGENT_PREFIX}iris"), "other".to_owned()]))
|
||||
.expect("a readable list");
|
||||
assert_eq!(names, vec!["iris"]);
|
||||
}
|
||||
|
||||
/// A failed list is not an empty hive: the error reaches the caller.
|
||||
#[test]
|
||||
fn agent_names_propagates_an_unreadable_list() {
|
||||
let err = agent_names(Err(anyhow::anyhow!("connect to hive-priv socket")))
|
||||
.expect_err("an unreadable list must not read as zero agents");
|
||||
assert!(format!("{err:#}").contains("hive-priv"), "{err:#}");
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue