vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

pin internal/meta's hyperhive on the controller's rev and rebuild agents when it moves

Browse source 
The swarm controller offers its own running rev (services.hyperhive.c0re.
hyperhiveRev) as the hyperhive pin of internal/meta and takes it only when
the forge compare API shows it descends from the committed pin; merges to
hyperhive main no longer move the pin, and the first commit pins main's head.
When a commit moves the pin, every placed agent gets a TriggerDeploy.

A deploy after a meta update no longer waits on it succeeding (after_any), so
a down builder does not hold back hives that render locally.
This commit is contained in:
atlas 2026-10-24 14:29:12 +02:00
commit 22e2d30581
10 changed files with 1027 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

364
swarm-controller/src/jobq.rs
View file

@ -134,12 +134,12 @@ pub(crate) enum SwarmNodeKind {
/// agent creation, where the hive deploys what it seeds.
rev: Option<String>,
},
/// Pin the swarm meta flake (`internal/meta`) to the forge's current
/// heads: render, lock on the swarm builder, commit. See `meta_flake`.
/// Pin the swarm meta flake (`internal/meta`): render, lock on the swarm
/// builder, commit. See `meta_flake`. When the commit moves the hyperhive
/// pin, a `TriggerDeploy` per placed agent follows.
///
/// Carries nothing: it reads every head when it runs, so one pending node
/// covers every trigger queued behind it. Hives build from that repo, so a
/// deploy waits for this node.
/// covers every trigger queued behind it.
UpdateMeta,
}
@ -395,7 +395,7 @@ pub(crate) async fn run_swarm_node(
),
Some(client) => publish_deploy(&client, &hive, &agent, rev).await,
},
SwarmNodeKind::UpdateMeta => update_meta(&deps, forge).await,
SwarmNodeKind::UpdateMeta => finish_update_meta(update_meta(&deps, forge).await, &builder),
};
(builder, outcome)
}
@ -565,64 +565,104 @@ pub(crate) async fn build_agent(
.await
}
/// The `UpdateMeta` arm, lifted out for the same reason.
/// The `UpdateMeta` arm, lifted out for the same reason: what the update
/// reported, and the placements it rendered for. `Err` is the node's failure.
async fn update_meta(
deps: &WorkerDeps,
forge: Option<Arc<forge::Client>>,
) -> hive_jobq::scheduler::Outcome {
use hive_jobq::scheduler::Outcome;
) -> Result<
(
meta_flake::Report,
std::collections::BTreeSet<(String, String)>,
),
String,
> {
let Some(client) = forge else {
return Outcome::Failed(
return Err(
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
not in the swarm secret store), so internal/meta cannot be written"
.to_owned(),
);
};
let Some(meta) = meta_deps() else {
return Outcome::Failed(format!(
return Err(format!(
"{} is unset, so there are no swarm values to render internal/meta from",
meta_flake::SETTINGS_ENV
));
};
let (Some(queue), Some(wanted)) = (&deps.queue, &deps.wanted) else {
return Outcome::Failed(
"no swarm queue is configured on this host, so neither the placed agents nor a \
lock can be had"
return Err(
"no swarm queue is configured on this host, so neither the placed agents \
nor a lock can be had"
.to_owned(),
);
};
let agents = match placed_agents(queue, wanted, &meta.hives).await {
Ok(agents) => agents,
Err(e) => return Outcome::Failed(format!("{e:#}")),
};
match forge_node(
let placements = placements(wanted, &meta.hives)
.await
.map_err(|e| format!("{e:#}"))?;
let agents = placed_agents(queue, &placements)
.await
.map_err(|e| format!("{e:#}"))?;
let report = forge_node(
"update_meta",
meta_flake::update(
client.as_ref(),
queue,
&meta.settings,
&agents,
&meta.hives,
),
meta_flake::update(client.as_ref(), queue, &meta.settings, &agents, &meta.hives),
)
.await
{
Ok(updated) => {
tracing::info!(?updated, agents = agents.len(), "update_meta: done");
.map_err(|e| format!("{e:#}"))?;
tracing::info!(?report, agents = agents.len(), "update_meta: done");
Ok((report, placements))
}
/// The `UpdateMeta` node's outcome from what [`update_meta`] reported, with the
/// deploys a moved pin calls for declared into `builder` (the scheduler inserts
/// them under the node when it completes). Synchronous: the builder is not
/// `Sync`, so it cannot be held across an await.
fn finish_update_meta(
result: Result<
(
meta_flake::Report,
std::collections::BTreeSet<(String, String)>,
),
String,
>,
builder: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
) -> hive_jobq::scheduler::Outcome {
use hive_jobq::scheduler::Outcome;
match result {
Ok((report, placements)) => {
fan_out_if_moved(builder, &report, &placements);
Outcome::Done
}
Err(e) => Outcome::Failed(format!("{e:#}")),
Err(e) => Outcome::Failed(e),
}
}
/// Every agent some hive in `hives` places, with its archetype.
async fn placed_agents(
queue: &async_nats::Client,
/// Declare one rebuild of each `(hive, agent)` placement when `report` says
/// the agents' hyperhive moved: nothing else tells them to build again.
fn fan_out_if_moved(
grown: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
report: &meta_flake::Report,
placements: &std::collections::BTreeSet<(String, String)>,
) {
if !report.hyperhive_moved {
return;
}
for (hive, agent) in placements {
let _ = grown.node(SwarmNodeKind::TriggerDeploy {
hive: hive.clone(),
agent: agent.clone(),
rev: None,
});
}
}
/// Every `(hive, agent)` a hive in `hives` places.
async fn placements(
wanted: &wanted::WantedWriter,
hives: &[String],
) -> Result<Vec<meta_flake::Placed>> {
let mut names = std::collections::BTreeSet::new();
) -> Result<std::collections::BTreeSet<(String, String)>> {
let mut placed = std::collections::BTreeSet::new();
for hive in hives {
let Some(declared) = wanted
.view(hive)
@ -631,14 +671,24 @@ async fn placed_agents(
else {
continue;
};
names.extend(
placed.extend(
declared
.agents
.into_iter()
.filter(|(_, w)| crate::routes::agents::places_agent(w.state))
.map(|(name, _)| name),
.map(|(name, _)| (hive.clone(), name)),
);
}
Ok(placed)
}
/// Every agent in `placements`, with its archetype.
async fn placed_agents(
queue: &async_nats::Client,
placements: &std::collections::BTreeSet<(String, String)>,
) -> Result<Vec<meta_flake::Placed>> {
let names: std::collections::BTreeSet<&String> =
placements.iter().map(|(_, agent)| agent).collect();
let archetypes = swarm_queue_client::agent_archetype::open_or_create(queue)
.await
.map_err(|e| anyhow::anyhow!("open the agent-archetype bucket: {e}"))?;
@ -646,10 +696,13 @@ async fn placed_agents(
for name in names {
let (archetype, _, _) = crate::schedules::read_or_default::<
Option<swarm_wire::agent_archetype::ArchetypeRef>,
>(&archetypes, &name)
>(&archetypes, name)
.await
.map_err(|e| e.context(format!("read the archetype of {name}")))?;
placed.push(meta_flake::Placed { name, archetype });
placed.push(meta_flake::Placed {
name: name.clone(),
archetype,
});
}
Ok(placed)
}
@ -983,22 +1036,7 @@ pub(crate) async fn queue_merged_config_deploy(
.jobq
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert_job(None, |b| {
// The hive builds the merged commit from `internal/meta`, so the
// flake must pin it before the hive is told.
let update_meta = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta);
let root = update_meta.guid();
let _deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.clone(),
agent: agent.clone(),
rev: Some(rev.clone()),
})
.after_ok(update_meta);
vec![root]
});
.insert_job(None, |b| merged_deploy_job(b, &hive, &agent, &rev));
match inserted {
Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"),
Err(e) => {
@ -1007,6 +1045,33 @@ pub(crate) async fn queue_merged_config_deploy(
}
}
/// The sub-DAG a merged config PR is: the meta update, then the deploy of
/// `rev` on `hive`.
///
/// `after_any`: a hive builds from `internal/meta` only while it pins the
/// agent's config commit, and renders locally otherwise, as a hive that
/// predates the repo always does. A failed update (no builder, an unreadable
/// config repo) must not hold those hives back.
fn merged_deploy_job(
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
hive: &str,
agent: &str,
rev: &str,
) -> Vec<hive_jobq::builder::NodeGuid> {
let update_meta = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta);
let root = update_meta.guid();
let _deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.to_owned(),
agent: agent.to_owned(),
rev: Some(rev.to_owned()),
})
.after_any(update_meta);
vec![root]
}
/// The sub-DAG one agent creation is: the nodes, and the edges between them.
///
/// A function rather than a closure inside [`create_agent`](crate::routes::agents::create_agent) so the endpoint's
@ -1095,8 +1160,10 @@ pub(crate) fn declare_agent_job(
hive: hive.to_owned(),
agent: agent.to_owned(),
});
// The hive builds the agent from `internal/meta`, which can only carry it
// once the agent is declared on a hive and its config repo has a commit.
// The hive builds the agent from `internal/meta` once it pins the agent's
// config commit, and renders locally until then. `after_any` on the
// update, so one that failed (no builder, an unreadable config repo)
// still lets the hive build the agent that way.
let update_meta = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta)
@ -1131,7 +1198,7 @@ pub(crate) fn declare_agent_job(
agent: agent.to_owned(),
rev: None,
})
.after_ok(update_meta)
.after_any(update_meta)
.after_any(mint_identity)
.after_any(mint_forge_token)
// `after_any` for the same reason: the container reads this token
@ -1907,6 +1974,187 @@ mod tests {
);
}
/// Whether the edge from the `from` node to the `on` node accepts a
/// failed `on`, in a graph built by `declare`.
fn edge_accepts_failure(
declare: impl FnOnce(&hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>),
from: &str,
on: &str,
) -> bool {
use hive_jobq_wire::WireNode as _;
let mut sched = hive_jobq::scheduler::Scheduler::new(
hive_jobq::Graph::new(),
hive_jobq::resources::ResourceTable::new(),
);
sched
.insert_job(None, |b| {
declare(b);
Vec::new()
})
.expect("the graph inserts");
let graph = sched.graph();
let id_of = |label: &str| {
graph
.nodes()
.find(|n| n.payload.label() == label)
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
.id
};
let (from, on) = (id_of(from), id_of(on));
graph
.node(from)
.expect("just found")
.deps
.iter()
.find_map(|d| match d {
hive_jobq::Dep::Node { id, when } if *id == on => Some(*when),
_ => None,
})
.expect("the edge exists")
.accepts(hive_jobq::TerminalState::Failed)
}
/// A deploy that waits for the meta update must still run when the update
/// fails: a hive that renders locally never needed it.
#[test]
fn a_failed_meta_update_does_not_hold_back_the_deploys() {
let create = |b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>| {
let _ = super::declare_agent_job(b, "atlas", "pr1ma", false);
};
let merge = |b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>| {
let _ = super::merged_deploy_job(b, "pr1ma", "atlas", "abc");
};
assert!(edge_accepts_failure(
create,
"trigger_deploy",
"update_meta"
));
assert!(edge_accepts_failure(merge, "trigger_deploy", "update_meta"));
// The same lookup finds the edges that do hold a deploy back.
assert!(!edge_accepts_failure(
create,
"trigger_deploy",
"set_agent_wanted"
));
assert!(!edge_accepts_failure(
create,
"update_meta",
"set_agent_wanted"
));
}
/// The scheduler's side of the same edge: with the update failed, the
/// deploy is the next node to run.
#[tokio::test]
async fn the_merged_deploy_runs_after_a_failed_meta_update() {
use hive_jobq::scheduler::{Outcome, Scheduler};
use hive_jobq_wire::WireNode as _;
let mut sched = Scheduler::new(
hive_jobq::Graph::new(),
hive_jobq::resources::ResourceTable::new(),
);
sched
.insert_job(None, |b| {
super::merged_deploy_job(b, "pr1ma", "atlas", "abc")
})
.expect("the graph inserts");
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
let mut ran = Vec::new();
while let Some(runner) = Scheduler::claim_next(&sched, |_id, kind, builder| async move {
let outcome = match kind {
SwarmNodeKind::UpdateMeta => Outcome::Failed("no builder".to_owned()),
_ => Outcome::Done,
};
(builder, outcome)
}) {
runner.await.1.expect("no growth declared");
ran.push(
sched
.lock()
.unwrap()
.graph()
.nodes()
.filter(|n| n.state.is_terminal())
.map(|n| (n.payload.label(), n.state))
.collect::<Vec<_>>(),
);
}
let last = ran.last().expect("both nodes ran");
assert!(
last.contains(&("update_meta".to_owned(), hive_jobq::State::Failed)),
"{last:?}"
);
assert!(
last.contains(&("trigger_deploy".to_owned(), hive_jobq::State::Done)),
"{last:?}"
);
}
/// Run an `UpdateMeta` node that reports `report`, and return the
/// `(hive, agent, rev)` of every deploy it grew.
async fn grown_deploys(report: meta_flake::Report) -> Vec<(String, String, Option<String>)> {
use hive_jobq::scheduler::{Outcome, Scheduler};
let mut sched = Scheduler::new(
hive_jobq::Graph::new(),
hive_jobq::resources::ResourceTable::new(),
);
let update = sched
.append(SwarmNodeKind::UpdateMeta, Vec::new(), None)
.expect("insert");
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
let placements: std::collections::BTreeSet<(String, String)> =
[("pr1ma", "atlas"), ("pr1ma", "argus"), ("sec0nd", "iris")]
.into_iter()
.map(|(h, a)| (h.to_owned(), a.to_owned()))
.collect();
let runner = Scheduler::claim_next(&sched, move |_id, _kind, builder| async move {
fan_out_if_moved(&builder, &report, &placements);
(builder, Outcome::Done)
})
.expect("the node just inserted is runnable");
runner.await.1.expect("the grown nodes insert");
let guard = sched.lock().unwrap();
guard
.graph()
.nodes()
.filter_map(|n| match &n.payload {
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
assert_eq!(n.parent, Some(update), "grown under the update");
Some((hive.clone(), agent.clone(), rev.clone()))
}
_ => None,
})
.collect()
}
#[tokio::test]
async fn a_moved_pin_rebuilds_every_placed_agent_and_an_unmoved_one_none() {
let moved = grown_deploys(meta_flake::Report {
updated: meta_flake::Updated::Committed,
hyperhive_moved: true,
})
.await;
let mut moved: Vec<_> = moved.into_iter().collect();
moved.sort();
assert_eq!(
moved,
vec![
("pr1ma".to_owned(), "argus".to_owned(), None),
("pr1ma".to_owned(), "atlas".to_owned(), None),
("sec0nd".to_owned(), "iris".to_owned(), None),
]
);
let unmoved = grown_deploys(meta_flake::Report {
updated: meta_flake::Updated::Committed,
hyperhive_moved: false,
})
.await;
assert!(unmoved.is_empty(), "{unmoved:?}");
}
/// Node labels of the graph [`super::declare_agent_job`] builds, and the
/// labels the deploy node waits on.
fn creation_graph(archetype: bool) -> (Vec<String>, Vec<String>) {