vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-controller: render, lock and commit internal/meta

Browse source 
A new update_meta node renders internal/meta's flake.nix with
hive-meta-flake, the renderer hive-c0re uses: hyperhive pinned to its
forge main, each custom agent a hive places pinned to its agent-configs
main, all over https, and the swarm-wide values from
SWARM_CONTROLLER_META_FLAKE (set by swarm-controller.nix from
services.hyperhive.swarm.*). The swarm builder locks it, and the
controller commits flake.nix and flake.lock in one commit and gives
every hive's forge account read on the repo. A render equal to the
committed flake.nix sends no lock and commits nothing.

It runs at start, when hyperhive's main moves, before the deploy of a
merged config PR and before the first deploy of a new agent; those
deploys now wait for it. agent_web_port moves into hive-meta-flake so
both callers compute the same port; hive-c0re re-exports it.

On deploy: internal/meta appears on the forge and follows every merge.
No hive builds from it yet.

Refs #4857
This commit is contained in:
atlas 2026-10-22 07:52:23 +02:00
commit 5950363a10
16 changed files with 970 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

196
swarm-controller/src/jobq.rs
View file

@ -10,7 +10,8 @@ use crate::routes::AppState;
use crate::routes::agents::{claiming_hives, read_declarations};
use crate::startup::PUBLIC_URL_ENV;
use crate::{
agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, wanted,
agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, meta_flake,
wanted,
};
/// Node payload for the swarm-level job graph. Named `Swarm*` rather than
@ -133,6 +134,13 @@ pub(crate) enum SwarmNodeKind {
/// agent creation, where the hive deploys what it seeds.
rev: Option<String>,
},
/// Pin the swarm meta flake (`internal/meta`) to the forge's current
/// heads: render, lock on the swarm builder, commit. See `meta_flake`.
///
/// Carries nothing: it reads every head when it runs, so one pending node
/// covers every trigger queued behind it. Hives build from that repo, so a
/// deploy waits for this node.
UpdateMeta,
}
impl hive_jobq_wire::WireNode for SwarmNodeKind {
@ -158,6 +166,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
}
SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(),
SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(),
SwarmNodeKind::UpdateMeta => "update_meta".to_owned(),
}
}
@ -185,7 +194,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
| SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
SwarmNodeKind::MintClassifierForgeToken
| SwarmNodeKind::MintBuilderForgeToken
| SwarmNodeKind::MintCiMetricsForgeToken => serde_json::json!({}),
| SwarmNodeKind::MintCiMetricsForgeToken
| SwarmNodeKind::UpdateMeta => serde_json::json!({}),
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
serde_json::json!({ "agent": agent, "rev": rev, "pr": pr })
}
@ -206,12 +216,16 @@ pub(crate) enum SwarmResourceKind {
/// The swarm builder: one `BuildAgent` node runs at a time, and the rest
/// wait pending, where a newer push can replace them.
Builder,
/// `internal/meta`: one `UpdateMeta` commits at a time, so two never race
/// on the same file's blob.
Meta,
}
impl hive_jobq_wire::WireResource for SwarmResourceKind {
fn name(&self) -> String {
match self {
SwarmResourceKind::Builder => "builder".to_owned(),
SwarmResourceKind::Meta => "meta".to_owned(),
}
}
}
@ -253,6 +267,14 @@ pub(crate) struct WorkerDeps {
pub(crate) matrix_homeserver: Option<std::sync::Arc<str>>,
}
/// What `UpdateMeta` renders from, fixed for the process.
struct MetaDeps {
settings: meta_flake::Settings,
/// Every hive's name: each places agents, and each hive's forge account
/// gets read on the repo.
hives: Vec<String>,
}
/// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be.
///
/// A named constant rather than the literal inline, because this *is* the
@ -373,6 +395,7 @@ pub(crate) async fn run_swarm_node(
),
Some(client) => publish_deploy(&client, &hive, &agent, rev).await,
},
SwarmNodeKind::UpdateMeta => update_meta(&deps, forge).await,
};
(builder, outcome)
}
@ -542,6 +565,95 @@ pub(crate) async fn build_agent(
.await
}
/// The `UpdateMeta` arm, lifted out for the same reason.
async fn update_meta(
deps: &WorkerDeps,
forge: Option<Arc<forge::Client>>,
) -> hive_jobq::scheduler::Outcome {
use hive_jobq::scheduler::Outcome;
let Some(client) = forge else {
return Outcome::Failed(
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
not in the swarm secret store), so internal/meta cannot be written"
.to_owned(),
);
};
let Some(meta) = meta_deps() else {
return Outcome::Failed(format!(
"{} is unset, so there are no swarm values to render internal/meta from",
meta_flake::SETTINGS_ENV
));
};
let (Some(queue), Some(wanted)) = (&deps.queue, &deps.wanted) else {
return Outcome::Failed(
"no swarm queue is configured on this host, so neither the placed agents nor a \
lock can be had"
.to_owned(),
);
};
let agents = match placed_agents(queue, wanted, &meta.hives).await {
Ok(agents) => agents,
Err(e) => return Outcome::Failed(format!("{e:#}")),
};
match forge_node(
"update_meta",
meta_flake::update(
client.as_ref(),
queue,
&meta.settings,
&agents,
&meta.hives,
),
)
.await
{
Ok(updated) => {
tracing::info!(?updated, agents = agents.len(), "update_meta: done");
Outcome::Done
}
Err(e) => Outcome::Failed(format!("{e:#}")),
}
}
/// Every agent some hive in `hives` places, with its archetype.
async fn placed_agents(
queue: &async_nats::Client,
wanted: &wanted::WantedWriter,
hives: &[String],
) -> Result<Vec<meta_flake::Placed>> {
let mut names = std::collections::BTreeSet::new();
for hive in hives {
let Some(declared) = wanted
.view(hive)
.await
.map_err(|e| e.context(format!("read where hive {hive} places agents")))?
else {
continue;
};
names.extend(
declared
.agents
.into_iter()
.filter(|(_, w)| crate::routes::agents::places_agent(w.state))
.map(|(name, _)| name),
);
}
let archetypes = swarm_queue_client::agent_archetype::open_or_create(queue)
.await
.map_err(|e| anyhow::anyhow!("open the agent-archetype bucket: {e}"))?;
let mut placed = Vec::with_capacity(names.len());
for name in names {
let (archetype, _, _) = crate::schedules::read_or_default::<
Option<swarm_wire::agent_archetype::ArchetypeRef>,
>(&archetypes, &name)
.await
.map_err(|e| e.context(format!("read the archetype of {name}")))?;
placed.push(meta_flake::Placed { name, archetype });
}
Ok(placed)
}
/// The `MintBuilderForgeToken` arm, lifted out for the same reason.
async fn mint_builder_forge_token(
forge: Option<Arc<forge::Client>>,
@ -872,14 +984,20 @@ pub(crate) async fn queue_merged_config_deploy(
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert_job(None, |b| {
vec![
b.node(SwarmNodeKind::TriggerDeploy {
// The hive builds the merged commit from `internal/meta`, so the
// flake must pin it before the hive is told.
let update_meta = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta);
let root = update_meta.guid();
let _deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.clone(),
agent: agent.clone(),
rev: Some(rev.clone()),
})
.guid(),
]
.after_ok(update_meta);
vec![root]
});
match inserted {
Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"),
@ -977,6 +1095,12 @@ pub(crate) fn declare_agent_job(
hive: hive.to_owned(),
agent: agent.to_owned(),
});
// The hive builds the agent from `internal/meta`, which can only carry it
// once the agent is declared on a hive and its config repo has a commit.
let update_meta = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta)
.after_ok(set_wanted);
// Last, and specifically after the config repo is seeded: the hive
// deploys by reading that repo, so a deploy asked for any earlier would
// find nothing to build. This is the edge that makes creating an agent at
@ -1007,6 +1131,7 @@ pub(crate) fn declare_agent_job(
agent: agent.to_owned(),
rev: None,
})
.after_ok(update_meta)
.after_any(mint_identity)
.after_any(mint_forge_token)
// `after_any` for the same reason: the container reads this token
@ -1017,6 +1142,7 @@ pub(crate) fn declare_agent_job(
.after_ok(set_wanted);
if let Some(init_config) = init_config {
let _ = trigger_deploy.after_ok(init_config);
let _ = update_meta.after_ok(init_config);
}
vec![create_identity.guid()]
}
@ -1183,6 +1309,64 @@ pub(crate) fn queue_builder_forge_token_mint(
.map_err(|e| anyhow::anyhow!("{e}"))
}
/// Insert one `UpdateMeta` job unless one is already pending, which reads the
/// same heads when it runs. Returns whether a job was inserted.
pub(crate) fn queue_meta_update(
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
) -> Result<bool> {
let mut sched = sched
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let pending = sched.graph().nodes().any(|n| {
n.state == hive_jobq::State::Pending && matches!(n.payload, SwarmNodeKind::UpdateMeta)
});
if pending {
return Ok(false);
}
sched
.insert_job(None, |b| {
vec![
b.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta)
.guid(),
]
})
.map_err(|e| anyhow::anyhow!("{e}"))?;
Ok(true)
}
/// [`MetaDeps`] from the environment, read on first use; `None`, logged, when
/// the settings are unset or do not parse, which `UpdateMeta` then fails on
/// by name.
fn meta_deps() -> Option<&'static MetaDeps> {
static DEPS: std::sync::OnceLock<Option<MetaDeps>> = std::sync::OnceLock::new();
DEPS.get_or_init(read_meta_deps).as_ref()
}
fn read_meta_deps() -> Option<MetaDeps> {
let settings = match meta_flake::Settings::from_env() {
Ok(Some(s)) => s,
Ok(None) => {
tracing::warn!(
"{} unset; internal/meta will not be written",
meta_flake::SETTINGS_ENV
);
return None;
}
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "internal/meta will not be written");
return None;
}
};
Some(MetaDeps {
settings,
hives: crate::startup::load_hives()
.into_iter()
.map(|h| h.name)
.collect(),
})
}
/// Insert one `MintCiMetricsForgeToken` job and return its node's id.
/// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller.
pub(crate) fn queue_ci_metrics_forge_token_mint(