Watch
0
0
Fork
You've already forked hyperhive
0

swarm-controller: render, lock and commit internal/meta

A new update_meta node renders internal/meta's flake.nix with
hive-meta-flake, the renderer hive-c0re uses: hyperhive pinned to its
forge main, each custom agent a hive places pinned to its agent-configs
main, all over https, and the swarm-wide values from
SWARM_CONTROLLER_META_FLAKE (set by swarm-controller.nix from
services.hyperhive.swarm.*). The swarm builder locks it, and the
controller commits flake.nix and flake.lock in one commit and gives
every hive's forge account read on the repo. A render equal to the
committed flake.nix sends no lock and commits nothing.

It runs at start, when hyperhive's main moves, before the deploy of a
merged config PR and before the first deploy of a new agent; those
deploys now wait for it. agent_web_port moves into hive-meta-flake so
both callers compute the same port; hive-c0re re-exports it.

On deploy: internal/meta appears on the forge and follows every merge.
No hive builds from it yet.

Refs #4857
This commit is contained in:
atlas 2026-10-22 07:52:23 +02:00
commit 5950363a10
16 changed files with 970 additions and 41 deletions

View file

@ -0,0 +1,583 @@
//! `internal/meta`: the swarm meta flake hives build their agents from.
//!
//! [`update`] renders `flake.nix` with `hive_meta_flake::render_flake`: hyperhive
//! pinned to its forge `main`, each custom agent's config input pinned to its
//! `agent-configs/<agent>` `main`, every agent placed on some hive. The swarm
//! builder locks it (`swarm_wire::builder::LockMeta`), and the flake and the
//! lock are committed together in one commit. A render equal to the committed
//! `flake.nix` sends no lock and commits nothing.
use std::collections::BTreeMap;
use std::future::Future;
use anyhow::{Context, Result, bail};
use serde::Deserialize;
use swarm_wire::agent_archetype::ArchetypeRef;
use swarm_wire::builder::{LOCK_REPLY_DEADLINE, LOCK_SUBJECT, LockMeta, LockReply};
use crate::forge::{CONFIG_ORG, DefaultBranchFiles};
/// The repo in `internal` the flake is committed to.
pub(crate) const META_REPO: &str = "meta";
/// The env var holding [`Settings`] as JSON, set by `swarm-controller.nix`.
pub(crate) const SETTINGS_ENV: &str = "SWARM_CONTROLLER_META_FLAKE";
/// Where hyperhive itself lives on the forge.
const HYPERHIVE_OWNER: &str = "hyperhive";
const HYPERHIVE_REPO: &str = "hyperhive";
/// What every agent of the swarm gets, the same on every hive.
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct Settings {
/// Values by `hive_meta_flake::FORWARDED_VARS` name. `HIVE_FORGE_URL` is
/// required.
vars: BTreeMap<String, String>,
/// The forge's `https` origin, which every pinned input is fetched from:
/// the swarm builder's credential helper answers only `https` for it.
git_base: String,
/// The swarm turned the agents' GitHub integration off.
github_disabled: bool,
}
impl Settings {
/// [`Settings`] from [`SETTINGS_ENV`], `None` when it is unset.
pub(crate) fn from_env() -> Result<Option<Self>> {
match std::env::var(SETTINGS_ENV) {
Ok(raw) => Self::parse(&raw).map(Some),
Err(std::env::VarError::NotPresent) => Ok(None),
Err(e) => Err(e).with_context(|| format!("read {SETTINGS_ENV}")),
}
}
fn parse(raw: &str) -> Result<Self> {
let settings: Self =
serde_json::from_str(raw).with_context(|| format!("parse {SETTINGS_ENV}"))?;
if let Some(unknown) = settings
.vars
.keys()
.find(|k| !hive_meta_flake::FORWARDED_VARS.contains(&k.as_str()))
{
bail!("{SETTINGS_ENV}: {unknown} is not a value the meta flake forwards");
}
if settings.vars.get("HIVE_FORGE_URL").is_none_or(String::is_empty) {
bail!("{SETTINGS_ENV}: HIVE_FORGE_URL is unset");
}
if !settings.git_base.starts_with("https://") {
bail!("{SETTINGS_ENV}: git_base must be an https origin");
}
Ok(settings)
}
/// The forwarded pairs with a value, in `FORWARDED_VARS` order.
fn forwarded(&self) -> Vec<(&'static str, String)> {
hive_meta_flake::FORWARDED_VARS
.iter()
.filter_map(|&name| {
self.vars
.get(name)
.filter(|v| !v.is_empty())
.map(|v| (name, v.clone()))
})
.collect()
}
fn git_url(&self, owner: &str, repo: &str) -> String {
format!("{}/{owner}/{repo}.git", self.git_base.trim_end_matches('/'))
}
}
/// One agent some hive places, with the archetype it is built from.
#[derive(Debug, Clone)]
pub(crate) struct Placed {
pub(crate) name: String,
pub(crate) archetype: Option<ArchetypeRef>,
}
/// One file of a meta commit: a blob sha replaces that blob, `None` creates it.
pub(crate) struct MetaFile {
pub(crate) path: &'static str,
pub(crate) bytes: Vec<u8>,
pub(crate) sha: Option<String>,
}
/// The forge, as [`update`] reads and writes it.
pub(crate) trait MetaForge {
fn head_files(
&self,
owner: &str,
repo: &str,
paths: &[&str],
) -> impl Future<Output = Result<Option<DefaultBranchFiles>>> + Send;
fn ensure_meta_repo(&self) -> impl Future<Output = Result<()>> + Send;
fn grant_meta_read(&self, login: &str) -> impl Future<Output = Result<()>> + Send;
fn commit_meta(
&self,
message: &str,
files: Vec<MetaFile>,
) -> impl Future<Output = Result<()>> + Send;
}
/// Where a lock is sent: the queue in production, a fake in the tests. `Err`
/// is a lock that never reached a builder or never came back.
pub(crate) trait LockSink {
fn lock(&self, request: Vec<u8>) -> impl Future<Output = Result<LockReply, String>> + Send;
}
impl LockSink for async_nats::Client {
async fn lock(&self, request: Vec<u8>) -> Result<LockReply, String> {
use async_nats::RequestErrorKind;
let sent = async_nats::Request::new()
.payload(request.into())
.timeout(Some(LOCK_REPLY_DEADLINE));
match self.send_request(LOCK_SUBJECT, sent).await {
Ok(msg) => serde_json::from_slice(&msg.payload)
.map_err(|e| format!("the builder's lock reply did not decode: {e}")),
Err(e) => Err(match e.kind() {
RequestErrorKind::NoResponders => "no swarm builder is listening for locks: \
deploy.builder.enable is set on no host, or its builder predates the lock \
request"
.to_owned(),
RequestErrorKind::TimedOut => format!(
"the swarm builder sent no lock within {} minutes",
LOCK_REPLY_DEADLINE.as_secs() / 60
),
_ => format!("sending the lock to the swarm builder failed: {e}"),
}),
}
}
}
/// What [`update`] did.
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum Updated {
/// The committed flake already is the render.
Unchanged,
/// A new flake and lock were committed.
Committed,
}
/// Bring `internal/meta` up to the forge's current heads for `agents`, and
/// give each of `hives`' forge accounts read on it.
///
/// # Errors
/// A head that cannot be read, a lock the builder refuses, or a refused commit.
/// A failed read grant is logged, not returned: the commit is what hives wait
/// on.
pub(crate) async fn update(
forge: &impl MetaForge,
locker: &impl LockSink,
settings: &Settings,
agents: &[Placed],
hives: &[String],
) -> Result<Updated> {
forge.ensure_meta_repo().await?;
for hive in hives {
let login = swarm_secret_client::forge::hive_login(hive);
if let Err(e) = forge.grant_meta_read(&login).await {
tracing::warn!(%hive, error = %format!("{e:#}"), "meta flake: read grant on internal/meta failed");
}
}
let flake = render(forge, settings, agents).await?;
let committed = forge
.head_files(
crate::forge::objects::SHARED_ORG,
META_REPO,
&["flake.nix", "flake.lock"],
)
.await?;
let sha_of = |path: &str| {
committed
.as_ref()
.and_then(|c| c.files.iter().find(|f| f.path == path))
};
let flake_file = sha_of("flake.nix");
if flake_file.is_some_and(|f| f.bytes == flake.as_bytes()) && sha_of("flake.lock").is_some() {
return Ok(Updated::Unchanged);
}
let request = serde_json::to_vec(&LockMeta {
flake_nix: flake.clone(),
})
.context("encode the lock request")?;
let flake_lock = match locker.lock(request).await {
Ok(LockReply::Locked { flake_lock }) => flake_lock,
Ok(LockReply::Failed { error_tail }) => bail!("the builder could not lock: {error_tail}"),
Err(e) => bail!("{e}"),
};
forge
.commit_meta(
&format!("meta: pin {} agent(s)", agents.len()),
vec![
MetaFile {
path: "flake.nix",
bytes: flake.into_bytes(),
sha: flake_file.map(|f| f.sha.clone()),
},
MetaFile {
path: "flake.lock",
bytes: flake_lock.into_bytes(),
sha: sha_of("flake.lock").map(|f| f.sha.clone()),
},
],
)
.await?;
Ok(Updated::Committed)
}
/// Whether a `push` webhook body moved hyperhive's `main`, which the flake
/// pins.
pub(crate) fn moves_hyperhive(body: &[u8]) -> bool {
let Ok(v) = serde_json::from_slice::<serde_json::Value>(body) else {
return false;
};
let repo = v
.get("repository")
.and_then(|r| r.get("full_name"))
.and_then(serde_json::Value::as_str);
let git_ref = v.get("ref").and_then(serde_json::Value::as_str);
repo == Some(&format!("{HYPERHIVE_OWNER}/{HYPERHIVE_REPO}"))
&& git_ref == Some("refs/heads/main")
}
/// The `flake.nix` for `agents` at the forge's current heads.
async fn render(forge: &impl MetaForge, settings: &Settings, agents: &[Placed]) -> Result<String> {
let hyperhive = forge
.head_files(HYPERHIVE_OWNER, HYPERHIVE_REPO, &[])
.await?
.with_context(|| format!("{HYPERHIVE_OWNER}/{HYPERHIVE_REPO} is empty or absent"))?;
let hyperhive_url = hive_meta_flake::pinned_git_url(
&settings.git_url(HYPERHIVE_OWNER, HYPERHIVE_REPO),
Some(&hyperhive.commit),
)?;
let mut specs = Vec::with_capacity(agents.len());
for agent in agents {
let pin = if agent.archetype.is_some() {
None
} else {
let head = forge
.head_files(CONFIG_ORG, &agent.name, &["flake.lock"])
.await?
.with_context(|| {
format!(
"agent {}: {CONFIG_ORG}/{} has no commit on the forge",
agent.name, agent.name
)
})?;
let lock = head
.files
.first()
.map(|f| String::from_utf8_lossy(&f.bytes).into_owned());
Some(hive_meta_flake::AgentPin {
url: hive_meta_flake::pinned_git_url(
&settings.git_url(CONFIG_ORG, &agent.name),
Some(&head.commit),
)?,
canonical_inputs: lock
.as_deref()
.map(hive_meta_flake::canonical_inputs_in_lock)
.unwrap_or_default(),
})
};
specs.push(hive_meta_flake::AgentSpec {
name: agent.name.clone(),
port: hive_meta_flake::agent_web_port(&agent.name),
archetype: agent.archetype.clone(),
pin,
});
}
specs.sort_by(|a, b| a.name.cmp(&b.name));
hive_meta_flake::render_flake(
&hyperhive_url,
&specs,
&settings.forwarded(),
settings.github_disabled,
)
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
use crate::forge::BranchFile;
const FORGE: &str = "https://forge.t.local";
fn settings() -> Settings {
Settings::parse(&format!(
r#"{{"vars":{{"HIVE_FORGE_URL":"http://forge.t.local","HYPERHIVE_SWARM_NAME":"t"}},"git_base":"{FORGE}/","github_disabled":true}}"#
))
.unwrap()
}
fn head(commit: &str, files: &[(&str, &str)]) -> DefaultBranchFiles {
DefaultBranchFiles {
branch: "main".to_owned(),
commit: commit.to_owned(),
web_url: None,
files: files
.iter()
.map(|(path, body)| BranchFile {
path: (*path).to_owned(),
sha: format!("blob-{path}"),
bytes: body.as_bytes().to_vec(),
})
.collect(),
}
}
/// One recorded commit: its message, and each file's path, text and sha.
type Commit = (String, Vec<(&'static str, String, Option<String>)>);
#[derive(Default)]
struct FakeForge {
repos: BTreeMap<String, DefaultBranchFiles>,
commits: Mutex<Vec<Commit>>,
grants: Mutex<Vec<String>>,
}
impl MetaForge for FakeForge {
async fn head_files(
&self,
owner: &str,
repo: &str,
paths: &[&str],
) -> Result<Option<DefaultBranchFiles>> {
Ok(self.repos.get(&format!("{owner}/{repo}")).map(|h| {
let mut h = h.clone();
h.files.retain(|f| paths.contains(&f.path.as_str()));
h
}))
}
async fn ensure_meta_repo(&self) -> Result<()> {
Ok(())
}
async fn grant_meta_read(&self, login: &str) -> Result<()> {
self.grants.lock().unwrap().push(login.to_owned());
Ok(())
}
async fn commit_meta(&self, message: &str, files: Vec<MetaFile>) -> Result<()> {
self.commits.lock().unwrap().push((
message.to_owned(),
files
.into_iter()
.map(|f| (f.path, String::from_utf8(f.bytes).unwrap(), f.sha))
.collect(),
));
Ok(())
}
}
/// Answers every lock with `reply`, recording each flake it was sent.
struct FakeLocker {
reply: Result<LockReply, String>,
sent: Mutex<Vec<String>>,
}
impl FakeLocker {
fn locked() -> Self {
Self {
reply: Ok(LockReply::Locked {
flake_lock: "{\"version\":7}".to_owned(),
}),
sent: Mutex::default(),
}
}
}
impl LockSink for FakeLocker {
async fn lock(&self, request: Vec<u8>) -> Result<LockReply, String> {
let req: LockMeta = serde_json::from_slice(&request).unwrap();
self.sent.lock().unwrap().push(req.flake_nix);
self.reply.clone()
}
}
fn forge_with(extra: &[(&str, DefaultBranchFiles)]) -> FakeForge {
let mut repos = BTreeMap::new();
repos.insert("hyperhive/hyperhive".to_owned(), head("hh0", &[]));
repos.insert(
"agent-configs/argus".to_owned(),
head(
"a1",
&[(
"flake.lock",
r#"{"root":"root","nodes":{"root":{"inputs":{"nixpkgs":"nixpkgs"}}}}"#,
)],
),
);
for (name, h) in extra {
repos.insert((*name).to_owned(), h.clone());
}
FakeForge {
repos,
..FakeForge::default()
}
}
fn argus_and_reviewer() -> Vec<Placed> {
vec![
Placed {
name: "argus".to_owned(),
archetype: None,
},
Placed {
name: "rev".to_owned(),
archetype: Some(ArchetypeRef {
source: "hyperhive".to_owned(),
name: "reviewer".to_owned(),
}),
},
]
}
#[tokio::test]
async fn a_new_render_is_locked_and_committed_with_its_lock_in_one_commit() {
let forge = forge_with(&[]);
let locker = FakeLocker::locked();
let got = update(
&forge,
&locker,
&settings(),
&argus_and_reviewer(),
&["pr1ma".to_owned()],
)
.await
.unwrap();
assert_eq!(got, Updated::Committed);
let sent = locker.sent.lock().unwrap();
let [flake] = sent.as_slice() else {
panic!("one lock: {sent:?}")
};
assert!(flake.contains(&format!(
"hyperhive.url = \"git+{FORGE}/hyperhive/hyperhive.git?rev=hh0\";"
)));
assert!(flake.contains(&format!(
"agent-argus.url = \"git+{FORGE}/agent-configs/argus.git?rev=a1\";"
)));
assert!(flake.contains("agent-argus.inputs.nixpkgs.follows = \"nixpkgs\";"));
assert!(!flake.contains("agent-rev.url"), "an archetype agent has no input");
assert!(flake.contains("services.hyperhive.agent.github.enable = false;"));
assert!(flake.contains("HYPERHIVE_SWARM_NAME = \"t\";"));
let commits = forge.commits.lock().unwrap();
let [(_, files)] = commits.as_slice() else {
panic!("one commit: {commits:?}")
};
assert_eq!(
files,
&vec![
("flake.nix", flake.clone(), None),
("flake.lock", "{\"version\":7}".to_owned(), None),
]
);
assert_eq!(
*forge.grants.lock().unwrap(),
vec![swarm_secret_client::forge::hive_login("pr1ma")]
);
}
#[tokio::test]
async fn an_unchanged_render_sends_no_lock_and_commits_nothing() {
let first = forge_with(&[]);
let locker = FakeLocker::locked();
update(&first, &locker, &settings(), &argus_and_reviewer(), &[])
.await
.unwrap();
let flake = locker.sent.lock().unwrap()[0].clone();
let forge = forge_with(&[(
"internal/meta",
head("m1", &[("flake.nix", &flake), ("flake.lock", "{}")]),
)]);
let locker = FakeLocker::locked();
let got = update(&forge, &locker, &settings(), &argus_and_reviewer(), &[])
.await
.unwrap();
assert_eq!(got, Updated::Unchanged);
assert!(locker.sent.lock().unwrap().is_empty());
assert!(forge.commits.lock().unwrap().is_empty());
}
#[tokio::test]
async fn a_moved_head_replaces_both_committed_blobs() {
let forge = forge_with(&[(
"internal/meta",
head("m1", &[("flake.nix", "old"), ("flake.lock", "{}")]),
)]);
let locker = FakeLocker::locked();
update(&forge, &locker, &settings(), &argus_and_reviewer(), &[])
.await
.unwrap();
let commits = forge.commits.lock().unwrap();
let shas: Vec<_> = commits[0].1.iter().map(|(_, _, sha)| sha.clone()).collect();
assert_eq!(
shas,
vec![
Some("blob-flake.nix".to_owned()),
Some("blob-flake.lock".to_owned())
]
);
}
#[tokio::test]
async fn a_refused_lock_commits_nothing() {
let forge = forge_with(&[]);
let locker = FakeLocker {
reply: Ok(LockReply::Failed {
error_tail: "no such input".to_owned(),
}),
sent: Mutex::default(),
};
let err = update(&forge, &locker, &settings(), &argus_and_reviewer(), &[])
.await
.unwrap_err();
assert!(format!("{err:#}").contains("no such input"));
assert!(forge.commits.lock().unwrap().is_empty());
}
#[tokio::test]
async fn a_custom_agent_without_a_config_commit_fails_the_render() {
let forge = forge_with(&[]);
let agents = vec![Placed {
name: "ghost".to_owned(),
archetype: None,
}];
let err = update(&forge, &FakeLocker::locked(), &settings(), &agents, &[])
.await
.unwrap_err();
assert!(format!("{err:#}").contains("agent-configs/ghost"));
assert!(forge.commits.lock().unwrap().is_empty());
}
#[test]
fn only_a_push_to_hyperhive_main_moves_the_pin() {
let push = |repo: &str, git_ref: &str| {
serde_json::to_vec(&serde_json::json!({
"ref": git_ref,
"repository": { "full_name": repo },
"commits": [],
}))
.unwrap()
};
assert!(moves_hyperhive(&push("hyperhive/hyperhive", "refs/heads/main")));
assert!(!moves_hyperhive(&push("hyperhive/hyperhive", "refs/heads/x")));
assert!(!moves_hyperhive(&push("agents/foo", "refs/heads/main")));
assert!(!moves_hyperhive(b"not json"));
}
#[test]
fn settings_refuse_an_unknown_value_a_missing_forge_and_a_plain_http_base() {
let parse = |vars: &str, git_base: &str| {
Settings::parse(&format!(
r#"{{"vars":{vars},"git_base":"{git_base}","github_disabled":false}}"#
))
};
assert!(parse(r#"{"HIVE_FORGE_URL":"x"}"#, "https://f").is_ok());
let unknown = parse(r#"{"HIVE_FORGE_URL":"x","HIVE_AGENT_NATS_URL":"y"}"#, "https://f")
.unwrap_err();
assert!(format!("{unknown:#}").contains("HIVE_AGENT_NATS_URL"));
assert!(parse("{}", "https://f").is_err());
assert!(parse(r#"{"HIVE_FORGE_URL":"x"}"#, "http://f").is_err());
}
}