swarm-controller: render, lock and commit internal/meta

A new update_meta node renders internal/meta's flake.nix with
hive-meta-flake, the renderer hive-c0re uses: hyperhive pinned to its
forge main, each custom agent a hive places pinned to its agent-configs
main, all over https, and the swarm-wide values from
SWARM_CONTROLLER_META_FLAKE (set by swarm-controller.nix from
services.hyperhive.swarm.*). The swarm builder locks it, and the
controller commits flake.nix and flake.lock in one commit and gives
every hive's forge account read on the repo. A render equal to the
committed flake.nix sends no lock and commits nothing.

It runs at start, when hyperhive's main moves, before the deploy of a
merged config PR and before the first deploy of a new agent; those
deploys now wait for it. agent_web_port moves into hive-meta-flake so
both callers compute the same port; hive-c0re re-exports it.

On deploy: internal/meta appears on the forge and follows every merge.
No hive builds from it yet.

Refs #4857

swarm-controller/src/forge.rs

@@ -52,6 +52,7 @@ pub mod classifier_account;
 pub mod docs_grant;
 pub mod hive_token;
 pub mod legacy_tokens;
+mod meta_repo;
 pub mod objects;
 pub mod operator_profiles;
 pub mod pull_diff;

swarm-controller/src/forge/meta_repo.rs

@@ -0,0 +1,77 @@
+//! `internal/meta` on the forge: the repo the swarm meta flake is committed to
+//! (`crate::meta_flake`), and the read grant each hive's account holds on it.
+
+use anyhow::{Context, Result};
+use base64::Engine as _;
+use forgejo_api::structs::{
+    AddCollaboratorOptionPermission, ChangeFileOperation, ChangeFileOperationOperation,
+    ChangeFilesOptions,
+};
+
+use super::objects::SHARED_ORG;
+use super::{Client, DefaultBranchFiles};
+use crate::meta_flake::{META_REPO, MetaFile, MetaForge};
+
+impl MetaForge for Client {
+    async fn head_files(
+        &self,
+        owner: &str,
+        repo: &str,
+        paths: &[&str],
+    ) -> Result<Option<DefaultBranchFiles>> {
+        self.default_branch_files(owner, repo, paths).await
+    }
+
+    async fn ensure_meta_repo(&self) -> Result<()> {
+        self.create_shared_repo(SHARED_ORG, META_REPO, true).await
+    }
+
+    async fn grant_meta_read(&self, login: &str) -> Result<()> {
+        self.add_collaborator_to(
+            SHARED_ORG,
+            META_REPO,
+            login,
+            AddCollaboratorOptionPermission::Read,
+        )
+        .await
+    }
+
+    async fn commit_meta(&self, message: &str, files: Vec<MetaFile>) -> Result<()> {
+        let ops: Vec<ChangeFileOperation> = files
+            .into_iter()
+            .map(|f| ChangeFileOperation {
+                content: Some(base64::engine::general_purpose::STANDARD.encode(f.bytes)),
+                from_path: None,
+                operation: if f.sha.is_some() {
+                    ChangeFileOperationOperation::Update
+                } else {
+                    ChangeFileOperationOperation::Create
+                },
+                path: f.path.to_owned(),
+                sha: f.sha,
+            })
+            .collect();
+        self.api
+            .call(|api| {
+                api.repo_change_files(
+                    SHARED_ORG,
+                    META_REPO,
+                    ChangeFilesOptions {
+                        author: None,
+                        branch: None,
+                        committer: None,
+                        dates: None,
+                        files: ops.clone(),
+                        force_overwrite_new_branch: None,
+                        message: Some(message.to_owned()),
+                        new_branch: None,
+                        signoff: None,
+                    },
+                )
+                .into_future()
+            })
+            .await
+            .with_context(|| format!("commit to {SHARED_ORG}/{META_REPO}"))?;
+        Ok(())
+    }
+}

swarm-controller/src/forge/objects.rs

@@ -48,7 +48,7 @@ const AGENTS_ORG: &str = "agents";
 
 /// Org hosting the operator-curated shared repos. Same org as the knowledge
 /// repo's, named separately for the docs repo it also holds.
-pub(super) const SHARED_ORG: &str = KNOWLEDGE_ORG;
+pub(crate) const SHARED_ORG: &str = KNOWLEDGE_ORG;
 
 /// The shared docs repo inside [`SHARED_ORG`]. **Private**: every agent gets
 /// read-only collaborator access, granted per agent by [`super::docs_grant`].
@@ -56,9 +56,8 @@ pub(super) const SHARED_ORG: &str = KNOWLEDGE_ORG;
 /// content into the system prompt.
 pub(super) const SHARED_DOCS_REPO: &str = "docs";
 
-/// The orgs ensured on every pass. The meta repo lives at `core/meta`, the
-/// `core` user's own namespace, so no org is needed for it (and it is not a
-/// swarm object: it stays with `hive-c0re`).
+/// The orgs ensured on every pass. `internal/meta` is in [`SHARED_ORG`] and is
+/// created by `crate::meta_flake`'s update, not by this pass.
 const SEEDED_ORGS: [&str; 3] = [CONFIG_ORG, SHARED_ORG, AGENTS_ORG];
 
 /// Where the [`OPERATORS_TEAM`] must exist. Gitea teams are org-scoped, so a
@@ -938,7 +937,12 @@ impl Client {
     }
 
     /// Create `owner/name`, empty, defaulting to `main`.
-    async fn create_shared_repo(&self, owner: &str, name: &str, private: bool) -> Result<()> {
+    pub(super) async fn create_shared_repo(
+        &self,
+        owner: &str,
+        name: &str,
+        private: bool,
+    ) -> Result<()> {
         match self
             .api
             .call(|api| {

swarm-controller/src/jobq.rs

@@ -10,7 +10,8 @@ use crate::routes::AppState;
 use crate::routes::agents::{claiming_hives, read_declarations};
 use crate::startup::PUBLIC_URL_ENV;
 use crate::{
-    agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, wanted,
+    agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, meta_flake,
+    wanted,
 };
 
 /// Node payload for the swarm-level job graph. Named `Swarm*` rather than
@@ -133,6 +134,13 @@ pub(crate) enum SwarmNodeKind {
         /// agent creation, where the hive deploys what it seeds.
         rev: Option<String>,
     },
+    /// Pin the swarm meta flake (`internal/meta`) to the forge's current
+    /// heads: render, lock on the swarm builder, commit. See `meta_flake`.
+    ///
+    /// Carries nothing: it reads every head when it runs, so one pending node
+    /// covers every trigger queued behind it. Hives build from that repo, so a
+    /// deploy waits for this node.
+    UpdateMeta,
 }
 
 impl hive_jobq_wire::WireNode for SwarmNodeKind {
@@ -158,6 +166,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
             }
             SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(),
             SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(),
+            SwarmNodeKind::UpdateMeta => "update_meta".to_owned(),
         }
     }
 
@@ -185,7 +194,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
             | SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
             SwarmNodeKind::MintClassifierForgeToken
             | SwarmNodeKind::MintBuilderForgeToken
-            | SwarmNodeKind::MintCiMetricsForgeToken => serde_json::json!({}),
+            | SwarmNodeKind::MintCiMetricsForgeToken
+            | SwarmNodeKind::UpdateMeta => serde_json::json!({}),
             SwarmNodeKind::BuildAgent { agent, rev, pr } => {
                 serde_json::json!({ "agent": agent, "rev": rev, "pr": pr })
             }
@@ -206,12 +216,16 @@ pub(crate) enum SwarmResourceKind {
     /// The swarm builder: one `BuildAgent` node runs at a time, and the rest
     /// wait pending, where a newer push can replace them.
     Builder,
+    /// `internal/meta`: one `UpdateMeta` commits at a time, so two never race
+    /// on the same file's blob.
+    Meta,
 }
 
 impl hive_jobq_wire::WireResource for SwarmResourceKind {
     fn name(&self) -> String {
         match self {
             SwarmResourceKind::Builder => "builder".to_owned(),
+            SwarmResourceKind::Meta => "meta".to_owned(),
         }
     }
 }
@@ -253,6 +267,14 @@ pub(crate) struct WorkerDeps {
     pub(crate) matrix_homeserver: Option<std::sync::Arc<str>>,
 }
 
+/// What `UpdateMeta` renders from, fixed for the process.
+struct MetaDeps {
+    settings: meta_flake::Settings,
+    /// Every hive's name: each places agents, and each hive's forge account
+    /// gets read on the repo.
+    hives: Vec<String>,
+}
+
 /// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be.
 ///
 /// A named constant rather than the literal inline, because this *is* the
@@ -373,6 +395,7 @@ pub(crate) async fn run_swarm_node(
             ),
             Some(client) => publish_deploy(&client, &hive, &agent, rev).await,
         },
+        SwarmNodeKind::UpdateMeta => update_meta(&deps, forge).await,
     };
     (builder, outcome)
 }
@@ -542,6 +565,95 @@ pub(crate) async fn build_agent(
     .await
 }
 
+/// The `UpdateMeta` arm, lifted out for the same reason.
+async fn update_meta(
+    deps: &WorkerDeps,
+    forge: Option<Arc<forge::Client>>,
+) -> hive_jobq::scheduler::Outcome {
+    use hive_jobq::scheduler::Outcome;
+
+    let Some(client) = forge else {
+        return Outcome::Failed(
+            "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
+             not in the swarm secret store), so internal/meta cannot be written"
+                .to_owned(),
+        );
+    };
+    let Some(meta) = meta_deps() else {
+        return Outcome::Failed(format!(
+            "{} is unset, so there are no swarm values to render internal/meta from",
+            meta_flake::SETTINGS_ENV
+        ));
+    };
+    let (Some(queue), Some(wanted)) = (&deps.queue, &deps.wanted) else {
+        return Outcome::Failed(
+            "no swarm queue is configured on this host, so neither the placed agents nor a \
+             lock can be had"
+                .to_owned(),
+        );
+    };
+    let agents = match placed_agents(queue, wanted, &meta.hives).await {
+        Ok(agents) => agents,
+        Err(e) => return Outcome::Failed(format!("{e:#}")),
+    };
+    match forge_node(
+        "update_meta",
+        meta_flake::update(
+            client.as_ref(),
+            queue,
+            &meta.settings,
+            &agents,
+            &meta.hives,
+        ),
+    )
+    .await
+    {
+        Ok(updated) => {
+            tracing::info!(?updated, agents = agents.len(), "update_meta: done");
+            Outcome::Done
+        }
+        Err(e) => Outcome::Failed(format!("{e:#}")),
+    }
+}
+
+/// Every agent some hive in `hives` places, with its archetype.
+async fn placed_agents(
+    queue: &async_nats::Client,
+    wanted: &wanted::WantedWriter,
+    hives: &[String],
+) -> Result<Vec<meta_flake::Placed>> {
+    let mut names = std::collections::BTreeSet::new();
+    for hive in hives {
+        let Some(declared) = wanted
+            .view(hive)
+            .await
+            .map_err(|e| e.context(format!("read where hive {hive} places agents")))?
+        else {
+            continue;
+        };
+        names.extend(
+            declared
+                .agents
+                .into_iter()
+                .filter(|(_, w)| crate::routes::agents::places_agent(w.state))
+                .map(|(name, _)| name),
+        );
+    }
+    let archetypes = swarm_queue_client::agent_archetype::open_or_create(queue)
+        .await
+        .map_err(|e| anyhow::anyhow!("open the agent-archetype bucket: {e}"))?;
+    let mut placed = Vec::with_capacity(names.len());
+    for name in names {
+        let (archetype, _, _) = crate::schedules::read_or_default::<
+            Option<swarm_wire::agent_archetype::ArchetypeRef>,
+        >(&archetypes, &name)
+        .await
+        .map_err(|e| e.context(format!("read the archetype of {name}")))?;
+        placed.push(meta_flake::Placed { name, archetype });
+    }
+    Ok(placed)
+}
+
 /// The `MintBuilderForgeToken` arm, lifted out for the same reason.
 async fn mint_builder_forge_token(
     forge: Option<Arc<forge::Client>>,
@@ -872,14 +984,20 @@ pub(crate) async fn queue_merged_config_deploy(
         .lock()
         .unwrap_or_else(std::sync::PoisonError::into_inner)
         .insert_job(None, |b| {
-            vec![
-                b.node(SwarmNodeKind::TriggerDeploy {
+            // The hive builds the merged commit from `internal/meta`, so the
+            // flake must pin it before the hive is told.
+            let update_meta = b
+                .node(SwarmNodeKind::UpdateMeta)
+                .needs(SwarmResourceKind::Meta);
+            let root = update_meta.guid();
+            let _deploy = b
+                .node(SwarmNodeKind::TriggerDeploy {
                     hive: hive.clone(),
                     agent: agent.clone(),
                     rev: Some(rev.clone()),
                 })
-                .guid(),
-            ]
+                .after_ok(update_meta);
+            vec![root]
         });
     match inserted {
         Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"),
@@ -977,6 +1095,12 @@ pub(crate) fn declare_agent_job(
         hive: hive.to_owned(),
         agent: agent.to_owned(),
     });
+    // The hive builds the agent from `internal/meta`, which can only carry it
+    // once the agent is declared on a hive and its config repo has a commit.
+    let update_meta = b
+        .node(SwarmNodeKind::UpdateMeta)
+        .needs(SwarmResourceKind::Meta)
+        .after_ok(set_wanted);
     // Last, and specifically after the config repo is seeded: the hive
     // deploys by reading that repo, so a deploy asked for any earlier would
     // find nothing to build. This is the edge that makes creating an agent at
@@ -1007,6 +1131,7 @@ pub(crate) fn declare_agent_job(
             agent: agent.to_owned(),
             rev: None,
         })
+        .after_ok(update_meta)
         .after_any(mint_identity)
         .after_any(mint_forge_token)
         // `after_any` for the same reason: the container reads this token
@@ -1017,6 +1142,7 @@ pub(crate) fn declare_agent_job(
         .after_ok(set_wanted);
     if let Some(init_config) = init_config {
         let _ = trigger_deploy.after_ok(init_config);
+        let _ = update_meta.after_ok(init_config);
     }
     vec![create_identity.guid()]
 }
@@ -1183,6 +1309,64 @@ pub(crate) fn queue_builder_forge_token_mint(
         .map_err(|e| anyhow::anyhow!("{e}"))
 }
 
+/// Insert one `UpdateMeta` job unless one is already pending, which reads the
+/// same heads when it runs. Returns whether a job was inserted.
+pub(crate) fn queue_meta_update(
+    sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
+) -> Result<bool> {
+    let mut sched = sched
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let pending = sched.graph().nodes().any(|n| {
+        n.state == hive_jobq::State::Pending && matches!(n.payload, SwarmNodeKind::UpdateMeta)
+    });
+    if pending {
+        return Ok(false);
+    }
+    sched
+        .insert_job(None, |b| {
+            vec![
+                b.node(SwarmNodeKind::UpdateMeta)
+                    .needs(SwarmResourceKind::Meta)
+                    .guid(),
+            ]
+        })
+        .map_err(|e| anyhow::anyhow!("{e}"))?;
+    Ok(true)
+}
+
+/// [`MetaDeps`] from the environment, read on first use; `None`, logged, when
+/// the settings are unset or do not parse, which `UpdateMeta` then fails on
+/// by name.
+fn meta_deps() -> Option<&'static MetaDeps> {
+    static DEPS: std::sync::OnceLock<Option<MetaDeps>> = std::sync::OnceLock::new();
+    DEPS.get_or_init(read_meta_deps).as_ref()
+}
+
+fn read_meta_deps() -> Option<MetaDeps> {
+    let settings = match meta_flake::Settings::from_env() {
+        Ok(Some(s)) => s,
+        Ok(None) => {
+            tracing::warn!(
+                "{} unset; internal/meta will not be written",
+                meta_flake::SETTINGS_ENV
+            );
+            return None;
+        }
+        Err(e) => {
+            tracing::warn!(error = %format!("{e:#}"), "internal/meta will not be written");
+            return None;
+        }
+    };
+    Some(MetaDeps {
+        settings,
+        hives: crate::startup::load_hives()
+            .into_iter()
+            .map(|h| h.name)
+            .collect(),
+    })
+}
+
 /// Insert one `MintCiMetricsForgeToken` job and return its node's id.
 /// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller.
 pub(crate) fn queue_ci_metrics_forge_token_mint(

swarm-controller/src/main.rs

@@ -50,6 +50,7 @@ mod jobq;
 mod linked_accounts;
 mod matrix_account;
 mod message_stream;
+mod meta_flake;
 mod notify_flows;
 mod otel_http_client;
 mod pulls;

swarm-controller/src/meta_flake.rs

@@ -0,0 +1,583 @@
+//! `internal/meta`: the swarm meta flake hives build their agents from.
+//!
+//! [`update`] renders `flake.nix` with `hive_meta_flake::render_flake`: hyperhive
+//! pinned to its forge `main`, each custom agent's config input pinned to its
+//! `agent-configs/<agent>` `main`, every agent placed on some hive. The swarm
+//! builder locks it (`swarm_wire::builder::LockMeta`), and the flake and the
+//! lock are committed together in one commit. A render equal to the committed
+//! `flake.nix` sends no lock and commits nothing.
+
+use std::collections::BTreeMap;
+use std::future::Future;
+
+use anyhow::{Context, Result, bail};
+use serde::Deserialize;
+use swarm_wire::agent_archetype::ArchetypeRef;
+use swarm_wire::builder::{LOCK_REPLY_DEADLINE, LOCK_SUBJECT, LockMeta, LockReply};
+
+use crate::forge::{CONFIG_ORG, DefaultBranchFiles};
+
+/// The repo in `internal` the flake is committed to.
+pub(crate) const META_REPO: &str = "meta";
+
+/// The env var holding [`Settings`] as JSON, set by `swarm-controller.nix`.
+pub(crate) const SETTINGS_ENV: &str = "SWARM_CONTROLLER_META_FLAKE";
+
+/// Where hyperhive itself lives on the forge.
+const HYPERHIVE_OWNER: &str = "hyperhive";
+const HYPERHIVE_REPO: &str = "hyperhive";
+
+/// What every agent of the swarm gets, the same on every hive.
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct Settings {
+    /// Values by `hive_meta_flake::FORWARDED_VARS` name. `HIVE_FORGE_URL` is
+    /// required.
+    vars: BTreeMap<String, String>,
+    /// The forge's `https` origin, which every pinned input is fetched from:
+    /// the swarm builder's credential helper answers only `https` for it.
+    git_base: String,
+    /// The swarm turned the agents' GitHub integration off.
+    github_disabled: bool,
+}
+
+impl Settings {
+    /// [`Settings`] from [`SETTINGS_ENV`], `None` when it is unset.
+    pub(crate) fn from_env() -> Result<Option<Self>> {
+        match std::env::var(SETTINGS_ENV) {
+            Ok(raw) => Self::parse(&raw).map(Some),
+            Err(std::env::VarError::NotPresent) => Ok(None),
+            Err(e) => Err(e).with_context(|| format!("read {SETTINGS_ENV}")),
+        }
+    }
+
+    fn parse(raw: &str) -> Result<Self> {
+        let settings: Self =
+            serde_json::from_str(raw).with_context(|| format!("parse {SETTINGS_ENV}"))?;
+        if let Some(unknown) = settings
+            .vars
+            .keys()
+            .find(|k| !hive_meta_flake::FORWARDED_VARS.contains(&k.as_str()))
+        {
+            bail!("{SETTINGS_ENV}: {unknown} is not a value the meta flake forwards");
+        }
+        if settings.vars.get("HIVE_FORGE_URL").is_none_or(String::is_empty) {
+            bail!("{SETTINGS_ENV}: HIVE_FORGE_URL is unset");
+        }
+        if !settings.git_base.starts_with("https://") {
+            bail!("{SETTINGS_ENV}: git_base must be an https origin");
+        }
+        Ok(settings)
+    }
+
+    /// The forwarded pairs with a value, in `FORWARDED_VARS` order.
+    fn forwarded(&self) -> Vec<(&'static str, String)> {
+        hive_meta_flake::FORWARDED_VARS
+            .iter()
+            .filter_map(|&name| {
+                self.vars
+                    .get(name)
+                    .filter(|v| !v.is_empty())
+                    .map(|v| (name, v.clone()))
+            })
+            .collect()
+    }
+
+    fn git_url(&self, owner: &str, repo: &str) -> String {
+        format!("{}/{owner}/{repo}.git", self.git_base.trim_end_matches('/'))
+    }
+}
+
+/// One agent some hive places, with the archetype it is built from.
+#[derive(Debug, Clone)]
+pub(crate) struct Placed {
+    pub(crate) name: String,
+    pub(crate) archetype: Option<ArchetypeRef>,
+}
+
+/// One file of a meta commit: a blob sha replaces that blob, `None` creates it.
+pub(crate) struct MetaFile {
+    pub(crate) path: &'static str,
+    pub(crate) bytes: Vec<u8>,
+    pub(crate) sha: Option<String>,
+}
+
+/// The forge, as [`update`] reads and writes it.
+pub(crate) trait MetaForge {
+    fn head_files(
+        &self,
+        owner: &str,
+        repo: &str,
+        paths: &[&str],
+    ) -> impl Future<Output = Result<Option<DefaultBranchFiles>>> + Send;
+    fn ensure_meta_repo(&self) -> impl Future<Output = Result<()>> + Send;
+    fn grant_meta_read(&self, login: &str) -> impl Future<Output = Result<()>> + Send;
+    fn commit_meta(
+        &self,
+        message: &str,
+        files: Vec<MetaFile>,
+    ) -> impl Future<Output = Result<()>> + Send;
+}
+
+/// Where a lock is sent: the queue in production, a fake in the tests. `Err`
+/// is a lock that never reached a builder or never came back.
+pub(crate) trait LockSink {
+    fn lock(&self, request: Vec<u8>) -> impl Future<Output = Result<LockReply, String>> + Send;
+}
+
+impl LockSink for async_nats::Client {
+    async fn lock(&self, request: Vec<u8>) -> Result<LockReply, String> {
+        use async_nats::RequestErrorKind;
+
+        let sent = async_nats::Request::new()
+            .payload(request.into())
+            .timeout(Some(LOCK_REPLY_DEADLINE));
+        match self.send_request(LOCK_SUBJECT, sent).await {
+            Ok(msg) => serde_json::from_slice(&msg.payload)
+                .map_err(|e| format!("the builder's lock reply did not decode: {e}")),
+            Err(e) => Err(match e.kind() {
+                RequestErrorKind::NoResponders => "no swarm builder is listening for locks: \
+                     deploy.builder.enable is set on no host, or its builder predates the lock \
+                     request"
+                    .to_owned(),
+                RequestErrorKind::TimedOut => format!(
+                    "the swarm builder sent no lock within {} minutes",
+                    LOCK_REPLY_DEADLINE.as_secs() / 60
+                ),
+                _ => format!("sending the lock to the swarm builder failed: {e}"),
+            }),
+        }
+    }
+}
+
+/// What [`update`] did.
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum Updated {
+    /// The committed flake already is the render.
+    Unchanged,
+    /// A new flake and lock were committed.
+    Committed,
+}
+
+/// Bring `internal/meta` up to the forge's current heads for `agents`, and
+/// give each of `hives`' forge accounts read on it.
+///
+/// # Errors
+/// A head that cannot be read, a lock the builder refuses, or a refused commit.
+/// A failed read grant is logged, not returned: the commit is what hives wait
+/// on.
+pub(crate) async fn update(
+    forge: &impl MetaForge,
+    locker: &impl LockSink,
+    settings: &Settings,
+    agents: &[Placed],
+    hives: &[String],
+) -> Result<Updated> {
+    forge.ensure_meta_repo().await?;
+    for hive in hives {
+        let login = swarm_secret_client::forge::hive_login(hive);
+        if let Err(e) = forge.grant_meta_read(&login).await {
+            tracing::warn!(%hive, error = %format!("{e:#}"), "meta flake: read grant on internal/meta failed");
+        }
+    }
+    let flake = render(forge, settings, agents).await?;
+    let committed = forge
+        .head_files(
+            crate::forge::objects::SHARED_ORG,
+            META_REPO,
+            &["flake.nix", "flake.lock"],
+        )
+        .await?;
+    let sha_of = |path: &str| {
+        committed
+            .as_ref()
+            .and_then(|c| c.files.iter().find(|f| f.path == path))
+    };
+    let flake_file = sha_of("flake.nix");
+    if flake_file.is_some_and(|f| f.bytes == flake.as_bytes()) && sha_of("flake.lock").is_some() {
+        return Ok(Updated::Unchanged);
+    }
+    let request = serde_json::to_vec(&LockMeta {
+        flake_nix: flake.clone(),
+    })
+    .context("encode the lock request")?;
+    let flake_lock = match locker.lock(request).await {
+        Ok(LockReply::Locked { flake_lock }) => flake_lock,
+        Ok(LockReply::Failed { error_tail }) => bail!("the builder could not lock: {error_tail}"),
+        Err(e) => bail!("{e}"),
+    };
+    forge
+        .commit_meta(
+            &format!("meta: pin {} agent(s)", agents.len()),
+            vec![
+                MetaFile {
+                    path: "flake.nix",
+                    bytes: flake.into_bytes(),
+                    sha: flake_file.map(|f| f.sha.clone()),
+                },
+                MetaFile {
+                    path: "flake.lock",
+                    bytes: flake_lock.into_bytes(),
+                    sha: sha_of("flake.lock").map(|f| f.sha.clone()),
+                },
+            ],
+        )
+        .await?;
+    Ok(Updated::Committed)
+}
+
+/// Whether a `push` webhook body moved hyperhive's `main`, which the flake
+/// pins.
+pub(crate) fn moves_hyperhive(body: &[u8]) -> bool {
+    let Ok(v) = serde_json::from_slice::<serde_json::Value>(body) else {
+        return false;
+    };
+    let repo = v
+        .get("repository")
+        .and_then(|r| r.get("full_name"))
+        .and_then(serde_json::Value::as_str);
+    let git_ref = v.get("ref").and_then(serde_json::Value::as_str);
+    repo == Some(&format!("{HYPERHIVE_OWNER}/{HYPERHIVE_REPO}"))
+        && git_ref == Some("refs/heads/main")
+}
+
+/// The `flake.nix` for `agents` at the forge's current heads.
+async fn render(forge: &impl MetaForge, settings: &Settings, agents: &[Placed]) -> Result<String> {
+    let hyperhive = forge
+        .head_files(HYPERHIVE_OWNER, HYPERHIVE_REPO, &[])
+        .await?
+        .with_context(|| format!("{HYPERHIVE_OWNER}/{HYPERHIVE_REPO} is empty or absent"))?;
+    let hyperhive_url = hive_meta_flake::pinned_git_url(
+        &settings.git_url(HYPERHIVE_OWNER, HYPERHIVE_REPO),
+        Some(&hyperhive.commit),
+    )?;
+    let mut specs = Vec::with_capacity(agents.len());
+    for agent in agents {
+        let pin = if agent.archetype.is_some() {
+            None
+        } else {
+            let head = forge
+                .head_files(CONFIG_ORG, &agent.name, &["flake.lock"])
+                .await?
+                .with_context(|| {
+                    format!(
+                        "agent {}: {CONFIG_ORG}/{} has no commit on the forge",
+                        agent.name, agent.name
+                    )
+                })?;
+            let lock = head
+                .files
+                .first()
+                .map(|f| String::from_utf8_lossy(&f.bytes).into_owned());
+            Some(hive_meta_flake::AgentPin {
+                url: hive_meta_flake::pinned_git_url(
+                    &settings.git_url(CONFIG_ORG, &agent.name),
+                    Some(&head.commit),
+                )?,
+                canonical_inputs: lock
+                    .as_deref()
+                    .map(hive_meta_flake::canonical_inputs_in_lock)
+                    .unwrap_or_default(),
+            })
+        };
+        specs.push(hive_meta_flake::AgentSpec {
+            name: agent.name.clone(),
+            port: hive_meta_flake::agent_web_port(&agent.name),
+            archetype: agent.archetype.clone(),
+            pin,
+        });
+    }
+    specs.sort_by(|a, b| a.name.cmp(&b.name));
+    hive_meta_flake::render_flake(
+        &hyperhive_url,
+        &specs,
+        &settings.forwarded(),
+        settings.github_disabled,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Mutex;
+
+    use super::*;
+    use crate::forge::BranchFile;
+
+    const FORGE: &str = "https://forge.t.local";
+
+    fn settings() -> Settings {
+        Settings::parse(&format!(
+            r#"{{"vars":{{"HIVE_FORGE_URL":"http://forge.t.local","HYPERHIVE_SWARM_NAME":"t"}},"git_base":"{FORGE}/","github_disabled":true}}"#
+        ))
+        .unwrap()
+    }
+
+    fn head(commit: &str, files: &[(&str, &str)]) -> DefaultBranchFiles {
+        DefaultBranchFiles {
+            branch: "main".to_owned(),
+            commit: commit.to_owned(),
+            web_url: None,
+            files: files
+                .iter()
+                .map(|(path, body)| BranchFile {
+                    path: (*path).to_owned(),
+                    sha: format!("blob-{path}"),
+                    bytes: body.as_bytes().to_vec(),
+                })
+                .collect(),
+        }
+    }
+
+    /// One recorded commit: its message, and each file's path, text and sha.
+    type Commit = (String, Vec<(&'static str, String, Option<String>)>);
+
+    #[derive(Default)]
+    struct FakeForge {
+        repos: BTreeMap<String, DefaultBranchFiles>,
+        commits: Mutex<Vec<Commit>>,
+        grants: Mutex<Vec<String>>,
+    }
+
+    impl MetaForge for FakeForge {
+        async fn head_files(
+            &self,
+            owner: &str,
+            repo: &str,
+            paths: &[&str],
+        ) -> Result<Option<DefaultBranchFiles>> {
+            Ok(self.repos.get(&format!("{owner}/{repo}")).map(|h| {
+                let mut h = h.clone();
+                h.files.retain(|f| paths.contains(&f.path.as_str()));
+                h
+            }))
+        }
+        async fn ensure_meta_repo(&self) -> Result<()> {
+            Ok(())
+        }
+        async fn grant_meta_read(&self, login: &str) -> Result<()> {
+            self.grants.lock().unwrap().push(login.to_owned());
+            Ok(())
+        }
+        async fn commit_meta(&self, message: &str, files: Vec<MetaFile>) -> Result<()> {
+            self.commits.lock().unwrap().push((
+                message.to_owned(),
+                files
+                    .into_iter()
+                    .map(|f| (f.path, String::from_utf8(f.bytes).unwrap(), f.sha))
+                    .collect(),
+            ));
+            Ok(())
+        }
+    }
+
+    /// Answers every lock with `reply`, recording each flake it was sent.
+    struct FakeLocker {
+        reply: Result<LockReply, String>,
+        sent: Mutex<Vec<String>>,
+    }
+
+    impl FakeLocker {
+        fn locked() -> Self {
+            Self {
+                reply: Ok(LockReply::Locked {
+                    flake_lock: "{\"version\":7}".to_owned(),
+                }),
+                sent: Mutex::default(),
+            }
+        }
+    }
+
+    impl LockSink for FakeLocker {
+        async fn lock(&self, request: Vec<u8>) -> Result<LockReply, String> {
+            let req: LockMeta = serde_json::from_slice(&request).unwrap();
+            self.sent.lock().unwrap().push(req.flake_nix);
+            self.reply.clone()
+        }
+    }
+
+    fn forge_with(extra: &[(&str, DefaultBranchFiles)]) -> FakeForge {
+        let mut repos = BTreeMap::new();
+        repos.insert("hyperhive/hyperhive".to_owned(), head("hh0", &[]));
+        repos.insert(
+            "agent-configs/argus".to_owned(),
+            head(
+                "a1",
+                &[(
+                    "flake.lock",
+                    r#"{"root":"root","nodes":{"root":{"inputs":{"nixpkgs":"nixpkgs"}}}}"#,
+                )],
+            ),
+        );
+        for (name, h) in extra {
+            repos.insert((*name).to_owned(), h.clone());
+        }
+        FakeForge {
+            repos,
+            ..FakeForge::default()
+        }
+    }
+
+    fn argus_and_reviewer() -> Vec<Placed> {
+        vec![
+            Placed {
+                name: "argus".to_owned(),
+                archetype: None,
+            },
+            Placed {
+                name: "rev".to_owned(),
+                archetype: Some(ArchetypeRef {
+                    source: "hyperhive".to_owned(),
+                    name: "reviewer".to_owned(),
+                }),
+            },
+        ]
+    }
+
+    #[tokio::test]
+    async fn a_new_render_is_locked_and_committed_with_its_lock_in_one_commit() {
+        let forge = forge_with(&[]);
+        let locker = FakeLocker::locked();
+        let got = update(
+            &forge,
+            &locker,
+            &settings(),
+            &argus_and_reviewer(),
+            &["pr1ma".to_owned()],
+        )
+        .await
+        .unwrap();
+        assert_eq!(got, Updated::Committed);
+        let sent = locker.sent.lock().unwrap();
+        let [flake] = sent.as_slice() else {
+            panic!("one lock: {sent:?}")
+        };
+        assert!(flake.contains(&format!(
+            "hyperhive.url = \"git+{FORGE}/hyperhive/hyperhive.git?rev=hh0\";"
+        )));
+        assert!(flake.contains(&format!(
+            "agent-argus.url = \"git+{FORGE}/agent-configs/argus.git?rev=a1\";"
+        )));
+        assert!(flake.contains("agent-argus.inputs.nixpkgs.follows = \"nixpkgs\";"));
+        assert!(!flake.contains("agent-rev.url"), "an archetype agent has no input");
+        assert!(flake.contains("services.hyperhive.agent.github.enable = false;"));
+        assert!(flake.contains("HYPERHIVE_SWARM_NAME = \"t\";"));
+        let commits = forge.commits.lock().unwrap();
+        let [(_, files)] = commits.as_slice() else {
+            panic!("one commit: {commits:?}")
+        };
+        assert_eq!(
+            files,
+            &vec![
+                ("flake.nix", flake.clone(), None),
+                ("flake.lock", "{\"version\":7}".to_owned(), None),
+            ]
+        );
+        assert_eq!(
+            *forge.grants.lock().unwrap(),
+            vec![swarm_secret_client::forge::hive_login("pr1ma")]
+        );
+    }
+
+    #[tokio::test]
+    async fn an_unchanged_render_sends_no_lock_and_commits_nothing() {
+        let first = forge_with(&[]);
+        let locker = FakeLocker::locked();
+        update(&first, &locker, &settings(), &argus_and_reviewer(), &[])
+            .await
+            .unwrap();
+        let flake = locker.sent.lock().unwrap()[0].clone();
+        let forge = forge_with(&[(
+            "internal/meta",
+            head("m1", &[("flake.nix", &flake), ("flake.lock", "{}")]),
+        )]);
+        let locker = FakeLocker::locked();
+        let got = update(&forge, &locker, &settings(), &argus_and_reviewer(), &[])
+            .await
+            .unwrap();
+        assert_eq!(got, Updated::Unchanged);
+        assert!(locker.sent.lock().unwrap().is_empty());
+        assert!(forge.commits.lock().unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn a_moved_head_replaces_both_committed_blobs() {
+        let forge = forge_with(&[(
+            "internal/meta",
+            head("m1", &[("flake.nix", "old"), ("flake.lock", "{}")]),
+        )]);
+        let locker = FakeLocker::locked();
+        update(&forge, &locker, &settings(), &argus_and_reviewer(), &[])
+            .await
+            .unwrap();
+        let commits = forge.commits.lock().unwrap();
+        let shas: Vec<_> = commits[0].1.iter().map(|(_, _, sha)| sha.clone()).collect();
+        assert_eq!(
+            shas,
+            vec![
+                Some("blob-flake.nix".to_owned()),
+                Some("blob-flake.lock".to_owned())
+            ]
+        );
+    }
+
+    #[tokio::test]
+    async fn a_refused_lock_commits_nothing() {
+        let forge = forge_with(&[]);
+        let locker = FakeLocker {
+            reply: Ok(LockReply::Failed {
+                error_tail: "no such input".to_owned(),
+            }),
+            sent: Mutex::default(),
+        };
+        let err = update(&forge, &locker, &settings(), &argus_and_reviewer(), &[])
+            .await
+            .unwrap_err();
+        assert!(format!("{err:#}").contains("no such input"));
+        assert!(forge.commits.lock().unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn a_custom_agent_without_a_config_commit_fails_the_render() {
+        let forge = forge_with(&[]);
+        let agents = vec![Placed {
+            name: "ghost".to_owned(),
+            archetype: None,
+        }];
+        let err = update(&forge, &FakeLocker::locked(), &settings(), &agents, &[])
+            .await
+            .unwrap_err();
+        assert!(format!("{err:#}").contains("agent-configs/ghost"));
+        assert!(forge.commits.lock().unwrap().is_empty());
+    }
+
+    #[test]
+    fn only_a_push_to_hyperhive_main_moves_the_pin() {
+        let push = |repo: &str, git_ref: &str| {
+            serde_json::to_vec(&serde_json::json!({
+                "ref": git_ref,
+                "repository": { "full_name": repo },
+                "commits": [],
+            }))
+            .unwrap()
+        };
+        assert!(moves_hyperhive(&push("hyperhive/hyperhive", "refs/heads/main")));
+        assert!(!moves_hyperhive(&push("hyperhive/hyperhive", "refs/heads/x")));
+        assert!(!moves_hyperhive(&push("agents/foo", "refs/heads/main")));
+        assert!(!moves_hyperhive(b"not json"));
+    }
+
+    #[test]
+    fn settings_refuse_an_unknown_value_a_missing_forge_and_a_plain_http_base() {
+        let parse = |vars: &str, git_base: &str| {
+            Settings::parse(&format!(
+                r#"{{"vars":{vars},"git_base":"{git_base}","github_disabled":false}}"#
+            ))
+        };
+        assert!(parse(r#"{"HIVE_FORGE_URL":"x"}"#, "https://f").is_ok());
+        let unknown = parse(r#"{"HIVE_FORGE_URL":"x","HIVE_AGENT_NATS_URL":"y"}"#, "https://f")
+            .unwrap_err();
+        assert!(format!("{unknown:#}").contains("HIVE_AGENT_NATS_URL"));
+        assert!(parse("{}", "https://f").is_err());
+        assert!(parse(r#"{"HIVE_FORGE_URL":"x"}"#, "http://f").is_err());
+    }
+}

swarm-controller/src/routes/agents.rs

@@ -221,7 +221,7 @@ pub(crate) async fn set_agent_state(
 ///
 /// Exhaustive for the reason [`revokes_queue_credential`] is. A `Destroyed`
 /// agent is gone from that hive, and the hive deploys nothing for it.
-fn places_agent(state: swarm_wire::wanted::AgentState) -> bool {
+pub(crate) fn places_agent(state: swarm_wire::wanted::AgentState) -> bool {
     use swarm_wire::wanted::AgentState;
     match state {
         AgentState::Up | AgentState::Offline | AgentState::Paused => true,

swarm-controller/src/startup.rs

@@ -285,6 +285,11 @@ fn spawn_forge_workers(
     auth: Option<Arc<auth::AuthBridge>>,
     queue: Option<async_nats::Client>,
 ) {
+    // `internal/meta` appears, and catches up with every head that moved while
+    // this daemon was down.
+    if let Err(e) = crate::jobq::queue_meta_update(jobq) {
+        tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed");
+    }
     forge::objects::spawn(Arc::clone(&client));
     if let (Some(bridge), Some(queue)) = (auth, queue) {
         forge::operator_profiles::spawn(Arc::clone(&client), bridge, queue);

swarm-controller/src/webhook.rs

@@ -443,6 +443,11 @@ pub(super) async fn post_webhook_forge(
             }
         }
         DeliveryKind::VcsActivity => {
+            if crate::meta_flake::moves_hyperhive(&body)
+                && let Err(e) = crate::jobq::queue_meta_update(&state.jobq)
+            {
+                tracing::warn!(error = %format!("{e:#}"), "webhook: queueing update_meta failed");
+            }
             if let Some(activity) = crate::vcs_metrics::parse_push(&body) {
                 crate::vcs_metrics::record_push(&activity);
             } else {