Watch
0
0
Fork
You've already forked hyperhive
0

build agents from internal/meta when it pins their config

A rebuild's MetaSync resolves the head of the swarm's internal/meta once and
records it; the prebuild and the swap build that flake instead of the hive's
own, and a first deploy does the same in Create. hive-priv takes the commit as
a validated meta_rev on UpdateContainer and CreateContainer and builds from its
own forge URL (HIVE_PRIV_FORGE_URL).

The head is used only when its flake lists the agent at the config commit the
forge's main is at; otherwise, and when the repo has no commit or this hive has
no core token file for nix's git, the hive renders locally as before.

hive-c0re's and hive-priv's gitconfig now share the forge credential helper and
rewrite the forge's https URLs to http, which hive-priv lacked.
This commit is contained in:
atlas 2026-10-24 14:59:43 +02:00
commit 7c0a622d41
19 changed files with 831 additions and 64 deletions

View file

@ -0,0 +1,108 @@
//! This hive's reads of `internal/meta` and of an agent's config head, as
//! [`crate::meta_rev::MetaSource`], under the hive's forge read token.
use forgejo_api::structs::RepoGetRawFileQuery;
use forgejo_api::{ApiErrorKind, Forgejo, ForgejoError};
use reqwest::StatusCode;
use super::reader::ReaderToken;
use super::{CONFIG_ORG, bounded};
use crate::meta_rev::MetaSource;
/// The org and repo the swarm commits its meta flake to.
const META_OWNER: &str = "internal";
const META_REPO: &str = "meta";
/// The branch an agent's config is deployed from, which `fetch_forge_main`
/// and the swarm controller's pins both read.
const CONFIG_BRANCH: &str = "main";
/// [`MetaSource`] over the forge API.
pub(crate) struct ForgeSource {
api: Forgejo,
}
/// A source reading as this hive, or `None` when it holds no forge token.
pub(crate) async fn internal_meta_source() -> Option<ForgeSource> {
let token: ReaderToken = super::reader_token().await?;
match token.api() {
Ok(api) => Some(ForgeSource { api }),
Err(e) => {
tracing::warn!(error = ?e, "internal/meta: cannot build a forge client");
None
}
}
}
fn is_not_found(e: &ForgejoError) -> bool {
match e {
ForgejoError::ApiError(api) => match api.error_kind() {
ApiErrorKind::NotFound { .. } => true,
ApiErrorKind::Other(s) => *s == StatusCode::NOT_FOUND,
_ => false,
},
ForgejoError::UnexpectedStatusCode(s) => *s == StatusCode::NOT_FOUND,
_ => false,
}
}
impl ForgeSource {
/// The head commit of `branch` in `owner/repo`; `None` when the repo or
/// the branch is not there.
async fn branch_head(
&self,
owner: &str,
repo: &str,
branch: &str,
) -> anyhow::Result<Option<String>> {
let op = format!("read branch {branch} of {owner}/{repo}");
match bounded(&op, self.api.repo_get_branch(owner, repo, branch)).await? {
Ok(branch) => Ok(branch.commit.and_then(|c| c.id)),
Err(e) if is_not_found(&e) => Ok(None),
Err(e) => Err(anyhow::Error::new(e).context(op)),
}
}
}
impl MetaSource for ForgeSource {
fn nix_can_fetch(&self) -> bool {
super::core_token().is_some()
}
async fn meta_head(&self) -> anyhow::Result<Option<String>> {
use anyhow::Context as _;
let op = format!("read repo {META_OWNER}/{META_REPO}");
let repo = match bounded(&op, self.api.repo_get(META_OWNER, META_REPO)).await? {
Ok(repo) => repo,
Err(e) if is_not_found(&e) => return Ok(None),
Err(e) => return Err(anyhow::Error::new(e).context(op)),
};
if repo.empty.unwrap_or(false) {
return Ok(None);
}
let branch = repo
.default_branch
.with_context(|| format!("{META_OWNER}/{META_REPO} names no default branch"))?;
self.branch_head(META_OWNER, META_REPO, &branch).await
}
async fn flake_at(&self, rev: &str) -> anyhow::Result<String> {
let op = format!("read flake.nix of {META_OWNER}/{META_REPO} at {rev}");
let query = RepoGetRawFileQuery {
r#ref: Some(rev.to_owned()),
};
let raw = bounded(
&op,
self.api
.repo_get_raw_file(META_OWNER, META_REPO, "flake.nix", query),
)
.await?
.map_err(|e| anyhow::Error::new(e).context(op))?;
Ok(String::from_utf8_lossy(&raw).into_owned())
}
async fn config_head(&self, agent: &str) -> anyhow::Result<Option<String>> {
self.branch_head(CONFIG_ORG, agent, CONFIG_BRANCH).await
}
}

View file

@ -5,11 +5,13 @@
//! `internal/docs`, are swarm-controller's. Full design:
//! `docs/integrations/forge.md`.
mod internal_meta;
mod reader;
mod reconcile;
mod repos;
mod users;
pub(crate) use internal_meta::internal_meta_source;
pub use reader::reader_token;
pub(crate) use reconcile::FORGE_MAIN_REF;
pub use reconcile::{fetch_forge_main, reconcile_config_apply, reconcile_config_status};

View file

@ -34,6 +34,11 @@ impl ReaderToken {
pub fn auth_header(&self) -> String {
basic_auth_header(&self.login, &self.value)
}
/// A forge API client acting as this token's account.
pub(crate) fn api(&self) -> anyhow::Result<forgejo_api::Forgejo> {
super::api(&self.value)
}
}
/// The token to read the forge with, or `None` when this hive has neither.

View file

@ -412,6 +412,11 @@ async fn run_meta_sync(coord: &Arc<Coordinator>, name: &str, relock: bool) -> Re
if relock && crate::archetypes::of(name)?.is_none() {
crate::meta::lock_update_for_rebuild(name).await?;
}
// The commit of the swarm's `internal/meta` that `Prebuild` and `Swap`
// build this agent from, resolved once here so both build the same flake;
// none leaves them on the hive's own flake rendered above.
let meta_rev = crate::meta_rev::resolve(name).await;
crate::meta_rev::record(name, meta_rev.as_deref())?;
Ok(())
}
@ -428,7 +433,10 @@ async fn run_prebuild(name: &str, id: NodeId) -> Result<()> {
// stopped agent has no uptime to preserve, so skip the (expensive)
// eval and let the downstream `Swap` build inline.
if crate::lifecycle::is_running(name).await {
let flake_ref = format!("{}#{name}", crate::paths::meta_root().display());
let flake_ref = match crate::meta_rev::recorded(name) {
Some(rev) => crate::meta_rev::flake_ref(name, &rev),
None => format!("{}#{name}", crate::paths::meta_root().display()),
};
crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(id.get())).await?;
}
Ok(())
@ -445,7 +453,10 @@ async fn run_swap(coord: &Arc<Coordinator>, name: &str, id: NodeId) -> Result<()
let agent_dir = crate::paths::agent_runtime_dir(name);
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir)?;
let result = crate::lifecycle::swap_update(name, &hive, &paths, Some(id.get())).await;
let meta_rev = crate::meta_rev::recorded(name);
let result =
crate::lifecycle::swap_update(name, &hive, &paths, meta_rev.as_deref(), Some(id.get()))
.await;
// On success the Ok-only bookkeeping tail (rev marker, forge/matrix
// sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node,
// which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded
@ -512,7 +523,8 @@ async fn run_provision(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
/// `Reconcile` (`converge_start_preamble` + `register_agent`) so this
/// node stays purely "create", not "create + start".
async fn run_create(name: &str) -> Result<()> {
crate::lifecycle::create_only(name).await?;
let meta_rev = crate::meta_rev::resolve(name).await;
crate::lifecycle::create_only(name, meta_rev.as_deref()).await?;
Ok(())
}

View file

@ -237,8 +237,10 @@ pub async fn provision_container(name: &str, hive: &HiveEnv, paths: &AgentPaths)
/// The `nixos-container create` proper — the job queue's `Create` node.
/// Assumes `provision_container` already registered the agent in meta.
pub async fn create_only(name: &str) -> Result<()> {
priv_run("create", name).await
/// `meta_rev` is the `internal/meta` commit to build from; none builds from the
/// hive's own meta flake (`crate::meta_rev`).
pub async fn create_only(name: &str, meta_rev: Option<&str>) -> Result<()> {
priv_run_inner("create", name, meta_rev, None).await
}
/// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on
@ -290,15 +292,17 @@ pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()>
/// drop-ins, then `nixos-container update`. The job queue's `Swap`
/// node. Requires the container stopped (the queue's `StopForUpdate`
/// upstream); does NOT start it — the DAG's tail `Reconcile` owns
/// bringing the agent back to its wanted power state.
/// bringing the agent back to its wanted power state. `meta_rev` is as for
/// [`create_only`].
pub async fn swap_update(
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
meta_rev: Option<&str>,
node_id: Option<u64>,
) -> Result<()> {
write_dropins(name, hive, paths).await?;
priv_run_inner("update", name, node_id).await
priv_run_inner("update", name, meta_rev, node_id).await
}
/// Build the `AgentSpec` list for the meta flake from `nixos-container
@ -972,7 +976,7 @@ fn make_log_callback(
/// is appended to the build-log row as it arrives, so the dashboard
/// shows live progress during long `nixos-container create` / `update` runs.
async fn priv_run(kind: &str, name: &str) -> Result<()> {
priv_run_inner(kind, name, None).await
priv_run_inner(kind, name, None, None).await
}
/// Like `priv_run` but stamps `node_id` onto the build-log row it opens, so
@ -982,7 +986,12 @@ async fn priv_run(kind: &str, name: &str) -> Result<()> {
/// This used to be a `Fn(i64)` callback that handed the row id *back* to the
/// queue, which then held it in a side map. The row carries the link itself
/// now, so the id only ever travels one way.
async fn priv_run_inner(kind: &str, name: &str, node_id: Option<u64>) -> Result<()> {
async fn priv_run_inner(
kind: &str,
name: &str,
meta_rev: Option<&str>,
node_id: Option<u64>,
) -> Result<()> {
let container = container_name(name);
let cmdline = format!("nixos-container {kind} {container}");
@ -1001,6 +1010,7 @@ async fn priv_run_inner(kind: &str, name: &str, node_id: Option<u64>) -> Result<
"create" => {
crate::priv_client::create_container_streaming(
name,
meta_rev,
make_log_callback(logs.clone(), log_id, cmdline.clone()),
)
.await
@ -1008,6 +1018,7 @@ async fn priv_run_inner(kind: &str, name: &str, node_id: Option<u64>) -> Result<
"update" => {
crate::priv_client::update_container_streaming(
name,
meta_rev,
make_log_callback(logs.clone(), log_id, cmdline.clone()),
)
.await

View file

@ -26,6 +26,7 @@ mod lifecycle;
mod loose_ends;
mod matrix;
mod meta;
mod meta_rev;
mod migrate;
mod paths;
mod priv_client;

325
hive-c0re/src/meta_rev.rs Normal file
View file

@ -0,0 +1,325 @@
//! The `internal/meta` commit a hive builds an agent from.
//!
//! `internal/meta` is the swarm's meta flake, committed by swarm-controller.
//! A rebuild's `MetaSync` node resolves its head once ([`resolve`]) and
//! records it ([`record`]); `Prebuild` and `Swap` read the record
//! ([`recorded`]), so both build the same flake. A create resolves and uses
//! the rev directly. No rev, whether because the repo or its commit is absent
//! or because the commit does not pin this agent's current config, leaves the
//! build on the hive's own meta flake.
use anyhow::{Context, Result};
use hive_priv_sock::{meta_flake_url, valid_meta_rev};
/// What the forge says about `internal/meta` and an agent's config repo.
pub(crate) trait MetaSource {
/// Whether nix's own `git` can authenticate to the forge: it reads as
/// `core`, from the token file, whatever token this hive reads with.
fn nix_can_fetch(&self) -> bool;
/// The default branch's head commit of `internal/meta`; `None` when the
/// repo is absent or has no commit.
async fn meta_head(&self) -> Result<Option<String>>;
/// `internal/meta`'s `flake.nix` at `rev`.
async fn flake_at(&self, rev: &str) -> Result<String>;
/// The head of `agent-configs/<agent>`'s `main`; `None` when it has none.
async fn config_head(&self, agent: &str) -> Result<Option<String>>;
}
/// The `internal/meta` commit to build `agent` from, or `None` for the hive's
/// own flake.
///
/// `internal/meta` is private, so a hive whose nix cannot authenticate
/// ([`MetaSource::nix_can_fetch`]) builds from its own flake. The head is
/// taken only when its `flake.nix` builds the agent from the
/// config commit the forge's `main` is at now (`archetype`: when it lists the
/// agent at all). A head that is behind (the swarm's update has not run, or
/// failed) or lacks the agent would build an older config than the one being
/// deployed.
///
/// # Errors
/// A forge read that failed.
pub(crate) async fn choose(
source: &impl MetaSource,
agent: &str,
archetype: bool,
) -> Result<Option<String>> {
if !source.nix_can_fetch() {
return Ok(None);
}
let Some(rev) = source.meta_head().await? else {
return Ok(None);
};
if !valid_meta_rev(&rev) {
return Ok(None);
}
let flake = source.flake_at(&rev).await?;
let config_head = if archetype {
None
} else {
let Some(head) = source.config_head(agent).await? else {
return Ok(None);
};
Some(head)
};
Ok(hive_meta_flake::builds_agent_at(&flake, agent, config_head.as_deref()).then_some(rev))
}
/// [`choose`] against the forge this hive reads. `None`, logged, whenever the
/// hive cannot tell: no forge credential, a failed read, no usable commit.
pub(crate) async fn resolve(agent: &str) -> Option<String> {
let archetype = match crate::archetypes::of(agent) {
Ok(archetype) => archetype.is_some(),
Err(e) => {
tracing::warn!(%agent, error = ?e, "internal/meta: cannot read the agent's archetype; building from the hive's own flake");
return None;
}
};
let Some(source) = crate::forge::internal_meta_source().await else {
tracing::info!(%agent, "internal/meta: no forge read token; building from the hive's own flake");
return None;
};
match choose(&source, agent, archetype).await {
Ok(Some(rev)) => {
tracing::info!(%agent, %rev, "internal/meta: building from the swarm's meta flake");
Some(rev)
}
Ok(None) => {
tracing::info!(%agent, "internal/meta: absent or not pinning this agent's config; building from the hive's own flake");
None
}
Err(e) => {
tracing::warn!(%agent, error = %format!("{e:#}"), "internal/meta: cannot read it; building from the hive's own flake");
None
}
}
}
/// The record file in `agent`'s runtime dir, which a reboot clears, so a
/// record never outlives the boot it was made in.
const RECORD_FILE: &str = "meta-rev";
/// Record `rev` as the commit `agent`'s build reads, or clear the record for
/// `None`.
///
/// # Errors
/// The record file could not be written or removed.
pub(crate) fn record(agent: &str, rev: Option<&str>) -> Result<()> {
record_in(&crate::paths::agent_runtime_dir(agent), rev)
}
/// The commit [`record`] stored for `agent`, if it is a valid one.
pub(crate) fn recorded(agent: &str) -> Option<String> {
recorded_in(&crate::paths::agent_runtime_dir(agent))
}
fn record_in(dir: &std::path::Path, rev: Option<&str>) -> Result<()> {
let path = dir.join(RECORD_FILE);
match rev {
Some(rev) => {
std::fs::write(&path, rev).with_context(|| format!("write {}", path.display()))
}
None => match std::fs::remove_file(&path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => {
Err(e).with_context(|| format!("remove {}", path.display()))
}
_ => Ok(()),
},
}
}
fn recorded_in(dir: &std::path::Path) -> Option<String> {
let rev = std::fs::read_to_string(dir.join(RECORD_FILE)).ok()?;
valid_meta_rev(rev.trim()).then(|| rev.trim().to_owned())
}
/// The `<flake>#<agent>` ref [`crate::lifecycle::prebuild_toplevel`] builds
/// for `rev`: the same flake hive-priv builds, from the same forge URL.
pub(crate) fn flake_ref(agent: &str, rev: &str) -> String {
flake_ref_at(crate::forge::forge_http_base(), agent, rev)
}
fn flake_ref_at(forge_base: &str, agent: &str, rev: &str) -> String {
format!("{}#{agent}", meta_flake_url(forge_base, rev))
}
#[cfg(test)]
mod tests {
use super::*;
const HYPERHIVE: &str = "1111111111111111111111111111111111111111";
const META: &str = "3333333333333333333333333333333333333333";
const ARGUS_HEAD: &str = "2222222222222222222222222222222222222222";
const STALE: &str = "4444444444444444444444444444444444444444";
/// A forge holding `internal/meta` at [`META`] with the given flake, and
/// `agent-configs/argus` at `config_head`.
struct Forge {
head: Option<&'static str>,
flake: String,
config_head: Option<&'static str>,
fetchable: bool,
}
impl MetaSource for Forge {
fn nix_can_fetch(&self) -> bool {
self.fetchable
}
async fn meta_head(&self) -> Result<Option<String>> {
Ok(self.head.map(str::to_owned))
}
async fn flake_at(&self, rev: &str) -> Result<String> {
assert_eq!(Some(rev), self.head, "the flake is read at the head");
Ok(self.flake.clone())
}
async fn config_head(&self, agent: &str) -> Result<Option<String>> {
assert_eq!(agent, "argus");
Ok(self.config_head.map(str::to_owned))
}
}
/// `internal/meta`'s flake as swarm-controller renders it: argus pinned at
/// [`ARGUS_HEAD`], and a reviewer from an archetype.
fn flake() -> String {
use swarm_wire::agent_archetype::ArchetypeRef;
let url = |repo: &str, rev: &str| {
hive_meta_flake::pinned_git_url(&format!("https://f.t/{repo}.git"), Some(rev)).unwrap()
};
let spec = |name: &str, archetype, pin| hive_meta_flake::AgentSpec {
name: name.to_owned(),
port: hive_meta_flake::agent_web_port(name),
archetype,
pin,
};
hive_meta_flake::render_flake(
&url("hyperhive/hyperhive", HYPERHIVE),
&[
spec(
"argus",
None,
Some(hive_meta_flake::AgentPin {
url: url("agent-configs/argus", ARGUS_HEAD),
canonical_inputs: vec![],
}),
),
spec(
"rev",
Some(ArchetypeRef {
source: "hyperhive".to_owned(),
name: "reviewer".to_owned(),
}),
None,
),
],
&[],
false,
)
.unwrap()
}
fn forge(config_head: Option<&'static str>) -> Forge {
Forge {
head: Some(META),
flake: flake(),
config_head,
fetchable: true,
}
}
#[tokio::test]
async fn the_head_is_taken_when_it_pins_the_agents_current_config() {
assert_eq!(
choose(&forge(Some(ARGUS_HEAD)), "argus", false)
.await
.unwrap(),
Some(META.to_owned())
);
}
#[tokio::test]
async fn a_head_behind_the_agents_config_is_not_taken() {
assert_eq!(
choose(&forge(Some(STALE)), "argus", false).await.unwrap(),
None
);
}
#[tokio::test]
async fn a_head_without_the_agent_is_not_taken() {
let mut source = forge(Some(ARGUS_HEAD));
source.flake = source.flake.replace("argus", "someone");
assert_eq!(choose(&source, "argus", false).await.unwrap(), None);
}
#[tokio::test]
async fn an_archetype_agent_needs_only_to_be_listed() {
struct Archetypes(Forge);
impl MetaSource for Archetypes {
fn nix_can_fetch(&self) -> bool {
self.0.nix_can_fetch()
}
async fn meta_head(&self) -> Result<Option<String>> {
self.0.meta_head().await
}
async fn flake_at(&self, rev: &str) -> Result<String> {
self.0.flake_at(rev).await
}
async fn config_head(&self, _: &str) -> Result<Option<String>> {
panic!("an archetype agent has no config repo to read")
}
}
let source = Archetypes(forge(None));
assert_eq!(
choose(&source, "rev", true).await.unwrap(),
Some(META.to_owned())
);
assert_eq!(choose(&source, "ghost", true).await.unwrap(), None);
}
/// A hive without the credential nix's git reads builds from its own flake
/// even though its own token reads the repo; the first test is the control.
#[tokio::test]
async fn a_hive_whose_nix_cannot_authenticate_builds_from_its_own_flake() {
let mut source = forge(Some(ARGUS_HEAD));
source.fetchable = false;
assert_eq!(choose(&source, "argus", false).await.unwrap(), None);
}
#[tokio::test]
async fn no_repo_or_no_commit_is_the_hives_own_flake() {
let mut source = forge(Some(ARGUS_HEAD));
source.head = None;
assert_eq!(choose(&source, "argus", false).await.unwrap(), None);
}
#[tokio::test]
async fn a_custom_agent_with_no_config_commit_is_the_hives_own_flake() {
assert_eq!(choose(&forge(None), "argus", false).await.unwrap(), None);
}
#[test]
fn a_record_is_read_back_and_cleared() {
let dir = tempfile::tempdir().unwrap();
assert_eq!(recorded_in(dir.path()), None);
record_in(dir.path(), Some(META)).unwrap();
assert_eq!(recorded_in(dir.path()), Some(META.to_owned()));
record_in(dir.path(), None).unwrap();
assert_eq!(recorded_in(dir.path()), None);
record_in(dir.path(), None).unwrap();
}
#[test]
fn a_record_that_is_not_a_commit_id_is_no_record() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(RECORD_FILE), "main\n").unwrap();
assert_eq!(recorded_in(dir.path()), None);
}
#[test]
fn the_prebuild_ref_is_the_flake_hive_priv_builds() {
assert_eq!(
flake_ref_at("http://forge.t.local", "argus", META),
format!("git+http://forge.t.local/internal/meta.git?rev={META}#argus")
);
}
}

View file

@ -235,12 +235,14 @@ pub async fn kill_container(name: &str) -> Result<()> {
/// appending lines to `build_logs` or otherwise capturing the output.
pub async fn update_container_streaming(
name: &str,
meta_rev: Option<&str>,
on_line: impl FnMut(PrivStream, &str),
) -> Result<()> {
ok(call_streaming(
&PrivRequest::UpdateContainer {
name: name.to_owned(),
stream: true,
meta_rev: meta_rev.map(str::to_owned),
},
on_line,
)
@ -251,12 +253,14 @@ pub async fn update_container_streaming(
/// arrive. Returns `Ok(())` on success.
pub async fn create_container_streaming(
name: &str,
meta_rev: Option<&str>,
on_line: impl FnMut(PrivStream, &str),
) -> Result<()> {
ok(call_streaming(
&PrivRequest::CreateContainer {
name: name.to_owned(),
stream: true,
meta_rev: meta_rev.map(str::to_owned),
},
on_line,
)