diff --git a/AGENTS.md b/AGENTS.md index 80dc74c39..62b2c7b21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,9 @@ Per-hive (host + in-container): both (under a unit, stdout already is the journal). Do not grow it into a utility crate. - **`hive-metric/`** — one-shot CLI pushing a labeled metric over OTLP. +- **`hive-meta-flake/`** — the meta flake's `flake.nix` text, a pure + render, and the reads of its pins. hive-c0re and swarm-controller both call + it. Wire types + socket plumbing: diff --git a/Cargo.lock b/Cargo.lock index d18e05460..2bf6f0e4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,6 +1781,7 @@ dependencies = [ "hive-jobq", "hive-jobq-wire", "hive-log", + "hive-meta-flake", "hive-priv-sock", "hive-sh4re", "hive-sock-client", @@ -1959,6 +1960,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "hive-meta-flake" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde_json", + "swarm-wire", +] + [[package]] name = "hive-metric" version = "0.1.0" @@ -1988,6 +1998,7 @@ name = "hive-priv-sock" version = "0.1.0" dependencies = [ "serde", + "serde_json", ] [[package]] @@ -4880,6 +4891,7 @@ dependencies = [ "hive-jobq-metrics", "hive-jobq-wire", "hive-log", + "hive-meta-flake", "hive-types", "hmac 0.13.0", "http", diff --git a/Cargo.toml b/Cargo.toml index 30ab2a15d..cbea8d235 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "hive-jobq-metrics", "hive-jobq-wire", "hive-log", + "hive-meta-flake", "hive-matrix-mcp", "hive-metric", "hive-priv", @@ -107,6 +108,7 @@ hive-jobq = { path = "hive-jobq" } hive-jobq-metrics = { path = "hive-jobq-metrics" } hive-jobq-wire = { path = "hive-jobq-wire" } hive-log = { path = "hive-log" } +hive-meta-flake = { path = "hive-meta-flake" } hive-core-agent-sock = { path = "hive-core-agent-sock" } hive-claude = "0.1.4" hive-host-sock = { path = "hive-host-sock" } diff --git a/docs/agent-lifecycle/config-changes.md b/docs/agent-lifecycle/config-changes.md index ac36494a9..5c1cca40a 100644 --- a/docs/agent-lifecycle/config-changes.md +++ b/docs/agent-lifecycle/config-changes.md @@ -119,10 +119,12 @@ declares one flake input per agent (`agent-.url = `nixosConfigurations.` output per agent. Each output wraps `inputs.agent-.nixosModules.default` with the identity + `HIVE_PORT` / `HIVE_LABEL` injection module. Values that differ per host -(the dashboard port, the pinned claude-code, extra OTEL labels) aren't in it: +(the dashboard port, the pinned claude-code, extra OTEL labels, the queue +address) aren't in it: `hive-priv` writes them to the container's `/etc/hyperhive-hive-facts` at start (see `docs/process/conventions.md`). -Containers run against `--flake /var/lib/hyperhive/meta#`. +Containers run against `--flake /var/lib/hyperhive/meta#`, or against +`internal/meta` when it pins the agent ([below](#the-swarms-meta-flake)). The declared input url is the head of `main` on the agent's forge config repo (`agent-configs/`, the one config PRs merge into), which `sync_agents` @@ -160,6 +162,46 @@ resolves which sha the flake pins each agent at right now. Dashboard surfaces the same info as a `deployed:` chip per container row. +## The swarm's meta flake + +swarm-controller also commits a meta flake of its own, `internal/meta`, with +the same renderer: one `flake.nix` and `flake.lock` for every agent any hive +places (the pins: [`docs/swarm/builder.md`](../swarm/builder.md#meta-flake-lock)). +A hive builds an agent from it when the repo has a commit that pins the +agent's current config, and from its own meta flake above otherwise, so a hive +older or newer than the controller keeps building. + +- **Choosing the commit.** The rebuild's `MetaSync` node reads the repo's + default-branch head with the hive's read token (`forge::reader_token()`) + and takes it when its `flake.nix` builds the agent the way the agent needs now: + a custom agent from the config commit at the head of `agent-configs/` + `main`, an archetype agent from the archetype it has now. An agent switched + between custom and archetype, or to another archetype, fails that test. + A hive without the `core` token file that nix's git reads, a repo with no + commit, a head that lacks the agent and a head behind the config all select + the hive's own flake. `MetaSync` records the commit in `hive-c0re`'s own state, + `/var/lib/hyperhive/meta-rev/`, which the agent can't write (the container mounts its runtime + directory read-write); `Prebuild` and `Swap` read that + record, so both build one flake. A first deploy resolves the commit in its `Create` + node, since it has no `MetaSync`. +- **The build.** `hive-c0re` prebuilds + `git+/internal/meta.git?rev=#nixosConfigurations.…` and + names the commit to `hive-priv` (`meta_rev` on `UpdateContainer` and + `CreateContainer`). `hive-priv` accepts only a 40-digit lowercase hex + commit and builds the flake from its own forge URL + (`HIVE_PRIV_FORGE_URL`); the URL never comes from a request. On the wire the + `hive_priv_sock::PrivRequest` variants carry an optional field: + `{"op":"update_container","name":"atlas","stream":true,"meta_rev":"<40 hex>"}`. + Without it, as from a hive-c0re that predates the field, hive-priv builds the + hive's own flake. +- **Fetching.** nix fetches the forge's inputs with `git`. Both `hive-c0re` + and `hive-priv` carry a `$HOME/.gitconfig` that answers for the forge host + through `git-credential-hive-forge` (the `core` token file) and rewrites the + forge's `https` URL, which `internal/meta`'s lock names its inputs by, to + `http`. +- **Hyperhive.** The commit pins one hyperhive rev for the swarm, so an agent + runs the hyperhive of the controller's deploy, not its hive's. + ## Two repos per agent ``` @@ -194,7 +236,7 @@ proposed tree. The applied repo is never bind-mounted (except the read-only `.git` exposure described below) so a destructive move inside the container can't reach it. -The container's `--flake` ref is `/var/lib/hyperhive/meta#` +The container's `--flake` ref is `/var/lib/hyperhive/meta#` (or `internal/meta`) (see "Meta flake" above). The agent's own `applied//flake.nix` is a fixed boilerplate that exports `nixosModules.default = import ./agent.nix`; the meta flake imports that module and diff --git a/docs/integrations/forge.md b/docs/integrations/forge.md index de6372e66..14b307076 100644 --- a/docs/integrations/forge.md +++ b/docs/integrations/forge.md @@ -80,6 +80,12 @@ forge container: - seeding `proposed` from `agent-configs/` on first deploy - cloning and pulling `/knowledge` +The builds themselves read as `core`. nix fetches the config inputs and +`internal/meta` with `git`, and `git` gets its credential from +`git-credential-hive-forge`, which reads the `core` token file. A hive without +that file fetches nothing private through nix, whatever its own token reads, so +it builds from its own meta flake rather than `internal/meta`. + Roll a change to any of this out store host first (its granter writes the controller's grant on `swarm/hives/+/forge/token`), then the controller host, then the hives. A hive that starts before its token exists reads with the diff --git a/docs/process/conventions.md b/docs/process/conventions.md index 2ab12c09c..9fe7b8eee 100644 --- a/docs/process/conventions.md +++ b/docs/process/conventions.md @@ -41,8 +41,9 @@ Four env vars cover the identity surface, read by Inside an agent container, the hive's name and domain don't come from the agent's build, and neither do the hive's other per-host values: the dashboard port (`HIVE_DASHBOARD_PORT`), the pinned claude-code -(`HYPERHIVE_CLAUDE_CODE_PATH`) and extra OTEL labels -(`HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`). `hive-priv` writes them to +(`HYPERHIVE_CLAUDE_CODE_PATH`), extra OTEL labels +(`HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`) and the queue's address +(`HIVE_AGENT_NATS_URL`, from `deploy.hive-controller.queue.agentNatsUrl`). `hive-priv` writes them to `/etc/hyperhive-hive-facts` before every container start, leaving out any the hive doesn't set; the agent then uses its own default. The agent's environment generator (`nix/agent-modules/hive-facts.nix`) passes them to every unit, diff --git a/docs/swarm/README.md b/docs/swarm/README.md index 86055ea6b..e9c4666ce 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -176,8 +176,12 @@ holds it in memory only. The one thing to configure is where the queue is: | ------------------------------------------- | ------------------------------------------------------------- | | `deploy.hive-controller.queue.agentNatsUrl` | where the queue listens, as an agent **container** reaches it | -On every hive, `agentNatsUrl` defaults to the same -`tls://:` as the hive's own. On the queue's +It reaches each container at start, as `HIVE_AGENT_NATS_URL` in +`/etc/hyperhive-hive-facts`, so no agent build carries it. On every hive, +`agentNatsUrl` defaults to the same +`tls://:` as the hive's own. The option takes one +`://:` (a hostname or IPv4 address), not an IPv6 literal or a +list: evaluation refuses anything else. On the queue's host the name resolves inside a container to the bridge address, where the firewall opens the port; on any other hive it resolves through the host's DNS, like the store's name. ⚠️ **Never a loopback address here**: an agent has its diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index 864afcfe4..506657f30 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -115,6 +115,46 @@ forge token, and answers with the JSON. An evaluation doesn't wait for a build in progress and has two minutes. What the controller does with the answer: [`archetypes.md`](archetypes.md). +## Meta-flake lock + +The builder also answers `$SWARM.builder.lock-meta`. The request carries a +rendered `flake.nix`. The builder writes it into a scratch directory, runs +`nix flake lock` there with the same forge credential helper its builds use, +answers with the `flake.lock` that `nix` wrote, and removes the directory. A +lock doesn't wait for a build in progress and has ten minutes. The builder +commits nothing. + +swarm-controller sends the request. It renders `internal/meta`'s `flake.nix` +with the same renderer hive-c0re uses (`hive-meta-flake`), all over `https`. +Each custom agent that a hive places pins to its `agent-configs/` +`main`; the controller omits and logs one whose repo has no commit on `main`, so it +doesn't block the rest. Hyperhive pins to one rev, a ratchet on the controller's own +running rev (`services.hyperhive.c0re.hyperhiveRev`, from `self.rev`): + +- the first commit pins hyperhive's forge `main`; +- later, the pin moves to the controller's rev only when the forge's compare + API shows that rev is a strict descendant of the committed pin: commits + ahead of it, and none the pin has that it lacks. The compare endpoint counts + histories with no common ancestor as zero commits ahead, so the check needs + both directions; +- an older controller, a rev on another line, a controller built from a dirty + tree or a forge that can't answer leaves the pin. A merge to hyperhive's + `main` moves nothing. + +The controller commits `flake.nix` and the returned `flake.lock` to +`internal/meta` in one commit, and gives each hive's forge account read on the +repo. It re-renders when it starts, before it deploys a merged config PR, and +before the first deploy of a new agent. Those deploys run whether or not the +update succeeded: a hive that builds from `internal/meta` does so only while it +pins the agent's current config (see +[Config changes](../agent-lifecycle/config-changes.md#the-swarms-meta-flake)). A render +equal to the committed `flake.nix` sends no lock. + +When a commit moves the pin from one rev to another, the controller sends a +deploy to every agent a hive places, so they rebuild on the new hyperhive. A +restart that moves nothing deploys nothing, and the first commit deploys +nothing. + ## Credentials | What | Where | Who reads it | diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index 6bbb7cf20..4678d5169 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -39,6 +39,7 @@ hive-host-sock.workspace = true hive-jobq.workspace = true hive-jobq-wire.workspace = true hive-log.workspace = true +hive-meta-flake.workspace = true hive-priv-sock.workspace = true hive-sock-client.workspace = true hive-types.workspace = true diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index a14451d05..3c1a3b4a9 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -367,7 +367,7 @@ fn read_active_model(name: &hive_types::Ident) -> Option { /// The agent-side /// `hive-agent::identity::{hive_name, swarm_name}` accessors read the /// same env vars after they're forwarded into each sub-agent's -/// harness service environment by `meta::render_flake`; surfacing +/// harness service environment by `hive_meta_flake::render_flake`; surfacing /// them here from c0re's own env keeps the manager + agent /// `GetAgentMeta` paths consistent without a round-trip to the /// target container. diff --git a/hive-c0re/src/forge/internal_meta.rs b/hive-c0re/src/forge/internal_meta.rs new file mode 100644 index 000000000..702b26f97 --- /dev/null +++ b/hive-c0re/src/forge/internal_meta.rs @@ -0,0 +1,108 @@ +//! This hive's reads of `internal/meta` and of an agent's config head, as +//! [`crate::meta_rev::MetaSource`], under the hive's forge read token. + +use forgejo_api::structs::RepoGetRawFileQuery; +use forgejo_api::{ApiErrorKind, Forgejo, ForgejoError}; +use reqwest::StatusCode; + +use super::reader::ReaderToken; +use super::{CONFIG_ORG, bounded}; +use crate::meta_rev::MetaSource; + +/// The org and repo the swarm commits its meta flake to. +const META_OWNER: &str = "internal"; +const META_REPO: &str = "meta"; + +/// The branch an agent's config is deployed from, which `fetch_forge_main` +/// and the swarm controller's pins both read. +const CONFIG_BRANCH: &str = "main"; + +/// [`MetaSource`] over the forge API. +pub(crate) struct ForgeSource { + api: Forgejo, +} + +/// A source reading as this hive, or `None` when it holds no forge token. +pub(crate) async fn internal_meta_source() -> Option { + let token: ReaderToken = super::reader_token().await?; + match token.api() { + Ok(api) => Some(ForgeSource { api }), + Err(e) => { + tracing::warn!(error = ?e, "internal/meta: cannot build a forge client"); + None + } + } +} + +fn is_not_found(e: &ForgejoError) -> bool { + match e { + ForgejoError::ApiError(api) => match api.error_kind() { + ApiErrorKind::NotFound { .. } => true, + ApiErrorKind::Other(s) => *s == StatusCode::NOT_FOUND, + _ => false, + }, + ForgejoError::UnexpectedStatusCode(s) => *s == StatusCode::NOT_FOUND, + _ => false, + } +} + +impl ForgeSource { + /// The head commit of `branch` in `owner/repo`; `None` when the repo or + /// the branch is not there. + async fn branch_head( + &self, + owner: &str, + repo: &str, + branch: &str, + ) -> anyhow::Result> { + let op = format!("read branch {branch} of {owner}/{repo}"); + match bounded(&op, self.api.repo_get_branch(owner, repo, branch)).await? { + Ok(branch) => Ok(branch.commit.and_then(|c| c.id)), + Err(e) if is_not_found(&e) => Ok(None), + Err(e) => Err(anyhow::Error::new(e).context(op)), + } + } +} + +impl MetaSource for ForgeSource { + fn nix_can_fetch(&self) -> bool { + super::core_token().is_some() + } + + async fn meta_head(&self) -> anyhow::Result> { + use anyhow::Context as _; + + let op = format!("read repo {META_OWNER}/{META_REPO}"); + let repo = match bounded(&op, self.api.repo_get(META_OWNER, META_REPO)).await? { + Ok(repo) => repo, + Err(e) if is_not_found(&e) => return Ok(None), + Err(e) => return Err(anyhow::Error::new(e).context(op)), + }; + if repo.empty.unwrap_or(false) { + return Ok(None); + } + let branch = repo + .default_branch + .with_context(|| format!("{META_OWNER}/{META_REPO} names no default branch"))?; + self.branch_head(META_OWNER, META_REPO, &branch).await + } + + async fn flake_at(&self, rev: &str) -> anyhow::Result { + let op = format!("read flake.nix of {META_OWNER}/{META_REPO} at {rev}"); + let query = RepoGetRawFileQuery { + r#ref: Some(rev.to_owned()), + }; + let raw = bounded( + &op, + self.api + .repo_get_raw_file(META_OWNER, META_REPO, "flake.nix", query), + ) + .await? + .map_err(|e| anyhow::Error::new(e).context(op))?; + Ok(String::from_utf8_lossy(&raw).into_owned()) + } + + async fn config_head(&self, agent: &str) -> anyhow::Result> { + self.branch_head(CONFIG_ORG, agent, CONFIG_BRANCH).await + } +} diff --git a/hive-c0re/src/forge/mod.rs b/hive-c0re/src/forge/mod.rs index 029888e5f..c4ae2733b 100644 --- a/hive-c0re/src/forge/mod.rs +++ b/hive-c0re/src/forge/mod.rs @@ -5,11 +5,13 @@ //! `internal/docs`, are swarm-controller's. Full design: //! `docs/integrations/forge.md`. +mod internal_meta; mod reader; mod reconcile; mod repos; mod users; +pub(crate) use internal_meta::internal_meta_source; pub use reader::reader_token; pub(crate) use reconcile::FORGE_MAIN_REF; pub use reconcile::{fetch_forge_main, reconcile_config_apply, reconcile_config_status}; diff --git a/hive-c0re/src/forge/reader.rs b/hive-c0re/src/forge/reader.rs index c977ce984..26a680d7d 100644 --- a/hive-c0re/src/forge/reader.rs +++ b/hive-c0re/src/forge/reader.rs @@ -34,6 +34,11 @@ impl ReaderToken { pub fn auth_header(&self) -> String { basic_auth_header(&self.login, &self.value) } + + /// A forge API client acting as this token's account. + pub(crate) fn api(&self) -> anyhow::Result { + super::api(&self.value) + } } /// The token to read the forge with, or `None` when this hive has neither. diff --git a/hive-c0re/src/forge/users.rs b/hive-c0re/src/forge/users.rs index 22e854bc9..979ba4e39 100644 --- a/hive-c0re/src/forge/users.rs +++ b/hive-c0re/src/forge/users.rs @@ -55,7 +55,7 @@ fn extract_token(output: &str) -> Option { } /// Canonical email address for a hive agent's Forgejo account. -/// Must match the `user.email` set by `meta::render_flake` so commits +/// Must match the `user.email` set by `hive_meta_flake::render_flake` so commits /// by the agent link back to their Forgejo profile page. fn agent_email(name: &str) -> String { format!("{name}@hyperhive.local") diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index b9d1b9250..5345963fe 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -412,6 +412,11 @@ async fn run_meta_sync(coord: &Arc, name: &str, relock: bool) -> Re if relock && crate::archetypes::of(name)?.is_none() { crate::meta::lock_update_for_rebuild(name).await?; } + // The commit of the swarm's `internal/meta` that `Prebuild` and `Swap` + // build this agent from, resolved once here so both build the same flake; + // none leaves them on the hive's own flake rendered above. + let meta_rev = crate::meta_rev::resolve(name).await; + crate::meta_rev::record(name, meta_rev.as_deref())?; Ok(()) } @@ -428,7 +433,10 @@ async fn run_prebuild(name: &str, id: NodeId) -> Result<()> { // stopped agent has no uptime to preserve, so skip the (expensive) // eval and let the downstream `Swap` build inline. if crate::lifecycle::is_running(name).await { - let flake_ref = format!("{}#{name}", crate::paths::meta_root().display()); + let flake_ref = match crate::meta_rev::recorded(name) { + Some(rev) => crate::meta_rev::flake_ref(name, &rev), + None => format!("{}#{name}", crate::paths::meta_root().display()), + }; crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(id.get())).await?; } Ok(()) @@ -445,7 +453,10 @@ async fn run_swap(coord: &Arc, name: &str, id: NodeId) -> Result<() let agent_dir = crate::paths::agent_runtime_dir(name); let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir)?; - let result = crate::lifecycle::swap_update(name, &hive, &paths, Some(id.get())).await; + let meta_rev = crate::meta_rev::recorded(name); + let result = + crate::lifecycle::swap_update(name, &hive, &paths, meta_rev.as_deref(), Some(id.get())) + .await; // On success the Ok-only bookkeeping tail (rev marker, forge/matrix // sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node, // which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded @@ -512,7 +523,8 @@ async fn run_provision(coord: &Arc, name: &str) -> Result<()> { /// `Reconcile` (`converge_start_preamble` + `register_agent`) so this /// node stays purely "create", not "create + start". async fn run_create(name: &str) -> Result<()> { - crate::lifecycle::create_only(name).await?; + let meta_rev = crate::meta_rev::resolve(name).await; + crate::lifecycle::create_only(name, meta_rev.as_deref()).await?; Ok(()) } diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index b70b440b5..0c5bc58ec 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -53,34 +53,7 @@ pub fn container_claude_mount(name: &str) -> String { /// willing to lose (other agents may delete them). pub const CONTAINER_SHARED_MOUNT: &str = "/shared"; -/// Web UI port range, shared by every agent including the manager. -/// Deterministic from the agent's name (FNV-1a hash mod range size), so the -/// dashboard can compute the same port without asking hive-c0re. -const WEB_PORT_BASE: u16 = 8100; -const WEB_PORT_RANGE: u16 = 900; - -/// FNV-1a hash of a string — used by `agent_web_port`. -fn fnv1a(s: &str) -> u32 { - let mut hash: u32 = 2_166_136_261; - for b in s.bytes() { - hash ^= u32::from(b); - hash = hash.wrapping_mul(16_777_619); - } - hash -} - -/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) % -/// WEB_PORT_RANGE` for every agent including the manager. The port -/// allocation rule reads the same for every name; collisions are -/// possible (birthday paradox at ~30 agents) and the operator -/// resolves them by renaming an agent (different hash → different -/// port). Stable across hosts, restarts, and dashboard renders — -/// no state-file dance. -#[must_use] -pub fn agent_web_port(name: &str) -> u16 { - // Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails. - WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0) -} +pub use hive_meta_flake::agent_web_port; /// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`. /// @@ -170,8 +143,8 @@ pub fn network_isolation_from_env() -> Result } /// The hive's facts every container is handed at runtime, so that no agent -/// build carries a value that differs between hosts. The name, domain and -/// OTEL resource labels come from hive-c0re's own unit environment +/// build carries a value that differs between hosts. The name, domain, OTEL +/// resource labels and queue address come from hive-c0re's own unit environment /// (`nix/host-modules/hive-c0re/environment.nix`), where an unset or empty /// variable is `None`; the dashboard port and pinned claude from `hive`. fn hive_facts(hive: &HiveEnv) -> hive_priv_sock::HiveFacts { @@ -190,6 +163,7 @@ fn hive_facts_from( dashboard_port: Some(hive.dashboard_port), claude_code_path: hive.claude_code_path.clone(), otel_extra_resource_attributes: read("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES"), + agent_nats_url: read("HIVE_AGENT_NATS_URL"), } } @@ -263,8 +237,10 @@ pub async fn provision_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) /// The `nixos-container create` proper — the job queue's `Create` node. /// Assumes `provision_container` already registered the agent in meta. -pub async fn create_only(name: &str) -> Result<()> { - priv_run("create", name).await +/// `meta_rev` is the `internal/meta` commit to build from; none builds from the +/// hive's own meta flake (`crate::meta_rev`). +pub async fn create_only(name: &str, meta_rev: Option<&str>) -> Result<()> { + priv_run_inner("create", name, meta_rev, None).await } /// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on @@ -316,15 +292,17 @@ pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> /// drop-ins, then `nixos-container update`. The job queue's `Swap` /// node. Requires the container stopped (the queue's `StopForUpdate` /// upstream); does NOT start it — the DAG's tail `Reconcile` owns -/// bringing the agent back to its wanted power state. +/// bringing the agent back to its wanted power state. `meta_rev` is as for +/// [`create_only`]. pub async fn swap_update( name: &str, hive: &HiveEnv, paths: &AgentPaths, + meta_rev: Option<&str>, node_id: Option, ) -> Result<()> { write_dropins(name, hive, paths).await?; - priv_run_inner("update", name, node_id).await + priv_run_inner("update", name, meta_rev, node_id).await } /// Build the `AgentSpec` list for the meta flake from `nixos-container @@ -998,7 +976,7 @@ fn make_log_callback( /// is appended to the build-log row as it arrives, so the dashboard /// shows live progress during long `nixos-container create` / `update` runs. async fn priv_run(kind: &str, name: &str) -> Result<()> { - priv_run_inner(kind, name, None).await + priv_run_inner(kind, name, None, None).await } /// Like `priv_run` but stamps `node_id` onto the build-log row it opens, so @@ -1008,7 +986,12 @@ async fn priv_run(kind: &str, name: &str) -> Result<()> { /// This used to be a `Fn(i64)` callback that handed the row id *back* to the /// queue, which then held it in a side map. The row carries the link itself /// now, so the id only ever travels one way. -async fn priv_run_inner(kind: &str, name: &str, node_id: Option) -> Result<()> { +async fn priv_run_inner( + kind: &str, + name: &str, + meta_rev: Option<&str>, + node_id: Option, +) -> Result<()> { let container = container_name(name); let cmdline = format!("nixos-container {kind} {container}"); @@ -1027,6 +1010,7 @@ async fn priv_run_inner(kind: &str, name: &str, node_id: Option) -> Result< "create" => { crate::priv_client::create_container_streaming( name, + meta_rev, make_log_callback(logs.clone(), log_id, cmdline.clone()), ) .await @@ -1034,6 +1018,7 @@ async fn priv_run_inner(kind: &str, name: &str, node_id: Option) -> Result< "update" => { crate::priv_client::update_container_streaming( name, + meta_rev, make_log_callback(logs.clone(), log_id, cmdline.clone()), ) .await diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs index b68ef27e1..d638cdb22 100644 --- a/hive-c0re/src/lifecycle/tests.rs +++ b/hive-c0re/src/lifecycle/tests.rs @@ -528,9 +528,9 @@ async fn run_with_deadline_returns_output_of_a_fast_child() { assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "hi"); } -/// The three per-host values reach the container as facts: the dashboard -/// port and pinned claude from the hive's config, the OTEL labels from the -/// daemon's environment. +/// The per-host values reach the container as facts: the dashboard port and +/// pinned claude from the hive's config, the OTEL labels and the queue +/// address from the daemon's environment. #[test] fn hive_facts_carry_the_per_host_values() { let hive = HiveEnv { @@ -538,8 +538,10 @@ fn hive_facts_carry_the_per_host_values() { claude_code_path: Some("/nix/store/0abc-claude-code".to_owned()), ..HiveEnv::default() }; - let facts = hive_facts_from(&hive, |name| { - (name == "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES").then(|| "team=a".to_owned()) + let facts = hive_facts_from(&hive, |name| match name { + "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES" => Some("team=a".to_owned()), + "HIVE_AGENT_NATS_URL" => Some("tls://nats.t.local:4222".to_owned()), + _ => None, }); assert_eq!(facts.dashboard_port, Some(7123)); assert_eq!( @@ -550,6 +552,10 @@ fn hive_facts_carry_the_per_host_values() { facts.otel_extra_resource_attributes.as_deref(), Some("team=a") ); + assert_eq!( + facts.agent_nats_url.as_deref(), + Some("tls://nats.t.local:4222") + ); } /// Unset and empty both leave a fact out, so the agent falls back to its @@ -561,5 +567,6 @@ fn hive_facts_leave_unset_values_out() { }); assert_eq!(facts.claude_code_path, None); assert_eq!(facts.otel_extra_resource_attributes, None); + assert_eq!(facts.agent_nats_url, None); assert_eq!(facts.name, None); } diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 1c91613aa..f3e77db7a 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -26,6 +26,7 @@ mod lifecycle; mod loose_ends; mod matrix; mod meta; +mod meta_rev; mod migrate; mod paths; mod priv_client; diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 48891760a..77b907fdc 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -13,6 +13,11 @@ use tokio::process::Command; use tokio::sync::{Mutex, MutexGuard}; use tokio::time::Instant; +pub use hive_meta_flake::{AgentPin, AgentSpec}; +#[cfg(test)] +use hive_meta_flake::{FORWARDED_VAR_OPTIONS, push_forwarded_var_options}; +use hive_meta_flake::{FORWARDED_VARS, canonical_inputs_in_lock, pinned_git_url}; + use crate::coordinator::HiveEnv; use crate::lifecycle; @@ -112,30 +117,6 @@ async fn lock_notifying<'a>( /// Where the manager sees this directory inside its container (RO bind). pub const CONTAINER_MANAGER_META_MOUNT: &str = "/meta"; -#[derive(Debug, Clone)] -pub struct AgentSpec { - pub name: String, - pub port: u16, - /// The archetype the agent is built from (`crate::archetypes`), `None` - /// for a custom agent built from its own config repo. - pub archetype: Option, - /// The pin of a custom agent's config input. `None` until [`sync_agents`] - /// resolves it; an archetype agent has no config input and keeps `None`. - pub pin: Option, -} - -/// What the rendered flake needs to know about one agent's config input, all -/// of it read from the agent's config repo at one commit. The rendered text -/// is a function of this and nothing on the host. -#[derive(Debug, Clone)] -pub struct AgentPin { - /// `git+/agent-configs/.git?rev=`, `` being the head of - /// `main` on the forge when the flake was rendered. - pub url: String, - /// The `CANONICAL_INPUTS` the agent's `flake.lock` declares at that commit. - pub canonical_inputs: Vec<&'static str>, -} - /// Stage topology.json if it exists: it is regenerated by `reconcile` /// whenever the agent set changed. Without staging, an /// existing-but-untracked file shows up as untracked in the meta repo, @@ -422,15 +403,6 @@ fn hyperhive_input_url(hive: &HiveEnv) -> Result { ) } -/// The pure half of [`hyperhive_input_url`], testable without `HIVE_FORGE_URL`. -fn pinned_git_url(forge_git_url: &str, rev: Option<&str>) -> Result { - let rev = rev.filter(|r| !r.is_empty()).context( - "hyperhive rev unknown: the host was built from a dirty tree \ - (`self.rev` unset) and `services.hyperhive.c0re.hyperhiveRev` is not set", - )?; - Ok(format!("git+{forge_git_url}?rev={rev}")) -} - /// [`render_flake`] with the hyperhive pin taken from `hive`, as /// [`sync_agents`] renders it. Nothing else in `hive` reaches the flake: the /// values that differ per host go to the container at start @@ -439,113 +411,15 @@ fn render_hive_flake(hive: &HiveEnv, agents: &[AgentSpec]) -> Result { render_flake(&hyperhive_input_url(hive)?, agents) } -/// Canonical inputs meta knows how to dedup. An agent that declares one -/// of these as a top-level input in its own `flake.nix` will get a -/// `follows = ""` line emitted in meta — collapsing the -/// otherwise-separate-but-identical `nixpkgs_N` nodes into a single -/// meta-level reference. -const CANONICAL_INPUTS: &[&str] = &["nixpkgs"]; - -/// Env vars hive-c0re forwards from its own systemd unit env into every -/// sub-agent's harness service env. Each entry is `(env_var_name, -/// host_value)`. Empty / unset vars are filtered out so absent options -/// don't render no-op `FOO = ""` lines into the meta flake. -/// -/// Returns owned `String` values so the result is `'static`-friendly + -/// trivial to stub from tests (which build their own slice instead of -/// touching process-wide env). -const FORWARDED_VARS: &[&str] = &[ - // Where the swarm queue is, as an agent container reaches it. An address - // the *host* computes, never a loopback address, which inside a container - // is the agent itself. - "HIVE_AGENT_NATS_URL", - // Where the swarm secret store listens. The hive's own `BAO_ADDR` value, - // forwarded under a separate name so the container never inherits the - // hive's store environment wholesale — the address is the only part of it - // an agent may have, the certificate beside it being the hive's own. - "HIVE_AGENT_BAO_ADDR", - // The mount path of the store's services `pki` engine, whose root an agent - // fetches to trust the swarm. Forwarded with the address, under the same gate. - "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", - // Where an agent's `swarm-logs` queries, computed by the host for the - // same reason as the queue's: it is a gateway address, and a container - // cannot derive it from anything it holds. - "HIVE_AGENT_LOGS_QUERY_URL", - // The metrics store's origin, for `swarm-metrics`, for the same reason. - "HIVE_AGENT_METRICS_URL", - "HIVE_FORGE_URL", - "HIVE_FORGE_PUBLIC_URL", - "HIVE_MATRIX_URL", - "HYPERHIVE_SWARM_NAME", -]; - -/// Map of forwarded env var -> the agent option carrying the same value. -/// -/// Both exist because they're consumed at different times: the option is baked -/// into scripts at build time (forge-avatar-sync bakes `FORGE_URL` from it), the env -/// var is read at runtime. Setting only one leaves the other on its default, -/// which is how a hive ends up with two disagreeing answers for one value. -/// -/// The swarm display name is here for that reason: `otel.nix` reads the option -/// at eval time for the `swarm=` resource label, and with the env var alone -/// that label evaluates to `unknown` while the process env holds the right -/// name. Half-wiring this map is a value that evaluates fine and is silently -/// wrong. -const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[ - // Read at build time to decide whether the in-container identity check is - // generated at all — see `nix/agent-modules/bao.nix`. Both halves wired - // for this map's own reason: the unit bakes the address into its own - // environment, so an agent whose option says no store and whose env says - // otherwise would have a check that is simply not there. - ("HIVE_AGENT_BAO_ADDR", "services.hyperhive.agent.bao.addr"), - // Read at build time to bake the root-fetch URL into - // `nix/agent-modules/hive-ca.nix`'s unit. - ( - "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", - "services.hyperhive.agent.bao.servicesPkiMount", - ), - // Read at build time to decide whether `swarm-logs` is installed at all — - // see `nix/agent-modules/logs.nix`. Both halves wired for this map's own - // reason: an agent whose option says no log store and whose env says - // otherwise has a binary that is not on PATH and coordinates that claim - // it is. - ( - "HIVE_AGENT_LOGS_QUERY_URL", - "services.hyperhive.agent.logs.queryUrl", - ), - // The same for `swarm-metrics`: see `nix/agent-modules/metrics.nix`. - ( - "HIVE_AGENT_METRICS_URL", - "services.hyperhive.agent.metrics.url", - ), - ("HIVE_FORGE_URL", "services.hyperhive.agent.forge.url"), - ("HIVE_MATRIX_URL", "services.hyperhive.agent.matrix.url"), - ("HYPERHIVE_SWARM_NAME", "services.hyperhive.agent.swarmName"), -]; - -/// Render the forwarded-var option assignments for one agent's module block. -/// -/// Split out of `render_flake` so it can be tested without touching process -/// env: the render-level tests have to `set_var`, which forces them to -/// serialise on the crate's `test_env` lock (a `#[cfg(test)]` module, so this -/// deliberately isn't an intra-doc link). A pure function over the -/// already-collected pairs has no such hazard. -/// -/// A var that isn't present emits nothing rather than a guess. For an optional -/// service that is the whole point — the agent option defaults to `null`, -/// meaning "not configured", and the units that would use it aren't generated. -/// For a service the hive cannot run without, silence would instead produce a -/// fleet of agents quietly missing an integration, so those are checked by -/// [`require_service_urls`] before this is called. -fn push_forwarded_var_options(out: &mut String, vars: &[(&'static str, String)]) { - use std::fmt::Write as _; - for (var, val) in vars { - let Some((_, option)) = FORWARDED_VAR_OPTIONS.iter().find(|(name, _)| name == var) else { - continue; - }; - let escaped = val.replace('\\', "\\\\").replace('"', "\\\""); - let _ = writeln!(out, " {option} = \"{escaped}\";"); - } +/// [`hive_meta_flake::render_flake`] with the forwarded values and the GitHub +/// switch read from this process's environment. +fn render_flake(hyperhive_url: &str, agents: &[AgentSpec]) -> Result { + hive_meta_flake::render_flake( + hyperhive_url, + agents, + &forwarded_env_vars(), + std::env::var_os("HYPERHIVE_GITHUB_DISABLED").is_some(), + ) } /// Service URLs a running hive must always supply, checked before rendering. @@ -562,7 +436,7 @@ const REQUIRED_SERVICE_URL_VARS: &[&str] = &["HIVE_FORGE_URL"]; /// `vars`. /// /// Pure over the already-collected pairs so it can be tested without touching -/// process env (the same reason [`push_forwarded_var_options`] is split out). +/// process env (the same reason [`hive_meta_flake::push_forwarded_var_options`] is split out). /// /// Checked by `sync_agents` — the point at which the hive commits a rendered /// flake to disk — rather than inside the renderer. Rendering is a pure string @@ -693,252 +567,6 @@ where }) } -/// The subset of `CANONICAL_INPUTS` that the `flake.lock` text `raw` declares -/// as direct (root-level) inputs. Empty when the lock is unparsable — safe -/// degradation, the worst case is no dedup for that agent. -fn canonical_inputs_in_lock(raw: &str) -> Vec<&'static str> { - let Ok(json) = serde_json::from_str::(raw) else { - return Vec::new(); - }; - let Some(nodes) = json.get("nodes").and_then(|v| v.as_object()) else { - return Vec::new(); - }; - let Some(root_name) = json.get("root").and_then(|v| v.as_str()) else { - return Vec::new(); - }; - let Some(root_inputs) = nodes - .get(root_name) - .and_then(|n| n.get("inputs")) - .and_then(|v| v.as_object()) - else { - return Vec::new(); - }; - CANONICAL_INPUTS - .iter() - .copied() - .filter(|canon| root_inputs.contains_key(*canon)) - .collect() -} - -/// Render the meta flake. Errors on a custom agent without a [`AgentPin`]. -#[allow( - clippy::too_many_lines, - reason = "templated string-builder for the meta flake — the length is one \ - contiguous fmt block, splitting it would just hide the shape" -)] -fn render_flake(hyperhive_url: &str, agents: &[AgentSpec]) -> Result { - use std::fmt::Write as _; - let mut out = String::new(); - out.push_str("{\n description = \"hyperhive deployed agents\";\n inputs = {\n"); - // `hyperhive` is pinned by forge URL and rev, so the flake resolves on - // any machine that can reach the forge. `nixpkgs` follows hyperhive's - // own pin, which cascades to every agent because - // `agent-.inputs.nixpkgs.follows = "nixpkgs"` resolves to this node. - let _ = writeln!(out, " hyperhive.url = \"{hyperhive_url}\";"); - out.push_str(" nixpkgs.follows = \"hyperhive/nixpkgs\";\n"); - // Each agent's config input is its forge repo at the pinned commit, so - // the flake text depends only on (agent, rev) and resolves anywhere that - // can reach the forge. - for spec in agents { - // An archetype agent has no config repo, so no input: `mkAgent` - // takes its module from the archetype. - if spec.archetype.is_some() { - continue; - } - let pin = spec.pin.as_ref().with_context(|| { - format!( - "agent {}: no pinned rev, refusing to render an unpinned config input", - spec.name - ) - })?; - let _ = writeln!( - out, - " agent-{name}.url = \"{url}\";", - name = spec.name, - url = pin.url, - ); - // For each canonical input the agent's `flake.lock` declares at the - // pinned commit, emit - // `inputs.agent-.inputs..follows = ""`. - // Collapses otherwise-separate-but-identical nixpkgs nodes - // (root + every agent's own nixpkgs) into one. Skipped - // silently for agents that don't declare the input — emitting - // follows on a non-existent input would error at - // `nix flake lock` time. - for canon in &pin.canonical_inputs { - let _ = writeln!( - out, - " agent-{}.inputs.{canon}.follows = \"{canon}\";", - spec.name, - ); - } - } - out.push_str( - " };\n outputs =\n { self, hyperhive, ... }@inputs:\n let\n mkAgent = { name, port, archetype ? null }:\n", - ); - out.push_str( - r#" let - base = hyperhive.nixosConfigurations.agent-base; - # Every flake an archetype can come from, by its source name. - archetypeSources = { inherit hyperhive; }; - agentModule = - if archetype == null then - inputs."agent-${name}".nixosModules.default - else - (archetypeSources.${archetype.source} - or (throw "agent ${name}: archetype source ${archetype.source} is not an input of this hive's meta flake") - ).hyperhiveArchetypes.${archetype.name}.module - or (throw "agent ${name}: this hive's ${archetype.source} input has no archetype ${archetype.name}; update that input"); - service = "hive-agent"; - in - base.extendModules { - modules = [ - agentModule - { -"#, - ); - // Agent-facing service URLs (`services.hyperhive.agent.forge.url`, `services.hyperhive.agent.matrix.url`): - // emit the host's real values as build-time agent config, the same - // host-state -> agent-module shape as the CA block above. - // - // These options exist already, but until now *nothing set them*, so every - // agent silently fell back to their `localhost:` defaults while the - // true value reached the container only as an env var (below). Two sources - // of truth that disagree, with the winner decided by which code path - // happens to read which one --- the option default is baked into scripts - // at build time, the env var is read at runtime. - // - // Rendering them here makes the option the single source, which is the - // precondition for removing those defaults: a loopback address is only - // ever correct when the callee shares the caller's netns, and the forge - // and homeserver are moving to swarm level, possibly on other hosts. - // - // Absent vars emit nothing rather than a guess, and the agent option - // treats "unset" as "this service is not configured" rather than - // substituting a loopback address — an absent integration instead of a - // misdirected one. - // - // That is the right default for an optional service and the wrong one for - // the forge, which a running hive always has — so `sync_agents` rejects a - // missing `HIVE_FORGE_URL` before it writes anything (`require_service_urls`). - // The check lives there rather than here because rendering is a pure string - // operation the tests exercise directly. - push_forwarded_var_options(&mut out, &forwarded_env_vars()); - // GitHub integration is on by default in every agent - // (`services.hyperhive.agent.github.enable`); the host turns it off swarm-wide via - // `services.hyperhive.swarm.github.enable = false`, surfaced here as the - // `HYPERHIVE_GITHUB_DISABLED` env on hive-c0re's unit. Only the OFF - // override is propagated — the enabled default needs no per-agent line. - if std::env::var_os("HYPERHIVE_GITHUB_DISABLED").is_some() { - out.push_str(" services.hyperhive.agent.github.enable = false;\n"); - } - out.push_str( - r#" # The harness service inside the container runs as a - # non-root unix user named after the agent (`damocles`, - # `iris`, `root`, …). UID auto-assigned by NixOS; the - # per-agent override here is what makes - # `services.hyperhive.agent.user.name` match the agent's identity - # instead of the harness default of `"agent"`. - services.hyperhive.agent.user.name = name; - # Without this an agent container is called `nixos` and ships - # every log line under that: nixpkgs sets the hostname from the - # attr name in the `containers.` submodule, and an - # agent's system is its own `nixosSystem` off this flake, which - # never evaluates that option. The machine name, not the - # logical one — it is what `container.name` on our metrics and - # nspawn itself already call this container, so logs join to - # both without a prefix transform. - networking.hostName = "h-${name}"; - programs.git.config.user = { - name = name; - email = "${name}@hyperhive.local"; - }; - # Container-wide env: every service + co-process daemon can - # resolve the agent's durable state dir without hard-coding it. - # `environment.variables` only writes /etc/environment (login - # shells); `systemd.globalEnvironment` is the analogue for - # systemd units so forge-avatar-sync / - # hive-matrix-daemon etc. can read `$HYPERHIVE_STATE_DIR` - # without each service having to redeclare it. - environment.variables = { - HIVE_LABEL = name; - HYPERHIVE_STATE_DIR = "/agents/${name}/state"; - HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness"; - }; - systemd.globalEnvironment = { - HIVE_LABEL = name; - HYPERHIVE_STATE_DIR = "/agents/${name}/state"; - HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness"; -"#, - ); - // Forwarded vars (HIVE_FORGE_URL etc.) also go into globalEnvironment, - // not just the harness service env below, so EVERY service + shell in - // the container inherits them — crucially the bash-task runner (where - // `hive-forge` + `git` actually run), plus the matrix daemon, forge-avatar-sync - // and interactive shells. Scoped to the harness service alone they were - // invisible to bash tasks: harmless in shared netns (the localhost - // default works) but broken under isolation, where the in-cluster - // `forge.` URL is the only reachable path. - for (var, val) in forwarded_env_vars() { - let escaped = val.replace('\\', "\\\\").replace('"', "\\\""); - let _ = writeln!(out, " {var} = \"{escaped}\";"); - } - out.push_str( - r" }; - systemd.services.${service}.environment = { - HIVE_PORT = toString port; - HIVE_LABEL = name; -", - ); - // Forwarded env vars — picked up from hive-c0re's own systemd unit - // env (`services.hyperhive.*` options flow through the hyperhive - // NixOS module into the host process). We copy whatever's set into - // each sub-agent's harness service env so the in-container surfaces - // (`identity.rs`, `forge_notify`) see a consistent view across the - // whole hive. Absent host-side env (option not set) → skip emission - // → in-container accessors fall back to None / defaults gracefully. - // - // - HIVE_FORGE_URL: agents poll this for Forgejo notifications. - // - HYPERHIVE_SWARM_NAME: human display name for the swarm. - for (var, val) in forwarded_env_vars() { - let escaped = val.replace('\\', "\\\\").replace('"', "\\\""); - let _ = writeln!(out, " {var} = \"{escaped}\";"); - } - out.push_str( - r#" HYPERHIVE_STATE_DIR = "/agents/${name}/state"; - HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness"; - }; - } - # Tells the harness the agent has no config repo of its own. - (if archetype == null then { } else { - systemd.services.${service}.environment.HIVE_ARCHETYPE = archetype.name; - }) - ]; - }; - in - { - nixosConfigurations = { -"#, - ); - for spec in agents { - // `source` and `name` are identifiers (`archetypes::store_read`), so - // they need no escaping. - let archetype = spec.archetype.as_ref().map_or(String::new(), |a| { - format!( - " archetype = {{ source = \"{}\"; name = \"{}\"; }};", - a.source, a.name - ) - }); - let _ = writeln!( - out, - " {} = mkAgent {{ name = \"{}\"; port = {};{archetype} }};", - spec.name, spec.name, spec.port, - ); - } - out.push_str(" };\n };\n}\n"); - Ok(out) -} - /// Return the list of file names that are currently staged (index differs /// from HEAD). On the initial commit (`HEAD` doesn't exist yet) falls back /// to `git diff --cached --name-only HEAD` failing gracefully by using @@ -1547,12 +1175,12 @@ mod tests { ); } - /// The queue address reaches the harness as env only. The agent module - /// declares no `services.hyperhive.agent.queue.*` option, so rendering one - /// into the meta flake fails every agent's evaluation. + /// The queue address reaches the container in its hive facts, so the render + /// carries neither it nor any `services.hyperhive.agent.queue.*` option, + /// which no agent module declares. #[test] - fn queue_address_is_forwarded_as_env_and_no_option() { - assert!(FORWARDED_VARS.contains(&"HIVE_AGENT_NATS_URL")); + fn queue_address_is_not_rendered_into_the_flake() { + assert!(!FORWARDED_VARS.contains(&"HIVE_AGENT_NATS_URL")); assert!( !FORWARDED_VARS.contains(&"HIVE_AGENT_OIDC_TOKEN_ENDPOINT"), "no agent reads a token endpoint" diff --git a/hive-c0re/src/meta_rev.rs b/hive-c0re/src/meta_rev.rs new file mode 100644 index 000000000..2ee47cc16 --- /dev/null +++ b/hive-c0re/src/meta_rev.rs @@ -0,0 +1,395 @@ +//! The `internal/meta` commit a hive builds an agent from. +//! +//! `internal/meta` is the swarm's meta flake, committed by swarm-controller. +//! A rebuild's `MetaSync` node resolves its head once ([`resolve`]) and +//! records it ([`record`]); `Prebuild` and `Swap` read the record +//! ([`recorded`]), so both build the same flake. A create resolves and uses +//! the rev directly. No rev, whether because the repo or its commit is absent +//! or because the commit does not pin this agent's current config, leaves the +//! build on the hive's own meta flake. + +use anyhow::{Context, Result}; +use hive_priv_sock::{meta_flake_url, valid_meta_rev}; +use swarm_queue_client::agent_archetype::ArchetypeRef; + +/// What the forge says about `internal/meta` and an agent's config repo. +pub(crate) trait MetaSource { + /// Whether nix's own `git` can authenticate to the forge: it reads as + /// `core`, from the token file, whatever token this hive reads with. + fn nix_can_fetch(&self) -> bool; + /// The default branch's head commit of `internal/meta`; `None` when the + /// repo is absent or has no commit. + async fn meta_head(&self) -> Result>; + /// `internal/meta`'s `flake.nix` at `rev`. + async fn flake_at(&self, rev: &str) -> Result; + /// The head of `agent-configs/`'s `main`; `None` when it has none. + async fn config_head(&self, agent: &str) -> Result>; +} + +/// The `internal/meta` commit to build `agent` from, or `None` for the hive's +/// own flake. +/// +/// `internal/meta` is private, so a hive whose nix cannot authenticate +/// ([`MetaSource::nix_can_fetch`]) builds from its own flake. The head is +/// taken only when its `flake.nix` builds the agent the way it is wanted now: +/// from `archetype` when that is set, otherwise from the config commit the +/// forge's `main` is at. A head that is behind (the swarm's update has not +/// run, or failed), lacks the agent or lists it under another archetype would +/// build an older module than the one being deployed. +/// +/// # Errors +/// A forge read that failed. +pub(crate) async fn choose( + source: &impl MetaSource, + agent: &str, + archetype: Option<&ArchetypeRef>, +) -> Result> { + if !source.nix_can_fetch() { + return Ok(None); + } + let Some(rev) = source.meta_head().await? else { + return Ok(None); + }; + if !valid_meta_rev(&rev) { + return Ok(None); + } + let flake = source.flake_at(&rev).await?; + let config_head = if archetype.is_some() { + None + } else { + let Some(head) = source.config_head(agent).await? else { + return Ok(None); + }; + Some(head) + }; + Ok( + hive_meta_flake::builds_agent_at(&flake, agent, archetype, config_head.as_deref()) + .then_some(rev), + ) +} + +/// [`choose`] against the forge this hive reads. `None`, logged, whenever the +/// hive cannot tell: no forge credential, a failed read, no usable commit. +pub(crate) async fn resolve(agent: &str) -> Option { + let archetype = match crate::archetypes::of(agent) { + Ok(archetype) => archetype, + Err(e) => { + tracing::warn!(%agent, error = ?e, "internal/meta: cannot read the agent's archetype; building from the hive's own flake"); + return None; + } + }; + let Some(source) = crate::forge::internal_meta_source().await else { + tracing::info!(%agent, "internal/meta: no forge read token; building from the hive's own flake"); + return None; + }; + match choose(&source, agent, archetype.as_ref()).await { + Ok(Some(rev)) => { + tracing::info!(%agent, %rev, "internal/meta: building from the swarm's meta flake"); + Some(rev) + } + Ok(None) => { + tracing::info!(%agent, "internal/meta: absent or not pinning this agent's config; building from the hive's own flake"); + None + } + Err(e) => { + tracing::warn!(%agent, error = %format!("{e:#}"), "internal/meta: cannot read it; building from the hive's own flake"); + None + } + } +} + +/// Record `rev` as the commit `agent`'s build reads, or clear the record for +/// `None`. The record is in hive-c0re's own state ([`crate::paths::meta_rev_dir`]), +/// where the agent cannot write. +/// +/// # Errors +/// The record file could not be written or removed. +pub(crate) fn record(agent: &str, rev: Option<&str>) -> Result<()> { + record_in(&crate::paths::meta_rev_dir(), agent, rev) +} + +/// The commit [`record`] stored for `agent`, if it is a valid one. +pub(crate) fn recorded(agent: &str) -> Option { + recorded_in(&crate::paths::meta_rev_dir(), agent) +} + +/// Written whole to a temp file beside the record and renamed over it, so a +/// reader sees the old record or the new one and a crash leaves no partial +/// file. +fn record_in(dir: &std::path::Path, agent: &str, rev: Option<&str>) -> Result<()> { + let path = dir.join(agent); + match rev { + Some(rev) => { + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + let tmp = dir.join(format!(".{agent}.tmp")); + std::fs::write(&tmp, rev).with_context(|| format!("write {}", tmp.display()))?; + std::fs::rename(&tmp, &path) + .with_context(|| format!("rename {} to {}", tmp.display(), path.display())) + } + None => match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + Err(e).with_context(|| format!("remove {}", path.display())) + } + _ => Ok(()), + }, + } +} + +fn recorded_in(dir: &std::path::Path, agent: &str) -> Option { + let rev = std::fs::read_to_string(dir.join(agent)).ok()?; + valid_meta_rev(rev.trim()).then(|| rev.trim().to_owned()) +} + +/// The `#` ref [`crate::lifecycle::prebuild_toplevel`] builds +/// for `rev`: the same flake hive-priv builds, from the same forge URL. +pub(crate) fn flake_ref(agent: &str, rev: &str) -> String { + flake_ref_at(crate::forge::forge_http_base(), agent, rev) +} + +fn flake_ref_at(forge_base: &str, agent: &str, rev: &str) -> String { + format!("{}#{agent}", meta_flake_url(forge_base, rev)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HYPERHIVE: &str = "1111111111111111111111111111111111111111"; + const META: &str = "3333333333333333333333333333333333333333"; + const ARGUS_HEAD: &str = "2222222222222222222222222222222222222222"; + const STALE: &str = "4444444444444444444444444444444444444444"; + + /// A forge holding `internal/meta` at [`META`] with the given flake, and + /// `agent-configs/argus` at `config_head`. + struct Forge { + head: Option<&'static str>, + flake: String, + config_head: Option<&'static str>, + fetchable: bool, + } + + impl MetaSource for Forge { + fn nix_can_fetch(&self) -> bool { + self.fetchable + } + async fn meta_head(&self) -> Result> { + Ok(self.head.map(str::to_owned)) + } + async fn flake_at(&self, rev: &str) -> Result { + assert_eq!(Some(rev), self.head, "the flake is read at the head"); + Ok(self.flake.clone()) + } + async fn config_head(&self, _: &str) -> Result> { + Ok(self.config_head.map(str::to_owned)) + } + } + + /// `internal/meta`'s flake as swarm-controller renders it: argus pinned at + /// [`ARGUS_HEAD`], and a reviewer from an archetype. + fn flake() -> String { + use swarm_wire::agent_archetype::ArchetypeRef; + + let url = |repo: &str, rev: &str| { + hive_meta_flake::pinned_git_url(&format!("https://f.t/{repo}.git"), Some(rev)).unwrap() + }; + let spec = |name: &str, archetype, pin| hive_meta_flake::AgentSpec { + name: name.to_owned(), + port: hive_meta_flake::agent_web_port(name), + archetype, + pin, + }; + hive_meta_flake::render_flake( + &url("hyperhive/hyperhive", HYPERHIVE), + &[ + spec( + "argus", + None, + Some(hive_meta_flake::AgentPin { + url: url("agent-configs/argus", ARGUS_HEAD), + canonical_inputs: vec![], + }), + ), + spec( + "rev", + Some(ArchetypeRef { + source: "hyperhive".to_owned(), + name: "reviewer".to_owned(), + }), + None, + ), + ], + &[], + false, + ) + .unwrap() + } + + fn forge(config_head: Option<&'static str>) -> Forge { + Forge { + head: Some(META), + flake: flake(), + config_head, + fetchable: true, + } + } + + #[tokio::test] + async fn the_head_is_taken_when_it_pins_the_agents_current_config() { + assert_eq!( + choose(&forge(Some(ARGUS_HEAD)), "argus", None) + .await + .unwrap(), + Some(META.to_owned()) + ); + } + + #[tokio::test] + async fn a_head_behind_the_agents_config_is_not_taken() { + assert_eq!( + choose(&forge(Some(STALE)), "argus", None).await.unwrap(), + None + ); + } + + #[tokio::test] + async fn a_head_without_the_agent_is_not_taken() { + let mut source = forge(Some(ARGUS_HEAD)); + source.flake = source.flake.replace("argus", "someone"); + assert_eq!(choose(&source, "argus", None).await.unwrap(), None); + } + + fn reviewer() -> ArchetypeRef { + ArchetypeRef { + source: "hyperhive".to_owned(), + name: "reviewer".to_owned(), + } + } + + #[tokio::test] + async fn an_archetype_agent_is_taken_when_the_head_lists_that_archetype() { + struct Archetypes(Forge); + impl MetaSource for Archetypes { + fn nix_can_fetch(&self) -> bool { + self.0.nix_can_fetch() + } + async fn meta_head(&self) -> Result> { + self.0.meta_head().await + } + async fn flake_at(&self, rev: &str) -> Result { + self.0.flake_at(rev).await + } + async fn config_head(&self, _: &str) -> Result> { + panic!("an archetype agent has no config repo to read") + } + } + let source = Archetypes(forge(None)); + assert_eq!( + choose(&source, "rev", Some(&reviewer())).await.unwrap(), + Some(META.to_owned()) + ); + let other = ArchetypeRef { + name: "planner".to_owned(), + ..reviewer() + }; + assert_eq!( + choose(&source, "rev", Some(&other)).await.unwrap(), + None, + "the head lists another archetype" + ); + assert_eq!( + choose(&source, "ghost", Some(&reviewer())).await.unwrap(), + None + ); + } + + /// The head lists argus as a custom agent and rev as an archetype agent; + /// an agent switched either way is not what the head builds. + #[tokio::test] + async fn an_agent_switched_between_custom_and_archetype_is_not_taken() { + let source = forge(Some(ARGUS_HEAD)); + assert_eq!( + choose(&source, "argus", Some(&reviewer())).await.unwrap(), + None, + "custom in the head, archetype now" + ); + assert_eq!( + choose(&source, "rev", None).await.unwrap(), + None, + "archetype in the head, custom now" + ); + } + + /// A hive without the credential nix's git reads builds from its own flake + /// even though its own token reads the repo; the first test is the control. + #[tokio::test] + async fn a_hive_whose_nix_cannot_authenticate_builds_from_its_own_flake() { + let mut source = forge(Some(ARGUS_HEAD)); + source.fetchable = false; + assert_eq!(choose(&source, "argus", None).await.unwrap(), None); + } + + #[tokio::test] + async fn no_repo_or_no_commit_is_the_hives_own_flake() { + let mut source = forge(Some(ARGUS_HEAD)); + source.head = None; + assert_eq!(choose(&source, "argus", None).await.unwrap(), None); + } + + #[tokio::test] + async fn a_custom_agent_with_no_config_commit_is_the_hives_own_flake() { + assert_eq!(choose(&forge(None), "argus", None).await.unwrap(), None); + } + + #[test] + fn a_record_is_read_back_and_cleared() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!(recorded_in(dir.path(), "argus"), None); + record_in(dir.path(), "argus", Some(META)).unwrap(); + assert_eq!(recorded_in(dir.path(), "argus"), Some(META.to_owned())); + assert_eq!( + recorded_in(dir.path(), "rev"), + None, + "a record is one agent's" + ); + record_in(dir.path(), "argus", Some(STALE)).unwrap(); + assert_eq!(recorded_in(dir.path(), "argus"), Some(STALE.to_owned())); + record_in(dir.path(), "argus", None).unwrap(); + assert_eq!(recorded_in(dir.path(), "argus"), None); + record_in(dir.path(), "argus", None).unwrap(); + } + + #[test] + fn a_record_leaves_no_temp_file_and_creates_its_dir() { + let dir = tempfile::tempdir().unwrap(); + let records = dir.path().join("meta-rev"); + record_in(&records, "argus", Some(META)).unwrap(); + let names: Vec<_> = std::fs::read_dir(&records) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(names, ["argus"]); + } + + #[test] + fn a_record_that_is_not_a_commit_id_is_no_record() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("argus"), "main\n").unwrap(); + assert_eq!(recorded_in(dir.path(), "argus"), None); + } + + /// The record is hive-c0re's own state, never the agent-writable runtime + /// dir bind-mounted into the container. + #[test] + fn the_record_is_outside_the_agents_runtime_dir() { + assert!(!crate::paths::meta_rev_dir().starts_with(crate::paths::runtime_root())); + assert!(crate::paths::meta_rev_dir().starts_with(crate::paths::state_root())); + } + + #[test] + fn the_prebuild_ref_is_the_flake_hive_priv_builds() { + assert_eq!( + flake_ref_at("http://forge.t.local", "argus", META), + format!("git+http://forge.t.local/internal/meta.git?rev={META}#argus") + ); + } +} diff --git a/hive-c0re/src/paths.rs b/hive-c0re/src/paths.rs index d7cbddff3..574be7aca 100644 --- a/hive-c0re/src/paths.rs +++ b/hive-c0re/src/paths.rs @@ -195,6 +195,17 @@ pub fn meta_git_index_lock() -> PathBuf { meta_root().join(".git/index.lock") } +/// `meta-rev/` — one file per agent naming the `internal/meta` commit its +/// rebuild builds from (`crate::meta_rev`). +/// +/// ⚠️ Deliberately **not** in the agent's runtime dir: that is bind-mounted +/// read-write into the container as `/run/hive`, and the commit decides which +/// flake the agent's own toplevel is built from. +#[must_use] +pub fn meta_rev_dir() -> PathBuf { + state_root().join("meta-rev") +} + /// `shared/` — the cross-agent `/shared` scratch space. A `&str`, so it /// stays a const; [`shared_root`] wraps it. // nix: bind-mounted into every agent container as `/shared` (the harness nix modules) — must match. diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index d772873bb..70f39ddd6 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -235,12 +235,14 @@ pub async fn kill_container(name: &str) -> Result<()> { /// appending lines to `build_logs` or otherwise capturing the output. pub async fn update_container_streaming( name: &str, + meta_rev: Option<&str>, on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { ok(call_streaming( &PrivRequest::UpdateContainer { name: name.to_owned(), stream: true, + meta_rev: meta_rev.map(str::to_owned), }, on_line, ) @@ -251,12 +253,14 @@ pub async fn update_container_streaming( /// arrive. Returns `Ok(())` on success. pub async fn create_container_streaming( name: &str, + meta_rev: Option<&str>, on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { ok(call_streaming( &PrivRequest::CreateContainer { name: name.to_owned(), stream: true, + meta_rev: meta_rev.map(str::to_owned), }, on_line, ) diff --git a/hive-meta-flake/Cargo.toml b/hive-meta-flake/Cargo.toml new file mode 100644 index 000000000..2402defdd --- /dev/null +++ b/hive-meta-flake/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "hive-meta-flake" +edition.workspace = true +version.workspace = true +readme = "README.md" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +serde_json.workspace = true +swarm-wire.workspace = true diff --git a/hive-meta-flake/README.md b/hive-meta-flake/README.md new file mode 100644 index 000000000..9e1141be8 --- /dev/null +++ b/hive-meta-flake/README.md @@ -0,0 +1,10 @@ +# hive-meta-flake + +The meta flake's `flake.nix` text: one `nixosConfigurations.` per agent, +each built from hyperhive's `agent-base` plus the agent's config input or +archetype. + +Pure string rendering, with no IO and no environment reads. The caller passes +the hyperhive pin, the agents with their pins, and the values every agent +container gets. hive-c0re and swarm-controller both call `render_flake`, so +there is one renderer. diff --git a/hive-meta-flake/src/lib.rs b/hive-meta-flake/src/lib.rs new file mode 100644 index 000000000..6dbc6c296 --- /dev/null +++ b/hive-meta-flake/src/lib.rs @@ -0,0 +1,626 @@ +//! The meta flake's text: one `nixosConfigurations.` per agent, built +//! from hyperhive's `agent-base` and the agent's config input or archetype. +//! +//! Pure: the output is a function of the arguments only. hive-c0re and +//! swarm-controller both render through [`render_flake`], so the flake a hive +//! builds has one renderer. + +use anyhow::{Context, Result}; +use swarm_wire::agent_archetype::ArchetypeRef; + +/// One agent of the meta flake. +#[derive(Debug, Clone)] +pub struct AgentSpec { + pub name: String, + pub port: u16, + /// The archetype the agent is built from, `None` + /// for a custom agent built from its own config repo. + pub archetype: Option, + /// The pin of a custom agent's config input. `None` until the caller + /// resolves it; an archetype agent has no config input and keeps `None`. + pub pin: Option, +} + +/// What the rendered flake needs to know about one agent's config input, all +/// of it read from the agent's config repo at one commit. The rendered text +/// is a function of this and nothing on the host. +#[derive(Debug, Clone)] +pub struct AgentPin { + /// `git+/agent-configs/.git?rev=`, `` being the head of + /// `main` on the forge when the flake was rendered. + pub url: String, + /// The `CANONICAL_INPUTS` the agent's `flake.lock` declares at that commit. + pub canonical_inputs: Vec<&'static str>, +} + +/// `git+?rev=`. Errors on a missing or empty rev rather +/// than rendering an unpinned URL. +pub fn pinned_git_url(forge_git_url: &str, rev: Option<&str>) -> Result { + let rev = rev.filter(|r| !r.is_empty()).context( + "hyperhive rev unknown: the host was built from a dirty tree \ + (`self.rev` unset) and `services.hyperhive.c0re.hyperhiveRev` is not set", + )?; + Ok(format!("git+{forge_git_url}?rev={rev}")) +} + +/// Canonical inputs meta knows how to dedup. An agent that declares one +/// of these as a top-level input in its own `flake.nix` will get a +/// `follows = ""` line emitted in meta — collapsing the +/// otherwise-separate-but-identical `nixpkgs_N` nodes into a single +/// meta-level reference. +pub const CANONICAL_INPUTS: &[&str] = &["nixpkgs"]; + +/// Env vars the meta flake sets in every agent container, from the +/// `(name, value)` pairs [`render_flake`] is given. A caller passes only the +/// names it has a non-empty value for, so absent options render no no-op +/// `FOO = ""` lines. +pub const FORWARDED_VARS: &[&str] = &[ + // Where the swarm secret store listens. The hive's own `BAO_ADDR` value, + // forwarded under a separate name so the container never inherits the + // hive's store environment wholesale — the address is the only part of it + // an agent may have, the certificate beside it being the hive's own. + "HIVE_AGENT_BAO_ADDR", + // The mount path of the store's services `pki` engine, whose root an agent + // fetches to trust the swarm. Forwarded with the address, under the same gate. + "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", + // Where an agent's `swarm-logs` queries, computed by the host: it is a + // gateway address, and a container cannot derive it from anything it holds. + "HIVE_AGENT_LOGS_QUERY_URL", + // The metrics store's origin, for `swarm-metrics`, for the same reason. + "HIVE_AGENT_METRICS_URL", + "HIVE_FORGE_URL", + "HIVE_FORGE_PUBLIC_URL", + "HIVE_MATRIX_URL", + "HYPERHIVE_SWARM_NAME", +]; + +/// Map of forwarded env var -> the agent option carrying the same value. +/// +/// Both exist because they're consumed at different times: the option is baked +/// into scripts at build time (forge-avatar-sync bakes `FORGE_URL` from it), the env +/// var is read at runtime. Setting only one leaves the other on its default, +/// which is how a hive ends up with two disagreeing answers for one value. +/// +/// The swarm display name is here for that reason: `otel.nix` reads the option +/// at eval time for the `swarm=` resource label, and with the env var alone +/// that label evaluates to `unknown` while the process env holds the right +/// name. Half-wiring this map is a value that evaluates fine and is silently +/// wrong. +pub const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[ + // Read at build time to decide whether the in-container identity check is + // generated at all — see `nix/agent-modules/bao.nix`. Both halves wired + // for this map's own reason: the unit bakes the address into its own + // environment, so an agent whose option says no store and whose env says + // otherwise would have a check that is simply not there. + ("HIVE_AGENT_BAO_ADDR", "services.hyperhive.agent.bao.addr"), + // Read at build time to bake the root-fetch URL into + // `nix/agent-modules/hive-ca.nix`'s unit. + ( + "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", + "services.hyperhive.agent.bao.servicesPkiMount", + ), + // Read at build time to decide whether `swarm-logs` is installed at all — + // see `nix/agent-modules/logs.nix`. Both halves wired for this map's own + // reason: an agent whose option says no log store and whose env says + // otherwise has a binary that is not on PATH and coordinates that claim + // it is. + ( + "HIVE_AGENT_LOGS_QUERY_URL", + "services.hyperhive.agent.logs.queryUrl", + ), + // The same for `swarm-metrics`: see `nix/agent-modules/metrics.nix`. + ( + "HIVE_AGENT_METRICS_URL", + "services.hyperhive.agent.metrics.url", + ), + ("HIVE_FORGE_URL", "services.hyperhive.agent.forge.url"), + ("HIVE_MATRIX_URL", "services.hyperhive.agent.matrix.url"), + ("HYPERHIVE_SWARM_NAME", "services.hyperhive.agent.swarmName"), +]; + +/// Render the forwarded-var option assignments for one agent's module block. +/// +/// Split out of `render_flake` so it can be tested without touching process +/// env: the render-level tests have to `set_var`, which forces them to +/// serialise on the crate's `test_env` lock (a `#[cfg(test)]` module, so this +/// deliberately isn't an intra-doc link). A pure function over the +/// already-collected pairs has no such hazard. +/// +/// A var that isn't present emits nothing rather than a guess. For an optional +/// service that is the whole point — the agent option defaults to `null`, +/// meaning "not configured", and the units that would use it aren't generated. +/// For a service the hive cannot run without, silence would instead produce a +/// fleet of agents quietly missing an integration, so the caller checks +/// those before rendering. +pub fn push_forwarded_var_options(out: &mut String, vars: &[(&'static str, String)]) { + use std::fmt::Write as _; + for (var, val) in vars { + let Some((_, option)) = FORWARDED_VAR_OPTIONS.iter().find(|(name, _)| name == var) else { + continue; + }; + let escaped = val.replace('\\', "\\\\").replace('"', "\\\""); + let _ = writeln!(out, " {option} = \"{escaped}\";"); + } +} + +/// The subset of `CANONICAL_INPUTS` that the `flake.lock` text `raw` declares +/// as direct (root-level) inputs. Empty when the lock is unparsable — safe +/// degradation, the worst case is no dedup for that agent. +#[must_use] +pub fn canonical_inputs_in_lock(raw: &str) -> Vec<&'static str> { + let Ok(json) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + let Some(nodes) = json.get("nodes").and_then(|v| v.as_object()) else { + return Vec::new(); + }; + let Some(root_name) = json.get("root").and_then(|v| v.as_str()) else { + return Vec::new(); + }; + let Some(root_inputs) = nodes + .get(root_name) + .and_then(|n| n.get("inputs")) + .and_then(|v| v.as_object()) + else { + return Vec::new(); + }; + CANONICAL_INPUTS + .iter() + .copied() + .filter(|canon| root_inputs.contains_key(*canon)) + .collect() +} + +/// Render the meta flake. `vars` are the [`FORWARDED_VARS`] pairs with a +/// value; `github_disabled` turns the agents' GitHub integration off. Errors +/// on a custom agent without an [`AgentPin`]. +pub fn render_flake( + hyperhive_url: &str, + agents: &[AgentSpec], + vars: &[(&'static str, String)], + github_disabled: bool, +) -> Result { + let mut out = String::new(); + push_inputs(&mut out, hyperhive_url, agents)?; + push_mk_agent_open(&mut out); + push_agent_service_options(&mut out, vars, github_disabled); + push_agent_environment(&mut out, vars); + push_agent_entries(&mut out, agents); + Ok(out) +} + +/// The flake header and `inputs` block: `hyperhive`, `nixpkgs`, and one input +/// per config-repo agent. +fn push_inputs(out: &mut String, hyperhive_url: &str, agents: &[AgentSpec]) -> Result<()> { + use std::fmt::Write as _; + out.push_str("{\n description = \"hyperhive deployed agents\";\n inputs = {\n"); + // `hyperhive` is pinned by forge URL and rev, so the flake resolves on + // any machine that can reach the forge. `nixpkgs` follows hyperhive's + // own pin, which cascades to every agent because + // `agent-.inputs.nixpkgs.follows = "nixpkgs"` resolves to this node. + let _ = writeln!(out, " hyperhive.url = \"{hyperhive_url}\";"); + out.push_str(" nixpkgs.follows = \"hyperhive/nixpkgs\";\n"); + // Each agent's config input is its forge repo at the pinned commit, so + // the flake text depends only on (agent, rev) and resolves anywhere that + // can reach the forge. + for spec in agents { + // An archetype agent has no config repo, so no input: `mkAgent` + // takes its module from the archetype. + if spec.archetype.is_some() { + continue; + } + let pin = spec.pin.as_ref().with_context(|| { + format!( + "agent {}: no pinned rev, refusing to render an unpinned config input", + spec.name + ) + })?; + let _ = writeln!( + out, + " agent-{name}.url = \"{url}\";", + name = spec.name, + url = pin.url, + ); + // For each canonical input the agent's `flake.lock` declares at the + // pinned commit, emit + // `inputs.agent-.inputs..follows = ""`. + // Collapses otherwise-separate-but-identical nixpkgs nodes + // (root + every agent's own nixpkgs) into one. Skipped + // silently for agents that don't declare the input — emitting + // follows on a non-existent input would error at + // `nix flake lock` time. + for canon in &pin.canonical_inputs { + let _ = writeln!( + out, + " agent-{}.inputs.{canon}.follows = \"{canon}\";", + spec.name, + ); + } + } + Ok(()) +} + +/// Closes `inputs`, opens `outputs` and the `mkAgent` function up to the +/// start of the per-agent module. +fn push_mk_agent_open(out: &mut String) { + out.push_str( + " };\n outputs =\n { self, hyperhive, ... }@inputs:\n let\n mkAgent = { name, port, archetype ? null }:\n", + ); + out.push_str( + r#" let + base = hyperhive.nixosConfigurations.agent-base; + # Every flake an archetype can come from, by its source name. + archetypeSources = { inherit hyperhive; }; + agentModule = + if archetype == null then + inputs."agent-${name}".nixosModules.default + else + (archetypeSources.${archetype.source} + or (throw "agent ${name}: archetype source ${archetype.source} is not an input of this hive's meta flake") + ).hyperhiveArchetypes.${archetype.name}.module + or (throw "agent ${name}: this hive's ${archetype.source} input has no archetype ${archetype.name}; update that input"); + service = "hive-agent"; + in + base.extendModules { + modules = [ + agentModule + { +"#, + ); +} + +/// Service URL options and the GitHub override inside the per-agent module. +fn push_agent_service_options( + out: &mut String, + vars: &[(&'static str, String)], + github_disabled: bool, +) { + // Agent-facing service URLs (`services.hyperhive.agent.forge.url`, `services.hyperhive.agent.matrix.url`): + // emit the host's real values as build-time agent config, the same + // host-state -> agent-module shape as the CA block above. + // + // These options exist already, but until now *nothing set them*, so every + // agent silently fell back to their `localhost:` defaults while the + // true value reached the container only as an env var (below). Two sources + // of truth that disagree, with the winner decided by which code path + // happens to read which one --- the option default is baked into scripts + // at build time, the env var is read at runtime. + // + // Rendering them here makes the option the single source, which is the + // precondition for removing those defaults: a loopback address is only + // ever correct when the callee shares the caller's netns, and the forge + // and homeserver are moving to swarm level, possibly on other hosts. + // + // Absent vars emit nothing rather than a guess, and the agent option + // treats "unset" as "this service is not configured" rather than + // substituting a loopback address — an absent integration instead of a + // misdirected one. + // + // That is the right default for an optional service and the wrong one for + // the forge, which a running hive always has, so the caller rejects a + // missing `HIVE_FORGE_URL` before it renders. The check is not here because + // rendering is a pure string operation the tests exercise directly. + push_forwarded_var_options(out, vars); + // GitHub integration is on by default in every agent + // (`services.hyperhive.agent.github.enable`); the host turns it off swarm-wide via + // `services.hyperhive.swarm.github.enable = false`, passed here as + // `github_disabled`. Only the OFF + // override is propagated — the enabled default needs no per-agent line. + if github_disabled { + out.push_str(" services.hyperhive.agent.github.enable = false;\n"); + } +} + +/// The rest of the per-agent module: user, hostname, git identity, and the +/// container-wide and harness-service environments. +fn push_agent_environment(out: &mut String, vars: &[(&'static str, String)]) { + use std::fmt::Write as _; + out.push_str( + r#" # The harness service inside the container runs as a + # non-root unix user named after the agent (`damocles`, + # `iris`, `root`, …). UID auto-assigned by NixOS; the + # per-agent override here is what makes + # `services.hyperhive.agent.user.name` match the agent's identity + # instead of the harness default of `"agent"`. + services.hyperhive.agent.user.name = name; + # Without this an agent container is called `nixos` and ships + # every log line under that: nixpkgs sets the hostname from the + # attr name in the `containers.` submodule, and an + # agent's system is its own `nixosSystem` off this flake, which + # never evaluates that option. The machine name, not the + # logical one — it is what `container.name` on our metrics and + # nspawn itself already call this container, so logs join to + # both without a prefix transform. + networking.hostName = "h-${name}"; + programs.git.config.user = { + name = name; + email = "${name}@hyperhive.local"; + }; + # Container-wide env: every service + co-process daemon can + # resolve the agent's durable state dir without hard-coding it. + # `environment.variables` only writes /etc/environment (login + # shells); `systemd.globalEnvironment` is the analogue for + # systemd units so forge-avatar-sync / + # hive-matrix-daemon etc. can read `$HYPERHIVE_STATE_DIR` + # without each service having to redeclare it. + environment.variables = { + HIVE_LABEL = name; + HYPERHIVE_STATE_DIR = "/agents/${name}/state"; + HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness"; + }; + systemd.globalEnvironment = { + HIVE_LABEL = name; + HYPERHIVE_STATE_DIR = "/agents/${name}/state"; + HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness"; +"#, + ); + // Forwarded vars (HIVE_FORGE_URL etc.) also go into globalEnvironment, + // not just the harness service env below, so EVERY service + shell in + // the container inherits them — crucially the bash-task runner (where + // `hive-forge` + `git` actually run), plus the matrix daemon, forge-avatar-sync + // and interactive shells. Scoped to the harness service alone they were + // invisible to bash tasks: harmless in shared netns (the localhost + // default works) but broken under isolation, where the in-cluster + // `forge.` URL is the only reachable path. + for (var, val) in vars { + let escaped = val.replace('\\', "\\\\").replace('"', "\\\""); + let _ = writeln!(out, " {var} = \"{escaped}\";"); + } + out.push_str( + r" }; + systemd.services.${service}.environment = { + HIVE_PORT = toString port; + HIVE_LABEL = name; +", + ); + // Forwarded env vars, as the caller passed them. We copy whatever's set into + // each sub-agent's harness service env so the in-container surfaces + // (`identity.rs`, `forge_notify`) see a consistent view across the + // whole hive. Absent host-side env (option not set) → skip emission + // → in-container accessors fall back to None / defaults gracefully. + // + // - HIVE_FORGE_URL: agents poll this for Forgejo notifications. + // - HYPERHIVE_SWARM_NAME: human display name for the swarm. + for (var, val) in vars { + let escaped = val.replace('\\', "\\\\").replace('"', "\\\""); + let _ = writeln!(out, " {var} = \"{escaped}\";"); + } + out.push_str( + r#" HYPERHIVE_STATE_DIR = "/agents/${name}/state"; + HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness"; + }; + } + # Tells the harness the agent has no config repo of its own. + (if archetype == null then { } else { + systemd.services.${service}.environment.HIVE_ARCHETYPE = archetype.name; + }) + ]; + }; + in + { + nixosConfigurations = { +"#, + ); +} + +/// One `mkAgent` line per agent, closing `outputs`. +fn push_agent_entries(out: &mut String, agents: &[AgentSpec]) { + for spec in agents { + out.push_str(&agent_entry(&spec.name, spec.port, spec.archetype.as_ref())); + out.push('\n'); + } + out.push_str(" };\n };\n}\n"); +} + +/// One agent's `mkAgent` line in `nixosConfigurations`, without its newline. +fn agent_entry(name: &str, port: u16, archetype: Option<&ArchetypeRef>) -> String { + // `source` and `name` are identifiers (`archetypes::store_read`), so + // they need no escaping. + let archetype = archetype.map_or(String::new(), |a| { + format!( + " archetype = {{ source = \"{}\"; name = \"{}\"; }};", + a.source, a.name + ) + }); + format!(" {name} = mkAgent {{ name = \"{name}\"; port = {port};{archetype} }};") +} + +/// Web UI port range, shared by every agent including the manager. +/// Deterministic from the agent's name (FNV-1a hash mod range size), so the +/// dashboard can compute the same port without asking hive-c0re. +const WEB_PORT_BASE: u16 = 8100; +const WEB_PORT_RANGE: u16 = 900; + +/// FNV-1a hash of a string — used by `agent_web_port`. +fn fnv1a(s: &str) -> u32 { + let mut hash: u32 = 2_166_136_261; + for b in s.bytes() { + hash ^= u32::from(b); + hash = hash.wrapping_mul(16_777_619); + } + hash +} + +/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) % +/// WEB_PORT_RANGE` for every agent including the manager. The port +/// allocation rule reads the same for every name; collisions are +/// possible (birthday paradox at ~30 agents) and the operator +/// resolves them by renaming an agent (different hash → different +/// port). Stable across hosts, restarts, and dashboard renders — +/// no state-file dance. +#[must_use] +pub fn agent_web_port(name: &str) -> u16 { + // Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails. + WEB_PORT_BASE + u16::try_from(fnv1a(name) % u32::from(WEB_PORT_RANGE)).unwrap_or(0) +} + +/// Whether `rev` is a full git commit id: 40 lowercase hex digits. +#[must_use] +pub fn is_rev(rev: &str) -> bool { + rev.len() == 40 && rev.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +/// The commit `input` is pinned to in a flake [`render_flake`] rendered: +/// the `` of its `.url = "git+?rev=";` line. `None` when +/// the flake has no such input, or its rev is not a commit id. +#[must_use] +pub fn pinned_rev<'a>(flake_nix: &'a str, input: &str) -> Option<&'a str> { + let prefix = format!(" {input}.url = \""); + let url = flake_nix + .lines() + .find_map(|l| l.strip_prefix(prefix.as_str()))? + .split_once('"')? + .0; + let (_, rev) = url.split_once("?rev=")?; + let rev = rev.split('&').next()?; + is_rev(rev).then_some(rev) +} + +/// Whether a flake [`render_flake`] rendered builds agent `name` the way the +/// caller wants it built: from `archetype` when that is set, otherwise from +/// its own config repo at the commit `config_head`. A flake that lists the +/// agent under another archetype, or as custom where `archetype` is set (or +/// the reverse), does not; nor does a custom agent with no `config_head`. +#[must_use] +pub fn builds_agent_at( + flake_nix: &str, + name: &str, + archetype: Option<&ArchetypeRef>, + config_head: Option<&str>, +) -> bool { + let entry = format!("\n{}\n", agent_entry(name, agent_web_port(name), archetype)); + if !flake_nix.contains(&entry) { + return false; + } + archetype.is_some() + || config_head + .is_some_and(|head| pinned_rev(flake_nix, &format!("agent-{name}")) == Some(head)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HYPERHIVE: &str = "1111111111111111111111111111111111111111"; + const ARGUS: &str = "2222222222222222222222222222222222222222"; + + fn spec(name: &str, archetype: Option, pin: Option) -> AgentSpec { + AgentSpec { + name: name.to_owned(), + port: agent_web_port(name), + archetype, + pin, + } + } + + fn rendered() -> String { + let url = |repo: &str, rev: &str| { + pinned_git_url(&format!("https://f.t/{repo}.git"), Some(rev)).unwrap() + }; + render_flake( + &url("hyperhive/hyperhive", HYPERHIVE), + &[ + spec( + "argus", + None, + Some(AgentPin { + url: url("agent-configs/argus", ARGUS), + canonical_inputs: vec![], + }), + ), + spec( + "rev", + Some(ArchetypeRef { + source: "hyperhive".to_owned(), + name: "reviewer".to_owned(), + }), + None, + ), + ], + &[], + false, + ) + .unwrap() + } + + #[test] + fn a_rendered_flake_names_the_commit_each_input_is_pinned_to() { + let flake = rendered(); + assert_eq!(pinned_rev(&flake, "hyperhive"), Some(HYPERHIVE)); + assert_eq!(pinned_rev(&flake, "agent-argus"), Some(ARGUS)); + assert_eq!( + pinned_rev(&flake, "agent-rev"), + None, + "an archetype has no input" + ); + assert_eq!(pinned_rev(&flake, "agent-ghost"), None); + } + + #[test] + fn a_pin_that_is_not_a_commit_id_is_no_pin() { + let flake = " hyperhive.url = \"git+https://f.t/h.git?rev=main\";\n"; + assert_eq!(pinned_rev(flake, "hyperhive"), None); + assert_eq!( + pinned_rev( + " hyperhive.url = \"git+https://f.t/h.git\";\n", + "hyperhive" + ), + None + ); + } + + fn reviewer() -> ArchetypeRef { + ArchetypeRef { + source: "hyperhive".to_owned(), + name: "reviewer".to_owned(), + } + } + + #[test] + fn a_custom_agent_is_built_only_at_the_config_head_the_flake_pins() { + let flake = rendered(); + assert!(builds_agent_at(&flake, "argus", None, Some(ARGUS))); + assert!( + !builds_agent_at(&flake, "argus", None, Some(HYPERHIVE)), + "a stale pin" + ); + assert!( + !builds_agent_at(&flake, "argus", None, None), + "a custom agent with no config commit" + ); + assert!( + !builds_agent_at(&flake, "ghost", None, Some(ARGUS)), + "an agent the flake lacks" + ); + } + + #[test] + fn an_archetype_agent_is_built_only_from_the_archetype_the_flake_lists() { + let flake = rendered(); + assert!(builds_agent_at(&flake, "rev", Some(&reviewer()), None)); + let other = ArchetypeRef { + name: "planner".to_owned(), + ..reviewer() + }; + assert!( + !builds_agent_at(&flake, "rev", Some(&other), None), + "another archetype of the same source" + ); + assert!( + !builds_agent_at(&flake, "rev", None, Some(ARGUS)), + "listed as an archetype agent, wanted as a custom one" + ); + assert!( + !builds_agent_at(&flake, "argus", Some(&reviewer()), None), + "listed as a custom agent, wanted as an archetype one" + ); + assert!(!builds_agent_at(&flake, "ghost", Some(&reviewer()), None)); + } + + #[test] + fn an_agent_whose_name_ends_another_s_is_not_that_agent() { + let flake = rendered(); + assert!(!builds_agent_at(&flake, "gus", None, Some(ARGUS))); + assert!(!builds_agent_at(&flake, "v", Some(&reviewer()), None)); + } +} diff --git a/hive-priv-sock/Cargo.toml b/hive-priv-sock/Cargo.toml index d147d12ea..ee48717aa 100644 --- a/hive-priv-sock/Cargo.toml +++ b/hive-priv-sock/Cargo.toml @@ -9,3 +9,6 @@ workspace = true [dependencies] serde.workspace = true + +[dev-dependencies] +serde_json.workspace = true diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 331c50d2b..129fdcf81 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -101,12 +101,36 @@ impl std::str::FromStr for InfraContainer { } } -/// Host path of the meta flake. The flake ref for agent `` is -/// `{META_DIR}#{name}`, derived by `hive-priv` — never passed over the wire. -/// Must stay in sync with `hive-c0re::paths::meta_root()` (`STATE_ROOT + -/// "/meta"`); the privsep boundary prevents importing across the crate. +/// Host path of the meta flake a hive renders itself. The flake ref for agent +/// `` is `{META_DIR}#{name}`, derived by `hive-priv` — never passed over +/// the wire. Must stay in sync with `hive-c0re::paths::meta_root()` +/// (`STATE_ROOT + "/meta"`); the privsep boundary prevents importing across +/// the crate. pub const META_DIR: &str = "/var/lib/hyperhive/meta"; +/// Whether `rev` is a git commit id: 40 lowercase hex digits. The only part of +/// the `internal/meta` flake ref that crosses the wire (`meta_rev` on +/// `PrivRequest::UpdateContainer` and `CreateContainer`), so `hive-priv` checks +/// it before it builds a ref. +#[must_use] +pub fn valid_meta_rev(rev: &str) -> bool { + rev.len() == 40 && rev.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +/// The flake the swarm publishes at `internal/meta`, at `rev`: +/// `git+/internal/meta.git?rev=`. `forge_base` is the forge's +/// `http` origin, which the hive's git credential helper answers for. +/// +/// One definition for `hive-c0re`'s prebuild and `hive-priv`'s build, so both +/// name the same flake. +#[must_use] +pub fn meta_flake_url(forge_base: &str, rev: &str) -> String { + format!( + "git+{}/internal/meta.git?rev={rev}", + forge_base.trim_end_matches('/') + ) +} + /// Root of per-agent state directories on the host. /// Subdirectory layout: `//state/`. /// Used by `WriteAgentStateFile` to derive the write path so the @@ -196,6 +220,9 @@ pub struct HiveFacts { /// `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`: comma-separated `k=v` /// entries the agent appends to its `OTEL_RESOURCE_ATTRIBUTES`. pub otel_extra_resource_attributes: Option, + /// `HIVE_AGENT_NATS_URL`: where the agent reaches the swarm queue, from + /// `deploy.hive-controller.queue.agentNatsUrl`. + pub agent_nats_url: Option, } /// Wall-clock bound hive-priv puts on one [`PrivRequest::MeasureAgentDisk`] @@ -224,23 +251,33 @@ pub enum PrivRequest { KillContainer { name: String }, /// `nixos-container update --flake ` - /// The flake ref is derived from `name` by hive-priv. + /// The flake ref is derived from `name` by hive-priv: the hive's own meta + /// flake, or `internal/meta` at `meta_rev` when that is set (the forge URL + /// is hive-priv's own and never on the wire). /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. UpdateContainer { name: String, #[serde(default)] stream: bool, + /// A commit of `internal/meta` to build from, checked by + /// `valid_meta_rev`. `serde(default)` so a request from a hive-c0re + /// that predates the field builds from the hive's own flake. + #[serde(default)] + meta_rev: Option, }, /// `nixos-container create --flake ` - /// The flake ref is derived from `name` by hive-priv. + /// The flake ref is derived from `name` (and `meta_rev`) as for + /// `UpdateContainer`. /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. CreateContainer { name: String, #[serde(default)] stream: bool, + #[serde(default)] + meta_rev: Option, }, /// `nixos-container destroy ` @@ -653,7 +690,7 @@ pub enum PrivEvent { #[cfg(test)] mod tests { - use super::InfraContainer; + use super::{InfraContainer, PrivRequest, meta_flake_url, valid_meta_rev}; #[test] fn infra_control_names_no_c0re_or_swarm_service() { @@ -682,4 +719,56 @@ mod tests { fn gateway_resolves_to_host_nginx() { assert_eq!(InfraContainer::Gateway.service_unit(), "nginx.service"); } + + #[test] + fn only_a_full_lowercase_commit_id_is_a_meta_rev() { + assert!(valid_meta_rev(&"a1".repeat(20))); + for bad in [ + "", + "main", + &"a".repeat(39), + &"a".repeat(41), + &"A".repeat(40), + &format!("{}?x", "a".repeat(38)), + &format!("{}#x", "a".repeat(39)), + &format!("{} ", "a".repeat(39)), + ] { + assert!(!valid_meta_rev(bad), "{bad:?}"); + } + } + + #[test] + fn the_meta_flake_url_names_internal_meta_at_the_rev() { + let rev = "a".repeat(40); + assert_eq!( + meta_flake_url("http://forge.t.local/", &rev), + format!("git+http://forge.t.local/internal/meta.git?rev={rev}") + ); + } + + /// A hive-c0re that predates `meta_rev` sends neither field. + #[test] + fn a_request_without_meta_rev_builds_from_the_hives_own_flake() { + for op in ["update_container", "create_container"] { + let req: PrivRequest = + serde_json::from_str(&format!(r#"{{"op":"{op}","name":"atlas"}}"#)).unwrap(); + match req { + PrivRequest::UpdateContainer { meta_rev, .. } + | PrivRequest::CreateContainer { meta_rev, .. } => assert_eq!(meta_rev, None), + other => panic!("{other:?}"), + } + } + let with: PrivRequest = serde_json::from_str(&format!( + r#"{{"op":"update_container","name":"atlas","meta_rev":"{}"}}"#, + "b".repeat(40) + )) + .unwrap(); + assert!(matches!( + with, + PrivRequest::UpdateContainer { + meta_rev: Some(_), + .. + } + )); + } } diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs index c25bbdf1f..ddaf18e41 100644 --- a/hive-priv/src/container.rs +++ b/hive-priv/src/container.rs @@ -8,7 +8,9 @@ //! [`PrivEvent::Line`]s when asked to. use anyhow::{Context as _, Result, anyhow, bail}; -use hive_priv_sock::{META_DIR, PrivEvent, PrivStream, PrivStreamLine}; +use hive_priv_sock::{ + META_DIR, PrivEvent, PrivStream, PrivStreamLine, meta_flake_url, valid_meta_rev, +}; use tokio::io::{AsyncWriteExt, BufReader}; use tokio::net::unix::OwnedWriteHalf; use tokio::process::Command; @@ -100,6 +102,7 @@ pub(crate) async fn destroy_container(name: &str) -> Result<(String, String)> { pub(crate) async fn container_flake_action( verb: &str, name: &str, + meta_rev: Option<&str>, stream: bool, writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { @@ -111,7 +114,7 @@ pub(crate) async fn container_flake_action( // (which is the whole point of this fix) would silently regress every // UI that shows build progress: nothing until the longest phase // finishes, then everything at once. - let toplevel = nix_build_toplevel(name, stream.then_some(&mut *writer)).await?; + let toplevel = nix_build_toplevel(name, meta_rev, stream.then_some(&mut *writer)).await?; let system_name = container_system_name(name); if verb == "update" { return swap_container_profile(&system_name, &toplevel, stream, writer).await; @@ -208,12 +211,34 @@ async fn log_and_forward(stdout: &str, stderr: &str, stream: bool, writer: &mut } } +/// The environment variable naming the forge's `http` origin, which +/// `internal/meta` is fetched from. Set by the unit; never taken from a +/// request. +pub(crate) const FORGE_URL_ENV: &str = "HIVE_PRIV_FORGE_URL"; + +/// The flake agent toplevels are built from: the hive's own meta flake, or +/// `internal/meta` at `meta_rev` when the request names one. The URL is +/// `forge_base`, hive-priv's own; only the rev comes from the request, and it +/// is checked here. +fn meta_flake(meta_rev: Option<&str>, forge_base: Option<&str>) -> Result { + let Some(rev) = meta_rev else { + return Ok(META_DIR.to_owned()); + }; + if !valid_meta_rev(rev) { + bail!("meta_rev {rev:?} is not a 40-digit lowercase hex commit id"); + } + let base = forge_base.filter(|b| !b.is_empty()).with_context(|| { + format!("{FORGE_URL_ENV} is unset, so there is no internal/meta to build from") + })?; + Ok(meta_flake_url(base, rev)) +} + /// The explicit `nixosConfigurations..config.system.build.toplevel` /// flake attr path — same construction `hive-c0re`'s own /// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so /// the exact string shape is unit-tested without needing to run `nix`. -fn toplevel_attr(name: &str) -> String { - format!("{META_DIR}#nixosConfigurations.{name}.config.system.build.toplevel") +fn toplevel_attr(flake: &str, name: &str) -> String { + format!("{flake}#nixosConfigurations.{name}.config.system.build.toplevel") } /// Wall-clock bound on [`nix_build_toplevel`]. Four times CI's observed @@ -350,8 +375,13 @@ async fn run_bounded( /// `writer` is `None` for the non-streaming call shape (`stream: false`); /// stderr still logs to journald either way, just without the /// `PrivEvent::Line` forwarding. Bounded by [`NIX_BUILD_TIMEOUT`]. -async fn nix_build_toplevel(name: &str, writer: Option<&mut OwnedWriteHalf>) -> Result { - let attr = toplevel_attr(name); +async fn nix_build_toplevel( + name: &str, + meta_rev: Option<&str>, + writer: Option<&mut OwnedWriteHalf>, +) -> Result { + let flake = meta_flake(meta_rev, std::env::var(FORGE_URL_ENV).ok().as_deref())?; + let attr = toplevel_attr(&flake, name); let mut cmd = Command::new("nix"); cmd.args([ "--extra-experimental-features", @@ -687,7 +717,9 @@ async fn container_run_streaming( } #[cfg(test)] mod tests { - use super::{BoundedRun, run_bounded, single_output_path, toplevel_attr}; + use super::{ + BoundedRun, FORGE_URL_ENV, meta_flake, run_bounded, single_output_path, toplevel_attr, + }; use crate::shared::test_support::scratch; /// Pins the exact attr path we hand to `nix build` — it has to match @@ -698,11 +730,56 @@ mod tests { #[test] fn toplevel_attr_matches_prebuild_toplevels_construction() { assert_eq!( - toplevel_attr("atlas"), + toplevel_attr(&meta_flake(None, None).unwrap(), "atlas"), "/var/lib/hyperhive/meta#nixosConfigurations.atlas.config.system.build.toplevel" ); } + /// The flake is the hive's own unless the request names an `internal/meta` + /// commit, and then its URL is the unit's, never the request's. + #[test] + fn a_meta_rev_builds_from_internal_meta_at_the_units_forge() { + let rev = "c".repeat(40); + let forge = Some("http://forge.t.local"); + assert_eq!( + toplevel_attr(&meta_flake(Some(&rev), forge).unwrap(), "atlas"), + format!( + "git+http://forge.t.local/internal/meta.git?rev={rev}\ + #nixosConfigurations.atlas.config.system.build.toplevel" + ) + ); + // No rev: the forge is not consulted at all. + assert_eq!(meta_flake(None, forge).unwrap(), hive_priv_sock::META_DIR); + } + + /// Nothing but a commit id gets into the ref, whatever the request says. + #[test] + fn a_meta_rev_that_is_not_a_commit_id_is_refused() { + let forge = Some("http://forge.t.local"); + for bad in [ + "main", + "", + "../../x", + &format!("{}#evil", "a".repeat(34)), + &format!("{}&ref=x", "a".repeat(34)), + ] { + let err = meta_flake(Some(bad), forge).unwrap_err(); + assert!( + format!("{err:#}").contains("not a 40-digit"), + "{bad}: {err:#}" + ); + } + } + + #[test] + fn a_meta_rev_without_a_forge_in_the_unit_is_refused() { + let rev = "c".repeat(40); + for forge in [None, Some("")] { + let err = meta_flake(Some(&rev), forge).unwrap_err(); + assert!(format!("{err:#}").contains(FORGE_URL_ENV), "{err:#}"); + } + } + /// `--print-out-paths`' exact per-shape table (measured against real /// `rustc` semantics, not just reasoned about) — a lone `"\n"` and a /// trailing-space path are the two shapes a bare `.lines().collect()` diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs index 0fc3e2976..2cf886913 100644 --- a/hive-priv/src/container_config.rs +++ b/hive-priv/src/container_config.rs @@ -402,6 +402,13 @@ fn render_hive_facts(facts: &HiveFacts) -> Result { 1024, &|b: u8| dns(b) || matches!(b, b'=' | b',' | b'/' | b':' | b'%' | b'+' | b'@'), ), + // `://:`. + ( + "HIVE_AGENT_NATS_URL", + facts.agent_nats_url.as_ref(), + 253, + &|b: u8| dns(b) || matches!(b, b':' | b'/'), + ), ] { let Some(value) = value else { continue }; if value.is_empty() || value.len() > max_len || !value.bytes().all(allowed) { @@ -774,8 +781,8 @@ mod tests { } } - /// The three per-host values agents used to get at build time arrive - /// here, one line each. + /// The per-host values agents used to get at build time arrive here, one + /// line each. #[test] fn hive_facts_render_the_per_host_values() { let facts = HiveFacts { @@ -784,13 +791,15 @@ mod tests { otel_extra_resource_attributes: Some( "deployment.environment=prod,team=a%2Fb".to_owned(), ), + agent_nats_url: Some("tls://nats.example.test:4222".to_owned()), ..HiveFacts::default() }; assert_eq!( render_hive_facts(&facts).unwrap(), "HIVE_DASHBOARD_PORT=7123\n\ HYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code-2.1.220\n\ - HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=deployment.environment=prod,team=a%2Fb\n" + HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=deployment.environment=prod,team=a%2Fb\n\ + HIVE_AGENT_NATS_URL=tls://nats.example.test:4222\n" ); } diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 1981918a8..a2e8c71fa 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -349,12 +349,22 @@ async fn exec( PrivRequest::KillContainer { ref name } => container::kill_container(name).await, - PrivRequest::UpdateContainer { ref name, stream } => { - container::container_flake_action("update", name, stream, writer).await + PrivRequest::UpdateContainer { + ref name, + stream, + ref meta_rev, + } => { + container::container_flake_action("update", name, meta_rev.as_deref(), stream, writer) + .await } - PrivRequest::CreateContainer { ref name, stream } => { - container::container_flake_action("create", name, stream, writer).await + PrivRequest::CreateContainer { + ref name, + stream, + ref meta_rev, + } => { + container::container_flake_action("create", name, meta_rev.as_deref(), stream, writer) + .await } PrivRequest::DestroyContainer { ref name } => container::destroy_container(name).await, diff --git a/nix/agent-modules/claude-settings.nix b/nix/agent-modules/claude-settings.nix index a5e38a2ff..4278c8e91 100644 --- a/nix/agent-modules/claude-settings.nix +++ b/nix/agent-modules/claude-settings.nix @@ -72,7 +72,7 @@ in internal = true; description = '' Human-readable swarm name, rendered per-agent by - `meta.rs::render_flake` from the host's + `hive_meta_flake::render_flake` from the host's `services.hyperhive.swarm.name`. Read during evaluation for the OTEL `swarm` resource label; `HYPERHIVE_SWARM_NAME` carries the same value at runtime. diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index cafc97c9d..d7f475409 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -36,7 +36,7 @@ let ''; queueRemoved = '' The harness reads the queue's address at runtime from HIVE_AGENT_NATS_URL, - which the hive forwards into every container, and authenticates with the + in /etc/hyperhive-hive-facts, and authenticates with the agent's own secret from the swarm secret store, so no agent build sets a queue option. Remove the setting. ''; diff --git a/nix/agent-modules/hive-facts.nix b/nix/agent-modules/hive-facts.nix index 864965bbf..e9010ebf0 100644 --- a/nix/agent-modules/hive-facts.nix +++ b/nix/agent-modules/hive-facts.nix @@ -10,6 +10,8 @@ # ./default.nix runs this container's own when absent); # - `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`, appended to # `OTEL_RESOURCE_ATTRIBUTES` by ./otel.nix (no extra labels when absent). +# - `HIVE_AGENT_NATS_URL`, where the harness reaches the swarm queue (it fails +# at boot when absent). # # Any of them may be absent: a hive-c0re older than the agent's build writes # fewer. Two readers hand the lines on: @@ -43,6 +45,7 @@ let HYPERHIVE_HIVE_NAME) hive=$value ;; HYPERHIVE_HIVE_DOMAIN | HIVE_DASHBOARD_PORT) ;; HYPERHIVE_CLAUDE_CODE_PATH | HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES) ;; + HIVE_AGENT_NATS_URL) ;; *) continue ;; esac printf '%s=%s\n' "$key" "$value" diff --git a/nix/agent-modules/matrix.nix b/nix/agent-modules/matrix.nix index 6cc1da063..d9fabf64e 100644 --- a/nix/agent-modules/matrix.nix +++ b/nix/agent-modules/matrix.nix @@ -275,7 +275,7 @@ in # homeserver is an account the daemon can never log in as, so declaring one # unconditionally would enable matrix everywhere and inject a tool surface # backed by a permanently no-opping daemon. On a real hive hive-c0re renders - # this URL per agent (meta.rs's `FORWARDED_VAR_OPTIONS`), so the common case + # this URL per agent (hive-meta-flake's `FORWARDED_VAR_OPTIONS`), so the common case # is still "every agent has `main`". services.hyperhive.agent.matrixAccounts = lib.mkIf (config.services.hyperhive.agent.matrix.url != null) diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 119bbb7a7..d33d66d9e 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -35,6 +35,11 @@ let gatewayCfg = config.services.hyperhive.gateway; }; + forgeGit = import ../lib/hive-forge-git.nix { + inherit pkgs; + forgeDomain = config.services.hyperhive.swarm.forge.domain; + }; + # Privsep splits ownership across users, so git/libgit2's dubious- # ownership guard trips on legitimate cross-user reads: hive-priv (root) # fetches the hive-core-owned meta/applied repos via nix, and hive-c0re @@ -44,24 +49,13 @@ let # (mid-path globs aren't supported, so per-agent repos can't be listed); # in practice these processes only ever touch hyperhive's own repos. # - # The agent config inputs are `git+http` forge URLs pinned by sha (meta.rs - # render). The `[credential]` stanza points git at the `hive-forge` helper - # below, scoped to the forge host, for any `git+http` fetch from it as the - # forge `core` user. hive-c0re's own forge reads use `forge::reader_token()` instead. - # - # git runs every matching `credential.helper`, system config first, so a - # host helper (e.g. `oauth`, which waits on a browser) would run ahead of - # ours. An empty `helper =` resets the list; it comes first at both the - # global and the forge level so no /etc/gitconfig helper survives. + # The forge sections (`forgeGit.config`) answer for the `git+http` forge + # fetches nix makes: the agent config inputs and `internal/meta`. hive-c0re's + # own forge reads use `forge::reader_token()` instead. safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" '' [safe] directory = * - [credential] - helper = - [credential "http://${config.services.hyperhive.swarm.forge.domain}"] - helper = - helper = hive-forge - username = core + ${forgeGit.config} [http] # Abort a stalled fetch instead of blocking startup indefinitely. If a # git http transfer from the forge (e.g. filling an applied mirror) @@ -73,21 +67,6 @@ let lowSpeedTime = 30 ''; - # git credential helper answering for the forge host as `core`. Reads the - # live forge-core admin token - # (`/var/lib/hyperhive/forge-core-token` — paths.rs `FORGE_CORE_TOKEN`) on - # every invocation, so it never holds a stale copy and survives token - # rotation; prints nothing where that file is absent. Scoped to the forge - # host by the `[credential]` stanza above; implements the git - # credential-helper protocol (only `get` answers). - forgeCredHelper = pkgs.writeShellScriptBin "git-credential-hive-forge" '' - [ "''${1:-}" = "get" ] || exit 0 - if [ -r /var/lib/hyperhive/forge-core-token ]; then - printf 'username=core\n' - printf 'password=%s\n' "$(cat /var/lib/hyperhive/forge-core-token)" - fi - ''; - # The `hive-c0re serve` config JSON. Keys are snake_case to match the # `ServeConfig` serde shape the daemon deserialises (the # container-injected HiveEnv fields, flattened, plus the hive-c0re-local @@ -276,7 +255,7 @@ in pkgs.git # `git-credential-hive-forge` on PATH so git finds it for git+http # fetches from the forge (helper = hive-forge). - forgeCredHelper + forgeGit.credentialHelper "/run/current-system/sw" ]; environment = import ./environment.nix { inherit lib config pkgs; }; diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 37b63d28b..53b106367 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -222,8 +222,8 @@ in HIVE_C0RE_OIDC_CLIENT_ID = "hive-${config.services.hyperhive.hiveName}"; } // - # The agents' half of the same queue, forwarded by `hive_c0re::meta` into - # every container. + # Where the agents reach the queue: the address goes into every container in + # its hive facts (`hive_c0re::lifecycle::hive_facts`). # # ⚠️ By name, like `HIVE_C0RE_NATS_URL` above, and never a loopback # address: inside a container `127.0.0.1` is the agent itself. @@ -256,7 +256,7 @@ in lib.optionalAttrs haveBaoClientIdentity { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; # The same address again under an agent-facing name, forwarded into every - # container's module block by `meta::render_flake`. Deliberately not the + # container's module block by `hive_meta_flake::render_flake`. Deliberately not the # `BAO_ADDR` line itself: a container that inherited this hive's store # environment wholesale would inherit the hive's certificate paths with it, # and those name a credential no agent may present. diff --git a/nix/host-modules/hive-c0re/options.nix b/nix/host-modules/hive-c0re/options.nix index 17267da89..61c34b165 100644 --- a/nix/host-modules/hive-c0re/options.nix +++ b/nix/host-modules/hive-c0re/options.nix @@ -89,9 +89,12 @@ Git rev of the hyperhive checkout this host runs. hive-c0re pins the generated meta flake's `hyperhive` input to `git+/hyperhive/hyperhive.git?rev=`, and `nixpkgs` - follows that input. Wired by `nixosModules.default` from + follows that input. The swarm controller offers the same rev as the + `hyperhive` pin of `internal/meta`, which it takes when the rev is a + descendant of the committed pin. Wired by `nixosModules.default` from `self.rev`, which a dirty tree does not have; with `null`, - hive-c0re refuses to render the meta flake. + hive-c0re refuses to render the meta flake and the controller leaves + the `internal/meta` pin where it is. ''; }; agentBaseToplevel = lib.mkOption { diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index 69bbee787..54359c452 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -10,12 +10,20 @@ let cfg = config.services.hyperhive.c0re; - # Same safe.directory gitconfig as the c0re unit (see ./hive-c0re) - # — hive-priv (root) runs nix, which fetches the hive-core-owned - # meta/applied repos; libgit2 refuses cross-user reads without it. + forgeGit = import ./lib/hive-forge-git.nix { + inherit pkgs; + forgeDomain = config.services.hyperhive.swarm.forge.domain; + }; + + # Same gitconfig as the c0re unit (see ./hive-c0re) apart from its stalled- + # transfer limit: hive-priv (root) runs nix, which fetches the hive-core-owned + # meta/applied repos (libgit2 refuses cross-user reads without + # safe.directory) and, from a fresh HOME, the forge inputs of `internal/meta` + # (the forge sections). safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" '' [safe] directory = * + ${forgeGit.config} ''; in { @@ -82,6 +90,7 @@ in pkgs.nixos-container pkgs.nix # nix, nix-env, nix-instantiate — create + update pkgs.gitMinimal # git — nix fetches/re-resolves git-source flake inputs + forgeGit.credentialHelper # git-credential-hive-forge — the forge's `git+http` inputs pkgs.openssh # ssh — nix dispatches builds to remote builders pkgs.util-linux # umount (nsenter is hardcoded in the script) pkgs.e2fsprogs # chattr @@ -100,6 +109,9 @@ in # routing so nixos-container update and the nix prebuild see the # same store and substituters as every other build context. NIX_REMOTE = "daemon"; + # The forge origin `internal/meta` is built from when a request names + # one of its commits. Fixed here; no request chooses it. + HIVE_PRIV_FORGE_URL = "http://${config.services.hyperhive.swarm.forge.domain}"; }; serviceConfig = { ExecStart = "${cfg.package}/bin/hive-priv"; diff --git a/nix/host-modules/lib/hive-forge-git.nix b/nix/host-modules/lib/hive-forge-git.nix new file mode 100644 index 000000000..d885ded87 --- /dev/null +++ b/nix/host-modules/lib/hive-forge-git.nix @@ -0,0 +1,44 @@ +# Git settings for the host processes that hand nix a forge URL: hive-c0re +# (prebuilds as `hive-core`) and hive-priv (builds as root, with a HOME of its +# own), each through its own `$HOME/.gitconfig`. nix runs `git` for a `git+…` +# input, so git's config is the only credential and URL plumbing it has. +# +# - `credential`: the forge host answers through `git-credential-hive-forge`, +# which reads the live `core` token file. An empty `helper =` first resets the +# list, so no /etc/gitconfig helper (e.g. `oauth`, which waits on a browser) +# runs ahead of ours. +# - `url`: the swarm's `internal/meta` lock names its inputs by the forge's +# `https` URL, which only the builder's credential helper answers for. They +# are fetched over the `http` URL the helper above is scoped to. +# +# Pure function — NOT a NixOS module (don't add it to the host-modules +# aggregator). Call it from a module's `let`: +# +# forgeGit = import ./lib/hive-forge-git.nix { inherit pkgs; forgeDomain = …; }; +# # then put `forgeGit.credentialHelper` on the unit's `path` and +# # `forgeGit.config` into its gitconfig. +{ pkgs, forgeDomain }: +{ + # Answers only `get`, for the forge `core` user, from the live token file + # (`/var/lib/hyperhive/forge-core-token`, hive-c0re's `paths::FORGE_CORE_TOKEN`) + # on every invocation, so it never holds a stale copy and survives token + # rotation; prints nothing where the file is absent. + credentialHelper = pkgs.writeShellScriptBin "git-credential-hive-forge" '' + [ "''${1:-}" = "get" ] || exit 0 + if [ -r /var/lib/hyperhive/forge-core-token ]; then + printf 'username=core\n' + printf 'password=%s\n' "$(cat /var/lib/hyperhive/forge-core-token)" + fi + ''; + + config = '' + [credential] + helper = + [credential "http://${forgeDomain}"] + helper = + helper = hive-forge + username = core + [url "http://${forgeDomain}/"] + insteadOf = https://${forgeDomain}/ + ''; +} diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 55f149830..c9933720c 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -252,6 +252,46 @@ let SWARM_CONTROLLER_MATRIX_HOMESERVER_URL = deployCfg.swarm-controller.matrixHomeserverUrl; }; + # What `internal/meta` renders into every agent's build + # (swarm-controller/src/meta_flake.rs `Settings`): the names and values + # ./hive-c0re/environment.nix forwards, minus the per-host ones. One flake + # serves every hive, so a host's own answer here would reach all of them. + # `HIVE_AGENT_NATS_URL` is per host and reaches the container through the + # hive facts instead. + # + # The store-backed values are gated on this controller's store identity, + # where a hive gates them on its own: the swarm has a store when its + # controller can reach one. + metaEnv = { + SWARM_CONTROLLER_META_FLAKE = builtins.toJSON { + vars = { + HIVE_FORGE_URL = "http://${forgeCfg.domain}"; + } + // lib.optionalAttrs (forgeCfg.publicUrl != null) { + HIVE_FORGE_PUBLIC_URL = forgeCfg.publicUrl; + } + // lib.optionalAttrs (matrixCfg.gatewayHost != null) { + HIVE_MATRIX_URL = "http://${matrixCfg.gatewayHost}"; + } + // lib.optionalAttrs (swarmCfg.name != null) { + HYPERHIVE_SWARM_NAME = swarmCfg.name; + } + // lib.optionalAttrs haveBaoIdentity { + HIVE_AGENT_LOGS_QUERY_URL = "https://${swarmCfg.victorialogs.domain}/select/logsql/query"; + HIVE_AGENT_METRICS_URL = "https://${swarmCfg.victoriametrics.domain}"; + HIVE_AGENT_BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = deployCfg.bao.servicesPkiMountPath; + }; + # Where the pinned inputs are fetched from: the builder's credential + # helper answers only `https` for the forge. + git_base = "https://${forgeCfg.domain}"; + github_disabled = !swarmCfg.github.enable; + # The rev this controller runs, which `internal/meta` pins hyperhive + # to once it descends from the committed pin; null on a dirty tree. + hyperhive_rev = config.services.hyperhive.c0re.hyperhiveRev; + }; + }; + # The swarm's own display name, for `GET /api/swarm` (swarm-ui's chrome). # Gated on the option resolving, not defaulted to an empty string: an # operator who never named the swarm gets `name: null` from the @@ -831,7 +871,8 @@ in // authBridgeEnv // swarmNameEnv // matrixHomeserverEnv - // baoEnv; + // baoEnv + // metaEnv; }; }; } diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 929874756..175c3f634 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -477,7 +477,7 @@ in # Only the address lives here. Each agent reads its own credential from the # store (`nix/agent-modules/queue-identity.nix`). options.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl = lib.mkOption { - type = lib.types.nullOr lib.types.str; + type = lib.types.nullOr (lib.types.strMatching "[a-z]{1,10}://[A-Za-z0-9._-]{1,200}:[0-9]{1,5}"); default = "tls://${swarmCfg.nats.domain}:${toString swarmCfg.nats.port}"; defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"''; example = "tls://nats.example.com:4222"; @@ -492,6 +492,10 @@ in agent's network namespace `127.0.0.1` is the agent, not this host, and the queue's certificate names no address anyway. + One `://:`: a hostname or IPv4 address, no IPv6 literal and no + list. The value reaches the container as `HIVE_AGENT_NATS_URL`, whose + character set hive-priv enforces at container start. + Null means this hive's agents have not been given the queue's address, and the harness is handed none. ''; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 974a6eb0c..38d66ad8e 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -215,6 +215,49 @@ let } && env shared.controllerAlone == { }; } + { + # nix fetches the forge inputs with `git`, so the credential helper and + # the https-to-http rewrite are git config in both processes that build: + # hive-c0re (prebuilds) and hive-priv (builds as root, from its own + # HOME). The controller's unit is the control that the lookup tells a + # unit with the helper from one without; the URL hive-priv builds + # `internal/meta` from is the one hive-c0re reads the forge at. + name = "hive-c0re and hive-priv answer the forge's git fetches and rewrite its https inputs to http"; + ok = + let + forge = allLocal.services.hyperhive.swarm.forge.domain; + git = import ../host-modules/lib/hive-forge-git.nix { + inherit pkgs; + forgeDomain = forge; + }; + helperOn = + unit: + lib.any ( + p: builtins.isAttrs p && (p.name or "") == "git-credential-hive-forge" + ) allLocal.systemd.services.${unit}.path; + env = unit: allLocal.systemd.services.${unit}.environment; + in + helperOn "hive-c0re" + && helperOn "hive-priv" + && !(helperOn "swarm-controller") + && lib.hasInfix "[credential \"http://${forge}\"]\nhelper =\nhelper = hive-forge" git.config + && lib.hasInfix "[url \"http://${forge}/\"]\ninsteadOf = https://${forge}/" git.config + && (env "hive-priv").HIVE_PRIV_FORGE_URL == "http://${forge}" + && (env "hive-priv").HIVE_PRIV_FORGE_URL == (env "hive-c0re").HIVE_FORGE_URL; + } + { + # `hyperhive_rev` defaults to none on the controller, so a settings JSON + # without the key is accepted and the pin would never move: the key has + # to be there, carrying the option it is wired from. + name = "the controller hands its running rev to the meta flake settings"; + ok = + let + settings = builtins.fromJSON allLocal.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; + in + settings ? hyperhive_rev + && settings.hyperhive_rev == allLocal.services.hyperhive.c0re.hyperhiveRev + && settings ? git_base; + } { # An agent connects to the queue with its own secret, so the hive hands # its agents no shared credential and no token endpoint to mint one at. @@ -265,6 +308,25 @@ let name = "a hive with no agent queue address forwards none"; ok = !(noAgentQueue.systemd.services.hive-c0re.environment ? HIVE_AGENT_NATS_URL); } + { + # hive-priv refuses a value outside the facts' character set at container + # start, so the option refuses an IPv6 literal or a list at evaluation. + # Only the option's own value is forced, not the rest of either host. The + # default (`allLocal`) is the control that the lookup accepts an address. + name = "the agents' queue address option takes one scheme://host:port and nothing else"; + ok = + let + accepts = + host: (builtins.tryEval host.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl).success; + in + accepts allLocal + && !(accepts (hiveWithController { + deploy.hive-controller.queue.agentNatsUrl = "tls://[fd00::1]:4222"; + })) + && !(accepts (hiveWithController { + deploy.hive-controller.queue.agentNatsUrl = "tls://a.t.local:4222,tls://b.t.local:4222"; + })); + } { # Control: `bare` sets none of them, and its serve.json carries the # dashboard port, so the key lookup reads a real document. diff --git a/nix/script-tests/agent-hive-facts.sh b/nix/script-tests/agent-hive-facts.sh index 58f57ac77..2d1b3517f 100644 --- a/nix/script-tests/agent-hive-facts.sh +++ b/nix/script-tests/agent-hive-facts.sh @@ -40,7 +40,7 @@ check() { for sh in $SHELLS; do # Only what the snippet exported reaches the child's environment. No PATH: # the snippet must not need one. - got=$(env -i "$sh" -c ". '$dir/profile'; exec '$envbin'" | grep -E '^(HYPERHIVE_|HIVE_DASHBOARD_PORT=|CLAUDE_|LD_PRELOAD=|__hyperhive)' | sort) + got=$(env -i "$sh" -c ". '$dir/profile'; exec '$envbin'" | grep -E '^(HYPERHIVE_|HIVE_DASHBOARD_PORT=|HIVE_AGENT_NATS_URL=|CLAUDE_|LD_PRELOAD=|__hyperhive)' | sort) if [ "$got" != "$(printf '%s\n' "$want" | sort)" ]; then fail "$sh: a child of the sourcing shell saw:"$'\n'"$got" fi @@ -61,10 +61,11 @@ check "a last line without a newline is kept" "HYPERHIVE_HIVE_DOMAIN=pr1ma.examp HYPERHIVE_HIVE_NAME=pr1ma CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=pr1ma-a1" -printf 'HIVE_DASHBOARD_PORT=7123\nHYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code\nHYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod\n' >"$facts" +printf 'HIVE_DASHBOARD_PORT=7123\nHYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code\nHYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod\nHIVE_AGENT_NATS_URL=tls://nats.example.test:4222\n' >"$facts" check "the per-host values, a value holding = kept whole" "HIVE_DASHBOARD_PORT=7123 HYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod +HIVE_AGENT_NATS_URL=tls://nats.example.test:4222 CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=unknown-a1" : >"$facts" diff --git a/swarm-builder/README.md b/swarm-builder/README.md index ae3b640bb..7b7893835 100644 --- a/swarm-builder/README.md +++ b/swarm-builder/README.md @@ -39,3 +39,10 @@ every `hyperhiveArchetypes` entry in this host's hyperhive source, with nixpkgs pointed at this host's. It fetches nothing and uses no token, and it doesn't wait for a running build. See [`docs/swarm/archetypes.md`](../docs/swarm/archetypes.md). + +## Meta-flake lock + +The service also locks the swarm meta flake on request: `nix flake lock` of +the `flake.nix` it was sent, in a scratch directory, with the build's forge +credential. It answers with the lock and commits nothing. See +[`docs/swarm/builder.md`](../docs/swarm/builder.md#meta-flake-lock). diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index c68a38645..86b4aad76 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -8,6 +8,10 @@ //! [`swarm_wire::builder::ARCHETYPES_SUBJECT`] with the agent //! archetypes its hyperhive source declares, which the controller stores. //! +//! Lock requests on [`swarm_wire::builder::LOCK_SUBJECT`] carry the swarm meta +//! flake's `flake.nix`, and the answer is the `flake.lock` `nix` wrote for it in +//! a scratch directory. The controller commits it. +//! //! The store identity comes from the `BAO_*` environment. The queue client //! secret is read at start and again when authelia refuses it, the forge token //! once per build, both from the store under that identity. @@ -23,8 +27,8 @@ use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::forge::Credential; use swarm_secret_client::stored_secret::StoredSecret; use swarm_wire::builder::{ - ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, QUEUE_GROUP, - REPLY_DEADLINE, Reply, SUBJECT, + ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, LOCK_DEADLINE, + LOCK_SUBJECT, LockMeta, LockReply, QUEUE_GROUP, REPLY_DEADLINE, Reply, SUBJECT, }; mod archetypes; @@ -71,6 +75,7 @@ async fn main() -> Result<()> { forge_url: forge_url()?, config_org: var("CONFIG_ORG")?, deadline: BUILD_DEADLINE, + lock_deadline: LOCK_DEADLINE, }); let secret = swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { @@ -109,6 +114,13 @@ async fn main() -> Result<()> { .with_context(|| format!("subscribing to {ARCHETYPES_SUBJECT}"))?; // Outside the build slot below: a list refresh does not wait on a build. tokio::spawn(answer_evaluations(evaluations, queue.clone(), evaluator)); + let locks = queue + .queue_subscribe(LOCK_SUBJECT, QUEUE_GROUP.to_owned()) + .await + .with_context(|| format!("subscribing to {LOCK_SUBJECT}"))?; + // Outside the build slot too: a lock builds nothing, so it does not wait + // behind an hour-long build. + tokio::spawn(answer_locks(locks, queue.clone(), Arc::clone(&builder))); // One build at a time. swarm-controller already sends one at a time; this // holds the bound for any request it did not serialise, and `in_slot` @@ -186,6 +198,57 @@ async fn answer_evaluations( tracing::error!("the subscription to {ARCHETYPES_SUBJECT} ended"); } +/// Answer every meta-flake lock request, one at a time, with the lock `nix` +/// wrote or why it did not. +async fn answer_locks( + mut requests: async_nats::Subscriber, + queue: async_nats::Client, + builder: Arc, +) { + while let Some(msg) = requests.next().await { + let Some(reply_to) = msg.reply.clone() else { + tracing::warn!("a meta-flake lock arrived with no reply subject; dropping it"); + continue; + }; + let reply = handle_lock(&msg.payload, &builder).await; + match serde_json::to_vec(&reply) { + Ok(body) => { + if let Err(e) = queue.publish(reply_to, body.into()).await { + tracing::warn!(error = %e, "publishing a lock's reply failed"); + } + } + Err(e) => tracing::warn!(error = %e, "encoding a lock's reply failed"), + } + } + tracing::error!("the subscription to {LOCK_SUBJECT} ended"); +} + +/// Decode one lock request, lock it and log the lock. +async fn handle_lock(payload: &[u8], builder: &nix_build::Builder) -> LockReply { + let job: LockMeta = match serde_json::from_slice(payload) { + Ok(job) => job, + Err(e) => { + tracing::warn!(error = %e, "an undecodable lock; answering with the error"); + return LockReply::Failed { + error_tail: format!("the builder could not decode the request: {e}"), + }; + } + }; + let reply = match within(TOKEN_DEADLINE, forge_token()).await { + Ok(token) => builder.lock(&job, &token).await, + Err(e) => LockReply::Failed { + error_tail: format!("{e:#}"), + }, + }; + match &reply { + LockReply::Locked { .. } => tracing::info!("swarm meta flake locked"), + LockReply::Failed { error_tail } => { + tracing::warn!(%error_tail, "swarm meta flake lock failed"); + } + } + reply +} + /// `build`'s reply once `job` holds the build slot, or a refusal, without /// running `build`, once it has waited [`MAX_SLOT_WAIT`] since `received`. /// `None` only when the slot is closed. diff --git a/swarm-builder/src/nix_build.rs b/swarm-builder/src/nix_build.rs index 414211fe2..b5c6b90d1 100644 --- a/swarm-builder/src/nix_build.rs +++ b/swarm-builder/src/nix_build.rs @@ -1,6 +1,7 @@ //! One build: `nix build` of the check flake with its three inputs pointed at //! this host's hyperhive and nixpkgs and at the agent's config -//! commit. +//! commit. One lock: `nix flake lock` of the swarm meta flake in a scratch +//! directory. //! //! The config repo is private, so `git` fetches it with the builder's forge //! token. The token reaches `git` through a credential helper that reads it @@ -16,7 +17,7 @@ use std::path::PathBuf; use std::process::Stdio; use std::time::Duration; -use swarm_wire::builder::{BuildAgent, Reply}; +use swarm_wire::builder::{BuildAgent, LockMeta, LockReply, Reply}; use tokio::io::AsyncReadExt as _; /// The most of `nix`'s stderr a failed build's reply carries, from its end. @@ -51,6 +52,8 @@ pub struct Builder { pub config_org: String, /// How long one build may run. pub deadline: Duration, + /// How long one lock may run. + pub lock_deadline: Duration, } impl Builder { @@ -59,9 +62,48 @@ impl Builder { if let Err(error) = validate(job) { return Reply::Failed { error_tail: error }; } + let mut cmd = self.nix_command(token); + cmd.args(self.args(job)); + run(cmd, self.deadline).await + } + + /// Lock `job`'s flake with `token` as the forge credential, in a scratch + /// directory removed afterwards, and answer with the lock `nix` wrote. + pub async fn lock(&self, job: &LockMeta, token: &str) -> LockReply { + let scratch = match Scratch::new() { + Ok(s) => s, + Err(e) => { + return LockReply::Failed { + error_tail: format!("creating the scratch directory: {e}"), + }; + } + }; + if let Err(e) = std::fs::write(scratch.0.join("flake.nix"), &job.flake_nix) { + return LockReply::Failed { + error_tail: format!("writing flake.nix: {e}"), + }; + } + let mut cmd = self.nix_command(token); + cmd.args(LOCK_ARGS).current_dir(&scratch.0); + match run_raw(cmd, self.lock_deadline).await { + Err(error_tail) => LockReply::Failed { error_tail }, + Ok((status, _, err)) if !status.success() => LockReply::Failed { + error_tail: tail(&String::from_utf8_lossy(&err), status), + }, + Ok(_) => match std::fs::read_to_string(scratch.0.join("flake.lock")) { + Ok(flake_lock) => LockReply::Locked { flake_lock }, + Err(e) => LockReply::Failed { + error_tail: format!("nix exited 0 but flake.lock is unreadable: {e}"), + }, + }, + } + } + + /// `nix` with the forge credential reachable by `git` alone, and its + /// output captured. + fn nix_command(&self, token: &str) -> tokio::process::Command { let mut cmd = tokio::process::Command::new(&self.nix); - cmd.args(self.args(job)) - .env(TOKEN_VAR, token) + cmd.env(TOKEN_VAR, token) .env("GIT_TERMINAL_PROMPT", "0") .env("GIT_CONFIG_COUNT", "1") .env(HOST_VAR, forge_host(&self.forge_url)) @@ -74,7 +116,7 @@ impl Builder { .stdout(Stdio::piped()) .stderr(Stdio::piped()) .kill_on_drop(true); - run(cmd, self.deadline).await + cmd } /// The `nix build` arguments for `job`. @@ -116,6 +158,33 @@ impl Builder { } } +/// `nix flake lock` of the flake in the working directory. The flake's config +/// is not trusted, and nothing is built. +const LOCK_ARGS: &[&str] = &["flake", "lock", "--option", "accept-flake-config", "false"]; + +/// A fresh directory under the temp dir, removed with everything in it on drop. +struct Scratch(PathBuf); + +impl Scratch { + fn new() -> std::io::Result { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + let dir = std::env::temp_dir().join(format!( + "lock-meta-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&dir)?; + Ok(Self(dir)) + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + /// The host part of `forge_url` (`https://[:port]`), as `git` names it /// to a credential helper. fn forge_host(forge_url: &str) -> &str { @@ -162,14 +231,33 @@ fn validate(job: &BuildAgent) -> Result<(), String> { /// Run `cmd` to its end or `deadline`, and read the store path it printed or /// the tail of what it said. -async fn run(mut cmd: tokio::process::Command, deadline: Duration) -> Reply { +async fn run(cmd: tokio::process::Command, deadline: Duration) -> Reply { + match run_raw(cmd, deadline).await { + Err(error_tail) => Reply::Failed { error_tail }, + Ok((status, out, err)) => { + let out_path = String::from_utf8_lossy(&out) + .lines() + .rfind(|l| !l.trim().is_empty()) + .map(|l| l.trim().to_owned()); + match out_path { + Some(out_path) if status.success() => Reply::Built { out_path }, + _ => Reply::Failed { + error_tail: tail(&String::from_utf8_lossy(&err), status), + }, + } + } + } +} + +/// Run `cmd` to its end or `deadline`: its status and the kept end of its +/// stdout and stderr, or why it did not finish. +async fn run_raw( + mut cmd: tokio::process::Command, + deadline: Duration, +) -> Result<(std::process::ExitStatus, Vec, Vec), String> { let mut child = match cmd.spawn() { Ok(child) => child, - Err(e) => { - return Reply::Failed { - error_tail: format!("starting nix: {e}"), - }; - } + Err(e) => return Err(format!("starting nix: {e}")), }; let stdout = child.stdout.take().expect("piped"); let stderr = child.stderr.take().expect("piped"); @@ -183,27 +271,12 @@ async fn run(mut cmd: tokio::process::Command, deadline: Duration) -> Reply { status.map(|s| (s, out, err)) }; match tokio::time::timeout(deadline, work).await { - Err(_) => Reply::Failed { - error_tail: format!( - "the build did not finish within {} minutes", - deadline.as_secs() / 60 - ), - }, - Ok(Err(e)) => Reply::Failed { - error_tail: format!("waiting for nix: {e}"), - }, - Ok(Ok((status, out, err))) => { - let out_path = String::from_utf8_lossy(&out) - .lines() - .rfind(|l| !l.trim().is_empty()) - .map(|l| l.trim().to_owned()); - match out_path { - Some(out_path) if status.success() => Reply::Built { out_path }, - _ => Reply::Failed { - error_tail: tail(&String::from_utf8_lossy(&err), status), - }, - } - } + Err(_) => Err(format!( + "nix did not finish within {} minutes", + deadline.as_secs() / 60 + )), + Ok(Err(e)) => Err(format!("waiting for nix: {e}")), + Ok(Ok(done)) => Ok(done), } } @@ -288,7 +361,7 @@ pub(crate) fn tail(stderr: &str, status: std::process::ExitStatus) -> String { #[cfg(test)] mod tests { use super::*; - use swarm_wire::builder::BUILD_DEADLINE; + use swarm_wire::builder::{BUILD_DEADLINE, LOCK_DEADLINE}; fn job() -> BuildAgent { BuildAgent { @@ -307,6 +380,7 @@ mod tests { config_org: "agent-configs".to_owned(), nix: PathBuf::from("nix"), deadline: BUILD_DEADLINE, + lock_deadline: LOCK_DEADLINE, } } @@ -600,6 +674,72 @@ mod tests { assert!(kept.starts_with(b"line\n")); } + /// A builder whose `nix` is the shell script `body`, in a directory the + /// returned guard removes. + fn scripted(body: &str) -> (Builder, Scratch) { + use std::os::unix::fs::PermissionsExt as _; + + let dir = Scratch::new().expect("a temp dir"); + let nix = dir.0.join("nix"); + std::fs::write(&nix, format!("#!/bin/sh\n{body}\n")).expect("the script is written"); + std::fs::set_permissions(&nix, std::fs::Permissions::from_mode(0o755)) + .expect("the script is executable"); + (Builder { nix, ..builder() }, dir) + } + + fn lock_job() -> LockMeta { + LockMeta { + flake_nix: "{ outputs = _: { }; }".to_owned(), + } + } + + #[tokio::test] + async fn a_lock_answers_with_the_lock_nix_wrote_and_leaves_no_scratch() { + // The lock records where it ran, so the test can look for it after. + let (b, _guard) = scripted(r#"test "$1 $2" = "flake lock" && pwd > flake.lock"#); + let reply = b.lock(&lock_job(), "t").await; + let LockReply::Locked { flake_lock } = reply else { + panic!("{reply:?}") + }; + let ran_in = PathBuf::from(flake_lock.trim()); + assert!(ran_in.starts_with(std::env::temp_dir()), "{ran_in:?}"); + assert!( + !ran_in.exists(), + "the scratch dir {ran_in:?} was left behind" + ); + } + + #[tokio::test] + async fn a_lock_sees_the_flake_it_was_sent() { + let (b, _guard) = scripted("cp flake.nix flake.lock"); + assert_eq!( + b.lock(&lock_job(), "t").await, + LockReply::Locked { + flake_lock: lock_job().flake_nix + } + ); + } + + #[tokio::test] + async fn a_failed_lock_answers_with_what_nix_said_last() { + let (b, _guard) = scripted("echo 'error: cannot fetch' >&2; exit 1"); + let reply = b.lock(&lock_job(), "t").await; + assert!( + matches!(&reply, LockReply::Failed { error_tail } if error_tail.ends_with("error: cannot fetch")), + "{reply:?}" + ); + } + + #[tokio::test] + async fn a_lock_that_wrote_no_lock_file_failed() { + let (b, _guard) = scripted("exit 0"); + let reply = b.lock(&lock_job(), "t").await; + assert!( + matches!(&reply, LockReply::Failed { error_tail } if error_tail.contains("flake.lock")), + "{reply:?}" + ); + } + #[tokio::test] async fn a_build_past_its_deadline_is_stopped() { let reply = sh("sleep 30", Duration::from_millis(200)).await; diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml index 9b70dfdce..02b61c7d2 100644 --- a/swarm-controller/Cargo.toml +++ b/swarm-controller/Cargo.toml @@ -46,6 +46,8 @@ problem_details = { version = "0.9.0", features = ["axum"] } hive-jobq.workspace = true hive-jobq-wire.workspace = true hive-log.workspace = true +# `internal/meta`'s `flake.nix`, rendered by the same code hive-c0re renders with. +hive-meta-flake.workspace = true # The jobq-rollup OTEL exporter, wired up in `main` via # `hive_jobq_metrics::spawn_exporter` — moved to its own crate (rather than # living here as `jobq_metrics.rs`) specifically so a future second caller diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index e8689b925..7685dc7a6 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -52,6 +52,7 @@ pub mod classifier_account; pub mod docs_grant; pub mod hive_token; pub mod legacy_tokens; +mod meta_repo; pub mod objects; pub mod operator_profiles; pub mod pull_diff; @@ -157,8 +158,8 @@ pub struct PullForReview { pub narrowed: Option, } -/// Files on a repo's default branch, as [`Client::default_branch_files`] reads -/// them. +/// Files on a repo's branch, as [`Client::default_branch_files`] and +/// [`Client::branch_files`] read them. #[derive(Clone, Debug, PartialEq, Eq)] pub struct DefaultBranchFiles { pub branch: String, @@ -1443,6 +1444,21 @@ impl Client { owner: &str, repo: &str, paths: &[&str], + ) -> Result> { + self.branch_files(owner, repo, None, paths).await + } + + /// [`Self::default_branch_files`] on `branch`, the default branch when it is + /// `None`. `None` as well when the branch has no commit. + /// + /// # Errors + /// As [`Self::default_branch_files`]. + pub async fn branch_files( + &self, + owner: &str, + repo: &str, + branch: Option<&str>, + paths: &[&str], ) -> Result> { let info = self .api @@ -1452,9 +1468,12 @@ impl Client { if info.empty.unwrap_or(false) { return Ok(None); } - let branch = info - .default_branch - .with_context(|| format!("{owner}/{repo} names no default branch"))?; + let branch = match branch { + Some(branch) => branch.to_owned(), + None => info + .default_branch + .with_context(|| format!("{owner}/{repo} names no default branch"))?, + }; let head = match self .api .call(|api| api.repo_get_branch(owner, repo, &branch).into_future()) diff --git a/swarm-controller/src/forge/meta_repo.rs b/swarm-controller/src/forge/meta_repo.rs new file mode 100644 index 000000000..b711e5fb7 --- /dev/null +++ b/swarm-controller/src/forge/meta_repo.rs @@ -0,0 +1,166 @@ +//! `internal/meta` on the forge: the repo the swarm meta flake is committed to +//! (`crate::meta_flake`), and the read grant each hive's account holds on it. + +use anyhow::{Context, Result}; +use base64::Engine as _; +use forgejo_api::structs::{ + AddCollaboratorOptionPermission, ChangeFileOperation, ChangeFileOperationOperation, + ChangeFilesOptions, +}; + +use super::objects::SHARED_ORG; +use super::{Client, DefaultBranchFiles, is_repo_not_found}; +use crate::meta_flake::{META_REPO, MetaFile, MetaForge}; + +impl MetaForge for Client { + async fn head_files( + &self, + owner: &str, + repo: &str, + branch: Option<&str>, + paths: &[&str], + ) -> Result> { + self.branch_files(owner, repo, branch, paths).await + } + + async fn ensure_meta_repo(&self) -> Result<()> { + self.create_shared_repo(SHARED_ORG, META_REPO, true).await + } + + async fn grant_meta_read(&self, login: &str) -> Result<()> { + self.add_collaborator_to( + SHARED_ORG, + META_REPO, + login, + AddCollaboratorOptionPermission::Read, + ) + .await + } + + async fn commits_ahead( + &self, + owner: &str, + repo: &str, + base: &str, + head: &str, + ) -> Result> { + let basehead = format!("{base}...{head}"); + match self + .api + .call(|api| api.repo_compare_diff(owner, repo, &basehead).into_future()) + .await + { + Ok(compare) => Ok(Some( + u64::try_from(compare.total_commits.unwrap_or(0)).unwrap_or(0), + )), + Err(e) if is_repo_not_found(&e) => Ok(None), + Err(e) => Err(e).with_context(|| format!("compare {basehead} in {owner}/{repo}")), + } + } + + async fn commit_meta(&self, message: &str, files: Vec) -> Result<()> { + let ops: Vec = files + .into_iter() + .map(|f| ChangeFileOperation { + content: Some(base64::engine::general_purpose::STANDARD.encode(f.bytes)), + from_path: None, + operation: if f.sha.is_some() { + ChangeFileOperationOperation::Update + } else { + ChangeFileOperationOperation::Create + }, + path: f.path.to_owned(), + sha: f.sha, + }) + .collect(); + self.api + .call(|api| { + api.repo_change_files( + SHARED_ORG, + META_REPO, + ChangeFilesOptions { + author: None, + branch: None, + committer: None, + dates: None, + files: ops.clone(), + force_overwrite_new_branch: None, + message: Some(message.to_owned()), + new_branch: None, + signoff: None, + }, + ) + .into_future() + }) + .await + .with_context(|| format!("commit to {SHARED_ORG}/{META_REPO}"))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use super::*; + + /// A stand-in forge answering Forgejo's compare endpoint: `total_commits` + /// of 3 for `aaa...bbb`, 0 for `bbb...aaa`, 404 for anything else. + /// Records every request's path. + async fn compare_forge() -> (Client, Arc>>) { + use axum::http::{StatusCode, Uri}; + + let seen = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&seen); + let app = axum::Router::new().fallback(move |uri: Uri| { + let recorded = Arc::clone(&recorded); + async move { + recorded.lock().unwrap().push(uri.path().to_owned()); + match uri.path() { + "/api/v1/repos/o/r/compare/aaa...bbb" => ( + StatusCode::OK, + r#"{"total_commits":3,"commits":[],"files":[]}"#, + ), + "/api/v1/repos/o/r/compare/bbb...aaa" => { + (StatusCode::OK, r#"{"total_commits":0,"commits":[]}"#) + } + _ => (StatusCode::NOT_FOUND, r#"{"message":"not found"}"#), + } + } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = url::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await }); + let api = crate::forge::api::tests::fixed(url.clone(), "stub-token"); + (Client { api, url }, seen) + } + + #[tokio::test] + async fn commits_ahead_reads_the_compare_endpoint_as_base_dots_head() { + let (client, seen) = compare_forge().await; + assert_eq!( + client.commits_ahead("o", "r", "aaa", "bbb").await.unwrap(), + Some(3) + ); + assert_eq!( + client.commits_ahead("o", "r", "bbb", "aaa").await.unwrap(), + Some(0) + ); + assert_eq!( + *seen.lock().unwrap(), + [ + "/api/v1/repos/o/r/compare/aaa...bbb", + "/api/v1/repos/o/r/compare/bbb...aaa" + ] + ); + } + + #[tokio::test] + async fn a_commit_the_forge_lacks_is_no_count_not_an_error() { + let (client, _) = compare_forge().await; + assert_eq!( + client.commits_ahead("o", "r", "aaa", "zzz").await.unwrap(), + None + ); + } +} diff --git a/swarm-controller/src/forge/objects.rs b/swarm-controller/src/forge/objects.rs index 8e8d5c7be..7d957c3a6 100644 --- a/swarm-controller/src/forge/objects.rs +++ b/swarm-controller/src/forge/objects.rs @@ -48,7 +48,7 @@ const AGENTS_ORG: &str = "agents"; /// Org hosting the operator-curated shared repos. Same org as the knowledge /// repo's, named separately for the docs repo it also holds. -pub(super) const SHARED_ORG: &str = KNOWLEDGE_ORG; +pub(crate) const SHARED_ORG: &str = KNOWLEDGE_ORG; /// The shared docs repo inside [`SHARED_ORG`]. **Private**: every agent gets /// read-only collaborator access, granted per agent by [`super::docs_grant`]. @@ -56,9 +56,8 @@ pub(super) const SHARED_ORG: &str = KNOWLEDGE_ORG; /// content into the system prompt. pub(super) const SHARED_DOCS_REPO: &str = "docs"; -/// The orgs ensured on every pass. The meta repo lives at `core/meta`, the -/// `core` user's own namespace, so no org is needed for it (and it is not a -/// swarm object: it stays with `hive-c0re`). +/// The orgs ensured on every pass. `internal/meta` is in [`SHARED_ORG`] and is +/// created by `crate::meta_flake`'s update, not by this pass. const SEEDED_ORGS: [&str; 3] = [CONFIG_ORG, SHARED_ORG, AGENTS_ORG]; /// Where the [`OPERATORS_TEAM`] must exist. Gitea teams are org-scoped, so a @@ -938,7 +937,12 @@ impl Client { } /// Create `owner/name`, empty, defaulting to `main`. - async fn create_shared_repo(&self, owner: &str, name: &str, private: bool) -> Result<()> { + pub(super) async fn create_shared_repo( + &self, + owner: &str, + name: &str, + private: bool, + ) -> Result<()> { match self .api .call(|api| { diff --git a/swarm-controller/src/jobq.rs b/swarm-controller/src/jobq.rs index 98d65b54e..a15c5f804 100644 --- a/swarm-controller/src/jobq.rs +++ b/swarm-controller/src/jobq.rs @@ -10,7 +10,8 @@ use crate::routes::AppState; use crate::routes::agents::{claiming_hives, read_declarations}; use crate::startup::PUBLIC_URL_ENV; use crate::{ - agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, wanted, + agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, meta_flake, + wanted, }; /// Node payload for the swarm-level job graph. Named `Swarm*` rather than @@ -133,6 +134,13 @@ pub(crate) enum SwarmNodeKind { /// agent creation, where the hive deploys what it seeds. rev: Option, }, + /// Pin the swarm meta flake (`internal/meta`): render, lock on the swarm + /// builder, commit. See `meta_flake`. When the commit moves the hyperhive + /// pin, a `TriggerDeploy` per placed agent follows. + /// + /// Carries nothing: it reads every head when it runs, so one pending node + /// covers every trigger queued behind it. + UpdateMeta, } impl hive_jobq_wire::WireNode for SwarmNodeKind { @@ -158,6 +166,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { } SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(), SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(), + SwarmNodeKind::UpdateMeta => "update_meta".to_owned(), } } @@ -185,7 +194,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { | SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }), SwarmNodeKind::MintClassifierForgeToken | SwarmNodeKind::MintBuilderForgeToken - | SwarmNodeKind::MintCiMetricsForgeToken => serde_json::json!({}), + | SwarmNodeKind::MintCiMetricsForgeToken + | SwarmNodeKind::UpdateMeta => serde_json::json!({}), SwarmNodeKind::BuildAgent { agent, rev, pr } => { serde_json::json!({ "agent": agent, "rev": rev, "pr": pr }) } @@ -206,12 +216,16 @@ pub(crate) enum SwarmResourceKind { /// The swarm builder: one `BuildAgent` node runs at a time, and the rest /// wait pending, where a newer push can replace them. Builder, + /// `internal/meta`: one `UpdateMeta` commits at a time, so two never race + /// on the same file's blob. + Meta, } impl hive_jobq_wire::WireResource for SwarmResourceKind { fn name(&self) -> String { match self { SwarmResourceKind::Builder => "builder".to_owned(), + SwarmResourceKind::Meta => "meta".to_owned(), } } } @@ -253,6 +267,14 @@ pub(crate) struct WorkerDeps { pub(crate) matrix_homeserver: Option>, } +/// What `UpdateMeta` renders from, fixed for the process. +struct MetaDeps { + settings: meta_flake::Settings, + /// Every hive's name: each places agents, and each hive's forge account + /// gets read on the repo. + hives: Vec, +} + /// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be. /// /// A named constant rather than the literal inline, because this *is* the @@ -373,6 +395,7 @@ pub(crate) async fn run_swarm_node( ), Some(client) => publish_deploy(&client, &hive, &agent, rev).await, }, + SwarmNodeKind::UpdateMeta => finish_update_meta(update_meta(&deps, forge).await, &builder), }; (builder, outcome) } @@ -542,6 +565,148 @@ pub(crate) async fn build_agent( .await } +/// The `UpdateMeta` arm, lifted out for the same reason: what the update +/// reported, and the placements it rendered for. `Err` is the node's failure. +async fn update_meta( + deps: &WorkerDeps, + forge: Option>, +) -> Result< + ( + meta_flake::Report, + std::collections::BTreeSet<(String, String)>, + ), + String, +> { + let Some(client) = forge else { + return Err( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store), so internal/meta cannot be written" + .to_owned(), + ); + }; + let Some(meta) = meta_deps() else { + return Err(format!( + "{} is unset, so there are no swarm values to render internal/meta from", + meta_flake::SETTINGS_ENV + )); + }; + let (Some(queue), Some(wanted)) = (&deps.queue, &deps.wanted) else { + return Err( + "no swarm queue is configured on this host, so neither the placed agents \ + nor a lock can be had" + .to_owned(), + ); + }; + let placements = placements(wanted, &meta.hives) + .await + .map_err(|e| format!("{e:#}"))?; + let agents = placed_agents(queue, &placements) + .await + .map_err(|e| format!("{e:#}"))?; + let report = forge_node( + "update_meta", + meta_flake::update(client.as_ref(), queue, &meta.settings, &agents, &meta.hives), + ) + .await + .map_err(|e| format!("{e:#}"))?; + tracing::info!(?report, agents = agents.len(), "update_meta: done"); + Ok((report, placements)) +} + +/// The `UpdateMeta` node's outcome from what [`update_meta`] reported, with the +/// deploys a moved pin calls for declared into `builder` (the scheduler inserts +/// them under the node when it completes). Synchronous: the builder is not +/// `Sync`, so it cannot be held across an await. +fn finish_update_meta( + result: Result< + ( + meta_flake::Report, + std::collections::BTreeSet<(String, String)>, + ), + String, + >, + builder: &hive_jobq::builder::JobBuilder, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + match result { + Ok((report, placements)) => { + fan_out_if_moved(builder, &report, &placements); + Outcome::Done + } + Err(e) => Outcome::Failed(e), + } +} + +/// Declare one rebuild of each `(hive, agent)` placement when `report` says +/// the agents' hyperhive moved: nothing else tells them to build again. +fn fan_out_if_moved( + grown: &hive_jobq::builder::JobBuilder, + report: &meta_flake::Report, + placements: &std::collections::BTreeSet<(String, String)>, +) { + if !report.hyperhive_moved { + return; + } + for (hive, agent) in placements { + let _ = grown.node(SwarmNodeKind::TriggerDeploy { + hive: hive.clone(), + agent: agent.clone(), + rev: None, + }); + } +} + +/// Every `(hive, agent)` a hive in `hives` places. +async fn placements( + wanted: &wanted::WantedWriter, + hives: &[String], +) -> Result> { + let mut placed = std::collections::BTreeSet::new(); + for hive in hives { + let Some(declared) = wanted + .view(hive) + .await + .map_err(|e| e.context(format!("read where hive {hive} places agents")))? + else { + continue; + }; + placed.extend( + declared + .agents + .into_iter() + .filter(|(_, w)| crate::routes::agents::places_agent(w.state)) + .map(|(name, _)| (hive.clone(), name)), + ); + } + Ok(placed) +} + +/// Every agent in `placements`, with its archetype. +async fn placed_agents( + queue: &async_nats::Client, + placements: &std::collections::BTreeSet<(String, String)>, +) -> Result> { + let names: std::collections::BTreeSet<&String> = + placements.iter().map(|(_, agent)| agent).collect(); + let archetypes = swarm_queue_client::agent_archetype::open_or_create(queue) + .await + .map_err(|e| anyhow::anyhow!("open the agent-archetype bucket: {e}"))?; + let mut placed = Vec::with_capacity(names.len()); + for name in names { + let (archetype, _, _) = crate::schedules::read_or_default::< + Option, + >(&archetypes, name) + .await + .map_err(|e| e.context(format!("read the archetype of {name}")))?; + placed.push(meta_flake::Placed { + name: name.clone(), + archetype, + }); + } + Ok(placed) +} + /// The `MintBuilderForgeToken` arm, lifted out for the same reason. async fn mint_builder_forge_token( forge: Option>, @@ -871,16 +1036,7 @@ pub(crate) async fn queue_merged_config_deploy( .jobq .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert_job(None, |b| { - vec![ - b.node(SwarmNodeKind::TriggerDeploy { - hive: hive.clone(), - agent: agent.clone(), - rev: Some(rev.clone()), - }) - .guid(), - ] - }); + .insert_job(None, |b| merged_deploy_job(b, &hive, &agent, &rev)); match inserted { Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"), Err(e) => { @@ -889,6 +1045,33 @@ pub(crate) async fn queue_merged_config_deploy( } } +/// The sub-DAG a merged config PR is: the meta update, then the deploy of +/// `rev` on `hive`. +/// +/// `after_any`: a hive builds from `internal/meta` only while it pins the +/// agent's config commit, and renders locally otherwise, as a hive that +/// predates the repo always does. A failed update (no builder, an unreadable +/// config repo) must not hold those hives back. +fn merged_deploy_job( + b: &hive_jobq::builder::JobBuilder, + hive: &str, + agent: &str, + rev: &str, +) -> Vec { + let update_meta = b + .node(SwarmNodeKind::UpdateMeta) + .needs(SwarmResourceKind::Meta); + let root = update_meta.guid(); + let _deploy = b + .node(SwarmNodeKind::TriggerDeploy { + hive: hive.to_owned(), + agent: agent.to_owned(), + rev: Some(rev.to_owned()), + }) + .after_any(update_meta); + vec![root] +} + /// The sub-DAG one agent creation is: the nodes, and the edges between them. /// /// A function rather than a closure inside [`create_agent`](crate::routes::agents::create_agent) so the endpoint's @@ -977,6 +1160,14 @@ pub(crate) fn declare_agent_job( hive: hive.to_owned(), agent: agent.to_owned(), }); + // The hive builds the agent from `internal/meta` once it pins the agent's + // config commit, and renders locally until then. `after_any` on the + // update, so one that failed (no builder, an unreadable config repo) + // still lets the hive build the agent that way. + let update_meta = b + .node(SwarmNodeKind::UpdateMeta) + .needs(SwarmResourceKind::Meta) + .after_ok(set_wanted); // Last, and specifically after the config repo is seeded: the hive // deploys by reading that repo, so a deploy asked for any earlier would // find nothing to build. This is the edge that makes creating an agent at @@ -1007,6 +1198,7 @@ pub(crate) fn declare_agent_job( agent: agent.to_owned(), rev: None, }) + .after_any(update_meta) .after_any(mint_identity) .after_any(mint_forge_token) // `after_any` for the same reason: the container reads this token @@ -1017,6 +1209,7 @@ pub(crate) fn declare_agent_job( .after_ok(set_wanted); if let Some(init_config) = init_config { let _ = trigger_deploy.after_ok(init_config); + let _ = update_meta.after_ok(init_config); } vec![create_identity.guid()] } @@ -1183,6 +1376,64 @@ pub(crate) fn queue_builder_forge_token_mint( .map_err(|e| anyhow::anyhow!("{e}")) } +/// Insert one `UpdateMeta` job unless one is already pending, which reads the +/// same heads when it runs. Returns whether a job was inserted. +pub(crate) fn queue_meta_update( + sched: &Mutex>, +) -> Result { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let pending = sched.graph().nodes().any(|n| { + n.state == hive_jobq::State::Pending && matches!(n.payload, SwarmNodeKind::UpdateMeta) + }); + if pending { + return Ok(false); + } + sched + .insert_job(None, |b| { + vec![ + b.node(SwarmNodeKind::UpdateMeta) + .needs(SwarmResourceKind::Meta) + .guid(), + ] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + Ok(true) +} + +/// [`MetaDeps`] from the environment, read on first use; `None`, logged, when +/// the settings are unset or do not parse, which `UpdateMeta` then fails on +/// by name. +fn meta_deps() -> Option<&'static MetaDeps> { + static DEPS: std::sync::OnceLock> = std::sync::OnceLock::new(); + DEPS.get_or_init(read_meta_deps).as_ref() +} + +fn read_meta_deps() -> Option { + let settings = match meta_flake::Settings::from_env() { + Ok(Some(s)) => s, + Ok(None) => { + tracing::warn!( + "{} unset; internal/meta will not be written", + meta_flake::SETTINGS_ENV + ); + return None; + } + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "internal/meta will not be written"); + return None; + } + }; + Some(MetaDeps { + settings, + hives: crate::startup::load_hives() + .into_iter() + .map(|h| h.name) + .collect(), + }) +} + /// Insert one `MintCiMetricsForgeToken` job and return its node's id. /// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller. pub(crate) fn queue_ci_metrics_forge_token_mint( @@ -1723,6 +1974,187 @@ mod tests { ); } + /// Whether the edge from the `from` node to the `on` node accepts a + /// failed `on`, in a graph built by `declare`. + fn edge_accepts_failure( + declare: impl FnOnce(&hive_jobq::builder::JobBuilder), + from: &str, + on: &str, + ) -> bool { + use hive_jobq_wire::WireNode as _; + + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + sched + .insert_job(None, |b| { + declare(b); + Vec::new() + }) + .expect("the graph inserts"); + let graph = sched.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let (from, on) = (id_of(from), id_of(on)); + graph + .node(from) + .expect("just found") + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == on => Some(*when), + _ => None, + }) + .expect("the edge exists") + .accepts(hive_jobq::TerminalState::Failed) + } + + /// A deploy that waits for the meta update must still run when the update + /// fails: a hive that renders locally never needed it. + #[test] + fn a_failed_meta_update_does_not_hold_back_the_deploys() { + let create = |b: &hive_jobq::builder::JobBuilder| { + let _ = super::declare_agent_job(b, "atlas", "pr1ma", false); + }; + let merge = |b: &hive_jobq::builder::JobBuilder| { + let _ = super::merged_deploy_job(b, "pr1ma", "atlas", "abc"); + }; + assert!(edge_accepts_failure( + create, + "trigger_deploy", + "update_meta" + )); + assert!(edge_accepts_failure(merge, "trigger_deploy", "update_meta")); + // The same lookup finds the edges that do hold a deploy back. + assert!(!edge_accepts_failure( + create, + "trigger_deploy", + "set_agent_wanted" + )); + assert!(!edge_accepts_failure( + create, + "update_meta", + "set_agent_wanted" + )); + } + + /// The scheduler's side of the same edge: with the update failed, the + /// deploy is the next node to run. + #[tokio::test] + async fn the_merged_deploy_runs_after_a_failed_meta_update() { + use hive_jobq::scheduler::{Outcome, Scheduler}; + use hive_jobq_wire::WireNode as _; + + let mut sched = Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + sched + .insert_job(None, |b| { + super::merged_deploy_job(b, "pr1ma", "atlas", "abc") + }) + .expect("the graph inserts"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + let mut ran = Vec::new(); + while let Some(runner) = Scheduler::claim_next(&sched, |_id, kind, builder| async move { + let outcome = match kind { + SwarmNodeKind::UpdateMeta => Outcome::Failed("no builder".to_owned()), + _ => Outcome::Done, + }; + (builder, outcome) + }) { + runner.await.1.expect("no growth declared"); + ran.push( + sched + .lock() + .unwrap() + .graph() + .nodes() + .filter(|n| n.state.is_terminal()) + .map(|n| (n.payload.label(), n.state)) + .collect::>(), + ); + } + let last = ran.last().expect("both nodes ran"); + assert!( + last.contains(&("update_meta".to_owned(), hive_jobq::State::Failed)), + "{last:?}" + ); + assert!( + last.contains(&("trigger_deploy".to_owned(), hive_jobq::State::Done)), + "{last:?}" + ); + } + + /// Run an `UpdateMeta` node that reports `report`, and return the + /// `(hive, agent, rev)` of every deploy it grew. + async fn grown_deploys(report: meta_flake::Report) -> Vec<(String, String, Option)> { + use hive_jobq::scheduler::{Outcome, Scheduler}; + + let mut sched = Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let update = sched + .append(SwarmNodeKind::UpdateMeta, Vec::new(), None) + .expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + let placements: std::collections::BTreeSet<(String, String)> = + [("pr1ma", "atlas"), ("pr1ma", "argus"), ("sec0nd", "iris")] + .into_iter() + .map(|(h, a)| (h.to_owned(), a.to_owned())) + .collect(); + let runner = Scheduler::claim_next(&sched, move |_id, _kind, builder| async move { + fan_out_if_moved(&builder, &report, &placements); + (builder, Outcome::Done) + }) + .expect("the node just inserted is runnable"); + runner.await.1.expect("the grown nodes insert"); + let guard = sched.lock().unwrap(); + guard + .graph() + .nodes() + .filter_map(|n| match &n.payload { + SwarmNodeKind::TriggerDeploy { hive, agent, rev } => { + assert_eq!(n.parent, Some(update), "grown under the update"); + Some((hive.clone(), agent.clone(), rev.clone())) + } + _ => None, + }) + .collect() + } + + #[tokio::test] + async fn a_moved_pin_rebuilds_every_placed_agent_and_an_unmoved_one_none() { + let moved = grown_deploys(meta_flake::Report { + updated: meta_flake::Updated::Committed, + hyperhive_moved: true, + }) + .await; + let mut moved: Vec<_> = moved.into_iter().collect(); + moved.sort(); + assert_eq!( + moved, + vec![ + ("pr1ma".to_owned(), "argus".to_owned(), None), + ("pr1ma".to_owned(), "atlas".to_owned(), None), + ("sec0nd".to_owned(), "iris".to_owned(), None), + ] + ); + let unmoved = grown_deploys(meta_flake::Report { + updated: meta_flake::Updated::Committed, + hyperhive_moved: false, + }) + .await; + assert!(unmoved.is_empty(), "{unmoved:?}"); + } + /// Node labels of the graph [`super::declare_agent_job`] builds, and the /// labels the deploy node waits on. fn creation_graph(archetype: bool) -> (Vec, Vec) { diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index b59030ebe..f76caf53b 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -50,6 +50,7 @@ mod jobq; mod linked_accounts; mod matrix_account; mod message_stream; +mod meta_flake; mod notify_flows; mod otel_http_client; mod pulls; diff --git a/swarm-controller/src/meta_flake.rs b/swarm-controller/src/meta_flake.rs new file mode 100644 index 000000000..518835adb --- /dev/null +++ b/swarm-controller/src/meta_flake.rs @@ -0,0 +1,1017 @@ +//! `internal/meta`: the swarm meta flake hives build from. +//! +//! [`update`] renders `flake.nix` with `hive_meta_flake::render_flake`: hyperhive +//! pinned by [`hyperhive_pin`], each custom agent's config input pinned to its +//! `agent-configs/` `main`, every agent placed on some hive. The swarm +//! builder locks it (`swarm_wire::builder::LockMeta`), and the flake and the +//! lock are committed together in one commit. A render equal to the committed +//! `flake.nix` sends no lock and commits nothing. +//! +//! The hyperhive pin is a ratchet on this controller's running rev: it moves +//! only to a rev that descends from the committed one, so deploying an older +//! controller never rolls the agents back. + +use std::collections::BTreeMap; +use std::future::Future; + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; +use swarm_wire::agent_archetype::ArchetypeRef; +use swarm_wire::builder::{LOCK_REPLY_DEADLINE, LOCK_SUBJECT, LockMeta, LockReply}; + +use crate::forge::{CONFIG_ORG, DefaultBranchFiles}; + +/// The branch an agent's config is deployed from, which hive-c0re's +/// `internal_meta` reads the config head from too. +const CONFIG_BRANCH: &str = "main"; + +/// The repo in `internal` the flake is committed to. +pub(crate) const META_REPO: &str = "meta"; + +/// The env var holding [`Settings`] as JSON, set by `swarm-controller.nix`. +pub(crate) const SETTINGS_ENV: &str = "SWARM_CONTROLLER_META_FLAKE"; + +/// Where hyperhive itself lives on the forge. +const HYPERHIVE_OWNER: &str = "hyperhive"; +const HYPERHIVE_REPO: &str = "hyperhive"; + +/// What every agent of the swarm gets, the same on every hive. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Settings { + /// Values by `hive_meta_flake::FORWARDED_VARS` name. `HIVE_FORGE_URL` is + /// required. + vars: BTreeMap, + /// The forge's `https` origin, which every pinned input is fetched from: + /// the swarm builder's credential helper answers only `https` for it. + git_base: String, + /// The swarm turned the agents' GitHub integration off. + github_disabled: bool, + /// The hyperhive commit this controller runs (`self.rev`); `None` for a + /// dirty tree, which leaves the pin where it is. + #[serde(default)] + hyperhive_rev: Option, +} + +impl Settings { + /// [`Settings`] from [`SETTINGS_ENV`], `None` when it is unset. + pub(crate) fn from_env() -> Result> { + match std::env::var(SETTINGS_ENV) { + Ok(raw) => Self::parse(&raw).map(Some), + Err(std::env::VarError::NotPresent) => Ok(None), + Err(e) => Err(e).with_context(|| format!("read {SETTINGS_ENV}")), + } + } + + fn parse(raw: &str) -> Result { + let settings: Self = + serde_json::from_str(raw).with_context(|| format!("parse {SETTINGS_ENV}"))?; + if let Some(unknown) = settings + .vars + .keys() + .find(|k| !hive_meta_flake::FORWARDED_VARS.contains(&k.as_str())) + { + bail!("{SETTINGS_ENV}: {unknown} is not a value the meta flake forwards"); + } + if settings + .vars + .get("HIVE_FORGE_URL") + .is_none_or(String::is_empty) + { + bail!("{SETTINGS_ENV}: HIVE_FORGE_URL is unset"); + } + if !settings.git_base.starts_with("https://") { + bail!("{SETTINGS_ENV}: git_base must be an https origin"); + } + if let Some(rev) = settings + .hyperhive_rev + .as_deref() + .filter(|r| !hive_meta_flake::is_rev(r)) + { + bail!("{SETTINGS_ENV}: hyperhive_rev {rev:?} is not a full commit id"); + } + Ok(settings) + } + + /// The forwarded pairs with a value, in `FORWARDED_VARS` order. + fn forwarded(&self) -> Vec<(&'static str, String)> { + hive_meta_flake::FORWARDED_VARS + .iter() + .filter_map(|&name| { + self.vars + .get(name) + .filter(|v| !v.is_empty()) + .map(|v| (name, v.clone())) + }) + .collect() + } + + fn git_url(&self, owner: &str, repo: &str) -> String { + format!("{}/{owner}/{repo}.git", self.git_base.trim_end_matches('/')) + } +} + +/// One agent some hive places, with the archetype it is built from. +#[derive(Debug, Clone)] +pub(crate) struct Placed { + pub(crate) name: String, + pub(crate) archetype: Option, +} + +/// One file of a meta commit: a blob sha replaces that blob, `None` creates it. +pub(crate) struct MetaFile { + pub(crate) path: &'static str, + pub(crate) bytes: Vec, + pub(crate) sha: Option, +} + +/// The forge, as [`update`] reads and writes it. +pub(crate) trait MetaForge { + /// The `paths` of `owner/repo` at the head of `branch`, the default + /// branch when it is `None`; `None` when the branch has no commit. + fn head_files( + &self, + owner: &str, + repo: &str, + branch: Option<&str>, + paths: &[&str], + ) -> impl Future>> + Send; + /// The number of commits `head` has that `base` lacks, as the forge's + /// compare endpoint counts them: `GET /repos/{owner}/{repo}/compare/ + /// {base}...{head}`, whose `total_commits` is the commits from the merge + /// base of the two to `head`. `None` when either commit is not on the + /// forge. + /// + /// ⚠️ Two histories with no common ancestor count 0, the same as a + /// `head` already contained in `base`. [`is_strict_descendant`] is the + /// reading that tells them apart. + fn commits_ahead( + &self, + owner: &str, + repo: &str, + base: &str, + head: &str, + ) -> impl Future>> + Send; + fn ensure_meta_repo(&self) -> impl Future> + Send; + fn grant_meta_read(&self, login: &str) -> impl Future> + Send; + fn commit_meta( + &self, + message: &str, + files: Vec, + ) -> impl Future> + Send; +} + +/// Where a lock is sent: the queue in production, a fake in the tests. `Err` +/// is a lock that never reached a builder or never came back. +pub(crate) trait LockSink { + fn lock(&self, request: Vec) -> impl Future> + Send; +} + +impl LockSink for async_nats::Client { + async fn lock(&self, request: Vec) -> Result { + use async_nats::RequestErrorKind; + + let sent = async_nats::Request::new() + .payload(request.into()) + .timeout(Some(LOCK_REPLY_DEADLINE)); + match self.send_request(LOCK_SUBJECT, sent).await { + Ok(msg) => serde_json::from_slice(&msg.payload) + .map_err(|e| format!("the builder's lock reply did not decode: {e}")), + Err(e) => Err(match e.kind() { + RequestErrorKind::NoResponders => "no swarm builder is listening for locks: \ + deploy.builder.enable is set on no host, or its builder predates the lock \ + request" + .to_owned(), + RequestErrorKind::TimedOut => format!( + "the swarm builder sent no lock within {} minutes", + LOCK_REPLY_DEADLINE.as_secs() / 60 + ), + _ => format!("sending the lock to the swarm builder failed: {e}"), + }), + } + } +} + +/// What [`update`] did. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Updated { + /// The committed flake already is the render. + Unchanged, + /// A new flake and lock were committed. + Committed, +} + +/// What [`update`] reports. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Report { + pub(crate) updated: Updated, + /// The commit moved the hyperhive pin from one rev to another, which is + /// what the placed agents are rebuilt for. The first commit moves nothing: + /// no agent was ever told to build from this repo. + pub(crate) hyperhive_moved: bool, +} + +/// Bring `internal/meta` up to the forge's current heads for `agents`, and +/// give each of `hives`' forge accounts read on it. +/// +/// # Errors +/// A head that cannot be read, a lock the builder refuses, or a refused commit. +/// A failed read grant is logged, not returned: the commit is what hives wait +/// on. +pub(crate) async fn update( + forge: &impl MetaForge, + locker: &impl LockSink, + settings: &Settings, + agents: &[Placed], + hives: &[String], +) -> Result { + forge.ensure_meta_repo().await?; + for hive in hives { + let login = swarm_secret_client::forge::hive_login(hive); + if let Err(e) = forge.grant_meta_read(&login).await { + tracing::warn!(%hive, error = %format!("{e:#}"), "meta flake: read grant on internal/meta failed"); + } + } + let committed = forge + .head_files( + crate::forge::objects::SHARED_ORG, + META_REPO, + None, + &["flake.nix", "flake.lock"], + ) + .await?; + let sha_of = |path: &str| { + committed + .as_ref() + .and_then(|c| c.files.iter().find(|f| f.path == path)) + }; + let flake_file = sha_of("flake.nix"); + let pin = hyperhive_pin( + forge, + settings, + flake_file + .map(|f| String::from_utf8_lossy(&f.bytes)) + .as_deref(), + ) + .await?; + let flake = render(forge, settings, &pin.rev, agents).await?; + if flake_file.is_some_and(|f| f.bytes == flake.as_bytes()) && sha_of("flake.lock").is_some() { + return Ok(Report { + updated: Updated::Unchanged, + hyperhive_moved: false, + }); + } + let request = serde_json::to_vec(&LockMeta { + flake_nix: flake.clone(), + }) + .context("encode the lock request")?; + let flake_lock = match locker.lock(request).await { + Ok(LockReply::Locked { flake_lock }) => flake_lock, + Ok(LockReply::Failed { error_tail }) => bail!("the builder could not lock: {error_tail}"), + Err(e) => bail!("{e}"), + }; + forge + .commit_meta( + &format!("meta: pin {} agent(s)", agents.len()), + vec![ + MetaFile { + path: "flake.nix", + bytes: flake.into_bytes(), + sha: flake_file.map(|f| f.sha.clone()), + }, + MetaFile { + path: "flake.lock", + bytes: flake_lock.into_bytes(), + sha: sha_of("flake.lock").map(|f| f.sha.clone()), + }, + ], + ) + .await?; + Ok(Report { + updated: Updated::Committed, + hyperhive_moved: pin.moved, + }) +} + +/// The hyperhive commit the flake pins, and whether that is a move off the +/// committed pin. +struct HyperhivePin { + rev: String, + moved: bool, +} + +/// The hyperhive pin for the next render, given the committed `flake.nix` +/// (`None` before the first commit). +/// +/// - No committed flake: the forge's `main` head, the one time the pin follows +/// `main`. +/// - Otherwise the committed pin, unless this controller's running rev is a +/// strict descendant of it ([`is_strict_descendant`]), which becomes the +/// pin. A merge to `main` moves nothing; a controller older than, or on +/// another line from, the pin leaves it. A forge that cannot answer the +/// comparison leaves it too, and the next update asks again. +async fn hyperhive_pin( + forge: &impl MetaForge, + settings: &Settings, + committed_flake: Option<&str>, +) -> Result { + let Some(flake) = committed_flake else { + let head = forge + .head_files(HYPERHIVE_OWNER, HYPERHIVE_REPO, None, &[]) + .await? + .with_context(|| format!("{HYPERHIVE_OWNER}/{HYPERHIVE_REPO} is empty or absent"))?; + return Ok(HyperhivePin { + rev: head.commit, + moved: false, + }); + }; + let pinned = hive_meta_flake::pinned_rev(flake, "hyperhive") + .context("the committed internal/meta/flake.nix pins no hyperhive commit")?; + let keep = || HyperhivePin { + rev: pinned.to_owned(), + moved: false, + }; + let Some(running) = settings.hyperhive_rev.as_deref() else { + tracing::info!(%pinned, "meta flake: this controller has no rev; keeping the hyperhive pin"); + return Ok(keep()); + }; + match is_strict_descendant(forge, HYPERHIVE_OWNER, HYPERHIVE_REPO, pinned, running).await { + Ok(true) => { + tracing::info!(%pinned, %running, "meta flake: moving the hyperhive pin to this controller's rev"); + Ok(HyperhivePin { + rev: running.to_owned(), + moved: true, + }) + } + Ok(false) => { + tracing::info!(%pinned, %running, "meta flake: this controller's rev is not newer than the hyperhive pin; keeping it"); + Ok(keep()) + } + Err(e) => { + tracing::warn!(%pinned, %running, error = %format!("{e:#}"), "meta flake: cannot compare this controller's rev with the hyperhive pin; keeping it"); + Ok(keep()) + } + } +} + +/// Whether `descendant` is a commit of `owner/repo` strictly after `ancestor`: +/// it has commits `ancestor` lacks ([`MetaForge::commits_ahead`]) and +/// `ancestor` has none it lacks. Equal commits, a `descendant` that is behind, +/// histories that diverged and histories with nothing in common are all +/// `false`; a commit the forge does not have is too. +async fn is_strict_descendant( + forge: &impl MetaForge, + owner: &str, + repo: &str, + ancestor: &str, + descendant: &str, +) -> Result { + if ancestor == descendant { + return Ok(false); + } + // Unrelated histories also count 0, so a zero here is `false` whatever + // the other side says. + let Some(ahead) = forge + .commits_ahead(owner, repo, ancestor, descendant) + .await? + else { + return Ok(false); + }; + if ahead == 0 { + return Ok(false); + } + let behind = forge + .commits_ahead(owner, repo, descendant, ancestor) + .await?; + Ok(behind == Some(0)) +} + +/// The `flake.nix` for `agents` at the forge's current config heads, with +/// hyperhive at `hyperhive_rev`. A custom agent whose config repo has no +/// commit on [`CONFIG_BRANCH`] is left out, so a hive builds it from its own +/// flake and no other agent waits on it. +async fn render( + forge: &impl MetaForge, + settings: &Settings, + hyperhive_rev: &str, + agents: &[Placed], +) -> Result { + let hyperhive_url = hive_meta_flake::pinned_git_url( + &settings.git_url(HYPERHIVE_OWNER, HYPERHIVE_REPO), + Some(hyperhive_rev), + )?; + let mut specs = Vec::with_capacity(agents.len()); + for agent in agents { + let pin = if agent.archetype.is_some() { + None + } else { + let Some(head) = forge + .head_files( + CONFIG_ORG, + &agent.name, + Some(CONFIG_BRANCH), + &["flake.lock"], + ) + .await? + else { + tracing::warn!(agent = %agent.name, "meta flake: its config repo has no commit on {CONFIG_BRANCH}; leaving the agent out"); + continue; + }; + let lock = head + .files + .first() + .map(|f| String::from_utf8_lossy(&f.bytes).into_owned()); + Some(hive_meta_flake::AgentPin { + url: hive_meta_flake::pinned_git_url( + &settings.git_url(CONFIG_ORG, &agent.name), + Some(&head.commit), + )?, + canonical_inputs: lock + .as_deref() + .map(hive_meta_flake::canonical_inputs_in_lock) + .unwrap_or_default(), + }) + }; + specs.push(hive_meta_flake::AgentSpec { + name: agent.name.clone(), + port: hive_meta_flake::agent_web_port(&agent.name), + archetype: agent.archetype.clone(), + pin, + }); + } + specs.sort_by(|a, b| a.name.cmp(&b.name)); + hive_meta_flake::render_flake( + &hyperhive_url, + &specs, + &settings.forwarded(), + settings.github_disabled, + ) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::sync::Mutex; + + use super::*; + use crate::forge::BranchFile; + + const FORGE: &str = "https://forge.t.local"; + + /// A full commit id made of one hex digit. + fn rev(digit: char) -> String { + digit.to_string().repeat(40) + } + + /// The hyperhive history the fake forge knows: `1 <- 2 <- 3`, `2 <- 4` + /// (4 diverged from 3) and `9` on its own. `8` is not on the forge. + fn history() -> BTreeMap> { + [ + ('1', vec![]), + ('2', vec!['1']), + ('3', vec!['2']), + ('4', vec!['2']), + ('9', vec![]), + ] + .into_iter() + .map(|(c, parents)| (rev(c), parents.into_iter().map(rev).collect())) + .collect() + } + + fn settings_running(running: Option<&str>) -> Settings { + let running = running.map_or("null".to_owned(), |r| format!("\"{r}\"")); + Settings::parse(&format!( + r#"{{"vars":{{"HIVE_FORGE_URL":"http://forge.t.local","HYPERHIVE_SWARM_NAME":"t"}},"git_base":"{FORGE}/","github_disabled":true,"hyperhive_rev":{running}}}"# + )) + .unwrap() + } + + fn settings() -> Settings { + settings_running(None) + } + + fn head(commit: &str, files: &[(&str, &str)]) -> DefaultBranchFiles { + DefaultBranchFiles { + branch: "main".to_owned(), + commit: commit.to_owned(), + web_url: None, + files: files + .iter() + .map(|(path, body)| BranchFile { + path: (*path).to_owned(), + sha: format!("blob-{path}"), + bytes: body.as_bytes().to_vec(), + }) + .collect(), + } + } + + /// One recorded commit: its message, and each file's path, text and sha. + type Commit = (String, Vec<(&'static str, String, Option)>); + + #[derive(Default)] + struct FakeForge { + repos: BTreeMap, + commits: Mutex>, + grants: Mutex>, + /// Hyperhive's commits and their parents. + history: BTreeMap>, + /// Every compare asked, as `(base, head)`. + compares: Mutex>, + /// Every compare fails, as a forge that is down does. + compare_fails: bool, + /// Every `head_files` read, as `("owner/repo", branch)`. + head_reads: Mutex)>>, + } + + impl FakeForge { + /// `commit` and everything it descends from; `None` for a commit not + /// on the forge. + fn ancestry(&self, commit: &str) -> Option> { + self.history.get(commit)?; + let mut seen = BTreeSet::new(); + let mut todo = vec![commit.to_owned()]; + while let Some(c) = todo.pop() { + if seen.insert(c.clone()) { + todo.extend(self.history.get(&c).cloned().unwrap_or_default()); + } + } + Some(seen) + } + } + + impl MetaForge for FakeForge { + async fn head_files( + &self, + owner: &str, + repo: &str, + branch: Option<&str>, + paths: &[&str], + ) -> Result> { + self.head_reads + .lock() + .unwrap() + .push((format!("{owner}/{repo}"), branch.map(str::to_owned))); + Ok(self.repos.get(&format!("{owner}/{repo}")).map(|h| { + let mut h = h.clone(); + h.files.retain(|f| paths.contains(&f.path.as_str())); + h + })) + } + /// Forgejo's `GetCompareInfo` for `base...head`: the commits `head` + /// has beyond the merge base, and none at all when the two share no + /// ancestor. + async fn commits_ahead( + &self, + _owner: &str, + _repo: &str, + base: &str, + head: &str, + ) -> Result> { + self.compares + .lock() + .unwrap() + .push((base.to_owned(), head.to_owned())); + if self.compare_fails { + bail!("the forge is down"); + } + let (Some(base), Some(head)) = (self.ancestry(base), self.ancestry(head)) else { + return Ok(None); + }; + if base.is_disjoint(&head) { + return Ok(Some(0)); + } + Ok(Some(head.difference(&base).count() as u64)) + } + async fn ensure_meta_repo(&self) -> Result<()> { + Ok(()) + } + async fn grant_meta_read(&self, login: &str) -> Result<()> { + self.grants.lock().unwrap().push(login.to_owned()); + Ok(()) + } + async fn commit_meta(&self, message: &str, files: Vec) -> Result<()> { + self.commits.lock().unwrap().push(( + message.to_owned(), + files + .into_iter() + .map(|f| (f.path, String::from_utf8(f.bytes).unwrap(), f.sha)) + .collect(), + )); + Ok(()) + } + } + + /// Answers every lock with `reply`, recording each flake it was sent. + struct FakeLocker { + reply: Result, + sent: Mutex>, + } + + impl FakeLocker { + fn locked() -> Self { + Self { + reply: Ok(LockReply::Locked { + flake_lock: "{\"version\":7}".to_owned(), + }), + sent: Mutex::default(), + } + } + } + + impl LockSink for FakeLocker { + async fn lock(&self, request: Vec) -> Result { + let req: LockMeta = serde_json::from_slice(&request).unwrap(); + self.sent.lock().unwrap().push(req.flake_nix); + self.reply.clone() + } + } + + /// A forge whose hyperhive `main` is at commit `1`, with no `internal/meta`. + fn forge_with(extra: &[(&str, DefaultBranchFiles)]) -> FakeForge { + let mut repos = BTreeMap::new(); + repos.insert("hyperhive/hyperhive".to_owned(), head(&rev('1'), &[])); + repos.insert( + "agent-configs/argus".to_owned(), + head( + "a1", + &[( + "flake.lock", + r#"{"root":"root","nodes":{"root":{"inputs":{"nixpkgs":"nixpkgs"}}}}"#, + )], + ), + ); + for (name, h) in extra { + repos.insert((*name).to_owned(), h.clone()); + } + FakeForge { + repos, + history: history(), + ..FakeForge::default() + } + } + + /// A forge with `internal/meta` committed, pinning hyperhive at `pin`. + async fn forge_pinned_at(pin: &str) -> FakeForge { + let base = forge_with(&[]); + let flake = render(&base, &settings(), pin, &argus_and_reviewer()) + .await + .unwrap(); + forge_with(&[( + "internal/meta", + head("m1", &[("flake.nix", &flake), ("flake.lock", "{}")]), + )]) + } + + fn argus_and_reviewer() -> Vec { + vec![ + Placed { + name: "argus".to_owned(), + archetype: None, + }, + Placed { + name: "rev".to_owned(), + archetype: Some(ArchetypeRef { + source: "hyperhive".to_owned(), + name: "reviewer".to_owned(), + }), + }, + ] + } + + /// The hyperhive commit the one lock sent to `locker` pins. + fn locked_pin(locker: &FakeLocker) -> String { + let sent = locker.sent.lock().unwrap(); + let [flake] = sent.as_slice() else { + panic!("one lock: {sent:?}") + }; + hive_meta_flake::pinned_rev(flake, "hyperhive") + .expect("a hyperhive pin") + .to_owned() + } + + async fn run(forge: &FakeForge, locker: &FakeLocker, settings: &Settings) -> Report { + update(forge, locker, settings, &argus_and_reviewer(), &[]) + .await + .unwrap() + } + + #[tokio::test] + async fn a_new_render_is_locked_and_committed_with_its_lock_in_one_commit() { + let forge = forge_with(&[]); + let locker = FakeLocker::locked(); + let got = update( + &forge, + &locker, + &settings(), + &argus_and_reviewer(), + &["pr1ma".to_owned()], + ) + .await + .unwrap(); + assert_eq!( + got, + Report { + updated: Updated::Committed, + hyperhive_moved: false + } + ); + let sent = locker.sent.lock().unwrap(); + let [flake] = sent.as_slice() else { + panic!("one lock: {sent:?}") + }; + assert!(flake.contains(&format!( + "hyperhive.url = \"git+{FORGE}/hyperhive/hyperhive.git?rev={}\";", + rev('1') + ))); + assert!(flake.contains(&format!( + "agent-argus.url = \"git+{FORGE}/agent-configs/argus.git?rev=a1\";" + ))); + assert!(flake.contains("agent-argus.inputs.nixpkgs.follows = \"nixpkgs\";")); + assert!( + !flake.contains("agent-rev.url"), + "an archetype agent has no input" + ); + assert!(flake.contains("services.hyperhive.agent.github.enable = false;")); + assert!(flake.contains("HYPERHIVE_SWARM_NAME = \"t\";")); + let commits = forge.commits.lock().unwrap(); + let [(_, files)] = commits.as_slice() else { + panic!("one commit: {commits:?}") + }; + assert_eq!( + files, + &vec![ + ("flake.nix", flake.clone(), None), + ("flake.lock", "{\"version\":7}".to_owned(), None), + ] + ); + assert_eq!( + *forge.grants.lock().unwrap(), + vec![swarm_secret_client::forge::hive_login("pr1ma")] + ); + } + + #[tokio::test] + async fn an_unchanged_render_sends_no_lock_and_commits_nothing() { + let forge = forge_pinned_at(&rev('1')).await; + let locker = FakeLocker::locked(); + let got = run(&forge, &locker, &settings()).await; + assert_eq!(got.updated, Updated::Unchanged); + assert!(!got.hyperhive_moved); + assert!(locker.sent.lock().unwrap().is_empty()); + assert!(forge.commits.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_moved_config_head_replaces_both_committed_blobs() { + let base = forge_with(&[]); + let old = render(&base, &settings(), &rev('1'), &[]).await.unwrap(); + let forge = forge_with(&[( + "internal/meta", + head("m1", &[("flake.nix", &old), ("flake.lock", "{}")]), + )]); + let locker = FakeLocker::locked(); + run(&forge, &locker, &settings()).await; + let commits = forge.commits.lock().unwrap(); + let shas: Vec<_> = commits[0].1.iter().map(|(_, _, sha)| sha.clone()).collect(); + assert_eq!( + shas, + vec![ + Some("blob-flake.nix".to_owned()), + Some("blob-flake.lock".to_owned()) + ] + ); + } + + #[tokio::test] + async fn a_refused_lock_commits_nothing() { + let forge = forge_with(&[]); + let locker = FakeLocker { + reply: Ok(LockReply::Failed { + error_tail: "no such input".to_owned(), + }), + sent: Mutex::default(), + }; + let err = update(&forge, &locker, &settings(), &argus_and_reviewer(), &[]) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("no such input")); + assert!(forge.commits.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_custom_agent_without_a_config_commit_is_left_out_of_the_render() { + let forge = forge_with(&[]); + let mut agents = argus_and_reviewer(); + agents.push(Placed { + name: "ghost".to_owned(), + archetype: None, + }); + let locker = FakeLocker::locked(); + let got = update(&forge, &locker, &settings(), &agents, &[]) + .await + .unwrap(); + assert_eq!(got.updated, Updated::Committed); + let sent = locker.sent.lock().unwrap(); + let [flake] = sent.as_slice() else { + panic!("one lock: {sent:?}") + }; + assert!( + flake.contains("\n argus = mkAgent"), + "the agents with a commit are still rendered" + ); + assert!(flake.contains("\n rev = mkAgent")); + assert!(!flake.contains("ghost")); + } + + #[tokio::test] + async fn a_config_head_is_read_from_main_not_the_default_branch() { + let forge = forge_with(&[]); + run(&forge, &FakeLocker::locked(), &settings()).await; + let reads = forge.head_reads.lock().unwrap(); + assert!( + reads.contains(&("agent-configs/argus".to_owned(), Some("main".to_owned()))), + "{reads:?}" + ); + assert!( + reads.contains(&("internal/meta".to_owned(), None)), + "the meta repo is read on its default branch, as hive-c0re reads it" + ); + } + + #[tokio::test] + async fn the_first_commit_pins_the_forge_main_head_whatever_the_controller_runs() { + let forge = forge_with(&[]); + let locker = FakeLocker::locked(); + let got = run(&forge, &locker, &settings_running(Some(&rev('3')))).await; + assert_eq!(locked_pin(&locker), rev('1')); + assert!( + !got.hyperhive_moved, + "no agent was told to build from it yet" + ); + assert!( + forge.compares.lock().unwrap().is_empty(), + "nothing to compare with before the first commit" + ); + } + + #[tokio::test] + async fn a_controller_on_a_descendant_moves_the_pin_and_says_so() { + let forge = forge_pinned_at(&rev('2')).await; + let locker = FakeLocker::locked(); + let got = run(&forge, &locker, &settings_running(Some(&rev('3')))).await; + assert_eq!(locked_pin(&locker), rev('3')); + assert_eq!( + got, + Report { + updated: Updated::Committed, + hyperhive_moved: true + } + ); + } + + /// Each running rev that must not move its pin; the control is the test + /// above, where a descendant does. + #[tokio::test] + async fn a_controller_that_is_not_strictly_newer_leaves_the_pin() { + for (name, pin, running) in [ + ("the same commit", rev('2'), rev('2')), + ("an ancestor", rev('2'), rev('1')), + ("a diverged commit", rev('3'), rev('4')), + ("an unrelated history", rev('2'), rev('9')), + ("a commit the forge lacks", rev('2'), rev('8')), + ] { + let forge = forge_pinned_at(&pin).await; + let locker = FakeLocker::locked(); + let got = run(&forge, &locker, &settings_running(Some(&running))).await; + assert_eq!(got.updated, Updated::Unchanged, "{name}"); + assert!(!got.hyperhive_moved, "{name}"); + assert!(locker.sent.lock().unwrap().is_empty(), "{name}"); + assert!(forge.commits.lock().unwrap().is_empty(), "{name}"); + } + } + + #[tokio::test] + async fn a_merge_to_hyperhive_main_does_not_move_the_pin() { + let mut forge = forge_pinned_at(&rev('2')).await; + forge + .repos + .insert("hyperhive/hyperhive".to_owned(), head(&rev('3'), &[])); + let locker = FakeLocker::locked(); + let got = run(&forge, &locker, &settings_running(Some(&rev('2')))).await; + assert_eq!(got.updated, Updated::Unchanged); + assert!(forge.commits.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_controller_without_a_rev_keeps_the_pin_and_asks_nothing() { + let forge = forge_pinned_at(&rev('2')).await; + let locker = FakeLocker::locked(); + let got = run(&forge, &locker, &settings()).await; + assert_eq!(got.updated, Updated::Unchanged); + assert!(forge.compares.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_forge_that_cannot_compare_keeps_the_pin_and_still_commits_the_rest() { + let mut forge = forge_pinned_at(&rev('2')).await; + forge.compare_fails = true; + // A new agent makes the render differ from the committed flake. + let mut agents = argus_and_reviewer(); + agents.push(Placed { + name: "newcomer".to_owned(), + archetype: Some(ArchetypeRef { + source: "hyperhive".to_owned(), + name: "reviewer".to_owned(), + }), + }); + let locker = FakeLocker::locked(); + let got = update( + &forge, + &locker, + &settings_running(Some(&rev('3'))), + &agents, + &[], + ) + .await + .unwrap(); + assert_eq!(locked_pin(&locker), rev('2')); + assert!(!got.hyperhive_moved); + assert_eq!(got.updated, Updated::Committed); + } + + #[tokio::test] + async fn a_committed_flake_without_a_hyperhive_pin_is_an_error_not_a_first_commit() { + let forge = forge_with(&[( + "internal/meta", + head("m1", &[("flake.nix", "{ }"), ("flake.lock", "{}")]), + )]); + let err = update(&forge, &FakeLocker::locked(), &settings(), &[], &[]) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("pins no hyperhive commit")); + } + + /// The fake is Forgejo's `GetCompareInfo`, so the descendant test rests + /// on these: a `base...head` count of 0 is both "head is contained in + /// base" and "no common ancestor", and only the reverse count tells a + /// divergence from a descendant. + #[tokio::test] + async fn descent_needs_commits_ahead_one_way_and_none_the_other() { + let forge = forge_with(&[]); + let ahead = |base: char, head: char| { + let forge = &forge; + async move { + forge + .commits_ahead("o", "r", &rev(base), &rev(head)) + .await + .unwrap() + } + }; + assert_eq!(ahead('2', '3').await, Some(1)); + assert_eq!(ahead('1', '3').await, Some(2)); + assert_eq!(ahead('3', '2').await, Some(0), "contained in base"); + assert_eq!(ahead('2', '9').await, Some(0), "no common ancestor"); + assert_eq!(ahead('3', '4').await, Some(1), "diverged: both ways"); + assert_eq!(ahead('4', '3').await, Some(1), "diverged: both ways"); + assert_eq!(ahead('2', '8').await, None); + let descends = |ancestor: char, descendant: char| { + let forge = &forge; + async move { + is_strict_descendant(forge, "o", "r", &rev(ancestor), &rev(descendant)) + .await + .unwrap() + } + }; + assert!(descends('2', '3').await); + assert!(descends('1', '3').await); + assert!(!descends('3', '2').await); + assert!(!descends('2', '2').await); + assert!(!descends('3', '4').await); + assert!(!descends('2', '9').await, "unrelated histories"); + assert!(!descends('2', '8').await); + } + + #[test] + fn settings_refuse_an_unknown_value_a_missing_forge_a_plain_http_base_and_a_short_rev() { + let parse = |vars: &str, git_base: &str, rev: &str| { + Settings::parse(&format!( + r#"{{"vars":{vars},"git_base":"{git_base}","github_disabled":false,"hyperhive_rev":{rev}}}"# + )) + }; + let ok = r#"{"HIVE_FORGE_URL":"x"}"#; + assert!(parse(ok, "https://f", "null").is_ok()); + let full = format!("\"{}\"", rev('a')); + assert!(parse(ok, "https://f", &full).is_ok()); + let unknown = parse( + r#"{"HIVE_FORGE_URL":"x","HIVE_AGENT_NATS_URL":"y"}"#, + "https://f", + "null", + ) + .unwrap_err(); + assert!(format!("{unknown:#}").contains("HIVE_AGENT_NATS_URL")); + assert!(parse("{}", "https://f", "null").is_err()); + assert!(parse(ok, "http://f", "null").is_err()); + assert!(parse(ok, "https://f", "\"main\"").is_err()); + } +} diff --git a/swarm-controller/src/routes/agents.rs b/swarm-controller/src/routes/agents.rs index 139ca5cf4..a8ffba65f 100644 --- a/swarm-controller/src/routes/agents.rs +++ b/swarm-controller/src/routes/agents.rs @@ -221,7 +221,7 @@ pub(crate) async fn set_agent_state( /// /// Exhaustive for the reason [`revokes_queue_credential`] is. A `Destroyed` /// agent is gone from that hive, and the hive deploys nothing for it. -fn places_agent(state: swarm_wire::wanted::AgentState) -> bool { +pub(crate) fn places_agent(state: swarm_wire::wanted::AgentState) -> bool { use swarm_wire::wanted::AgentState; match state { AgentState::Up | AgentState::Offline | AgentState::Paused => true, diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index f2529e1e8..5bb724e41 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -285,6 +285,12 @@ fn spawn_forge_workers( auth: Option>, queue: Option, ) { + // `internal/meta` appears, catches up with every config head that moved + // while this daemon was down, and takes this daemon's rev as the hyperhive + // pin when that is newer. + if let Err(e) = crate::jobq::queue_meta_update(jobq) { + tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed"); + } forge::objects::spawn(Arc::clone(&client)); if let (Some(bridge), Some(queue)) = (auth, queue) { forge::operator_profiles::spawn(Arc::clone(&client), bridge, queue); diff --git a/swarm-nats-auth/src/policy.rs b/swarm-nats-auth/src/policy.rs index 1a824a29c..c2d43a5e2 100644 --- a/swarm-nats-auth/src/policy.rs +++ b/swarm-nats-auth/src/policy.rs @@ -243,8 +243,8 @@ impl Policy { } } - /// What the builder may do: hear build and archetype evaluation requests - /// and answer each on its reply subject, for as long as a build may take. + /// What the builder may do: hear build, archetype evaluation and meta-flake + /// lock requests and answer each on its reply subject, for as long as a build may take. /// Its publish list is its own inbox, for /// [`Self::review_classifier_permissions`]' reason. fn builder_permissions(client_id: &str) -> Permissions { @@ -253,6 +253,7 @@ impl Policy { subscribe: Some(vec![ swarm_wire::builder::SUBJECT.to_owned(), swarm_wire::builder::ARCHETYPES_SUBJECT.to_owned(), + swarm_wire::builder::LOCK_SUBJECT.to_owned(), Self::inbox(client_id), ]), responses: Some(swarm_wire::builder::REPLY_DEADLINE), @@ -693,6 +694,8 @@ impl Policy { swarm_wire::builder::SUBJECT.to_owned(), // An archetype evaluation, sent as a request the same way. swarm_wire::builder::ARCHETYPES_SUBJECT.to_owned(), + // A swarm meta-flake lock, sent as a request the same way. + swarm_wire::builder::LOCK_SUBJECT.to_owned(), // An operator's claude login step, sent as a request the same way. swarm_wire::claude_login::SUBJECT.to_owned(), // Verified forge deliveries, kept in the forge events stream for @@ -1614,7 +1617,7 @@ mod tests { #[test] fn the_builder_hears_its_subject_and_inbox_only_and_answers_by_reply() { use swarm_queue_client::client_inbox_prefix; - use swarm_wire::builder::{ARCHETYPES_SUBJECT, REPLY_DEADLINE, SUBJECT}; + use swarm_wire::builder::{ARCHETYPES_SUBJECT, LOCK_SUBJECT, REPLY_DEADLINE, SUBJECT}; let p = policy() .with_builders(vec![BUILDER.to_owned()]) @@ -1623,6 +1626,7 @@ mod tests { let inbox = client_inbox_prefix(BUILDER); assert!(may_subscribe(&p, SUBJECT)); assert!(may_subscribe(&p, ARCHETYPES_SUBJECT)); + assert!(may_subscribe(&p, LOCK_SUBJECT)); assert!(may_subscribe(&p, &reply(&inbox))); for refused in [ ">".to_owned(), @@ -1643,11 +1647,11 @@ mod tests { #[test] fn only_the_reader_may_send_a_build() { - use swarm_wire::builder::{ARCHETYPES_SUBJECT, SUBJECT}; + use swarm_wire::builder::{ARCHETYPES_SUBJECT, LOCK_SUBJECT, SUBJECT}; let p = with_classifier().with_builders(vec![BUILDER.to_owned()]); let reader = p.permissions("swarm-controller").expect("the reader"); - for subject in [SUBJECT, ARCHETYPES_SUBJECT] { + for subject in [SUBJECT, ARCHETYPES_SUBJECT, LOCK_SUBJECT] { assert!(reader.publish.iter().any(|g| covers(g, subject))); for (who, grant) in [ ("a hive", p.permissions("hive-alpha").expect("a hive")), diff --git a/swarm-wire/src/builder.rs b/swarm-wire/src/builder.rs index aeb66f54e..f3def4298 100644 --- a/swarm-wire/src/builder.rs +++ b/swarm-wire/src/builder.rs @@ -1,5 +1,5 @@ -//! The swarm builder's requests: the subjects swarm-controller sends a build -//! and an archetype evaluation on, and what travels each way. +//! The swarm builder's requests: the subjects swarm-controller sends a build, +//! an archetype evaluation and a meta-flake lock on, and what travels each way. //! //! Three crates name these: the controller sends the requests, the builder //! answers them, and the auth-callout responder grants the subjects to each. @@ -80,6 +80,38 @@ pub enum ArchetypesReply { Failed { error_tail: String }, } +/// The subject a swarm meta-flake lock goes to, with the same one publisher and +/// one subscriber as [`SUBJECT`]. +pub const LOCK_SUBJECT: &str = "$SWARM.builder.lock-meta"; + +/// The longest one lock may take. A lock fetches every input's source and +/// builds nothing. +pub const LOCK_DEADLINE: Duration = Duration::from_mins(10); + +/// How long the controller waits for a lock's reply: the lock at its deadline, +/// plus a minute for the rest. +pub const LOCK_REPLY_DEADLINE: Duration = Duration::from_mins(11); + +/// One lock: the swarm meta flake's `flake.nix`, locked on its own in a scratch +/// directory. The builder writes nothing anywhere else and commits nothing; the +/// controller commits the returned lock beside the flake. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LockMeta { + /// The rendered `flake.nix`. + pub flake_nix: String, +} + +/// The builder's answer to one [`LockMeta`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "kebab-case")] +pub enum LockReply { + /// The flake locked; `flake_lock` is the `flake.lock` `nix` wrote. + Locked { flake_lock: String }, + /// The lock did not finish: the last lines `nix` printed, or why the + /// builder could not start it. + Failed { error_tail: String }, +} + #[cfg(test)] mod tests { use serde_json::json; @@ -168,6 +200,7 @@ mod tests { fn the_subjects_are_pinned() { assert_eq!(SUBJECT, "$SWARM.builder.build"); assert_eq!(ARCHETYPES_SUBJECT, "$SWARM.builder.archetypes"); + assert_eq!(LOCK_SUBJECT, "$SWARM.builder.lock-meta"); assert_eq!(QUEUE_GROUP, "builder"); } @@ -175,4 +208,42 @@ mod tests { fn the_reply_deadline_outlasts_the_build() { assert!(REPLY_DEADLINE > BUILD_DEADLINE); } + + #[test] + fn the_lock_reply_deadline_outlasts_the_lock() { + assert!(LOCK_REPLY_DEADLINE > LOCK_DEADLINE); + } + + #[test] + fn the_lock_request_and_reply_serialize_as_pinned() { + let req = LockMeta { + flake_nix: "{ }".to_owned(), + }; + let value = serde_json::to_value(&req).expect("encodes"); + assert_eq!(value, json!({"flake_nix": "{ }"})); + assert_eq!( + serde_json::from_value::(value).expect("decodes"), + req + ); + for (reply, wire) in [ + ( + LockReply::Locked { + flake_lock: "{}".to_owned(), + }, + json!({"state": "locked", "flake_lock": "{}"}), + ), + ( + LockReply::Failed { + error_tail: "e".to_owned(), + }, + json!({"state": "failed", "error_tail": "e"}), + ), + ] { + assert_eq!(serde_json::to_value(&reply).expect("encodes"), wire); + assert_eq!( + serde_json::from_value::(wire).expect("decodes"), + reply + ); + } + } }