From 4590b04cac2c889195345d683dae4a7d000b4b4c Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 22:17:07 +0100 Subject: [PATCH] remove the hive's local build of agent closures A rebuild or create downloads the closure the swarm built and applies it; with none to deploy it fails and changes nothing. hive-priv takes a store path instead of an internal/meta commit and builds nothing. --- docs/agent-lifecycle/config-changes.md | 36 +- docs/process/gotchas.md | 6 - docs/scheduler/coordinator.md | 57 +-- docs/swarm/builder.md | 59 ++- docs/trust-boundary/security.md | 2 +- hive-c0re/src/attic/closure.rs | 59 +-- hive-c0re/src/coordinator.rs | 12 +- hive-c0re/src/forge/internal_meta.rs | 37 +- hive-c0re/src/job_queue/exec.rs | 244 +++++----- hive-c0re/src/job_queue/exec/tests.rs | 521 +++++++++++---------- hive-c0re/src/job_queue/model.rs | 4 +- hive-c0re/src/job_queue/templates.rs | 6 +- hive-c0re/src/lifecycle/mod.rs | 233 +-------- hive-c0re/src/lifecycle/tests.rs | 62 --- hive-c0re/src/main.rs | 2 +- hive-c0re/src/meta_rev.rs | 82 +--- hive-c0re/src/priv_client.rs | 8 +- hive-c0re/src/stores/build_logs.rs | 2 +- hive-priv-sock/src/lib.rs | 60 +-- hive-priv/README.md | 70 +-- hive-priv/src/closure.rs | 6 +- hive-priv/src/container.rs | 318 +++---------- hive-priv/src/main.rs | 8 +- nix/host-modules/hive-c0re/default.nix | 13 +- nix/host-modules/hive-c0re/environment.nix | 6 +- nix/host-modules/hive-c0re/options.nix | 8 +- nix/host-modules/hive-priv.nix | 49 +- nix/module-eval/core-toggle.nix | 19 +- 28 files changed, 635 insertions(+), 1354 deletions(-) diff --git a/docs/agent-lifecycle/config-changes.md b/docs/agent-lifecycle/config-changes.md index 4a8b9c64..85b23111 100644 --- a/docs/agent-lifecycle/config-changes.md +++ b/docs/agent-lifecycle/config-changes.md @@ -145,9 +145,10 @@ commit the agent's last successful build pinned. swarm-controller commits `internal/meta`: one `flake.nix` and `flake.lock` for every agent any hive places (the pins: -[`docs/swarm/builder.md`](../swarm/builder.md#meta-flake-lock)). A hive builds -every agent from it. While the repo has no commit that pins an agent's current -config, that agent's build fails and names the agent. The controller queues +[`docs/swarm/builder.md`](../swarm/builder.md#meta-flake-lock)). The swarm +builder builds every agent from it, and a hive deploys that build. While the +repo has no commit that pins an agent's current config, that agent's deploy +fails and names the agent. The controller queues its update of the repo before the deploy request for a merged config commit and for a new agent, and sends the deploy only when that update succeeded and the swarm builder built the agent from the updated repo (the @@ -159,28 +160,23 @@ the swarm builder built the agent from the updated repo (the a custom agent from the config commit at the head of `agent-configs/` `main`, an archetype agent from the archetype it has now. An agent switched between custom and archetype, or to another archetype, fails that test. - A hive without the forge credential file that nix's git reads, a repo with no - commit, a head that lacks the agent and a head behind the config each fail - the build with an error that names the agent. `MetaSync` records the commit, + A repo with no commit, a head that lacks the agent and a head behind the + config each fail the deploy with an error that names the agent. `MetaSync` records the commit, with the hyperhive and config commits the head pins, in `hive-c0re`'s own state, `/var/lib/hyperhive/meta-rev/`, which the agent can't write (the container mounts its runtime directory read-write); `Prebuild` and `Swap` read that - record, so both build one flake. A first deploy resolves the commit in its `Create` + record, so both expect the closure of one commit. A first deploy resolves the commit in its `Create` node, since it has no `MetaSync`. -- **The build.** `hive-c0re` prebuilds - `git+/internal/meta.git?rev=#nixosConfigurations.…` and - names the commit to `hive-priv` (`meta_rev` on `UpdateContainer` and - `CreateContainer`). `hive-priv` accepts only a 40-digit lowercase hex - commit and builds the flake from its own forge URL - (`HIVE_PRIV_FORGE_URL`); the URL never comes from a request. On the wire the +- **The closure.** The hive builds nothing. `Prebuild`, `Swap` and `Create` read + the agent's record in the swarm's `agent-closures` bucket (the controller + writes it from the builder's reply) and have `hive-priv` download the closure + it names from the swarm cache (`RealiseClosure`). `Swap` and `Create` then + give `hive-priv` the store path (`toplevel` on `UpdateContainer` and + `CreateContainer`), which accepts only a store path. On the wire the `hive_priv_sock::PrivRequest` variants carry a required field: - `{"op":"update_container","name":"atlas","stream":true,"meta_rev":"<40 hex>"}`. -- **Fetching.** nix fetches the forge's inputs with `git`. Both `hive-c0re` - and `hive-priv` carry a `$HOME/.gitconfig` that answers for the forge host - through `git-credential-hive-forge` (the hive's forge read token, published to - `/run/hyperhive/forge-hive-token`) and rewrites the - forge's `https` URL, which `internal/meta`'s lock names its inputs by, to - `http`. + `{"op":"update_container","name":"atlas","stream":true,"toplevel":"/nix/store/-nixos-system-atlas"}`. + The three cases when the swarm's closure isn't the one this hive expects + are in [`docs/swarm/builder.md`](../swarm/builder.md#rebuilds-download-the-closure). - **Hyperhive.** The commit pins one hyperhive rev for the swarm, so an agent runs the hyperhive of the controller's deploy, not its hive's. - **Markers.** After a successful build `hive-c0re` writes the hyperhive diff --git a/docs/process/gotchas.md b/docs/process/gotchas.md index 1e575c54..03812067 100644 --- a/docs/process/gotchas.md +++ b/docs/process/gotchas.md @@ -301,12 +301,6 @@ nix build "git+/internal/meta.git?rev=#argus.config.system.build.top nix build "git+/internal/meta.git?rev=#nixosConfigurations.argus.config.system.build.toplevel" ``` -`lifecycle::prebuild_toplevel` hit this once by constructing the attr -path as `{flake_ref}.config…` — which produced `#argus.config…` -instead of `#nixosConfigurations.argus.config…`. The fix: -`split_once('#')` to separate flake path from name, then template -`{path}#nixosConfigurations.{name}.config.system.build.toplevel`. - ### Containerized nix-daemon needs `sandbox-fallback = true` Agent containers bind-mount the host's nix-daemon socket. nspawn diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 665aa52a..7b5b3dff 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -42,11 +42,11 @@ because there is no malformed spec to reject. Nix-heavy — hold one of the `buildSlots` permits for the node's duration: -| Node | Wraps | -| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `Prebuild` | `lifecycle::prebuild_toplevel` — build the toplevel out-of-band while the container keeps serving (its meta preamble is the upstream `MetaSync` node). Skipped when the container is already down; `Swap` builds inline instead | -| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `RebuildBookkeeping` node | -| `Create` | first-spawn `nixos-container create` proper; assumes the upstream `Provision` node already registered the agent; downloads the closure the swarm cached for the `internal/meta` commit it resolves, else builds it locally, and marks the agent's pins | +| Node | Wraps | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Prebuild` | download the swarm's closure for the agent into the store while the container keeps serving (its meta preamble is the upstream `MetaSync` node). With no closure to deploy it fails here, before the stop. Skipped when the container is already down; `Swap` downloads it instead | +| `Swap` | drop-in rewrite + profile-swap onto the swarm's closure (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `RebuildBookkeeping` node | +| `Create` | first-spawn `nixos-container create` proper; assumes the upstream `Provision` node already registered the agent; downloads the closure the swarm built for the agent and creates the container from it (with none to deploy it fails and creates nothing), then marks the agent's pins | Cheap — no build slot: @@ -54,7 +54,7 @@ Cheap — no build slot: | Node | Behavior | | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MetaSync` | the rebuild's preamble — rebuild-dir prep, the agent-set refresh (`agent_set::refresh`), and the `internal/meta` commit `Prebuild` and `Swap` build from, resolved once and recorded; fails when the repo doesn't pin the agent's current config. Declares no resources | +| `MetaSync` | the rebuild's preamble — rebuild-dir prep, the agent-set refresh (`agent_set::refresh`), and the `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to be built from, resolved once and recorded; fails when the repo doesn't pin the agent's current config. Declares no resources | | `Provision` | first-spawn pre-create provisioning — proposed/applied repos, state subvolume, agent-set refresh (`agent_set::refresh`); runs ahead of `Create`. Store-only, no container yet | | `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop | | `Start` | mechanical container start — runtime dir + drop-ins, `start_with_fallback`, MCP listener registration, the manager kick. Fanned out by a `Reconcile` that observed `wanted = Up` and the container down | @@ -186,7 +186,7 @@ resources are free. Resources: clobber intent before either reconciles.) **Lease-exempt**: `MetaSync`, `Prebuild`, `Provision` — they touch the store, not the running container, which is exactly why a - stop can land while another DAG's prebuild is still building. Also exempt, for a different + stop can land while another DAG's prebuild is still downloading. Also exempt, for a different reason, are the rebuild subtree's own members (`StopForUpdate`, `Swap`, `Signal`, `Drain`, `RebuildBookkeeping`): they genuinely do touch the container, but their `AgentWindow` brace holds the lease above them — see @@ -344,17 +344,17 @@ one go, rather than the double-bounce a live `update` would trigger. Sequence for a rebuild DAG (each step is its own queue node): 1. `MetaSync` — rebuild-dir prep, the agent-set refresh, and the choice of the - `internal/meta` commit to build from. Short, and it declares no resources. -2. `Prebuild` — build the new `system.build.toplevel` **before** stopping. - The container keeps serving the previous generation while eval + fetch + - build happen out-of-band. `nixos-container update` then finds the result - cached and skips straight to the profile-swap. Build failures surface - here, before `Swap` touches the running container. Skipped when the - container is already down — a stopped agent has no uptime to preserve, - so `Swap` builds inline instead of paying the double eval. + `internal/meta` commit the swarm's closure should come from. Short, + and it declares no resources. +2. `Prebuild` — download the swarm's closure for the agent **before** stopping. + The container keeps serving the previous generation while the download + happens out-of-band. A missing or unpullable closure fails the DAG here, + before `Swap` touches the running container, and the container keeps + running. Skipped when the container is already down — a stopped agent has + no uptime to preserve, so `Swap` downloads it. 3. `StopForUpdate` — bring the container down (noop when already stopped). -4. `Swap` — `nixos-container update` profile-swap from `internal/meta` - (near-instant after the prebuild). +4. `Swap` — profile-swap onto the swarm's closure (near-instant after the + prebuild). 5. `Reconcile` — boot into the new generation when `wanted = Up`; the in-container activation script transitions old → new. Holds no build slot, so the next DAG's `Prebuild` overlaps the container boot. @@ -374,18 +374,10 @@ fallback. ### Spawn path (new container) -For a first-time `create`, `nixos-container create` is atomic: if the build fails, -it leaves no container record to clean up. A separate prebuild would just duplicate -the eval, so it's skipped. Sequence: `create` from `internal/meta` → write nspawn -flags → `systemctl daemon-reload` → `start`. - -### Prebuild attr path - -`nix build` doesn't autoresolve `#` against `nixosConfigurations` the -way `nixos-container` does internally. nix requires the explicit attr path -`#nixosConfigurations..config.system.build.toplevel`; -using the bare `#` ref would make nix look in `packages`, `legacyPackages`, -or the flake root directly — none of which exist in the rendered meta flake. +For a first-time `create`, `nixos-container create` is atomic: it leaves no container +record to clean up if it fails. The `Create` node downloads the swarm's closure first, +so a spawn with no closure to deploy fails before it creates the container. Sequence: +download the closure → `create` from it → write nspawn flags → `systemctl daemon-reload` → `start`. --- @@ -396,11 +388,10 @@ limits, build parallelism, and first-spawn latency. ### Build slots -`buildSlots` (default `1`) sets how many nix-heavy job-queue nodes +`buildSlots` (default `1`) sets how many closure-fetching job-queue nodes (prebuilds, profile swaps, first-spawn creates, meta lock bumps) run -concurrently. The default serializes all heavy nix work like the pre-DAG -rebuild queue did; raise it on hosts with the cores/RAM to build multiple -agent toplevels at once. Per-agent correctness is independent of the count — +concurrently. The default serializes all of them; raise it on hosts with +the bandwidth and disk to take more agents at once. Per-agent correctness is independent of the count — each agent's container-affecting ops serialize on its lifecycle lease regardless. diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index d49864f7..f75fb109 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -200,12 +200,9 @@ minutes. The builder takes the same one-build slot as PR checks, and refuses a request that waited 30 seconds for it. It runs `nix build` of `nixosConfigurations..config.system.build.toplevel` from `git+https:///internal/meta.git?rev=`, with no input overridden: -the committed `flake.lock` decides every input. Like the hive's build, it uses -that attribute, so a green build here means the flake evaluates and builds for -the builder's platform. A hive [downloads the closure instead of building it](#what-hives-do-with-the-cache) -when the record is for the commit it builds from; otherwise it builds locally, -from whatever head it reads at deploy time, which can be newer than the one the -swarm built. +the committed `flake.lock` decides every input. A green build here means the +flake evaluates and builds for the builder's platform. A hive [downloads the closure](#rebuilds-download-the-closure) +this build produced and never builds one itself. The node ends `done` or `failed`. Every failure skips the deploy and shows on the Jobs page as a failed `build_meta` node, and the timer builds it again: @@ -478,11 +475,9 @@ means check attic's fingerprint format; it says nothing about an attacker. A ### Rebuilds download the closure -A rebuild's `Prebuild` node, which runs while the agent is still up, tries the -cache before it builds. It reads the agent's record in `agent-closures` and -goes on only when the record is for the `internal/meta` commit `MetaSync` -recorded for this rebuild and says the builder pushed the closure. It then -asks `hive-priv` to realise the record's store path (`RealiseClosure`). `hive-priv` runs +A hive builds nothing. A rebuild's `Prebuild` node, which runs while the agent is +still up, reads the agent's record in `agent-closures` and asks `hive-priv` to +realise the record's store path (`RealiseClosure`). `hive-priv` runs `nix-store --realise` with `--max-jobs 0`, so nix can't build the path, and with the host's configured cache address, and the public key and netrc from the root-owned `/run/hyperhive-priv/attic/`, as per-call settings. The settings replace the host's `substituters` and @@ -490,32 +485,32 @@ root-owned `/run/hyperhive-priv/attic/`, as per-call settings. The settings repl source, and nix stores the path only when the cache's key signed it. The host's nix configuration stays as it was. -The deploy log says which way it went: +`Swap` then does the same lookup and download, a no-op once the path is in the +store, and gives `hive-priv` the store path to apply. A stopped agent skips +`Prebuild`, so `Swap` downloads it. A first deploy's `Create` node does the +same and creates the container from that path. -- `prebuild: substituted `: the closure is in the store, and the local build found it there and had nothing to build. -- `prebuild: the downloaded closure is not the one this hive evaluates, ; built the latter locally`, at warn level: the builder and the hive evaluate different closures for the agent, which is worth a look. The download was no use and the rebuild built locally, with the agent still up. -- `prebuild: built locally: `: the reason is the first check that failed, - such as `no-record`, `rev-mismatch`, `not-cached`, or what `hive-priv` answered - (`not available: …`, for a cache that's down or has no credentials). The rebuild builds as before. -- The same line at error level, with `no signature by the swarm cache's key` in - the reason: nix refused the path because no trusted key signed it. The rebuild builds locally - and `hive-priv` logs the refusal as well. +A rebuild or create has three outcomes, by what the lookup and the download find: -After a download, `Prebuild` still runs the local `nix build` of the agent's toplevel, -while the agent is up. It's an evaluation and no build when the hive evaluates the path the -swarm built, and a full build, as without a cache, when it doesn't, so `Swap` never meets a -build it didn't expect. `Swap` still evaluates and builds in `hive-priv`, and finds the path in -the store. A stopped agent skips `Prebuild`, so it still builds in `Swap`. +| Found | What happens | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| a record that the builder pushed, for the commit this hive chose | The hive downloads the closure and deploys it. The hive logs `prebuild: substituted ` (`create: substituted ` for a create) at info level. | +| a record that the builder pushed, for another commit (the hive and the builder disagree) | The hive downloads the builder's closure and deploys it, with a warning that names the agent, both commits and the path. The hive evaluates nothing. | +| no record, or one the hive can't use (not pushed, queue or cache unreachable, download or signature failed) | The rebuild or create fails with an error that names the agent and the reason, logged at error level, and nothing changes. A running container keeps running unchanged, because `Prebuild` fails before `StopForUpdate`. A create makes no container. | + +The hive never falls back to a local build. The third row has one owner in the +code, `without_closure` in `hive-c0re/src/job_queue/exec.rs`, so a change to what +an unreachable cache costs is one function. Mixed versions: -| hive | controller or record | what happens | -| ---- | -------------------- | ---------------------------------------------------------------------------------- | -| old | any | builds as before | -| new | old, or no record | builds locally and logs why | -| new | cache down | the download fails, the hive builds locally and logs why | -| new | signature mismatch | nix refuses the path, the hive builds locally and logs an error | -| new | old `hive-priv` | the verb is unknown to it, so the call fails and the hive builds locally, as above | +| hive | controller or record | what happens | +| ---- | -------------------- | ------------------------------------------------------------------------------------------ | +| old | any | builds locally, as before | +| new | old, or no record | refuses the create; a running agent keeps running, and the rebuild logs an error naming it | +| new | cache down | the download fails; same as no record, with the download's error in the log | +| new | signature mismatch | nix refuses the path; same as no record, and `hive-priv` logs the refusal as well | +| new | old `hive-priv` | it refuses the request, which has no `meta_rev`, so the swap or create fails | ## Credentials diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index 3d7412e0..65c4f573 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -256,7 +256,7 @@ known operations; there is no arbitrary command pass-through: | ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `StartContainer` / `StopContainer` | `nixos-container start/stop ` | | `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | -| `CreateContainer` / `UpdateContainer` | `nixos-container create/update --flake ` | +| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must be a store path (`/nix/store/<32 chars>-`), and hive-priv builds nothing | | `RealiseClosure` | `nix-store --realise --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-` and the agent name valid. It downloads and never builds | | `DestroyContainer` | `nixos-container destroy ` | | `ListContainers` | `nixos-container list` | diff --git a/hive-c0re/src/attic/closure.rs b/hive-c0re/src/attic/closure.rs index f8e2c577..52b5af5c 100644 --- a/hive-c0re/src/attic/closure.rs +++ b/hive-c0re/src/attic/closure.rs @@ -1,62 +1,19 @@ -//! The closure the swarm built for an agent at a given `internal/meta` commit, -//! as a rebuild's `Prebuild` downloads it instead of building. +//! The closure the swarm built for an agent, as a deploy reads its record +//! before downloading it. use anyhow::{Result, anyhow}; use swarm_wire::agent_closures::AgentClosure; -use super::probe::{self, Unpullable}; +use super::probe; -/// The store path of the closure `agent`'s record names, when the record is a -/// build of `rev` that the builder pushed to the cache. +/// `agent`'s record in `agent-closures`; `None` when the builder has recorded +/// no build of it. /// /// # Errors -/// No record, one for another commit, or one the builder did not push; or the -/// queue cannot be reached. The message is what a deploy log says. -pub async fn path_for(agent: &str, rev: &str) -> Result { +/// The queue cannot be reached, or the record cannot be read. +pub async fn record(agent: &str) -> Result> { let client = crate::swarm_queue::client() .await .ok_or_else(|| anyhow!("this hive has no connection to the swarm queue"))?; - let record = probe::read_record(&client, agent).await?; - path_in(record.as_ref(), rev) -} - -fn path_in(record: Option<&AgentClosure>, rev: &str) -> Result { - probe::current_record(record, Some(rev)) - .map(|record| record.store_path.clone()) - .map_err(|Unpullable { reason, detail }| anyhow!("{}: {detail}", reason.label())) -} - -#[cfg(test)] -mod tests { - use super::*; - - const REV: &str = "a1b2c3"; - const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; - - fn record(rev: &str, cached: bool) -> AgentClosure { - AgentClosure { - meta_rev: rev.to_owned(), - store_path: PATH.to_owned(), - system: "x86_64-linux".to_owned(), - built_unix: 1, - cached, - } - } - - #[test] - fn a_pushed_build_of_the_rev_names_its_path() { - assert_eq!(path_in(Some(&record(REV, true)), REV).expect("path"), PATH); - } - - #[test] - fn anything_else_names_why_not() { - for (record, why) in [ - (None, "no-record"), - (Some(record("other", true)), "rev-mismatch"), - (Some(record(REV, false)), "not-cached"), - ] { - let err = path_in(record.as_ref(), REV).expect_err(why); - assert!(format!("{err}").starts_with(why), "{err}"); - } - } + probe::read_record(&client, agent).await } diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index c3cc36bc..2f1ad3cc 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -29,9 +29,8 @@ const LAST_STOPPED_RUNNING_KEY: &str = "last_stopped_running"; pub struct Coordinator { pub broker: Arc, - /// Full build-log capture. `lifecycle::run` / - /// `lifecycle::prebuild_toplevel` `start()` a row per attempt, - /// pipe every stdout/stderr line into it, and `finish()` it on + /// Full build-log capture. `lifecycle::run` `start()`s a row per attempt, + /// pipes every stdout/stderr line into it, and `finish()`es it on /// child exit. Dashboard reads it via `list_recent_for_agent` / /// `get_full` for the per-card chip + side-panel viewer. See /// `build_logs.rs` for retention. @@ -228,7 +227,7 @@ pub struct ServeConfig { #[serde(flatten)] pub env: HiveEnv, pub model_prices: crate::hive_stats::PriceTable, - /// Number of concurrent nix-heavy job-queue nodes (prebuild / + /// Number of concurrent closure-fetching job-queue nodes (prebuild / /// profile-swap / create / meta lock). hive-c0re-local like /// `model_prices` — never injected into containers. Set via /// `services.hyperhive.deploy.hive-controller.buildSlots`. @@ -374,9 +373,8 @@ impl Coordinator { "build_logs: closed rows left in-flight by a previous run" ); } - // Install the process-wide handle so `lifecycle::run` / - // `lifecycle::prebuild_toplevel` can write without us having - // to thread an `Arc` through every public entry + // Install the process-wide handle so `lifecycle::run` can write + // without us having to thread an `Arc` through every public entry // point in the lifecycle surface. crate::build_logs::install(build_logs.clone()); let power = Arc::new(crate::power::PowerStore::open(db_path).context("open agent_power")?); diff --git a/hive-c0re/src/forge/internal_meta.rs b/hive-c0re/src/forge/internal_meta.rs index 93e3193e..f13859e2 100644 --- a/hive-c0re/src/forge/internal_meta.rs +++ b/hive-c0re/src/forge/internal_meta.rs @@ -20,20 +20,13 @@ const CONFIG_BRANCH: &str = "main"; /// [`MetaSource`] over the forge API. pub(crate) struct ForgeSource { api: Forgejo, - /// Whether the credential file nix's `git` reads holds this hive's token. - nix_credential: bool, } -/// A source reading as this hive, or `None` when it holds no forge token. It -/// also refreshes the credential file nix's `git` fetches with, which the -/// build after this read needs to be current. +/// A source reading as this hive, or `None` when it holds no forge token. pub(crate) async fn internal_meta_source() -> Option { let token: ReaderToken = super::reader_token().await?; match token.api() { - Ok(api) => Some(ForgeSource { - api, - nix_credential: token.publish_for_nix(), - }), + Ok(api) => Some(ForgeSource { api }), Err(e) => { tracing::warn!(error = ?e, "internal/meta: cannot build a forge client"); None @@ -72,10 +65,6 @@ impl ForgeSource { } impl MetaSource for ForgeSource { - fn nix_can_fetch(&self) -> bool { - self.nix_credential - } - async fn meta_head(&self) -> anyhow::Result> { use anyhow::Context as _; @@ -113,25 +102,3 @@ impl MetaSource for ForgeSource { self.branch_head(CONFIG_ORG, agent, CONFIG_BRANCH).await } } - -#[cfg(test)] -mod tests { - use super::*; - use forgejo_api::Auth; - - fn source(nix_credential: bool) -> ForgeSource { - let url = url::Url::parse("http://forge.test").expect("url"); - ForgeSource { - api: Forgejo::new(Auth::None, url).expect("client"), - nix_credential, - } - } - - /// The control (`true`) shows the answer follows the field rather than - /// being a constant. - #[test] - fn nix_fetches_only_where_its_credential_was_published() { - assert!(source(true).nix_can_fetch()); - assert!(!source(false).nix_can_fetch()); - } -} diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 10654171..cbfce208 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context as _, Result}; +use swarm_wire::agent_closures::AgentClosure; use hive_jobq::NodeId; @@ -34,7 +35,7 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT; /// The container-state calls the executors below make on the host: systemd, -/// `hive-priv`, the nix build, the `internal/meta` record. Production uses +/// `hive-priv`, the swarm's closure record, the `internal/meta` record. Production uses /// [`RealHost`], which forwards to `lifecycle` / `meta_rev` / `Coordinator` /// unchanged; tests substitute a recording fake. pub(super) trait Host: Sync { @@ -44,28 +45,19 @@ pub(super) trait Host: Sync { async fn kill(&self, name: &str) -> Result<()>; /// The `internal/meta` commit `MetaSync` recorded for the agent. fn recorded_rev(&self, name: &str) -> Result; - /// The store path of the closure the swarm built for the agent at the - /// `internal/meta` commit `rev` and pushed to its cache. The error says why - /// there is none. - async fn cached_closure(&self, name: &str, rev: &str) -> Result; + /// The record the swarm's builder wrote for the agent; `None` when it wrote + /// none. + async fn closure_record(&self, name: &str) -> Result>; /// Download `store_path` from the swarm's cache into the store, never /// building it. async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>; - /// Warm the agent's toplevel build from the `internal/meta` commit `rev`. - /// The toplevel it printed, when it printed one. - async fn prebuild_toplevel( - &self, - name: &str, - rev: &str, - node_id: u64, - ) -> Result>; - /// Re-apply drop-ins, then `nixos-container update` at `rev`. + /// Re-apply drop-ins, then point the container at the closure `toplevel`. async fn swap_update( &self, name: &str, hive: &HiveEnv, paths: &AgentPaths, - rev: &str, + toplevel: &str, node_id: u64, ) -> Result<()>; /// Write or remove the agent's pause marker. @@ -80,7 +72,8 @@ pub(super) trait Host: Sync { async fn refresh_agent_set(&self) -> Result<()>; /// Pin the `internal/meta` commit the agent builds from. async fn resolve_meta(&self, name: &str) -> Result; - /// Record `build` as what `Prebuild` and `Swap` build from. + /// Record `build` as the commit `Prebuild` and `Swap` expect the swarm's + /// closure to be built from. fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()>; /// The build `MetaSync` recorded, if any. fn recorded_build(&self, name: &str) -> Option; @@ -88,8 +81,8 @@ pub(super) trait Host: Sync { fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()>; /// First-spawn provisioning: repos, state subvolume, agent-set refresh. async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()>; - /// `nixos-container create` at `rev`. - async fn create_only(&self, name: &str, rev: &str) -> Result<()>; + /// `nixos-container create` from the closure `toplevel`. + async fn create_only(&self, name: &str, toplevel: &str) -> Result<()>; /// Write the nspawn and resource-limit drop-ins, then reload systemd. async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>; /// Start preamble, then start with the cold-start fallback. @@ -128,33 +121,23 @@ impl Host for RealHost { Ok(crate::meta_rev::recorded_or_err(name)?.rev) } - async fn cached_closure(&self, name: &str, rev: &str) -> Result { - crate::attic::closure::path_for(name, rev).await + async fn closure_record(&self, name: &str) -> Result> { + crate::attic::closure::record(name).await } async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> { crate::priv_client::realise_closure(name, store_path).await } - async fn prebuild_toplevel( - &self, - name: &str, - rev: &str, - node_id: u64, - ) -> Result> { - let flake_ref = crate::meta_rev::flake_ref(name, rev); - crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(node_id)).await - } - async fn swap_update( &self, name: &str, hive: &HiveEnv, paths: &AgentPaths, - rev: &str, + toplevel: &str, node_id: u64, ) -> Result<()> { - crate::lifecycle::swap_update(name, hive, paths, rev, Some(node_id)).await + crate::lifecycle::swap_update(name, hive, paths, toplevel, Some(node_id)).await } async fn set_paused(&self, agent: &hive_types::Ident, paused: bool) -> Result<()> { @@ -198,8 +181,8 @@ impl Host for RealHost { crate::lifecycle::provision_container(name, paths).await } - async fn create_only(&self, name: &str, rev: &str) -> Result<()> { - crate::lifecycle::create_only(name, rev).await + async fn create_only(&self, name: &str, toplevel: &str) -> Result<()> { + crate::lifecycle::create_only(name, toplevel).await } async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { @@ -300,11 +283,11 @@ async fn dispatch( // growth executors return what to grow rather than taking the builder. let result = match kind { NodeKind::MetaSync { .. } => run_meta_sync(host, agent).await, - NodeKind::Prebuild { .. } => run_prebuild(host, agent, id).await, + NodeKind::Prebuild { .. } => run_prebuild(host, agent).await, NodeKind::Swap { .. } => run_swap(host, coord, agent, id).await, NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(host, coord, agent).await, NodeKind::Provision { .. } => run_provision(host, agent).await, - NodeKind::Create { .. } => run_create(host, agent, id).await, + NodeKind::Create { .. } => run_create(host, agent).await, NodeKind::DestroyContainer { .. } => run_destroy_container(host, coord, agent).await, NodeKind::PurgeState { .. } => { run_purge_state(host, agent).await; @@ -533,10 +516,11 @@ fn run_set_wanted(coord: &Arc, agent: &str, up: bool) -> Result<()> } /// The rebuild's preamble: runtime-dir prep, the agent-set refresh, and the -/// `internal/meta` commit `Prebuild` and `Swap` build from. +/// `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to +/// be built from. /// /// Fails when `internal/meta` does not pin the agent's current config yet: -/// there is no other flake to build it from. +/// there is no commit to deploy. async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> { // Runs while the agent is still up — the runtime dir and MCP listener // already exist. Use the pure path accessor; no need to re-register the @@ -550,98 +534,95 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> { } host.prepare_rebuild_dirs(name, &paths).await?; host.refresh_agent_set().await?; - // Resolved once here so `Prebuild` and `Swap` build the same flake. + // Resolved once here so `Prebuild` and `Swap` expect the same commit. let build = host.resolve_meta(name).await?; host.record_meta(name, &build)?; Ok(()) } -/// Out-of-band toplevel build while the container keeps serving: warm -/// `system.build.toplevel` so the later `Swap` hits cache and skips -/// straight to the profile-swap, from the `internal/meta` commit the upstream -/// `MetaSync` node recorded. The warm build is skipped when the -/// container is already down: its only purpose is to shrink the swap's -/// downtime window, so a stopped agent (no uptime to preserve) doesn't -/// pay the double eval — `Swap` builds inline instead. -/// -/// The swarm's cache is tried first: when the swarm built this agent at the -/// recorded commit and pushed the closure, it is downloaded. The local build -/// runs either way. After a download it is an evaluation and a no-op build when -/// this hive evaluates the path the swarm built, and builds it, with the agent -/// still up, when it doesn't; `Swap` never meets a build it didn't expect. -async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> { - // Warm the toplevel build only when the container is up — the whole - // point of prebuild is to shrink the swap's downtime window. A - // stopped agent has no uptime to preserve, so skip the (expensive) - // eval and let the downstream `Swap` build inline. +/// Download the swarm's closure for the agent while the container keeps +/// serving, so a rebuild with no closure fails here, before `StopForUpdate` +/// takes a healthy container down. `Swap` downloads the same closure again +/// (a no-op once it is in the store) and applies it. Skipped when the container +/// is already down: there is no uptime to protect, and `Swap` fails the same way. +async fn run_prebuild(host: &impl Host, name: &str) -> Result<()> { if host.is_running(name).await { let rev = host.recorded_rev(name)?; - let downloaded = match substitute(host, name, &rev).await { - Ok(path) => Some(path), - Err(fallback) => { - let reason = &fallback.reason; - if fallback.signature_refused { - tracing::error!(%name, %rev, "prebuild: built locally: {reason}"); - } else { - tracing::warn!(%name, %rev, "prebuild: built locally: {reason}"); - } - None - } - }; - let evaluated = host.prebuild_toplevel(name, &rev, id.get()).await?; - if let Some(path) = downloaded { - match drift(&path, evaluated.as_deref()) { - None => tracing::info!(%name, %rev, "prebuild: substituted {path}"), - Some(note) => tracing::warn!(%name, %rev, "prebuild: {note}"), - } - } + let path = closure_for(host, name, &rev, "prebuild").await?; + tracing::info!(%name, %rev, "prebuild: substituted {path}"); } Ok(()) } -/// What to log when the closure downloaded for a rebuild isn't the one this -/// hive's evaluation produced, which means the builder and the hive disagree -/// about the agent's flake. `None` when they agree. -fn drift(downloaded: &str, evaluated: Option<&str>) -> Option { - match evaluated { - Some(path) if path == downloaded => None, - Some(path) => Some(format!( - "the downloaded closure {downloaded} is not the one this hive evaluates, {path}; built the latter locally" - )), - None => Some(format!( - "downloaded {downloaded}, but the local build printed no path to compare it with" - )), - } +/// Why the swarm's closure for an agent cannot be deployed. +enum NoClosure { + /// The builder has recorded no build of the agent. + NoRecord, + /// A build is recorded, but its closure cannot be had: the record could not + /// be read, the builder did not push the closure, or the download failed. + Unpullable(String), } -/// Why a rebuild or create builds instead of downloading the swarm's closure. -struct LocalBuild { - reason: String, - /// The cache served a path no trusted key signed. - signature_refused: bool, -} - -/// Download the closure the swarm built for `name` at `rev`; its store path. -async fn substitute(host: &impl Host, name: &str, rev: &str) -> Result { - let path = host - .cached_closure(name, rev) +/// The store path of the closure the swarm built for `name`, downloaded into +/// the store. The deploy applies the swarm's closure whatever commit it was +/// built from: when that is not the `rev` this hive chose, it is logged. +/// +/// # Errors +/// [`without_closure`]'s refusal, after logging it. +async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result { + pull_closure(host, name, rev, step) .await - .map_err(|e| LocalBuild { - reason: format!("{e:#}"), - signature_refused: false, - })?; - host.realise_closure(name, &path).await.map_err(|e| { - let reason = format!("{e:#}"); - LocalBuild { - signature_refused: reason.contains(hive_priv_sock::SIGNATURE_REFUSED), - reason, - } - })?; - Ok(path) + .map_err(|why| without_closure(name, rev, step, &why)) } -/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), -/// `nixos-container update`, then the post-rebuild bookkeeping tail +async fn pull_closure( + host: &impl Host, + name: &str, + rev: &str, + step: &str, +) -> Result { + let record = host + .closure_record(name) + .await + .map_err(|e| NoClosure::Unpullable(format!("{e:#}")))? + .ok_or(NoClosure::NoRecord)?; + if !record.cached { + return Err(NoClosure::Unpullable( + "the builder did not push the closure".to_owned(), + )); + } + host.realise_closure(name, &record.store_path) + .await + .map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?; + if record.meta_rev != rev { + tracing::warn!( + %name, %rev, built_from = %record.meta_rev, + "{step}: the swarm built {} from internal/meta {}, not the {rev} this hive chose; deploying the swarm's", + record.store_path, record.meta_rev + ); + } + Ok(record.store_path) +} + +/// What a rebuild or create does when there is no closure to deploy: it +/// fails, and nothing changes, so a running container keeps running as it is +/// and a create makes none. The failure is logged at error level with the +/// agent's name. +/// +/// This is the one place that decides what an unpullable closure (the cache +/// unreachable, a TLS error, a failed download) costs. A rebuild never builds +/// locally on the hive. +fn without_closure(name: &str, rev: &str, step: &str, why: &NoClosure) -> anyhow::Error { + let reason = match why { + NoClosure::NoRecord => "the swarm has recorded no build of this agent".to_owned(), + NoClosure::Unpullable(e) => format!("the swarm's closure cannot be pulled: {e}"), + }; + tracing::error!(%name, %rev, "{step}: {reason}; nothing was changed"); + anyhow::anyhow!("{name}: {reason}") +} + +/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), point the +/// container at the swarm's closure, then the post-rebuild bookkeeping tail /// (rev marker, `Rebuilt` event, forge sync, kick, rescan). /// The recovery-start on failure is NOT here — the DAG's tail /// `Reconcile` runs after this node terminal ok *or* fail. @@ -657,7 +638,11 @@ async fn run_swap( let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir)?; let rev = host.recorded_rev(name)?; - let result = host.swap_update(name, &hive, &paths, &rev, id.get()).await; + let toplevel = closure_for(host, name, &rev, "swap").await?; + tracing::info!(%name, %rev, "swap: applying {toplevel}"); + let result = host + .swap_update(name, &hive, &paths, &toplevel, id.get()) + .await; // On success the Ok-only bookkeeping tail (rev marker, forge // sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node, // which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded @@ -719,30 +704,13 @@ async fn run_provision(host: &impl Host, name: &str) -> Result<()> { /// node stays purely "create", not "create + start". Marks the agent as /// running the pins it was created from. /// -/// The swarm's cache is tried first, as in [`run_prebuild`]: when it holds the -/// closure for the resolved commit it is downloaded, and the local evaluation -/// that follows is a no-op build on a path match and builds the evaluated path -/// on a mismatch. Without a cached closure the create builds it locally. -async fn run_create(host: &impl Host, name: &str, id: NodeId) -> Result<()> { +/// The container is created from the swarm's closure for the resolved commit; +/// with none to deploy the create fails and makes no container. +async fn run_create(host: &impl Host, name: &str) -> Result<()> { let build = host.resolve_meta(name).await?; - match substitute(host, name, &build.rev).await { - Ok(path) => { - let evaluated = host.prebuild_toplevel(name, &build.rev, id.get()).await?; - match drift(&path, evaluated.as_deref()) { - None => tracing::info!(%name, rev = %build.rev, "create: substituted {path}"), - Some(note) => tracing::warn!(%name, rev = %build.rev, "create: {note}"), - } - } - Err(fallback) => { - let reason = &fallback.reason; - if fallback.signature_refused { - tracing::error!(%name, rev = %build.rev, "create: built locally: {reason}"); - } else { - tracing::warn!(%name, rev = %build.rev, "create: built locally: {reason}"); - } - } - } - host.create_only(name, &build.rev).await?; + let toplevel = closure_for(host, name, &build.rev, "create").await?; + tracing::info!(%name, rev = %build.rev, "create: substituted {toplevel}"); + host.create_only(name, &toplevel).await?; host.mark_deployed(name, &build)?; Ok(()) } diff --git a/hive-c0re/src/job_queue/exec/tests.rs b/hive-c0re/src/job_queue/exec/tests.rs index 90fb9566..e1b6154d 100644 --- a/hive-c0re/src/job_queue/exec/tests.rs +++ b/hive-c0re/src/job_queue/exec/tests.rs @@ -11,16 +11,17 @@ use hive_jobq::NodeId; use hive_jobq::scheduler::Scheduler; use super::{ - Host, dispatch, drift, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain, + Host, dispatch, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain, run_emit_rebuilt, run_forge_sweep, run_knowledge_pull, run_meta_sync, run_pause_drain, run_pause_signal, run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping, run_reconcile, run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap, - run_wanted_pull, run_write_dropin, substitute, + run_wanted_pull, run_write_dropin, }; use crate::coordinator::{AgentPaths, Coordinator, HiveEnv}; use crate::job_queue::model::NodeKind; use crate::meta_rev::MetaBuild; use crate::power::Wanted; +use swarm_wire::agent_closures::AgentClosure; const REV: &str = "2222222222222222222222222222222222222222"; const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-alice"; @@ -30,18 +31,12 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system- struct FakeHost { running: bool, rev: Option, - /// The store path the swarm's record names, when it has a pushed build of - /// `REV`; `None` reads as no record. - closure: Option, - /// A failing `realise` says nix refused the path for want of a signature. - refusal: bool, - /// The toplevel this hive's evaluation produces; `None` is `CLOSURE`. - evaluates: Option, + /// What the swarm's builder recorded for the agent; `None` reads as no + /// record. + built: Option, /// Store paths a download put in the store. realised: Mutex>, - /// Store paths a prebuild had to build because the store lacked them. - builds: Mutex>, - /// Names of the calls that fail (`kill`, `realise`, `prebuild`, `swap_update`, + /// Names of the calls that fail (`kill`, `realise`, `closure_record`, `swap_update`, /// `set_paused`, ...). `recorded_build` fails by finding nothing. failing: Vec<&'static str>, /// The sweeps and pulls never finish, so only a deadline ends them. @@ -56,6 +51,7 @@ impl FakeHost { Self { running: true, rev: Some(REV.to_owned()), + built: Some(closure(REV, true)), ..Self::default() } } @@ -64,12 +60,19 @@ impl FakeHost { Self { running: false, rev: Some(REV.to_owned()), + built: Some(closure(REV, true)), ..Self::default() } } - fn with_closure(mut self) -> Self { - self.closure = Some(CLOSURE.to_owned()); + /// The builder recorded `meta_rev` for the agent, pushed or not. + fn built(mut self, meta_rev: &str, cached: bool) -> Self { + self.built = Some(closure(meta_rev, cached)); + self + } + + fn without_record(mut self) -> Self { + self.built = None; self } @@ -98,10 +101,6 @@ impl FakeHost { fn calls(&self) -> Vec { self.calls.lock().unwrap().clone() } - - fn builds(&self) -> Vec { - self.builds.lock().unwrap().clone() - } } impl Host for FakeHost { @@ -125,54 +124,29 @@ impl Host for FakeHost { .ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded")) } - async fn cached_closure(&self, name: &str, rev: &str) -> anyhow::Result { - self.record(format!("cached_closure {name} {rev}")); - self.closure - .clone() - .ok_or_else(|| anyhow::anyhow!("no-record: no build is recorded for the agent")) + async fn closure_record(&self, name: &str) -> anyhow::Result> { + self.step("closure_record", format!("closure_record {name}"))?; + Ok(self.built.clone()) } async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> { self.record(format!("realise {name} {store_path}")); if self.fails("realise") { - if self.refusal { - anyhow::bail!( - "not available: {}: lacks a signature", - hive_priv_sock::SIGNATURE_REFUSED - ); - } anyhow::bail!("not available: Couldn't connect to server"); } self.realised.lock().unwrap().push(store_path.to_owned()); Ok(()) } - async fn prebuild_toplevel( - &self, - name: &str, - rev: &str, - node_id: u64, - ) -> anyhow::Result> { - self.record(format!("prebuild {name} {rev} node={node_id}")); - if self.fails("prebuild") { - anyhow::bail!("prebuild failed"); - } - let path = self.evaluates.clone().unwrap_or_else(|| CLOSURE.to_owned()); - if !self.realised.lock().unwrap().contains(&path) { - self.builds.lock().unwrap().push(path.clone()); - } - Ok(Some(path)) - } - async fn swap_update( &self, name: &str, _hive: &HiveEnv, _paths: &AgentPaths, - rev: &str, + toplevel: &str, node_id: u64, ) -> anyhow::Result<()> { - self.record(format!("swap_update {name} {rev} node={node_id}")); + self.record(format!("swap_update {name} {toplevel} node={node_id}")); if self.fails("swap_update") { anyhow::bail!("nixos-container update failed"); } @@ -231,8 +205,8 @@ impl Host for FakeHost { self.step("provision", format!("provision {name}")) } - async fn create_only(&self, name: &str, rev: &str) -> anyhow::Result<()> { - self.step("create_only", format!("create_only {name} {rev}")) + async fn create_only(&self, name: &str, toplevel: &str) -> anyhow::Result<()> { + self.step("create_only", format!("create_only {name} {toplevel}")) } async fn write_dropins( @@ -287,6 +261,17 @@ impl Host for FakeHost { } } +/// The builder's record of a closure built from `internal/meta` at `meta_rev`. +fn closure(meta_rev: &str, cached: bool) -> AgentClosure { + AgentClosure { + meta_rev: meta_rev.to_owned(), + store_path: CLOSURE.to_owned(), + system: "x86_64-linux".to_owned(), + built_unix: 1, + cached, + } +} + fn build() -> MetaBuild { MetaBuild { rev: REV.to_owned(), @@ -323,188 +308,185 @@ fn node_id() -> NodeId { // --- Prebuild --------------------------------------------------------------- +const OTHER_REV: &str = "3333333333333333333333333333333333333333"; + +/// What a block logged, level and text: the message and each field. +#[derive(Clone, Default)] +struct Logged(Arc>>); + +impl tracing::Subscriber for Logged { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + use std::fmt::Write as _; + + let mut text = String::new(); + event.record( + &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| { + let _ = write!(text, "{}={value:?} ", field.name()); + }, + ); + self.0 + .lock() + .unwrap() + .push((*event.metadata().level(), text)); + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + +impl Logged { + fn at(&self, level: tracing::Level) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .filter(|(l, _)| *l == level) + .map(|(_, text)| text.clone()) + .collect() + } +} + +/// `fut`'s output and what it logged. The test runtime is single-threaded, so +/// the subscriber set here sees every event the future emits. +async fn logging(fut: impl std::future::Future) -> (T, Logged) { + let logged = Logged::default(); + let _guard = tracing::subscriber::set_default(logged.clone()); + let out = fut.await; + (out, logged) +} + +/// Every way the swarm's closure for `alice` cannot be had, with the text the +/// error log must carry and the host calls that must not happen. +fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] { + [ + ("no record", FakeHost::up().without_record(), "no build"), + ( + "not pushed", + FakeHost::up().built(REV, false), + "did not push", + ), + ( + "download fails", + FakeHost::up().failing("realise"), + "Couldn't connect", + ), + ( + "record unreadable", + FakeHost::up().failing("closure_record"), + "closure_record failed", + ), + ] +} + #[tokio::test] -async fn prebuild_warms_the_recorded_rev_when_the_container_is_up() { +async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() { let host = FakeHost::up(); - let id = node_id(); - run_prebuild(&host, "alice", id).await.expect("prebuild ok"); + let (result, log) = logging(run_prebuild(&host, "alice")).await; + result.expect("prebuild ok"); assert_eq!( host.calls(), [ "is_running alice".to_owned(), "recorded_rev alice".to_owned(), - format!("cached_closure alice {REV}"), - format!("prebuild alice {REV} node={}", id.get()), - ] - ); -} - -/// The download is followed by the local build either way. When this hive -/// evaluates the path the swarm built, the build finds it in the store. -#[tokio::test] -async fn a_downloaded_closure_the_hive_also_evaluates_is_found_not_built() { - let host = FakeHost::up().with_closure(); - let id = node_id(); - run_prebuild(&host, "alice", id).await.expect("prebuild ok"); - assert_eq!( - host.calls(), - [ - "is_running alice".to_owned(), - "recorded_rev alice".to_owned(), - format!("cached_closure alice {REV}"), + "closure_record alice".to_owned(), format!("realise alice {CLOSURE}"), - format!("prebuild alice {REV} node={}", id.get()), ] ); - assert_eq!(host.builds(), Vec::::new()); -} - -/// When it evaluates another path, the build happens in `Prebuild`, after the -/// check that the agent is up and before `Swap` could stop it. -#[tokio::test] -async fn a_downloaded_closure_the_hive_does_not_evaluate_is_built_before_the_swap() { - let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice"; - let host = FakeHost { - evaluates: Some(other.to_owned()), - ..FakeHost::up().with_closure() - }; - let id = node_id(); - run_prebuild(&host, "alice", id).await.expect("prebuild ok"); - let calls = host.calls(); - assert_eq!(calls[0], "is_running alice", "{calls:?}"); - assert_eq!( - calls.last().map(String::as_str), - Some(format!("prebuild alice {REV} node={}", id.get()).as_str()), - "{calls:?}" - ); - assert_eq!(host.builds(), [other.to_owned()]); -} - -#[test] -fn only_a_different_path_is_drift() { - assert_eq!(drift(CLOSURE, Some(CLOSURE)), None); - let note = drift(CLOSURE, Some("/nix/store/other")).expect("drift"); + let info = log.at(tracing::Level::INFO); assert!( - note.contains(CLOSURE) && note.contains("/nix/store/other"), - "{note}" + info.iter() + .any(|l| l.contains("prebuild: substituted") && l.contains(CLOSURE)), + "{info:?}" ); - assert!(drift(CLOSURE, None).expect("no path").contains(CLOSURE)); + assert!(log.at(tracing::Level::WARN).is_empty()); + assert!(log.at(tracing::Level::ERROR).is_empty()); } +/// No closure to deploy fails the node before `StopForUpdate`, so the running +/// container is untouched, and the log names the agent and why. #[tokio::test] -async fn a_failed_download_builds_locally() { - let host = FakeHost::up().with_closure().failing("realise"); - let id = node_id(); - run_prebuild(&host, "alice", id) - .await - .expect("the build covers it"); - let calls = host.calls(); - assert_eq!(calls[3], format!("realise alice {CLOSURE}"), "{calls:?}"); - assert_eq!( - calls[4], - format!("prebuild alice {REV} node={}", id.get()), - "{calls:?}" - ); - assert_eq!(calls.len(), 5, "{calls:?}"); +async fn a_rebuild_without_a_closure_fails_the_prebuild_and_logs_an_error() { + for (case, host, why) in without_a_closure() { + let (result, log) = logging(run_prebuild(&host, "alice")).await; + let err = result.expect_err(case); + assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}"); + let errors = log.at(tracing::Level::ERROR); + assert_eq!(errors.len(), 1, "{case}: {errors:?}"); + assert!( + errors[0].contains("alice") && errors[0].contains(why), + "{case}: {errors:?}" + ); + let calls = host.calls(); + assert!( + !calls.iter().any(|c| c.starts_with("kill")), + "{case}: {calls:?}" + ); + assert!( + !calls + .iter() + .any(|c| c.starts_with("swap_update") || c.starts_with("create_only")), + "{case}: {calls:?}" + ); + } } -/// The fallback is the build, so a build that fails still fails the node. +/// The builder's record is not for the commit this hive chose: its closure is +/// deployed and the difference is a warning. #[tokio::test] -async fn a_failed_download_and_a_failed_build_fail_the_node() { - let host = FakeHost::up() - .with_closure() - .failing("realise") - .failing("prebuild"); - let err = run_prebuild(&host, "alice", node_id()) - .await - .expect_err("nothing produced the closure"); - assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}"); +async fn a_closure_built_from_another_commit_is_deployed_with_a_warning() { + let host = FakeHost::up().built(OTHER_REV, true); + let (result, log) = logging(run_prebuild(&host, "alice")).await; + result.expect("drift is not a refusal"); + assert!( + host.calls().contains(&format!("realise alice {CLOSURE}")), + "{:?}", + host.calls() + ); + let warnings = log.at(tracing::Level::WARN); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!( + [REV, OTHER_REV, CLOSURE] + .iter() + .all(|part| warnings[0].contains(part)), + "{warnings:?}" + ); + assert!(log.at(tracing::Level::ERROR).is_empty()); } #[tokio::test] async fn a_stopped_container_looks_up_nothing() { - let host = FakeHost::down().with_closure(); - run_prebuild(&host, "alice", node_id()) - .await - .expect("skipped"); - assert_eq!(host.calls(), ["is_running alice"]); -} - -/// The error log is for a path the cache served unsigned, not for a cache that -/// is down or a missing record. -#[tokio::test] -async fn only_a_signature_refusal_is_flagged() { - let down = substitute( - &FakeHost::up().with_closure().failing("realise"), - "alice", - REV, - ) - .await - .expect_err("down"); - assert!(!down.signature_refused, "{}", down.reason); - assert!(down.reason.contains("Couldn't connect"), "{}", down.reason); - - let no_record = substitute(&FakeHost::up(), "alice", REV) - .await - .expect_err("no record"); - assert!(!no_record.signature_refused); - assert!( - no_record.reason.starts_with("no-record"), - "{}", - no_record.reason - ); - - let refused = FakeHost { - refusal: true, - ..FakeHost::up().with_closure().failing("realise") - }; - let refused = substitute(&refused, "alice", REV) - .await - .expect_err("refused"); - assert!(refused.signature_refused, "{}", refused.reason); - - let ok = substitute(&FakeHost::up().with_closure(), "alice", REV).await; - assert_eq!(ok.ok().as_deref(), Some(CLOSURE)); -} - -#[tokio::test] -async fn prebuild_is_skipped_when_the_container_is_down() { let host = FakeHost::down(); - run_prebuild(&host, "alice", node_id()) - .await - .expect("a skipped prebuild is ok"); + run_prebuild(&host, "alice").await.expect("skipped"); assert_eq!(host.calls(), ["is_running alice"]); } #[tokio::test] -async fn prebuild_without_a_recorded_rev_fails_before_building() { +async fn prebuild_without_a_recorded_rev_fails_before_looking_up_a_closure() { let host = FakeHost { rev: None, ..FakeHost::up() }; - let err = run_prebuild(&host, "alice", node_id()) + let err = run_prebuild(&host, "alice") .await - .expect_err("nothing to build from"); + .expect_err("nothing to deploy"); assert!( format!("{err:#}").contains("no internal/meta commit"), "{err:#}" ); - assert!(!host.calls().iter().any(|c| c.starts_with("prebuild"))); -} - -#[tokio::test] -async fn prebuild_failure_fails_the_node() { - let host = FakeHost::up().failing("prebuild"); - let err = run_prebuild(&host, "alice", node_id()) - .await - .expect_err("build failure propagates"); - assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}"); + assert!(!host.calls().iter().any(|c| c.starts_with("closure_record"))); } // --- Swap ------------------------------------------------------------------- #[tokio::test] -async fn swap_updates_at_the_recorded_rev() { +async fn swap_applies_the_swarms_closure() { let (_dir, coord) = coordinator(); let host = FakeHost::down(); let id = node_id(); @@ -513,11 +495,50 @@ async fn swap_updates_at_the_recorded_rev() { host.calls(), [ "recorded_rev alice".to_owned(), - format!("swap_update alice {REV} node={}", id.get()), + "closure_record alice".to_owned(), + format!("realise alice {CLOSURE}"), + format!("swap_update alice {CLOSURE} node={}", id.get()), ] ); } +/// A swap with no closure to deploy never updates: the container stays as it +/// was, and the log names the agent and why. +#[tokio::test] +async fn a_swap_without_a_closure_never_updates_and_logs_an_error() { + for (case, host, why) in without_a_closure() { + let (_dir, coord) = coordinator(); + let (result, log) = logging(run_swap(&host, &coord, "alice", node_id())).await; + result.expect_err(case); + let errors = log.at(tracing::Level::ERROR); + assert_eq!(errors.len(), 1, "{case}: {errors:?}"); + assert!( + errors[0].contains("alice") && errors[0].contains(why), + "{case}: {errors:?}" + ); + assert!( + !host.calls().iter().any(|c| c.starts_with("swap_update")), + "{case}: {:?}", + host.calls() + ); + } +} + +#[tokio::test] +async fn swap_applies_a_closure_built_from_another_commit_with_a_warning() { + let (_dir, coord) = coordinator(); + let host = FakeHost::down().built(OTHER_REV, true); + let id = node_id(); + let (result, log) = logging(run_swap(&host, &coord, "alice", id)).await; + result.expect("drift is not a refusal"); + assert_eq!( + host.calls().last(), + Some(&format!("swap_update alice {CLOSURE} node={}", id.get())) + ); + assert_eq!(log.at(tracing::Level::WARN).len(), 1); + assert!(log.at(tracing::Level::ERROR).is_empty()); +} + /// A failed update fails the node and nothing else: the swap never restarts or /// stops the container itself — recovery is the DAG's tail `Reconcile`. #[tokio::test] @@ -529,8 +550,8 @@ async fn a_failed_swap_reports_the_error_and_touches_no_container() { .expect_err("update failure propagates"); assert!(format!("{err:#}").contains("update failed"), "{err:#}"); let calls = host.calls(); - assert_eq!(calls.len(), 2, "{calls:?}"); - assert!(calls[1].starts_with("swap_update alice"), "{calls:?}"); + assert_eq!(calls.len(), 4, "{calls:?}"); + assert!(calls[3].starts_with("swap_update alice"), "{calls:?}"); } #[tokio::test] @@ -824,84 +845,73 @@ async fn provision_and_write_dropin_forward_and_propagate_failure() { } #[tokio::test] -async fn create_without_a_record_builds_locally_at_the_resolved_rev() { +async fn create_downloads_the_swarms_closure_and_creates_from_it() { let host = FakeHost::up(); - run_create(&host, "alice", node_id()).await.expect("ok"); + let (result, log) = logging(run_create(&host, "alice")).await; + result.expect("ok"); assert_eq!( host.calls(), [ "resolve_meta alice".to_owned(), - format!("cached_closure alice {REV}"), - format!("create_only alice {REV}"), - format!("mark_deployed alice {REV}"), - ] - ); -} - -#[tokio::test] -async fn create_downloads_a_cached_closure_and_builds_nothing() { - let host = FakeHost::up().with_closure(); - let id = node_id(); - run_create(&host, "alice", id).await.expect("ok"); - assert_eq!( - host.calls(), - [ - "resolve_meta alice".to_owned(), - format!("cached_closure alice {REV}"), + "closure_record alice".to_owned(), format!("realise alice {CLOSURE}"), - format!("prebuild alice {REV} node={}", id.get()), - format!("create_only alice {REV}"), + format!("create_only alice {CLOSURE}"), format!("mark_deployed alice {REV}"), ] ); - assert_eq!(host.builds(), Vec::::new()); -} - -#[tokio::test] -async fn create_builds_locally_when_the_download_fails() { - let host = FakeHost::up().with_closure().failing("realise"); - run_create(&host, "alice", node_id()).await.expect("ok"); - assert_eq!( - host.calls(), - [ - "resolve_meta alice".to_owned(), - format!("cached_closure alice {REV}"), - format!("realise alice {CLOSURE}"), - format!("create_only alice {REV}"), - format!("mark_deployed alice {REV}"), - ] + assert!( + log.at(tracing::Level::INFO) + .iter() + .any(|l| l.contains("create: substituted") && l.contains(CLOSURE)) ); + assert!(log.at(tracing::Level::WARN).is_empty()); +} + +/// A create with no closure to deploy is refused: no container is made, none is +/// marked deployed, and the log names the agent and why. +#[tokio::test] +async fn a_create_without_a_closure_is_refused_and_logs_an_error() { + for (case, host, why) in without_a_closure() { + let (result, log) = logging(run_create(&host, "alice")).await; + let err = result.expect_err(case); + assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}"); + let errors = log.at(tracing::Level::ERROR); + assert_eq!(errors.len(), 1, "{case}: {errors:?}"); + assert!( + errors[0].contains("alice") && errors[0].contains(why), + "{case}: {errors:?}" + ); + let calls = host.calls(); + assert!( + !calls + .iter() + .any(|c| c.starts_with("create_only") || c.starts_with("mark_deployed")), + "{case}: {calls:?}" + ); + } } #[tokio::test] -async fn create_builds_the_evaluated_path_when_it_differs_from_the_download() { - let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice"; - let host = FakeHost { - evaluates: Some(other.to_owned()), - ..FakeHost::up().with_closure() - }; - run_create(&host, "alice", node_id()).await.expect("ok"); - assert_eq!(host.builds(), [other.to_owned()]); +async fn create_uses_a_closure_built_from_another_commit_with_a_warning() { + let host = FakeHost::up().built(OTHER_REV, true); + let (result, log) = logging(run_create(&host, "alice")).await; + result.expect("drift is not a refusal"); let calls = host.calls(); assert!( - calls - .iter() - .any(|c| c == &format!("create_only alice {REV}")), + calls.contains(&format!("create_only alice {CLOSURE}")), "{calls:?}" ); + assert_eq!(log.at(tracing::Level::WARN).len(), 1); + assert!(log.at(tracing::Level::ERROR).is_empty()); } #[tokio::test] async fn a_failed_create_is_never_marked_deployed() { let host = FakeHost::up().failing("create_only"); - run_create(&host, "alice", node_id()) - .await - .expect_err("create fails"); - assert_eq!(host.calls().len(), 3); + run_create(&host, "alice").await.expect_err("create fails"); + assert_eq!(host.calls().len(), 4); let host = FakeHost::up().failing("resolve_meta"); - run_create(&host, "alice", node_id()) - .await - .expect_err("resolve fails"); + run_create(&host, "alice").await.expect_err("resolve fails"); assert_eq!(host.calls(), ["resolve_meta alice"]); } @@ -1165,11 +1175,19 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() { ), ( alice(|agent| NodeKind::Prebuild { agent }), - &["is_running alice", "recorded_rev alice", "prebuild alice"], + &[ + "is_running alice", + "recorded_rev alice", + "closure_record alice", + ], ), ( alice(|agent| NodeKind::Swap { agent }), - &["recorded_rev alice", "swap_update alice"], + &[ + "recorded_rev alice", + "closure_record alice", + "swap_update alice", + ], ), ( alice(|agent| NodeKind::RebuildBookkeeping { agent }), @@ -1183,6 +1201,7 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() { alice(|agent| NodeKind::Create { agent }), &[ "resolve_meta alice", + "closure_record alice", "create_only alice", "mark_deployed alice", ], diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 69a9cead..8eb0fe0d 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -32,9 +32,9 @@ pub enum NodeKind { /// The rebuild's preamble: runtime-dir prep and the `internal/meta` /// commit the build reads. MetaSync { agent: String }, - /// Out-of-band toplevel build while the container keeps serving. + /// Download of the swarm's closure while the container keeps serving. Prebuild { agent: String }, - /// `nixos-container update` profile-swap. + /// Profile-swap onto the swarm's closure. Swap { agent: String }, /// The post-`Swap` bookkeeping tail as a first-class node. RebuildBookkeeping { agent: String }, diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index f1817bbe..7e4bef86 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -190,7 +190,7 @@ fn rebuild_subtree<'a>( .needs(Resource::Agent(a())) .after_ok(meta_sync); - // Siblings under the brace: the build and the quiesce chain run + // Siblings under the brace: the download and the quiesce chain run // concurrently, borrowing the brace's grants rather than declaring their // own. Declaring the lease on both would make them mutually exclusive — // it is single-unit — which is exactly what this shape exists to avoid. @@ -200,8 +200,8 @@ fn rebuild_subtree<'a>( let drain = graceful.then(|| quiesce(builder, agent, agent_window)); - // The container goes down here, not earlier: `AfterOk` the build so a - // failed build never stops a healthy container, and `AfterOk` the drain so + // The container goes down here, not earlier: `AfterOk` the download so a + // missing closure never stops a healthy container, and `AfterOk` the drain so // the agent has checkpointed. let mut stop_for_update = builder .node(NodeKind::StopForUpdate { agent: a() }) diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index beb68060..15633a7c 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -234,9 +234,9 @@ pub async fn provision_container(name: &str, paths: &AgentPaths) -> Result<()> { /// The `nixos-container create` proper — the job queue's `Create` node. /// Assumes `provision_container` already registered the agent. -/// `meta_rev` is the `internal/meta` commit to build from (`crate::meta_rev`). -pub async fn create_only(name: &str, meta_rev: &str) -> Result<()> { - priv_run_inner("create", name, Some(meta_rev), None).await +/// `toplevel` is the store path of the system closure to create it from. +pub async fn create_only(name: &str, toplevel: &str) -> Result<()> { + priv_run_inner("create", name, Some(toplevel), None).await } /// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on @@ -285,20 +285,20 @@ pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> } /// Profile-swap for an existing, stopped container: re-apply the -/// drop-ins, then `nixos-container update`. The job queue's `Swap` -/// node. Requires the container stopped (the queue's `StopForUpdate` -/// upstream); does NOT start it — the DAG's tail `Reconcile` owns -/// bringing the agent back to its wanted power state. `meta_rev` is as for +/// drop-ins, then point the container at the closure `toplevel`. The job +/// queue's `Swap` node. Requires the container stopped (the queue's +/// `StopForUpdate` upstream); does NOT start it — the DAG's tail `Reconcile` owns +/// bringing the agent back to its wanted power state. `toplevel` is as for /// [`create_only`]. pub async fn swap_update( name: &str, hive: &HiveEnv, paths: &AgentPaths, - meta_rev: &str, + toplevel: &str, node_id: Option, ) -> Result<()> { write_dropins(name, hive, paths).await?; - priv_run_inner("update", name, Some(meta_rev), node_id).await + priv_run_inner("update", name, Some(toplevel), node_id).await } /// Build the `AgentSpec` list for [`crate::agent_set::refresh`] from @@ -720,195 +720,6 @@ fn confirm_gone_after_failed_destroy( } } -/// Wall-clock bound on [`prebuild_toplevel`]'s `nix build`. Four times CI's -/// observed cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`), -/// which compiles this workspace's crates, the part of an agent toplevel no -/// binary cache holds. A stalled download is bounded by nix's own -/// `stalled-download-timeout`; this covers the rest, such as a wedged -/// nix-daemon holding the rebuild's build slot forever. -const PREBUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1); - -/// Wait for `child` until `limit`. `None` means it was still running, and its -/// whole process group was `SIGKILL`ed and the child reaped, so a grandchild -/// holding the daemon connection dies with it. The child must have been -/// spawned with `process_group(0)`. -async fn wait_bounded( - child: &mut tokio::process::Child, - limit: std::time::Duration, -) -> std::io::Result> { - // Group leader, so its pid is the pgid. Read before waiting: a reaped - // child has no id. - let pgid = child.id().and_then(|pid| i32::try_from(pid).ok()); - if let Ok(status) = tokio::time::timeout(limit, child.wait()).await { - return status.map(Some); - } - if let Some(pgid) = pgid { - // SAFETY: plain `kill(2)`; `-pgid` targets the child's own process - // group, which `process_group(0)` made it lead. - unsafe { - libc::kill(-pgid, libc::SIGKILL); - } - } - child.wait().await?; - Ok(None) -} - -/// Log and record a prebuild's stdout; the last non-empty line is the -/// toplevel `nix build --print-out-paths` printed. -async fn pump_prebuild_stdout( - stdout: tokio::process::ChildStdout, - cmdline: String, - logs: Option>, - log_id: Option, -) -> Option { - use tokio::io::{AsyncBufReadExt, BufReader}; - let mut lines = BufReader::new(stdout).lines(); - let mut out_path = None; - while let Ok(Some(line)) = lines.next_line().await { - tracing::info!(target: "nix-prebuild", cmdline = %cmdline, "{line}"); - if !line.trim().is_empty() { - out_path = Some(line.trim().to_owned()); - } - if let (Some(h), Some(id)) = (&logs, log_id) { - h.append_stdout(id, &line); - } - } - out_path -} - -/// Pre-build `system.build.toplevel` against `meta#` so the -/// subsequent `nixos-container update` finds the result cached and -/// skips straight to the profile-swap. Store-warming only — container -/// is untouched. See `docs/scheduler/coordinator.md::Rebuild path` for why -/// the prebuild happens before stop, and `docs/scheduler/coordinator.md::Prebuild -/// attr path` for why the explicit nixosConfigurations attr is required. -/// -/// `node_id` is the queue node this build belongs to, when there is one — -/// it is stored on the `build_logs` row so the dashboard can find the log -/// from the node. Pass `None` for builds that run outside the queue. -/// -/// Returns the toplevel `nix build` printed, `None` if it printed nothing. -pub async fn prebuild_toplevel( - name: &str, - flake_ref: &str, - node_id: Option, -) -> Result> { - use tokio::io::{AsyncBufReadExt, BufReader}; - // Split `#` so we can re-emit with the explicit - // `nixosConfigurations.` segment. The flake_ref shape is - // constructed by the caller and always contains exactly one - // `#`; `split_once` returning None here would be a programmer - // error we'd want to surface loudly rather than paper over. - let (flake_root, fragment) = flake_ref - .split_once('#') - .with_context(|| format!("flake_ref {flake_ref:?} missing '#' fragment"))?; - // Sanity-check the fragment matches the agent name we were - // passed — guards against future calls that pass a divergent - // pair (no current callsite does, but the pair is redundant - // and worth checking once). - if fragment != name { - anyhow::bail!("prebuild_toplevel: flake_ref fragment '{fragment}' ≠ agent name '{name}'"); - } - let attr = format!("{flake_root}#nixosConfigurations.{name}.config.system.build.toplevel"); - let args = vec![ - "--extra-experimental-features", - "nix-command flakes", - "build", - "--no-link", - "--print-out-paths", - &attr, - ]; - let cmdline = format!("nix {}", args.join(" ")); - tracing::info!(%name, %cmdline, "prebuild: warming system toplevel"); - - // Open a build_logs row for this attempt (best-effort — None when - // the global handle hasn't been installed, e.g. early startup - // or standalone tests). Lines pumped from stdout/stderr append - // into the row; `finish` lands the terminal status before we bail. - let logs = crate::build_logs::global(); - let log_id = logs.as_ref().and_then(|h| { - h.start(name, "prebuild", &cmdline, node_id) - .map_err(|e| { - tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)"); - }) - .ok() - }); - - let mut child = Command::new("nix") - .args(&args) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .process_group(0) - .spawn() - .with_context(|| format!("spawn {cmdline}"))?; - - let stdout = child.stdout.take().expect("piped stdout"); - let stderr = child.stderr.take().expect("piped stderr"); - - let pump_stdout = tokio::spawn(pump_prebuild_stdout( - stdout, - cmdline.clone(), - logs.clone(), - log_id, - )); - - let stderr_cmdline = cmdline.clone(); - let stderr_logs = logs.clone(); - let pump_stderr = tokio::spawn(async move { - let mut lines = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = lines.next_line().await { - if is_actionable_nix_line(&line) { - tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}"); - } else { - tracing::info!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}"); - } - if let (Some(h), Some(id)) = (&stderr_logs, log_id) { - h.append_stderr(id, &line); - } - } - }); - - let Some(status) = wait_bounded(&mut child, PREBUILD_TIMEOUT) - .await - .with_context(|| format!("wait {cmdline}"))? - else { - // A process that left the group can still hold the pipes open. - pump_stdout.abort(); - pump_stderr.abort(); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.finish(id, crate::build_logs::BuildStatus::Fail); - } - let secs = PREBUILD_TIMEOUT.as_secs(); - match log_id { - Some(id) => { - bail!("prebuild {cmdline} timed out after {secs}s; killed it; see build log #{id}") - } - None => bail!("prebuild {cmdline} timed out after {secs}s; killed it"), - } - }; - let out_path = pump_stdout.await.ok().flatten(); - let _ = pump_stderr.await; - - let ok = status.success(); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.finish( - id, - if ok { - crate::build_logs::BuildStatus::Ok - } else { - crate::build_logs::BuildStatus::Fail - }, - ); - } - if !ok { - match log_id { - Some(id) => bail!("prebuild {cmdline} failed ({status}); see build log #{id}"), - None => bail!("prebuild {cmdline} failed ({status})"), - } - } - Ok(out_path) -} - pub async fn list() -> Result> { let stdout = crate::priv_client::list_containers().await?; Ok(stdout @@ -990,11 +801,11 @@ fn make_log_callback( } } -/// A `create` or `update` asked for without an `internal/meta` commit. The -/// public entry points take the commit, so this guards the string dispatch +/// A `create` or `update` asked for without a closure. The +/// public entry points take the closure, so this guards the string dispatch /// below only. -fn missing_meta_rev(kind: &str, name: &str) -> anyhow::Error { - anyhow::anyhow!("{name}: {kind} needs an internal/meta commit to build from") +fn missing_toplevel(kind: &str, name: &str) -> anyhow::Error { + anyhow::anyhow!("{name}: {kind} needs a closure to apply") } /// Execute a container operation via hive-priv and integrate with @@ -1016,7 +827,7 @@ async fn priv_run(kind: &str, name: &str) -> Result<()> { async fn priv_run_inner( kind: &str, name: &str, - meta_rev: Option<&str>, + toplevel: Option<&str>, node_id: Option, ) -> Result<()> { let container = container_name(name); @@ -1034,27 +845,27 @@ async fn priv_run_inner( // For long-running ops use the streaming protocol so build_logs // receives lines in real time rather than as a batch at completion. let result: Result<()> = match kind { - "create" => match meta_rev { - Some(rev) => { + "create" => match toplevel { + Some(toplevel) => { crate::priv_client::create_container_streaming( name, - rev, + toplevel, make_log_callback(logs.clone(), log_id, cmdline.clone()), ) .await } - None => Err(missing_meta_rev(kind, name)), + None => Err(missing_toplevel(kind, name)), }, - "update" => match meta_rev { - Some(rev) => { + "update" => match toplevel { + Some(toplevel) => { crate::priv_client::update_container_streaming( name, - rev, + toplevel, make_log_callback(logs.clone(), log_id, cmdline.clone()), ) .await } - None => Err(missing_meta_rev(kind, name)), + None => Err(missing_toplevel(kind, name)), }, "start" => crate::priv_client::start_container(name).await, "stop" => crate::priv_client::stop_container(name).await, diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs index 9a7ec33a..1124da33 100644 --- a/hive-c0re/src/lifecycle/tests.rs +++ b/hive-c0re/src/lifecycle/tests.rs @@ -349,68 +349,6 @@ fn agent_names_propagates_an_unreadable_list() { assert!(format!("{err:#}").contains("hive-priv"), "{err:#}"); } -fn is_dead(pid: &str) -> bool { - // A killed process that is not our child stays a zombie until its own - // parent reaps it, so a zombie counts as dead. - match std::fs::read_to_string(format!("/proc/{pid}/stat")) { - Err(_) => true, - Ok(stat) => stat - .rsplit_once(") ") - .is_some_and(|(_, rest)| rest.starts_with('Z')), - } -} - -/// A child still running at the limit comes back `None`, and the kill reaches -/// its whole process group: the backgrounded grandchild dies too, not just the -/// shell that started it. -#[tokio::test] -async fn a_prebuild_past_its_limit_is_killed_with_its_group() { - use std::time::{Duration, Instant}; - let dir = tempfile::tempdir().expect("tempdir"); - let pidfile = dir.path().join("grandchild"); - let mut child = Command::new("sh") - .arg("-c") - .arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display())) - .process_group(0) - .spawn() - .expect("spawn sh"); - - let started = Instant::now(); - let status = wait_bounded(&mut child, Duration::from_secs(2)) - .await - .expect("wait_bounded"); - assert!(status.is_none(), "must time out, got {status:?}"); - assert!( - started.elapsed() < Duration::from_mins(1), - "the limit did not bound the wait" - ); - - let pid = std::fs::read_to_string(&pidfile).expect("grandchild pid"); - let pid = pid.trim(); - let deadline = Instant::now() + Duration::from_secs(5); - while !is_dead(pid) && Instant::now() < deadline { - tokio::time::sleep(Duration::from_millis(50)).await; - } - assert!(is_dead(pid), "grandchild {pid} outlived the timeout"); -} - -/// The control: a child that exits inside its limit comes back with its -/// status. -#[tokio::test] -async fn a_prebuild_inside_its_limit_returns_its_status() { - let mut child = Command::new("sh") - .arg("-c") - .arg("exit 3") - .process_group(0) - .spawn() - .expect("spawn sh"); - let status = wait_bounded(&mut child, std::time::Duration::from_mins(1)) - .await - .expect("wait_bounded") - .expect("a fast child must not time out"); - assert_eq!(status.code(), Some(3)); -} - /// The setup path creates the agent's `work/` dir beside `state/`, so the /// bind source exists before nspawn starts the container. #[test] diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index ff32c531..c8568b9d 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -115,7 +115,7 @@ enum Cmd { /// option. #[arg(long)] model_prices: Option, - /// Override: number of concurrent nix-heavy job-queue nodes + /// Override: number of concurrent closure-fetching job-queue nodes /// (prebuild / profile-swap / create / meta lock). Set via the /// `services.hyperhive.deploy.hive-controller.buildSlots` NixOS option. #[arg(long)] diff --git a/hive-c0re/src/meta_rev.rs b/hive-c0re/src/meta_rev.rs index 0d692cbe..92af161f 100644 --- a/hive-c0re/src/meta_rev.rs +++ b/hive-c0re/src/meta_rev.rs @@ -1,27 +1,25 @@ -//! The `internal/meta` commit a hive builds an agent from. +//! The `internal/meta` commit a hive deploys an agent from. //! //! `internal/meta` is the swarm's meta flake, committed by swarm-controller; -//! a hive has no meta flake of its own. A rebuild's `MetaSync` node resolves -//! the head ([`resolve`]) and records it ([`record`]); `Prebuild` and `Swap` -//! read the record ([`recorded`]), so both build the same flake. A create -//! resolves and uses the commit directly. When the head does not pin the -//! agent's current config, the build fails rather than falling back: the -//! swarm re-pins before it asks a hive to deploy. +//! a hive has no meta flake of its own and builds nothing from it. A rebuild's +//! `MetaSync` node resolves the head ([`resolve`]) and records it ([`record`]); +//! `Prebuild` and `Swap` read the record ([`recorded`]) for the commit the +//! swarm's closure is expected to come from. A create resolves and uses the +//! commit directly. When the head does not pin the agent's current config, the +//! deploy fails rather than falling back: the swarm re-pins before it asks a +//! hive to deploy. //! -//! The record also names the pins a successful build deploys. [`mark_deployed`] +//! The record also names the pins a successful deploy applies. [`mark_deployed`] //! copies them to the agent's markers, which the boot reconcile //! ([`crate::auto_update`]) and the dashboard's update chip read. use anyhow::{Context, Result, bail}; -use hive_priv_sock::{meta_flake_url, valid_meta_rev}; +use hive_priv_sock::valid_meta_rev; use serde::{Deserialize, Serialize}; use swarm_queue_client::agent_archetype::ArchetypeRef; /// What the forge says about `internal/meta` and an agent's config repo. pub(crate) trait MetaSource { - /// Whether nix's own `git` can authenticate to the forge: it reads as this - /// hive's account, from the credential file the source publishes. - fn nix_can_fetch(&self) -> bool; /// The default branch's head commit of `internal/meta`; `None` when the /// repo is absent or has no commit. async fn meta_head(&self) -> Result>; @@ -42,15 +40,13 @@ pub(crate) struct MetaBuild { pub config: Option, } -/// The `internal/meta` commit to build `agent` from. +/// The `internal/meta` commit to deploy `agent` from. /// -/// `internal/meta` is private, so a hive whose nix cannot authenticate -/// ([`MetaSource::nix_can_fetch`]) cannot build. The head is taken only when -/// its `flake.nix` builds the agent the way it is wanted now: from -/// `archetype` when that is set, otherwise from the config commit the forge's -/// `main` is at. A head that is behind (the swarm's update has not run, or -/// failed), lacks the agent or lists it under another archetype would build an -/// older module than the one being deployed. +/// The head is taken only when its `flake.nix` builds the agent the way it is +/// wanted now: from `archetype` when that is set, otherwise from the config +/// commit the forge's `main` is at. A head that is behind (the swarm's update +/// has not run, or failed), lacks the agent or lists it under another archetype +/// would build an older module than the one being deployed. /// /// # Errors /// No usable head, a head that does not pin the agent's current config, or a @@ -60,9 +56,6 @@ pub(crate) async fn choose( agent: &str, archetype: Option<&ArchetypeRef>, ) -> Result { - if !source.nix_can_fetch() { - bail!("{agent}: this hive has no forge credential for nix to fetch internal/meta with"); - } let Some(rev) = source.meta_head().await? else { bail!("{agent}: the swarm has not published internal/meta yet"); }; @@ -174,7 +167,7 @@ fn recorded_in(dir: &std::path::Path, agent: &str) -> Option { } /// The record of `agent`'s build, or the error naming why a `Prebuild` or -/// `Swap` has nothing to build from. +/// `Swap` has no commit to expect. /// /// # Errors /// No valid record: its `MetaSync` did not run, or ran in an older hive-c0re. @@ -219,21 +212,11 @@ fn mark_deployed_in( /// Whether an agent's `stored` marker says it runs the hyperhive commit `pin` /// names. With no `pin`, because the hive can't read `internal/meta`, nothing -/// is stale: a rebuild would have no flake to build from either. +/// is stale: a rebuild would have no commit to deploy either. pub(crate) fn marker_is_fresh(pin: Option<&str>, stored: Option<&str>) -> bool { pin.is_none_or(|pin| stored == Some(pin)) } -/// The `#` ref [`crate::lifecycle::prebuild_toplevel`] builds -/// for `rev`: the same flake hive-priv builds, from the same forge URL. -pub(crate) fn flake_ref(agent: &str, rev: &str) -> String { - flake_ref_at(crate::forge::forge_http_base(), agent, rev) -} - -fn flake_ref_at(forge_base: &str, agent: &str, rev: &str) -> String { - format!("{}#{agent}", meta_flake_url(forge_base, rev)) -} - #[cfg(test)] mod tests { use super::*; @@ -249,13 +232,9 @@ mod tests { head: Option<&'static str>, flake: String, config_head: Option<&'static str>, - fetchable: bool, } impl MetaSource for Forge { - fn nix_can_fetch(&self) -> bool { - self.fetchable - } async fn meta_head(&self) -> Result> { Ok(self.head.map(str::to_owned)) } @@ -313,7 +292,6 @@ mod tests { head: Some(META), flake: flake(), config_head, - fetchable: true, } } @@ -369,9 +347,6 @@ mod tests { async fn an_archetype_agent_is_taken_when_the_head_lists_that_archetype() { struct Archetypes(Forge); impl MetaSource for Archetypes { - fn nix_can_fetch(&self) -> bool { - self.0.nix_can_fetch() - } async fn meta_head(&self) -> Result> { self.0.meta_head().await } @@ -422,19 +397,6 @@ mod tests { assert_unpinned(&choose(&source, "rev", None).await.unwrap_err(), "rev"); } - /// A hive without the credential nix's git reads cannot build from the - /// repo even though its own token reads it; the first test is the control. - #[tokio::test] - async fn a_hive_whose_nix_cannot_authenticate_cannot_build() { - let mut source = forge(Some(ARGUS_HEAD)); - source.fetchable = false; - let err = choose(&source, "argus", None).await.unwrap_err(); - assert!( - format!("{err:#}").contains("no forge credential"), - "{err:#}" - ); - } - #[tokio::test] async fn no_repo_or_no_commit_is_an_error_naming_the_agent() { let mut source = forge(Some(ARGUS_HEAD)); @@ -568,12 +530,4 @@ mod tests { assert!(marker_is_fresh(None, Some(STALE)), "no pin to compare with"); assert!(marker_is_fresh(None, None)); } - - #[test] - fn the_prebuild_ref_is_the_flake_hive_priv_builds() { - assert_eq!( - flake_ref_at("http://forge.t.local", "argus", META), - format!("git+http://forge.t.local/internal/meta.git?rev={META}#argus") - ); - } } diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index 35c31c53..48741238 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -235,14 +235,14 @@ pub async fn kill_container(name: &str) -> Result<()> { /// appending lines to `build_logs` or otherwise capturing the output. pub async fn update_container_streaming( name: &str, - meta_rev: &str, + toplevel: &str, on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { ok(call_streaming( &PrivRequest::UpdateContainer { name: name.to_owned(), stream: true, - meta_rev: meta_rev.to_owned(), + toplevel: toplevel.to_owned(), }, on_line, ) @@ -253,14 +253,14 @@ pub async fn update_container_streaming( /// arrive. Returns `Ok(())` on success. pub async fn create_container_streaming( name: &str, - meta_rev: &str, + toplevel: &str, on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { ok(call_streaming( &PrivRequest::CreateContainer { name: name.to_owned(), stream: true, - meta_rev: meta_rev.to_owned(), + toplevel: toplevel.to_owned(), }, on_line, ) diff --git a/hive-c0re/src/stores/build_logs.rs b/hive-c0re/src/stores/build_logs.rs index 6af05a71..203ed4b6 100644 --- a/hive-c0re/src/stores/build_logs.rs +++ b/hive-c0re/src/stores/build_logs.rs @@ -16,7 +16,7 @@ use utoipa::ToSchema; use crate::db::Migration; /// Process-singleton handle, set once at coordinator startup. Lets -/// the `lifecycle` module's `run` / `prebuild_toplevel` access the +/// the `lifecycle` module's `run` access the /// writer without threading an `Arc` through every /// `pub async fn` entry point in the lifecycle surface — there are /// 10+ callsites and the handle is the same `Arc` everywhere diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 623fe470..fce863f9 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -101,10 +101,7 @@ impl std::str::FromStr for InfraContainer { } } -/// Whether `rev` is a git commit id: 40 lowercase hex digits. The only part of -/// the `internal/meta` flake ref that crosses the wire (`meta_rev` on -/// `PrivRequest::UpdateContainer` and `CreateContainer`), so `hive-priv` checks -/// it before it builds a ref. +/// Whether `rev` is a git commit id: 40 lowercase hex digits. #[must_use] pub fn valid_meta_rev(rev: &str) -> bool { rev.len() == 40 && rev.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) @@ -113,7 +110,8 @@ pub fn valid_meta_rev(rev: &str) -> bool { /// Whether `path` is a store path in the shape nix prints one: `/nix/store/`, /// 32 lowercase letters or digits, `-`, then a name of the characters store /// names allow. The path crosses the wire in [`PrivRequest::RealiseClosure`], -/// so `hive-priv` checks it before it hands it to nix. +/// [`PrivRequest::UpdateContainer`] and [`PrivRequest::CreateContainer`], so +/// `hive-priv` checks it before it hands it to nix. #[must_use] pub fn valid_closure_path(path: &str) -> bool { let Some(rest) = path.strip_prefix("/nix/store/") else { @@ -137,20 +135,6 @@ pub fn valid_closure_path(path: &str) -> bool { /// cache that is down. pub const SIGNATURE_REFUSED: &str = "no signature by the swarm cache's key"; -/// The flake the swarm publishes at `internal/meta`, at `rev`: -/// `git+/internal/meta.git?rev=`. `forge_base` is the forge's -/// `http` origin, which the hive's git credential helper answers for. -/// -/// One definition for `hive-c0re`'s prebuild and `hive-priv`'s build, so both -/// name the same flake. -#[must_use] -pub fn meta_flake_url(forge_base: &str, rev: &str) -> String { - format!( - "git+{}/internal/meta.git?rev={rev}", - forge_base.trim_end_matches('/') - ) -} - /// Root of per-agent state directories on the host. /// Subdirectory layout: `//state/`. /// Used by `WriteAgentStateFile` to derive the write path so the @@ -278,31 +262,28 @@ pub enum PrivRequest { /// in the container. nixos-container has no kill verb. KillContainer { name: String }, - /// `nixos-container update --flake ` - /// The flake ref is derived from `name` and `meta_rev` by hive-priv: the - /// swarm's `internal/meta` at that commit (the forge URL is hive-priv's - /// own and never on the wire). + /// Point `name`'s container profile at `toplevel`, then reload the unit if + /// it runs. hive-priv never builds: `toplevel` is a closure already in the + /// store (see `RealiseClosure`). /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. UpdateContainer { name: String, #[serde(default)] stream: bool, - /// The commit of `internal/meta` to build from, checked by - /// `valid_meta_rev`. - meta_rev: String, + /// The agent's system closure, checked by `valid_closure_path`. + toplevel: String, }, - /// `nixos-container create --flake ` - /// The flake ref is derived from `name` (and `meta_rev`) as for - /// `UpdateContainer`. + /// `nixos-container create --system-path ` + /// `toplevel` is as for `UpdateContainer`. /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. CreateContainer { name: String, #[serde(default)] stream: bool, - meta_rev: String, + toplevel: String, }, /// `nix-store --realise ` from the swarm's binary cache, never @@ -718,7 +699,7 @@ pub enum PrivEvent { #[cfg(test)] mod tests { - use super::{InfraContainer, PrivRequest, meta_flake_url, valid_closure_path, valid_meta_rev}; + use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev}; #[test] fn infra_control_names_no_c0re_or_swarm_service() { @@ -765,15 +746,6 @@ mod tests { } } - #[test] - fn the_meta_flake_url_names_internal_meta_at_the_rev() { - let rev = "a".repeat(40); - assert_eq!( - meta_flake_url("http://forge.t.local/", &rev), - format!("git+http://forge.t.local/internal/meta.git?rev={rev}") - ); - } - #[test] fn only_a_store_path_is_a_closure_path() { let hash = "0123456789abcdfghijklmnpqrsvwxyz"; @@ -814,9 +786,9 @@ mod tests { assert!(serde_json::from_str::(without).is_err()); } - /// `meta_rev` is required: there is no local flake to fall back to. + /// `toplevel` is required: hive-priv has no flake to build one from. #[test] - fn a_request_without_meta_rev_is_refused() { + fn a_request_without_a_toplevel_is_refused() { for op in ["update_container", "create_container"] { let without = format!(r#"{{"op":"{op}","name":"atlas"}}"#); assert!( @@ -824,8 +796,8 @@ mod tests { "{op}" ); let with = format!( - r#"{{"op":"{op}","name":"atlas","meta_rev":"{}"}}"#, - "b".repeat(40) + r#"{{"op":"{op}","name":"atlas","toplevel":"/nix/store/{}-nixos-system-atlas"}}"#, + "b".repeat(32) ); assert!(serde_json::from_str::(&with).is_ok(), "{op}"); } diff --git a/hive-priv/README.md b/hive-priv/README.md index 45f2b27b..10864609 100644 --- a/hive-priv/README.md +++ b/hive-priv/README.md @@ -42,60 +42,22 @@ Each module's `//!` header lists its ops. ## Implementation notes -### Container toplevel builds (create/update) +### Container toplevels (create/update) -`container_flake_action` (in `src/container.rs`) builds -`nixosConfigurations..config.system.build.toplevel` itself -(`nix_build_toplevel`) and passes the resolved store path to -`nixos-container create`/`update` via `--system-path`, for both verbs. +`container_toplevel_action` (in `src/container.rs`) applies a closure that is +already in the store; hive-priv builds nothing. hive-c0re names the closure +(`toplevel` on `CreateContainer` / `UpdateContainer`), downloaded beforehand +with `RealiseClosure`, and hive-priv accepts it only in the shape of a store +path. -**Why not let `nixos-container` build it (the old `create`-only -behaviour, `update` used its own `--flake` path)**: `nixos-container`'s -own `buildFlake()` — invoked whenever `--system-path` isn't passed — -builds to a _hardcoded relative path_. `$systemPath` is only ever -assigned from the CLI flag or from `buildFlake()`'s own result, so with -no flag it stays `undef` and `"$systemPath.tmp"` interpolates to the -bare string `.tmp` in whatever the caller's cwd happens to be. -`buildFlake()` itself takes no lock at all: `create` wraps its _whole -action_ in an exclusive `flock` before calling it, but `update` used to -call it with no lock whatsoever — so `create`'s lock never protected -against a concurrent `update` clobbering the same `.tmp`. hive-priv never -sets a per-call cwd, so with `services.hyperhive.deploy.hive-controller.buildSlots > 1`, -two concurrent calls (any mix of `create`/`update`) could share that one -`.tmp`: one's `readlink(".tmp")` resolving to the _other's_ build -output, handing an agent's container the wrong agent's closure — the -"agent container gets closure of other agent" bug. +`create` passes the path to `nixos-container create --system-path`, so +`nixos-container`'s own `buildFlake()` never runs. Without the flag, it builds +to a hardcoded relative `.tmp` out-link: `$systemPath` stays `undef` and +`"$systemPath.tmp"` interpolates to the bare string `.tmp` in the caller's cwd. +With `services.hyperhive.deploy.hive-controller.buildSlots > 1`, two concurrent +calls could share that one `.tmp`, and one agent's container got another +agent's closure. -Building the toplevel here and passing the resolved store path via -`--system-path` for _every_ call means `buildFlake()` never runs at all, -for either verb — no shared `.tmp` left to race on, no locking invariant -of a script we don't own to keep track of. `--no-link` avoids a -competing out-link race of our own; we only need the store path, not a -GC root (it's safe from collection for as long as it takes -`nixos-container` to register it against the container's own profile, -same window every other `--print-out-paths` consumer already relies on). - -**Forwards stderr live, captures stdout silently — deliberately not -symmetric.** This build is the multi-minute phase of a `create`/ -`update`, and it used to run _inside_ `nixos-container`'s own `--flake` -invocation, which streams every line to the caller in real time. -Buffering it instead (`Command::output()`, as this function first -shipped) regressed that: nothing on the wire — dashboard or -`journalctl -f` alike — until the whole build finishes, then everything -at once. So both pipes are drained concurrently (needed to avoid -deadlocking if either pipe fills while the other is being read), but -only stderr — where nix's own progress goes — is forwarded live, same -shape as `container_run_streaming`. stdout is different: -`--print-out-paths` writes _only_ the final store path there, once, at -the end — forwarding it the same way would risk interleaving a progress -line into the value this function hands back as `--system-path`, trading -a closure-mixup bug for a corrupted-argument one. So stdout lines are -accumulated silently and only consulted after the exit status is known -to be success — and even then, exactly one non-empty, trimmed line is -required (`nix build --print-out-paths` prints one line _per output_, -not one line total; `config.system.build.toplevel` is single-output -today, but a bare whole-buffer `.trim()` would silently hand a -multi-line string to `--system-path` the day that ever changes, and a -bare untrimmed/unfiltered `.lines()` turns a lone `"\n"` into a bogus -empty-string "path" — a wrong line count, or a blank/whitespace one, -`bail!`s instead). +`update` points the container's profile at the path with `nix-env --set` and +reloads the unit if it runs, because `nixos-container update` dies on its +version probe before it honors `--system-path`. diff --git a/hive-priv/src/closure.rs b/hive-priv/src/closure.rs index 5c1bbc3a..d54e4e00 100644 --- a/hive-priv/src/closure.rs +++ b/hive-priv/src/closure.rs @@ -170,7 +170,7 @@ async fn realise_with( let cache = Cache::read(url, dir, owner)?; let mut cmd = tokio::process::Command::new("nix-store"); cmd.args(realise_args(store_path, &cache)); - match run_bounded(cmd, REALISE_TIMEOUT, None) + match run_bounded(cmd, REALISE_TIMEOUT) .await .context("running nix-store")? { @@ -342,8 +342,8 @@ mod tests { assert!(Cache::read(Some(URL), &dir.join("nowhere"), me).is_err()); } - /// Missing facts reach the caller as `not available`, which its prebuild - /// answers by building. + /// Missing facts reach the caller as `not available`, which a rebuild + /// reports as a closure that cannot be pulled. #[tokio::test] async fn without_the_root_files_the_answer_is_not_available() { let nowhere = scratch().join("nowhere"); diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs index fc25a6c7..50ac3737 100644 --- a/hive-priv/src/container.rs +++ b/hive-priv/src/container.rs @@ -3,12 +3,11 @@ //! and `ListContainers`. Every op takes a logical agent name and reaches only //! the `h-`-prefixed container [`container_system_name`] derives from it. //! -//! `CreateContainer` / `UpdateContainer` build the toplevel here -//! ([`nix_build_toplevel`]) and stream progress to the caller as -//! [`PrivEvent::Line`]s when asked to. +//! `CreateContainer` / `UpdateContainer` apply a closure already in the store +//! and stream progress to the caller as [`PrivEvent::Line`]s when asked to. -use anyhow::{Context as _, Result, anyhow, bail}; -use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine, meta_flake_url, valid_meta_rev}; +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine, valid_closure_path}; use tokio::io::{AsyncWriteExt, BufReader}; use tokio::net::unix::OwnedWriteHalf; use tokio::process::Command; @@ -72,52 +71,43 @@ pub(crate) async fn destroy_container(name: &str) -> Result<(String, String)> { container_run(&["destroy", &container_system_name(name)]).await } -/// Shared body for `CreateContainer` / `UpdateContainer`: validate the -/// name, build the toplevel ourselves, and apply it (streaming line -/// events to `writer` when `stream` is set). +/// `toplevel` as a store path, or the error saying it is not one. The path +/// crosses the socket from hive-c0re, which is not root. +fn toplevel_to_apply(toplevel: &str) -> Result<&str> { + if !valid_closure_path(toplevel) { + bail!("{toplevel:?} is not a store path"); + } + Ok(toplevel) +} + +/// Shared body for `CreateContainer` / `UpdateContainer`: validate the name +/// and the closure, and apply it (streaming line events to `writer` when +/// `stream` is set). Nothing is built here: `toplevel` is a closure already in +/// the store, which `RealiseClosure` put there. /// -/// **Both verbs build explicitly now — not just `update`.** The first cut -/// of this fix only rewrote `update`, reasoning that `create` was already -/// safe: it wraps its whole action in an exclusive `flock` before calling -/// `nixos-container`'s own `buildFlake()`, and once `update` stopped -/// writing to `buildFlake()`'s shared `.tmp` out-link, concurrent -/// `create`s were the only remaining writers — mutually excluded by that -/// lock. True, but it leaves `create`'s safety resting on an internal -/// implementation detail of a script we don't own (its current locking -/// behavior, which could change upstream without notice) instead of on -/// something we control. Building here for both verbs removes `buildFlake()` -/// from the picture entirely — there's no shared `.tmp` left to race on, -/// so there's nothing left to reason about staying in sync with. +/// `create` passes `--system-path` so `nixos-container`'s own `buildFlake()` +/// never runs; it builds to a relative `.tmp` out-link that concurrent calls +/// share, which handed an agent another agent's closure. /// -/// **`update` no longer calls `nixos-container update` at all.** That -/// action's own version-compat probe runs unconditionally before -/// `--system-path` is ever honored, dying on every agent's update. -/// [`swap_container_profile`] replicates exactly what `update`'s own -/// action does *past* that probe (confirmed against its source): `nix-env -/// --set` the per-container profile, then `systemctl reload` if the -/// container is running. `create` is untouched — it isn't the failing -/// verb. -pub(crate) async fn container_flake_action( +/// `update` does not call `nixos-container update` at all: that action's +/// version-compat probe runs before `--system-path` is honored and dies on every +/// agent. [`swap_container_profile`] does what the action does past that probe: +/// `nix-env --set` the per-container profile, then `systemctl reload` if the +/// container is running. +pub(crate) async fn container_toplevel_action( verb: &str, name: &str, - meta_rev: &str, + toplevel: &str, stream: bool, writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { validate_agent_name(name)?; - // The build is the multi-minute phase of this operation — give it the - // same live-line treatment `container_run_streaming` gives - // `nixos-container` itself when the caller asked for it. Without - // this, moving the build out of the streamed `nixos-container` call - // (which is the whole point of this fix) would silently regress every - // UI that shows build progress: nothing until the longest phase - // finishes, then everything at once. - let toplevel = nix_build_toplevel(name, meta_rev, stream.then_some(&mut *writer)).await?; + let toplevel = toplevel_to_apply(toplevel)?; let system_name = container_system_name(name); if verb == "update" { - return swap_container_profile(&system_name, &toplevel, stream, writer).await; + return swap_container_profile(&system_name, toplevel, stream, writer).await; } - let args = [verb, &system_name, "--system-path", &toplevel]; + let args = [verb, &system_name, "--system-path", toplevel]; if stream { container_run_streaming(&args, writer).await } else { @@ -126,7 +116,7 @@ pub(crate) async fn container_flake_action( } /// Apply a prebuilt toplevel to an existing container directly, without -/// going through `nixos-container update` (see `container_flake_action`'s +/// going through `nixos-container update` (see `container_toplevel_action`'s /// doc comment for why). Mirrors `nixos-container.pl`'s own `update` /// action verbatim, past its version probe: point the per-container `nix- /// env` profile at the new toplevel, then reload the container unit *if* @@ -209,46 +199,10 @@ async fn log_and_forward(stdout: &str, stderr: &str, stream: bool, writer: &mut } } -/// The environment variable naming the forge's `http` origin, which -/// `internal/meta` is fetched from. Set by the unit; never taken from a -/// request. -pub(crate) const FORGE_URL_ENV: &str = "HIVE_PRIV_FORGE_URL"; - -/// The flake agent toplevels are built from: `internal/meta` at `rev`. The URL -/// is `forge_base`, hive-priv's own; only the rev comes from the request, and -/// it is checked here. -fn meta_flake(rev: &str, forge_base: Option<&str>) -> Result { - if !valid_meta_rev(rev) { - bail!("meta_rev {rev:?} is not a 40-digit lowercase hex commit id"); - } - let base = forge_base.filter(|b| !b.is_empty()).with_context(|| { - format!("{FORGE_URL_ENV} is unset, so there is no internal/meta to build from") - })?; - Ok(meta_flake_url(base, rev)) -} - -/// The explicit `nixosConfigurations..config.system.build.toplevel` -/// flake attr path — same construction `hive-c0re`'s own -/// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so -/// the exact string shape is unit-tested without needing to run `nix`. -fn toplevel_attr(flake: &str, name: &str) -> String { - format!("{flake}#nixosConfigurations.{name}.config.system.build.toplevel") -} - -/// Wall-clock bound on [`nix_build_toplevel`]. Four times CI's observed -/// cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`), which -/// compiles this workspace's crates, the part of an agent toplevel no binary -/// cache holds; the rebuild path has usually warmed this exact attr already -/// (`hive-c0re`'s `prebuild_toplevel`). A stalled download is bounded by nix's -/// own `stalled-download-timeout`; this covers the rest, such as a wedged -/// nix-daemon holding the caller's build slot forever. -const NIX_BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1); - /// How a [`run_bounded`] child ended. pub(crate) enum BoundedRun { Exited { status: std::process::ExitStatus, - stdout: String, stderr: String, }, /// Still running at the limit; its whole process group was killed. @@ -256,8 +210,7 @@ pub(crate) enum BoundedRun { } /// Run `cmd` to completion or until `limit`, draining both pipes: stdout is -/// captured silently, stderr is logged and, when `writer` is set, forwarded -/// live as [`PrivEvent::Line`]s. +/// discarded, stderr is kept and also logged as it arrives. /// /// The child leads its own process group and a timeout `SIGKILL`s the whole /// group, so a grandchild still holding the pipes (or the daemon connection) @@ -265,7 +218,6 @@ pub(crate) enum BoundedRun { pub(crate) async fn run_bounded( mut cmd: Command, limit: std::time::Duration, - mut writer: Option<&mut OwnedWriteHalf>, ) -> std::io::Result { use tokio::io::AsyncBufReadExt as _; @@ -282,13 +234,10 @@ pub(crate) async fn run_bounded( let run = async { let mut stdout_lines = BufReader::new(stdout).lines(); let mut stderr_lines = BufReader::new(stderr).lines(); - let mut stdout_buf = String::new(); let mut stderr_buf = String::new(); - // ⚠️ Both pipes are drained concurrently even though only one is - // streamed: reading stderr alone would let stdout fill its pipe - // buffer and deadlock the child on a build with enough stdout output - // to fill it. + // ⚠️ Both pipes are drained concurrently: reading one alone would let + // the other fill its pipe buffer and deadlock the child. // // ⚠️ Each stream's EOF is tracked separately rather than breaking on // the first `None`: `next_line()` on a closed stream returns @@ -300,27 +249,18 @@ pub(crate) async fn run_bounded( tokio::select! { line = stdout_lines.next_line(), if !stdout_done => { match line { - // Captured, never streamed — this is the store path. - Ok(Some(l)) => { - stdout_buf.push_str(&l); - stdout_buf.push('\n'); - } + Ok(Some(_)) => {} Ok(None) => stdout_done = true, Err(e) => { - tracing::warn!(error = %e, "nix build stdout read error"); + tracing::warn!(error = %e, "child stdout read error"); stdout_done = true; } } } line = stderr_lines.next_line(), if !stderr_done => { match line { - // Streamed as it arrives, so the dashboard and - // `journalctl -f` show build progress live. Ok(Some(l)) => { - tracing::info!(target: "nix-build-toplevel", "{l}"); - if let Some(w) = writer.as_deref_mut() { - write_line_event(w, PrivStream::Stderr, &l).await; - } + tracing::info!(target: "nix-store", "{l}"); if !stderr_buf.is_empty() { stderr_buf.push('\n'); } @@ -328,7 +268,7 @@ pub(crate) async fn run_bounded( } Ok(None) => stderr_done = true, Err(e) => { - tracing::warn!(error = %e, "nix build stderr read error"); + tracing::warn!(error = %e, "child stderr read error"); stderr_done = true; } } @@ -338,7 +278,6 @@ pub(crate) async fn run_bounded( let status = child.wait().await?; Ok::<_, std::io::Error>(BoundedRun::Exited { status, - stdout: stdout_buf, stderr: stderr_buf, }) }; @@ -356,90 +295,6 @@ pub(crate) async fn run_bounded( Ok(BoundedRun::TimedOut) } -/// Build `nixosConfigurations..config.system.build.toplevel` and -/// return the resulting store path, so `create`/`update` can hand -/// `nixos-container` an explicit `--system-path` instead of letting its -/// own `buildFlake()` build to a racy shared out-link. See "Container -/// toplevel builds" in this crate's README for the full story — the -/// concurrency bug this closes (the "agent container gets closure of -/// other agent" mystery bug) and why stdout/stderr are drained -/// concurrently but handled asymmetrically (stderr streamed live, -/// stdout captured and required to be exactly one line). -/// -/// `writer` is `None` for the non-streaming call shape (`stream: false`); -/// stderr still logs to journald either way, just without the -/// `PrivEvent::Line` forwarding. Bounded by [`NIX_BUILD_TIMEOUT`]. -async fn nix_build_toplevel( - name: &str, - meta_rev: &str, - writer: Option<&mut OwnedWriteHalf>, -) -> Result { - let flake = meta_flake(meta_rev, std::env::var(FORGE_URL_ENV).ok().as_deref())?; - let attr = toplevel_attr(&flake, name); - let mut cmd = Command::new("nix"); - cmd.args([ - "--extra-experimental-features", - "nix-command flakes", - "build", - "--no-link", - "--print-out-paths", - &attr, - ]); - let (status, stdout_buf, stderr_buf) = match run_bounded(cmd, NIX_BUILD_TIMEOUT, writer) - .await - .with_context(|| format!("run nix build {attr}"))? - { - BoundedRun::Exited { - status, - stdout, - stderr, - } => (status, stdout, stderr), - BoundedRun::TimedOut => bail!( - "nix build {attr} timed out after {}s; killed it", - NIX_BUILD_TIMEOUT.as_secs() - ), - }; - - // ⚠️ Success is decided by the exit status, not by "we parsed a - // path" — a build can print to stdout and still fail. - if !status.success() { - bail!( - "nix build {attr} failed ({status}): {}", - stderr_buf.lines().last().unwrap_or("").trim() - ); - } - single_output_path(&stdout_buf) - .map(str::to_owned) - .map_err(|count| { - anyhow!("nix build {attr} produced {count} output path(s), expected exactly 1: {stdout_buf:?}") - }) -} - -/// Parse `nix build --print-out-paths`' stdout down to the single output -/// path this function's caller expects. `--print-out-paths` prints one -/// line *per output*, not one line total (`nix build --no-link -/// --print-out-paths nixpkgs#openssl` prints two: `…-bin`, `…-man`); -/// `config.system.build.toplevel` is single-output today, so this is one -/// line in practice — but a bare whole-buffer `.trim()` would silently -/// hand a multi-line string on to `--system-path` the day that ever -/// changes, the same corrupted-argument failure this function exists to -/// avoid. Trims and drops empty lines *before* counting, so a lone -/// `"\n"` (or trailing whitespace on the real line) can't be mistaken -/// for a present-but-blank path — see the unit tests below for the exact -/// table this closes. `Err` carries the surviving line count, for the -/// caller's error message. -fn single_output_path(stdout: &str) -> Result<&str, usize> { - let lines: Vec<&str> = stdout - .lines() - .map(str::trim) - .filter(|l| !l.is_empty()) - .collect(); - match lines[..] { - [path] => Ok(path), - _ => Err(lines.len()), - } -} - /// Invoke `nixos-container` with the given args, log output to journald. pub(crate) async fn container_run(args: &[&str]) -> Result<(String, String)> { let out = Command::new("nixos-container") @@ -709,85 +564,37 @@ async fn container_run_streaming( } Ok((String::new(), String::new())) } + #[cfg(test)] mod tests { - use super::{ - BoundedRun, FORGE_URL_ENV, meta_flake, run_bounded, single_output_path, toplevel_attr, - }; + use super::{BoundedRun, run_bounded, toplevel_to_apply}; use crate::shared::test_support::scratch; - /// Pins the exact attr path we hand to `nix build` — it has to match - /// `hive-c0re`'s own `lifecycle::prebuild_toplevel` construction, since - /// that step's whole point is warming the store for this later build. - /// A drifted attr path defeats the cache-warming silently — no error, - /// just a slower `update`. The flake is `internal/meta` at the rev, and its - /// URL is the unit's, never the request's. - #[test] - fn a_meta_rev_builds_from_internal_meta_at_the_units_forge() { - let rev = "c".repeat(40); - let forge = Some("http://forge.t.local"); - assert_eq!( - toplevel_attr(&meta_flake(&rev, forge).unwrap(), "atlas"), - format!( - "git+http://forge.t.local/internal/meta.git?rev={rev}\ - #nixosConfigurations.atlas.config.system.build.toplevel" - ) - ); - } + const TOPLEVEL: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; - /// Nothing but a commit id gets into the ref, whatever the request says. + /// A store path passes through unchanged; anything else, whatever the + /// request says, never reaches `nix-env` or `nixos-container`. #[test] - fn a_meta_rev_that_is_not_a_commit_id_is_refused() { - let forge = Some("http://forge.t.local"); + fn only_a_store_path_is_applied() { + assert_eq!(toplevel_to_apply(TOPLEVEL).unwrap(), TOPLEVEL); for bad in [ - "main", "", + "main", "../../x", - &format!("{}#evil", "a".repeat(34)), - &format!("{}&ref=x", "a".repeat(34)), + "/etc/passwd", + "/nix/store/x", + &format!("{TOPLEVEL}/bin"), + &format!("{TOPLEVEL};id"), + &"c".repeat(40), ] { - let err = meta_flake(bad, forge).unwrap_err(); + let err = toplevel_to_apply(bad).unwrap_err(); assert!( - format!("{err:#}").contains("not a 40-digit"), + format!("{err:#}").contains("not a store path"), "{bad}: {err:#}" ); } } - #[test] - fn a_meta_rev_without_a_forge_in_the_unit_is_refused() { - let rev = "c".repeat(40); - for forge in [None, Some("")] { - let err = meta_flake(&rev, forge).unwrap_err(); - assert!(format!("{err:#}").contains(FORGE_URL_ENV), "{err:#}"); - } - } - - /// `--print-out-paths`' exact per-shape table (measured against real - /// `rustc` semantics, not just reasoned about) — a lone `"\n"` and a - /// trailing-space path are the two shapes a bare `.lines().collect()` - /// gets wrong, both catchable only by trimming + dropping empties - /// before counting rather than after. - #[test] - fn single_output_path_rejects_blank_and_trims_whitespace() { - assert_eq!(single_output_path(""), Err(0)); - assert_eq!(single_output_path("\n"), Err(0)); - assert_eq!(single_output_path("\n\n"), Err(0)); - assert_eq!(single_output_path("/nix/store/abc\n"), Ok("/nix/store/abc")); - assert_eq!( - single_output_path("/nix/store/abc \n"), - Ok("/nix/store/abc") - ); - assert_eq!( - single_output_path("/nix/store/abc\r\n"), - Ok("/nix/store/abc") - ); - assert_eq!( - single_output_path("/nix/store/abc\n/nix/store/def\n"), - Err(2) - ); - } - /// Whether `pid` is gone, a zombie (`Z`), or being reaped (`X`): none of /// these runs again, and a process passes through them in that order, /// so once this is true it stays true. @@ -817,7 +624,7 @@ mod tests { /// reaches its whole process group: the backgrounded grandchild dies too, /// not just the shell that started it. #[tokio::test] - async fn a_build_past_its_limit_is_killed_with_its_group() { + async fn a_child_past_its_limit_is_killed_with_its_group() { use std::time::{Duration, Instant}; let dir = scratch(); let pidfile = dir.join("grandchild"); @@ -826,9 +633,7 @@ mod tests { .arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display())); let started = Instant::now(); - let run = run_bounded(cmd, Duration::from_secs(2), None) - .await - .unwrap(); + let run = run_bounded(cmd, Duration::from_secs(2)).await.unwrap(); assert!(matches!(run, BoundedRun::TimedOut), "must time out"); assert!( started.elapsed() < Duration::from_mins(1), @@ -846,23 +651,18 @@ mod tests { } /// The control: a child that exits inside its limit is `Exited`, with its - /// status and both streams kept apart. + /// status and its stderr. #[tokio::test] - async fn a_build_inside_its_limit_returns_its_output() { + async fn a_child_inside_its_limit_returns_its_status_and_stderr() { let mut cmd = tokio::process::Command::new("sh"); cmd.arg("-c") .arg("echo progress >&2; echo /nix/store/abc-toplevel"); - match run_bounded(cmd, std::time::Duration::from_mins(1), None) + match run_bounded(cmd, std::time::Duration::from_mins(1)) .await .unwrap() { - BoundedRun::Exited { - status, - stdout, - stderr, - } => { + BoundedRun::Exited { status, stderr } => { assert!(status.success()); - assert_eq!(stdout, "/nix/store/abc-toplevel\n"); assert_eq!(stderr, "progress"); } BoundedRun::TimedOut => panic!("a fast child must not time out"), diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 48ca9f7b..10971f6a 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -366,14 +366,14 @@ async fn exec( PrivRequest::UpdateContainer { ref name, stream, - ref meta_rev, - } => container::container_flake_action("update", name, meta_rev, stream, writer).await, + ref toplevel, + } => container::container_toplevel_action("update", name, toplevel, stream, writer).await, PrivRequest::CreateContainer { ref name, stream, - ref meta_rev, - } => container::container_flake_action("create", name, meta_rev, stream, writer).await, + ref toplevel, + } => container::container_toplevel_action("create", name, toplevel, stream, writer).await, PrivRequest::RealiseClosure { ref agent, diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 4de6106e..6cd79d13 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -41,13 +41,12 @@ let }; # Privsep splits ownership across users, so git/libgit2's dubious- - # ownership guard trips on legitimate cross-user reads: hive-priv (root) - # fetches the hive-core-owned meta/applied repos via nix, and hive-c0re - # (hive-core) fetches the agent-owned proposed-config repos. Both - # processes are trusted and can already read the files; this gitconfig - # only satisfies the ownership guard. libgit2 honours the literal `*` - # (mid-path globs aren't supported, so per-agent repos can't be listed); - # in practice these processes only ever touch hyperhive's own repos. + # ownership guard trips on legitimate cross-user reads: hive-c0re + # (hive-core) fetches the agent-owned proposed-config repos. The process is + # trusted and can already read the files; this gitconfig only satisfies the + # ownership guard. libgit2 honours the literal `*` (mid-path globs aren't + # supported, so per-agent repos can't be listed); in practice this process + # only ever touches hyperhive's own repos. # # The forge sections (`forgeGit.config`) answer for the `git+http` forge # fetches nix makes: the agent config inputs and `internal/meta`. hive-c0re's diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 8f395357..bad8fc36 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -41,11 +41,9 @@ let ); in { - # nix (the prebuild `nix build`, flake-check, and meta eval) writes - # its fetcher/eval cache under $HOME/.cache. As a system user + # git writes its config and caches under $HOME. As a system user # hive-core has no home, so HOME defaults to the unwritable - # /var/empty and Lix fails to initialise its cache. Point HOME at - # the writable StateDirectory. + # /var/empty. Point HOME at the writable StateDirectory. HOME = "/var/lib/hyperhive"; HYPERHIVE_GIT = "${pkgs.git}/bin/git"; # Never let git block on an interactive credential prompt. hive-core is a diff --git a/nix/host-modules/hive-c0re/options.nix b/nix/host-modules/hive-c0re/options.nix index 5d2eefc0..75fdfe0e 100644 --- a/nix/host-modules/hive-c0re/options.nix +++ b/nix/host-modules/hive-c0re/options.nix @@ -290,11 +290,11 @@ default = 1; example = 2; description = '' - Number of nix-heavy job-queue nodes (container prebuilds, + Number of closure-fetching job-queue nodes (container prebuilds, profile swaps, first-spawn creates, meta lock bumps) hive-c0re - runs concurrently. The default of 1 serializes all heavy nix - work; raise it on hosts with the cores/RAM to build several - agent toplevels at once. Per-agent correctness is independent + runs concurrently. The default of 1 serializes all of them; + raise it on hosts with the bandwidth and disk to take several + agents at once. Per-agent correctness is independent of this count — each agent's container-affecting operations are serialized by its lifecycle lease regardless. ''; diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index 5f865f57..3975e926 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -9,22 +9,6 @@ }: let controllerCfg = config.services.hyperhive.deploy.hive-controller; - - forgeGit = import ./lib/hive-forge-git.nix { - inherit pkgs; - forgeDomain = config.services.hyperhive.swarm.forge.domain; - }; - - # Same gitconfig as the c0re unit (see ./hive-c0re) apart from its stalled- - # transfer limit: hive-priv (root) runs nix, which fetches the hive-core-owned - # meta/applied repos (libgit2 refuses cross-user reads without - # safe.directory) and, from a fresh HOME, the forge inputs of `internal/meta` - # (the forge sections). - safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" '' - [safe] - directory = * - ${forgeGit.config} - ''; in { config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable { @@ -77,41 +61,25 @@ in # findutils already in the default unit PATH. Without `nixos-container` # on PATH every container op fails ENOENT, which `build_all` silently # swallows into an empty list ("no managed containers"). - # - # `nix` itself shells out by bare name too: `git` whenever it has to - # fetch/re-resolve a git-source flake input (an agent.nix with a - # `git+https://…` input, or a stale flake.lock whose node URL no longer - # matches the flake's declared input → nix re-resolves at eval), and - # `ssh` to dispatch to remote builders (`nix.buildMachines` / - # `ssh-ng://`). Without these on PATH `nixos-container update` dies with - # `executing "git": No such file or directory` / `Could not find - # executable 'ssh'` — the agent build fails before it starts. path = [ pkgs.nixos-container pkgs.nix # nix, nix-env, nix-instantiate — create + update - pkgs.gitMinimal # git — nix fetches/re-resolves git-source flake inputs - forgeGit.credentialHelper # git-credential-hive-forge — the forge's `git+http` inputs - pkgs.openssh # ssh — nix dispatches builds to remote builders pkgs.util-linux # umount (nsenter is hardcoded in the script) pkgs.e2fsprogs # chattr pkgs.btrfs-progs # btrfs subvolume create/delete — Ensure/DeleteAgentSubvolume ]; environment = { - # `nixos-container update/create` runs `nix`, which writes its - # fetcher/eval cache under $HOME/.cache. With ProtectHome and no - # explicit HOME this lands on the unwritable /var/empty and Lix - # errors out. Point HOME at the StateDirectory below (persistent, - # so the cache survives across rebuilds). + # nix (`nix-env`, `nix-store`) writes its cache under $HOME/.cache. + # With ProtectHome and no explicit HOME this lands on the unwritable + # /var/empty and Lix errors out. Point HOME at the StateDirectory + # below (persistent, so the cache survives across restarts). HOME = "/var/lib/hive-priv"; # hive-priv runs as root. Root nix defaults to store=auto which # resolves to the LOCAL store — bypassing the host daemon, its # remote builders, and prebuilt derivation outputs. Force daemon - # routing so nixos-container update and the nix prebuild see the - # same store and substituters as every other build context. + # routing so the closure `RealiseClosure` downloads is in the same + # store the container is then pointed at. NIX_REMOTE = "daemon"; - # The forge origin `internal/meta` is built from when a request names - # one of its commits. Fixed here; no request chooses it. - HIVE_PRIV_FORGE_URL = "http://${config.services.hyperhive.swarm.forge.domain}"; }; serviceConfig = { ExecStart = "${controllerCfg.package}/bin/hive-priv"; @@ -159,11 +127,6 @@ in # Point the working directory at the writable StateDirectory so nix # drops its temp symlink there instead. WorkingDirectory = "/var/lib/hive-priv"; - # nix (run here as root for `nixos-container update --flake - # /var/lib/hyperhive/meta#`) fetches the hive-core-owned - # meta/applied repos; libgit2 refuses them without safe.directory. - # See safeDirGitconfig above. - ExecStartPre = "+-${pkgs.coreutils}/bin/cp ${safeDirGitconfig} /var/lib/hive-priv/.gitconfig"; # Kernel surfaces neither hive-priv nor its children use: no # personality(2), no real-time scheduling, no /dev/kmsg or # /proc/kmsg, no explicit module loading (kernel-initiated autoload diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index bcdb300f..3c6bb9a5 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -234,13 +234,12 @@ let && env shared.controllerAlone == { }; } { - # nix fetches the forge inputs with `git`, so the credential helper and - # the https-to-http rewrite are git config in both processes that build: - # hive-c0re (prebuilds) and hive-priv (builds as root, from its own - # HOME). The controller's unit is the control that the lookup tells a - # unit with the helper from one without; the URL hive-priv builds - # `internal/meta` from is the one hive-c0re reads the forge at. - name = "hive-c0re and hive-priv answer the forge's git fetches and rewrite its https inputs to http"; + # hive-c0re carries the credential helper and the https-to-http rewrite as + # git config. hive-priv builds nothing and fetches nothing from the forge, + # so it carries neither and has no forge URL. The hive-c0re arm is the + # control that the lookups find a helper and a forge URL where there is + # one, so the absences on hive-priv and the controller are real. + name = "hive-c0re answers the forge's git fetches and rewrites its https inputs to http; hive-priv has neither"; ok = let forge = allLocal.services.hyperhive.swarm.forge.domain; @@ -256,7 +255,7 @@ let env = unit: allLocal.systemd.services.${unit}.environment; in helperOn "hive-c0re" - && helperOn "hive-priv" + && !(helperOn "hive-priv") # The helper answers from the file hive-c0re publishes the hive's own # token to. The first arm is the control for the second: the lookup # finds the helper's text, so the absent `core` strings are really absent. @@ -267,8 +266,8 @@ let && !(helperOn "swarm-controller") && lib.hasInfix "[credential \"http://${forge}\"]\nhelper =\nhelper = hive-forge" git.config && lib.hasInfix "[url \"http://${forge}/\"]\ninsteadOf = https://${forge}/" git.config - && (env "hive-priv").HIVE_PRIV_FORGE_URL == "http://${forge}" - && (env "hive-priv").HIVE_PRIV_FORGE_URL == (env "hive-c0re").HIVE_FORGE_URL; + && (env "hive-c0re") ? HIVE_FORGE_URL + && !((env "hive-priv") ? HIVE_PRIV_FORGE_URL); } { # `hyperhive_rev` defaults to none on the controller, so a settings JSON