From 4590b04cac2c889195345d683dae4a7d000b4b4c Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 22:17:07 +0100 Subject: [PATCH 1/3] remove the hive's local build of agent closures A rebuild or create downloads the closure the swarm built and applies it; with none to deploy it fails and changes nothing. hive-priv takes a store path instead of an internal/meta commit and builds nothing. --- docs/agent-lifecycle/config-changes.md | 36 +- docs/process/gotchas.md | 6 - docs/scheduler/coordinator.md | 57 +-- docs/swarm/builder.md | 59 ++- docs/trust-boundary/security.md | 2 +- hive-c0re/src/attic/closure.rs | 59 +-- hive-c0re/src/coordinator.rs | 12 +- hive-c0re/src/forge/internal_meta.rs | 37 +- hive-c0re/src/job_queue/exec.rs | 244 +++++----- hive-c0re/src/job_queue/exec/tests.rs | 521 +++++++++++---------- hive-c0re/src/job_queue/model.rs | 4 +- hive-c0re/src/job_queue/templates.rs | 6 +- hive-c0re/src/lifecycle/mod.rs | 233 +-------- hive-c0re/src/lifecycle/tests.rs | 62 --- hive-c0re/src/main.rs | 2 +- hive-c0re/src/meta_rev.rs | 82 +--- hive-c0re/src/priv_client.rs | 8 +- hive-c0re/src/stores/build_logs.rs | 2 +- hive-priv-sock/src/lib.rs | 60 +-- hive-priv/README.md | 70 +-- hive-priv/src/closure.rs | 6 +- hive-priv/src/container.rs | 318 +++---------- hive-priv/src/main.rs | 8 +- nix/host-modules/hive-c0re/default.nix | 13 +- nix/host-modules/hive-c0re/environment.nix | 6 +- nix/host-modules/hive-c0re/options.nix | 8 +- nix/host-modules/hive-priv.nix | 49 +- nix/module-eval/core-toggle.nix | 19 +- 28 files changed, 635 insertions(+), 1354 deletions(-) diff --git a/docs/agent-lifecycle/config-changes.md b/docs/agent-lifecycle/config-changes.md index 4a8b9c648..85b231112 100644 --- a/docs/agent-lifecycle/config-changes.md +++ b/docs/agent-lifecycle/config-changes.md @@ -145,9 +145,10 @@ commit the agent's last successful build pinned. swarm-controller commits `internal/meta`: one `flake.nix` and `flake.lock` for every agent any hive places (the pins: -[`docs/swarm/builder.md`](../swarm/builder.md#meta-flake-lock)). A hive builds -every agent from it. While the repo has no commit that pins an agent's current -config, that agent's build fails and names the agent. The controller queues +[`docs/swarm/builder.md`](../swarm/builder.md#meta-flake-lock)). The swarm +builder builds every agent from it, and a hive deploys that build. While the +repo has no commit that pins an agent's current config, that agent's deploy +fails and names the agent. The controller queues its update of the repo before the deploy request for a merged config commit and for a new agent, and sends the deploy only when that update succeeded and the swarm builder built the agent from the updated repo (the @@ -159,28 +160,23 @@ the swarm builder built the agent from the updated repo (the a custom agent from the config commit at the head of `agent-configs/` `main`, an archetype agent from the archetype it has now. An agent switched between custom and archetype, or to another archetype, fails that test. - A hive without the forge credential file that nix's git reads, a repo with no - commit, a head that lacks the agent and a head behind the config each fail - the build with an error that names the agent. `MetaSync` records the commit, + A repo with no commit, a head that lacks the agent and a head behind the + config each fail the deploy with an error that names the agent. `MetaSync` records the commit, with the hyperhive and config commits the head pins, in `hive-c0re`'s own state, `/var/lib/hyperhive/meta-rev/`, which the agent can't write (the container mounts its runtime directory read-write); `Prebuild` and `Swap` read that - record, so both build one flake. A first deploy resolves the commit in its `Create` + record, so both expect the closure of one commit. A first deploy resolves the commit in its `Create` node, since it has no `MetaSync`. -- **The build.** `hive-c0re` prebuilds - `git+/internal/meta.git?rev=#nixosConfigurations.…` and - names the commit to `hive-priv` (`meta_rev` on `UpdateContainer` and - `CreateContainer`). `hive-priv` accepts only a 40-digit lowercase hex - commit and builds the flake from its own forge URL - (`HIVE_PRIV_FORGE_URL`); the URL never comes from a request. On the wire the +- **The closure.** The hive builds nothing. `Prebuild`, `Swap` and `Create` read + the agent's record in the swarm's `agent-closures` bucket (the controller + writes it from the builder's reply) and have `hive-priv` download the closure + it names from the swarm cache (`RealiseClosure`). `Swap` and `Create` then + give `hive-priv` the store path (`toplevel` on `UpdateContainer` and + `CreateContainer`), which accepts only a store path. On the wire the `hive_priv_sock::PrivRequest` variants carry a required field: - `{"op":"update_container","name":"atlas","stream":true,"meta_rev":"<40 hex>"}`. -- **Fetching.** nix fetches the forge's inputs with `git`. Both `hive-c0re` - and `hive-priv` carry a `$HOME/.gitconfig` that answers for the forge host - through `git-credential-hive-forge` (the hive's forge read token, published to - `/run/hyperhive/forge-hive-token`) and rewrites the - forge's `https` URL, which `internal/meta`'s lock names its inputs by, to - `http`. + `{"op":"update_container","name":"atlas","stream":true,"toplevel":"/nix/store/-nixos-system-atlas"}`. + The three cases when the swarm's closure isn't the one this hive expects + are in [`docs/swarm/builder.md`](../swarm/builder.md#rebuilds-download-the-closure). - **Hyperhive.** The commit pins one hyperhive rev for the swarm, so an agent runs the hyperhive of the controller's deploy, not its hive's. - **Markers.** After a successful build `hive-c0re` writes the hyperhive diff --git a/docs/process/gotchas.md b/docs/process/gotchas.md index 1e575c543..03812067a 100644 --- a/docs/process/gotchas.md +++ b/docs/process/gotchas.md @@ -301,12 +301,6 @@ nix build "git+/internal/meta.git?rev=#argus.config.system.build.top nix build "git+/internal/meta.git?rev=#nixosConfigurations.argus.config.system.build.toplevel" ``` -`lifecycle::prebuild_toplevel` hit this once by constructing the attr -path as `{flake_ref}.config…` — which produced `#argus.config…` -instead of `#nixosConfigurations.argus.config…`. The fix: -`split_once('#')` to separate flake path from name, then template -`{path}#nixosConfigurations.{name}.config.system.build.toplevel`. - ### Containerized nix-daemon needs `sandbox-fallback = true` Agent containers bind-mount the host's nix-daemon socket. nspawn diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 665aa52ac..7b5b3dff4 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -42,11 +42,11 @@ because there is no malformed spec to reject. Nix-heavy — hold one of the `buildSlots` permits for the node's duration: -| Node | Wraps | -| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `Prebuild` | `lifecycle::prebuild_toplevel` — build the toplevel out-of-band while the container keeps serving (its meta preamble is the upstream `MetaSync` node). Skipped when the container is already down; `Swap` builds inline instead | -| `Swap` | drop-in rewrite + `nixos-container update` profile-swap (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `RebuildBookkeeping` node | -| `Create` | first-spawn `nixos-container create` proper; assumes the upstream `Provision` node already registered the agent; downloads the closure the swarm cached for the `internal/meta` commit it resolves, else builds it locally, and marks the agent's pins | +| Node | Wraps | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Prebuild` | download the swarm's closure for the agent into the store while the container keeps serving (its meta preamble is the upstream `MetaSync` node). With no closure to deploy it fails here, before the stop. Skipped when the container is already down; `Swap` downloads it instead | +| `Swap` | drop-in rewrite + profile-swap onto the swarm's closure (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `RebuildBookkeeping` node | +| `Create` | first-spawn `nixos-container create` proper; assumes the upstream `Provision` node already registered the agent; downloads the closure the swarm built for the agent and creates the container from it (with none to deploy it fails and creates nothing), then marks the agent's pins | Cheap — no build slot: @@ -54,7 +54,7 @@ Cheap — no build slot: | Node | Behavior | | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MetaSync` | the rebuild's preamble — rebuild-dir prep, the agent-set refresh (`agent_set::refresh`), and the `internal/meta` commit `Prebuild` and `Swap` build from, resolved once and recorded; fails when the repo doesn't pin the agent's current config. Declares no resources | +| `MetaSync` | the rebuild's preamble — rebuild-dir prep, the agent-set refresh (`agent_set::refresh`), and the `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to be built from, resolved once and recorded; fails when the repo doesn't pin the agent's current config. Declares no resources | | `Provision` | first-spawn pre-create provisioning — proposed/applied repos, state subvolume, agent-set refresh (`agent_set::refresh`); runs ahead of `Create`. Store-only, no container yet | | `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop | | `Start` | mechanical container start — runtime dir + drop-ins, `start_with_fallback`, MCP listener registration, the manager kick. Fanned out by a `Reconcile` that observed `wanted = Up` and the container down | @@ -186,7 +186,7 @@ resources are free. Resources: clobber intent before either reconciles.) **Lease-exempt**: `MetaSync`, `Prebuild`, `Provision` — they touch the store, not the running container, which is exactly why a - stop can land while another DAG's prebuild is still building. Also exempt, for a different + stop can land while another DAG's prebuild is still downloading. Also exempt, for a different reason, are the rebuild subtree's own members (`StopForUpdate`, `Swap`, `Signal`, `Drain`, `RebuildBookkeeping`): they genuinely do touch the container, but their `AgentWindow` brace holds the lease above them — see @@ -344,17 +344,17 @@ one go, rather than the double-bounce a live `update` would trigger. Sequence for a rebuild DAG (each step is its own queue node): 1. `MetaSync` — rebuild-dir prep, the agent-set refresh, and the choice of the - `internal/meta` commit to build from. Short, and it declares no resources. -2. `Prebuild` — build the new `system.build.toplevel` **before** stopping. - The container keeps serving the previous generation while eval + fetch + - build happen out-of-band. `nixos-container update` then finds the result - cached and skips straight to the profile-swap. Build failures surface - here, before `Swap` touches the running container. Skipped when the - container is already down — a stopped agent has no uptime to preserve, - so `Swap` builds inline instead of paying the double eval. + `internal/meta` commit the swarm's closure should come from. Short, + and it declares no resources. +2. `Prebuild` — download the swarm's closure for the agent **before** stopping. + The container keeps serving the previous generation while the download + happens out-of-band. A missing or unpullable closure fails the DAG here, + before `Swap` touches the running container, and the container keeps + running. Skipped when the container is already down — a stopped agent has + no uptime to preserve, so `Swap` downloads it. 3. `StopForUpdate` — bring the container down (noop when already stopped). -4. `Swap` — `nixos-container update` profile-swap from `internal/meta` - (near-instant after the prebuild). +4. `Swap` — profile-swap onto the swarm's closure (near-instant after the + prebuild). 5. `Reconcile` — boot into the new generation when `wanted = Up`; the in-container activation script transitions old → new. Holds no build slot, so the next DAG's `Prebuild` overlaps the container boot. @@ -374,18 +374,10 @@ fallback. ### Spawn path (new container) -For a first-time `create`, `nixos-container create` is atomic: if the build fails, -it leaves no container record to clean up. A separate prebuild would just duplicate -the eval, so it's skipped. Sequence: `create` from `internal/meta` → write nspawn -flags → `systemctl daemon-reload` → `start`. - -### Prebuild attr path - -`nix build` doesn't autoresolve `#` against `nixosConfigurations` the -way `nixos-container` does internally. nix requires the explicit attr path -`#nixosConfigurations..config.system.build.toplevel`; -using the bare `#` ref would make nix look in `packages`, `legacyPackages`, -or the flake root directly — none of which exist in the rendered meta flake. +For a first-time `create`, `nixos-container create` is atomic: it leaves no container +record to clean up if it fails. The `Create` node downloads the swarm's closure first, +so a spawn with no closure to deploy fails before it creates the container. Sequence: +download the closure → `create` from it → write nspawn flags → `systemctl daemon-reload` → `start`. --- @@ -396,11 +388,10 @@ limits, build parallelism, and first-spawn latency. ### Build slots -`buildSlots` (default `1`) sets how many nix-heavy job-queue nodes +`buildSlots` (default `1`) sets how many closure-fetching job-queue nodes (prebuilds, profile swaps, first-spawn creates, meta lock bumps) run -concurrently. The default serializes all heavy nix work like the pre-DAG -rebuild queue did; raise it on hosts with the cores/RAM to build multiple -agent toplevels at once. Per-agent correctness is independent of the count — +concurrently. The default serializes all of them; raise it on hosts with +the bandwidth and disk to take more agents at once. Per-agent correctness is independent of the count — each agent's container-affecting ops serialize on its lifecycle lease regardless. diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index d49864f76..f75fb1095 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -200,12 +200,9 @@ minutes. The builder takes the same one-build slot as PR checks, and refuses a request that waited 30 seconds for it. It runs `nix build` of `nixosConfigurations..config.system.build.toplevel` from `git+https:///internal/meta.git?rev=`, with no input overridden: -the committed `flake.lock` decides every input. Like the hive's build, it uses -that attribute, so a green build here means the flake evaluates and builds for -the builder's platform. A hive [downloads the closure instead of building it](#what-hives-do-with-the-cache) -when the record is for the commit it builds from; otherwise it builds locally, -from whatever head it reads at deploy time, which can be newer than the one the -swarm built. +the committed `flake.lock` decides every input. A green build here means the +flake evaluates and builds for the builder's platform. A hive [downloads the closure](#rebuilds-download-the-closure) +this build produced and never builds one itself. The node ends `done` or `failed`. Every failure skips the deploy and shows on the Jobs page as a failed `build_meta` node, and the timer builds it again: @@ -478,11 +475,9 @@ means check attic's fingerprint format; it says nothing about an attacker. A ### Rebuilds download the closure -A rebuild's `Prebuild` node, which runs while the agent is still up, tries the -cache before it builds. It reads the agent's record in `agent-closures` and -goes on only when the record is for the `internal/meta` commit `MetaSync` -recorded for this rebuild and says the builder pushed the closure. It then -asks `hive-priv` to realise the record's store path (`RealiseClosure`). `hive-priv` runs +A hive builds nothing. A rebuild's `Prebuild` node, which runs while the agent is +still up, reads the agent's record in `agent-closures` and asks `hive-priv` to +realise the record's store path (`RealiseClosure`). `hive-priv` runs `nix-store --realise` with `--max-jobs 0`, so nix can't build the path, and with the host's configured cache address, and the public key and netrc from the root-owned `/run/hyperhive-priv/attic/`, as per-call settings. The settings replace the host's `substituters` and @@ -490,32 +485,32 @@ root-owned `/run/hyperhive-priv/attic/`, as per-call settings. The settings repl source, and nix stores the path only when the cache's key signed it. The host's nix configuration stays as it was. -The deploy log says which way it went: +`Swap` then does the same lookup and download, a no-op once the path is in the +store, and gives `hive-priv` the store path to apply. A stopped agent skips +`Prebuild`, so `Swap` downloads it. A first deploy's `Create` node does the +same and creates the container from that path. -- `prebuild: substituted `: the closure is in the store, and the local build found it there and had nothing to build. -- `prebuild: the downloaded closure is not the one this hive evaluates, ; built the latter locally`, at warn level: the builder and the hive evaluate different closures for the agent, which is worth a look. The download was no use and the rebuild built locally, with the agent still up. -- `prebuild: built locally: `: the reason is the first check that failed, - such as `no-record`, `rev-mismatch`, `not-cached`, or what `hive-priv` answered - (`not available: …`, for a cache that's down or has no credentials). The rebuild builds as before. -- The same line at error level, with `no signature by the swarm cache's key` in - the reason: nix refused the path because no trusted key signed it. The rebuild builds locally - and `hive-priv` logs the refusal as well. +A rebuild or create has three outcomes, by what the lookup and the download find: -After a download, `Prebuild` still runs the local `nix build` of the agent's toplevel, -while the agent is up. It's an evaluation and no build when the hive evaluates the path the -swarm built, and a full build, as without a cache, when it doesn't, so `Swap` never meets a -build it didn't expect. `Swap` still evaluates and builds in `hive-priv`, and finds the path in -the store. A stopped agent skips `Prebuild`, so it still builds in `Swap`. +| Found | What happens | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| a record that the builder pushed, for the commit this hive chose | The hive downloads the closure and deploys it. The hive logs `prebuild: substituted ` (`create: substituted ` for a create) at info level. | +| a record that the builder pushed, for another commit (the hive and the builder disagree) | The hive downloads the builder's closure and deploys it, with a warning that names the agent, both commits and the path. The hive evaluates nothing. | +| no record, or one the hive can't use (not pushed, queue or cache unreachable, download or signature failed) | The rebuild or create fails with an error that names the agent and the reason, logged at error level, and nothing changes. A running container keeps running unchanged, because `Prebuild` fails before `StopForUpdate`. A create makes no container. | + +The hive never falls back to a local build. The third row has one owner in the +code, `without_closure` in `hive-c0re/src/job_queue/exec.rs`, so a change to what +an unreachable cache costs is one function. Mixed versions: -| hive | controller or record | what happens | -| ---- | -------------------- | ---------------------------------------------------------------------------------- | -| old | any | builds as before | -| new | old, or no record | builds locally and logs why | -| new | cache down | the download fails, the hive builds locally and logs why | -| new | signature mismatch | nix refuses the path, the hive builds locally and logs an error | -| new | old `hive-priv` | the verb is unknown to it, so the call fails and the hive builds locally, as above | +| hive | controller or record | what happens | +| ---- | -------------------- | ------------------------------------------------------------------------------------------ | +| old | any | builds locally, as before | +| new | old, or no record | refuses the create; a running agent keeps running, and the rebuild logs an error naming it | +| new | cache down | the download fails; same as no record, with the download's error in the log | +| new | signature mismatch | nix refuses the path; same as no record, and `hive-priv` logs the refusal as well | +| new | old `hive-priv` | it refuses the request, which has no `meta_rev`, so the swap or create fails | ## Credentials diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index 3d7412e08..65c4f5738 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -256,7 +256,7 @@ known operations; there is no arbitrary command pass-through: | ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `StartContainer` / `StopContainer` | `nixos-container start/stop ` | | `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | -| `CreateContainer` / `UpdateContainer` | `nixos-container create/update --flake ` | +| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must be a store path (`/nix/store/<32 chars>-`), and hive-priv builds nothing | | `RealiseClosure` | `nix-store --realise --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-` and the agent name valid. It downloads and never builds | | `DestroyContainer` | `nixos-container destroy ` | | `ListContainers` | `nixos-container list` | diff --git a/hive-c0re/src/attic/closure.rs b/hive-c0re/src/attic/closure.rs index f8e2c577c..52b5af5c0 100644 --- a/hive-c0re/src/attic/closure.rs +++ b/hive-c0re/src/attic/closure.rs @@ -1,62 +1,19 @@ -//! The closure the swarm built for an agent at a given `internal/meta` commit, -//! as a rebuild's `Prebuild` downloads it instead of building. +//! The closure the swarm built for an agent, as a deploy reads its record +//! before downloading it. use anyhow::{Result, anyhow}; use swarm_wire::agent_closures::AgentClosure; -use super::probe::{self, Unpullable}; +use super::probe; -/// The store path of the closure `agent`'s record names, when the record is a -/// build of `rev` that the builder pushed to the cache. +/// `agent`'s record in `agent-closures`; `None` when the builder has recorded +/// no build of it. /// /// # Errors -/// No record, one for another commit, or one the builder did not push; or the -/// queue cannot be reached. The message is what a deploy log says. -pub async fn path_for(agent: &str, rev: &str) -> Result { +/// The queue cannot be reached, or the record cannot be read. +pub async fn record(agent: &str) -> Result> { let client = crate::swarm_queue::client() .await .ok_or_else(|| anyhow!("this hive has no connection to the swarm queue"))?; - let record = probe::read_record(&client, agent).await?; - path_in(record.as_ref(), rev) -} - -fn path_in(record: Option<&AgentClosure>, rev: &str) -> Result { - probe::current_record(record, Some(rev)) - .map(|record| record.store_path.clone()) - .map_err(|Unpullable { reason, detail }| anyhow!("{}: {detail}", reason.label())) -} - -#[cfg(test)] -mod tests { - use super::*; - - const REV: &str = "a1b2c3"; - const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; - - fn record(rev: &str, cached: bool) -> AgentClosure { - AgentClosure { - meta_rev: rev.to_owned(), - store_path: PATH.to_owned(), - system: "x86_64-linux".to_owned(), - built_unix: 1, - cached, - } - } - - #[test] - fn a_pushed_build_of_the_rev_names_its_path() { - assert_eq!(path_in(Some(&record(REV, true)), REV).expect("path"), PATH); - } - - #[test] - fn anything_else_names_why_not() { - for (record, why) in [ - (None, "no-record"), - (Some(record("other", true)), "rev-mismatch"), - (Some(record(REV, false)), "not-cached"), - ] { - let err = path_in(record.as_ref(), REV).expect_err(why); - assert!(format!("{err}").starts_with(why), "{err}"); - } - } + probe::read_record(&client, agent).await } diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index c3cc36bcc..2f1ad3cc1 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -29,9 +29,8 @@ const LAST_STOPPED_RUNNING_KEY: &str = "last_stopped_running"; pub struct Coordinator { pub broker: Arc, - /// Full build-log capture. `lifecycle::run` / - /// `lifecycle::prebuild_toplevel` `start()` a row per attempt, - /// pipe every stdout/stderr line into it, and `finish()` it on + /// Full build-log capture. `lifecycle::run` `start()`s a row per attempt, + /// pipes every stdout/stderr line into it, and `finish()`es it on /// child exit. Dashboard reads it via `list_recent_for_agent` / /// `get_full` for the per-card chip + side-panel viewer. See /// `build_logs.rs` for retention. @@ -228,7 +227,7 @@ pub struct ServeConfig { #[serde(flatten)] pub env: HiveEnv, pub model_prices: crate::hive_stats::PriceTable, - /// Number of concurrent nix-heavy job-queue nodes (prebuild / + /// Number of concurrent closure-fetching job-queue nodes (prebuild / /// profile-swap / create / meta lock). hive-c0re-local like /// `model_prices` — never injected into containers. Set via /// `services.hyperhive.deploy.hive-controller.buildSlots`. @@ -374,9 +373,8 @@ impl Coordinator { "build_logs: closed rows left in-flight by a previous run" ); } - // Install the process-wide handle so `lifecycle::run` / - // `lifecycle::prebuild_toplevel` can write without us having - // to thread an `Arc` through every public entry + // Install the process-wide handle so `lifecycle::run` can write + // without us having to thread an `Arc` through every public entry // point in the lifecycle surface. crate::build_logs::install(build_logs.clone()); let power = Arc::new(crate::power::PowerStore::open(db_path).context("open agent_power")?); diff --git a/hive-c0re/src/forge/internal_meta.rs b/hive-c0re/src/forge/internal_meta.rs index 93e3193e6..f13859e23 100644 --- a/hive-c0re/src/forge/internal_meta.rs +++ b/hive-c0re/src/forge/internal_meta.rs @@ -20,20 +20,13 @@ const CONFIG_BRANCH: &str = "main"; /// [`MetaSource`] over the forge API. pub(crate) struct ForgeSource { api: Forgejo, - /// Whether the credential file nix's `git` reads holds this hive's token. - nix_credential: bool, } -/// A source reading as this hive, or `None` when it holds no forge token. It -/// also refreshes the credential file nix's `git` fetches with, which the -/// build after this read needs to be current. +/// A source reading as this hive, or `None` when it holds no forge token. pub(crate) async fn internal_meta_source() -> Option { let token: ReaderToken = super::reader_token().await?; match token.api() { - Ok(api) => Some(ForgeSource { - api, - nix_credential: token.publish_for_nix(), - }), + Ok(api) => Some(ForgeSource { api }), Err(e) => { tracing::warn!(error = ?e, "internal/meta: cannot build a forge client"); None @@ -72,10 +65,6 @@ impl ForgeSource { } impl MetaSource for ForgeSource { - fn nix_can_fetch(&self) -> bool { - self.nix_credential - } - async fn meta_head(&self) -> anyhow::Result> { use anyhow::Context as _; @@ -113,25 +102,3 @@ impl MetaSource for ForgeSource { self.branch_head(CONFIG_ORG, agent, CONFIG_BRANCH).await } } - -#[cfg(test)] -mod tests { - use super::*; - use forgejo_api::Auth; - - fn source(nix_credential: bool) -> ForgeSource { - let url = url::Url::parse("http://forge.test").expect("url"); - ForgeSource { - api: Forgejo::new(Auth::None, url).expect("client"), - nix_credential, - } - } - - /// The control (`true`) shows the answer follows the field rather than - /// being a constant. - #[test] - fn nix_fetches_only_where_its_credential_was_published() { - assert!(source(true).nix_can_fetch()); - assert!(!source(false).nix_can_fetch()); - } -} diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 10654171a..cbfce208e 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context as _, Result}; +use swarm_wire::agent_closures::AgentClosure; use hive_jobq::NodeId; @@ -34,7 +35,7 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT; /// The container-state calls the executors below make on the host: systemd, -/// `hive-priv`, the nix build, the `internal/meta` record. Production uses +/// `hive-priv`, the swarm's closure record, the `internal/meta` record. Production uses /// [`RealHost`], which forwards to `lifecycle` / `meta_rev` / `Coordinator` /// unchanged; tests substitute a recording fake. pub(super) trait Host: Sync { @@ -44,28 +45,19 @@ pub(super) trait Host: Sync { async fn kill(&self, name: &str) -> Result<()>; /// The `internal/meta` commit `MetaSync` recorded for the agent. fn recorded_rev(&self, name: &str) -> Result; - /// The store path of the closure the swarm built for the agent at the - /// `internal/meta` commit `rev` and pushed to its cache. The error says why - /// there is none. - async fn cached_closure(&self, name: &str, rev: &str) -> Result; + /// The record the swarm's builder wrote for the agent; `None` when it wrote + /// none. + async fn closure_record(&self, name: &str) -> Result>; /// Download `store_path` from the swarm's cache into the store, never /// building it. async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>; - /// Warm the agent's toplevel build from the `internal/meta` commit `rev`. - /// The toplevel it printed, when it printed one. - async fn prebuild_toplevel( - &self, - name: &str, - rev: &str, - node_id: u64, - ) -> Result>; - /// Re-apply drop-ins, then `nixos-container update` at `rev`. + /// Re-apply drop-ins, then point the container at the closure `toplevel`. async fn swap_update( &self, name: &str, hive: &HiveEnv, paths: &AgentPaths, - rev: &str, + toplevel: &str, node_id: u64, ) -> Result<()>; /// Write or remove the agent's pause marker. @@ -80,7 +72,8 @@ pub(super) trait Host: Sync { async fn refresh_agent_set(&self) -> Result<()>; /// Pin the `internal/meta` commit the agent builds from. async fn resolve_meta(&self, name: &str) -> Result; - /// Record `build` as what `Prebuild` and `Swap` build from. + /// Record `build` as the commit `Prebuild` and `Swap` expect the swarm's + /// closure to be built from. fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()>; /// The build `MetaSync` recorded, if any. fn recorded_build(&self, name: &str) -> Option; @@ -88,8 +81,8 @@ pub(super) trait Host: Sync { fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()>; /// First-spawn provisioning: repos, state subvolume, agent-set refresh. async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()>; - /// `nixos-container create` at `rev`. - async fn create_only(&self, name: &str, rev: &str) -> Result<()>; + /// `nixos-container create` from the closure `toplevel`. + async fn create_only(&self, name: &str, toplevel: &str) -> Result<()>; /// Write the nspawn and resource-limit drop-ins, then reload systemd. async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>; /// Start preamble, then start with the cold-start fallback. @@ -128,33 +121,23 @@ impl Host for RealHost { Ok(crate::meta_rev::recorded_or_err(name)?.rev) } - async fn cached_closure(&self, name: &str, rev: &str) -> Result { - crate::attic::closure::path_for(name, rev).await + async fn closure_record(&self, name: &str) -> Result> { + crate::attic::closure::record(name).await } async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> { crate::priv_client::realise_closure(name, store_path).await } - async fn prebuild_toplevel( - &self, - name: &str, - rev: &str, - node_id: u64, - ) -> Result> { - let flake_ref = crate::meta_rev::flake_ref(name, rev); - crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(node_id)).await - } - async fn swap_update( &self, name: &str, hive: &HiveEnv, paths: &AgentPaths, - rev: &str, + toplevel: &str, node_id: u64, ) -> Result<()> { - crate::lifecycle::swap_update(name, hive, paths, rev, Some(node_id)).await + crate::lifecycle::swap_update(name, hive, paths, toplevel, Some(node_id)).await } async fn set_paused(&self, agent: &hive_types::Ident, paused: bool) -> Result<()> { @@ -198,8 +181,8 @@ impl Host for RealHost { crate::lifecycle::provision_container(name, paths).await } - async fn create_only(&self, name: &str, rev: &str) -> Result<()> { - crate::lifecycle::create_only(name, rev).await + async fn create_only(&self, name: &str, toplevel: &str) -> Result<()> { + crate::lifecycle::create_only(name, toplevel).await } async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> { @@ -300,11 +283,11 @@ async fn dispatch( // growth executors return what to grow rather than taking the builder. let result = match kind { NodeKind::MetaSync { .. } => run_meta_sync(host, agent).await, - NodeKind::Prebuild { .. } => run_prebuild(host, agent, id).await, + NodeKind::Prebuild { .. } => run_prebuild(host, agent).await, NodeKind::Swap { .. } => run_swap(host, coord, agent, id).await, NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(host, coord, agent).await, NodeKind::Provision { .. } => run_provision(host, agent).await, - NodeKind::Create { .. } => run_create(host, agent, id).await, + NodeKind::Create { .. } => run_create(host, agent).await, NodeKind::DestroyContainer { .. } => run_destroy_container(host, coord, agent).await, NodeKind::PurgeState { .. } => { run_purge_state(host, agent).await; @@ -533,10 +516,11 @@ fn run_set_wanted(coord: &Arc, agent: &str, up: bool) -> Result<()> } /// The rebuild's preamble: runtime-dir prep, the agent-set refresh, and the -/// `internal/meta` commit `Prebuild` and `Swap` build from. +/// `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to +/// be built from. /// /// Fails when `internal/meta` does not pin the agent's current config yet: -/// there is no other flake to build it from. +/// there is no commit to deploy. async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> { // Runs while the agent is still up — the runtime dir and MCP listener // already exist. Use the pure path accessor; no need to re-register the @@ -550,98 +534,95 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> { } host.prepare_rebuild_dirs(name, &paths).await?; host.refresh_agent_set().await?; - // Resolved once here so `Prebuild` and `Swap` build the same flake. + // Resolved once here so `Prebuild` and `Swap` expect the same commit. let build = host.resolve_meta(name).await?; host.record_meta(name, &build)?; Ok(()) } -/// Out-of-band toplevel build while the container keeps serving: warm -/// `system.build.toplevel` so the later `Swap` hits cache and skips -/// straight to the profile-swap, from the `internal/meta` commit the upstream -/// `MetaSync` node recorded. The warm build is skipped when the -/// container is already down: its only purpose is to shrink the swap's -/// downtime window, so a stopped agent (no uptime to preserve) doesn't -/// pay the double eval — `Swap` builds inline instead. -/// -/// The swarm's cache is tried first: when the swarm built this agent at the -/// recorded commit and pushed the closure, it is downloaded. The local build -/// runs either way. After a download it is an evaluation and a no-op build when -/// this hive evaluates the path the swarm built, and builds it, with the agent -/// still up, when it doesn't; `Swap` never meets a build it didn't expect. -async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> { - // Warm the toplevel build only when the container is up — the whole - // point of prebuild is to shrink the swap's downtime window. A - // stopped agent has no uptime to preserve, so skip the (expensive) - // eval and let the downstream `Swap` build inline. +/// Download the swarm's closure for the agent while the container keeps +/// serving, so a rebuild with no closure fails here, before `StopForUpdate` +/// takes a healthy container down. `Swap` downloads the same closure again +/// (a no-op once it is in the store) and applies it. Skipped when the container +/// is already down: there is no uptime to protect, and `Swap` fails the same way. +async fn run_prebuild(host: &impl Host, name: &str) -> Result<()> { if host.is_running(name).await { let rev = host.recorded_rev(name)?; - let downloaded = match substitute(host, name, &rev).await { - Ok(path) => Some(path), - Err(fallback) => { - let reason = &fallback.reason; - if fallback.signature_refused { - tracing::error!(%name, %rev, "prebuild: built locally: {reason}"); - } else { - tracing::warn!(%name, %rev, "prebuild: built locally: {reason}"); - } - None - } - }; - let evaluated = host.prebuild_toplevel(name, &rev, id.get()).await?; - if let Some(path) = downloaded { - match drift(&path, evaluated.as_deref()) { - None => tracing::info!(%name, %rev, "prebuild: substituted {path}"), - Some(note) => tracing::warn!(%name, %rev, "prebuild: {note}"), - } - } + let path = closure_for(host, name, &rev, "prebuild").await?; + tracing::info!(%name, %rev, "prebuild: substituted {path}"); } Ok(()) } -/// What to log when the closure downloaded for a rebuild isn't the one this -/// hive's evaluation produced, which means the builder and the hive disagree -/// about the agent's flake. `None` when they agree. -fn drift(downloaded: &str, evaluated: Option<&str>) -> Option { - match evaluated { - Some(path) if path == downloaded => None, - Some(path) => Some(format!( - "the downloaded closure {downloaded} is not the one this hive evaluates, {path}; built the latter locally" - )), - None => Some(format!( - "downloaded {downloaded}, but the local build printed no path to compare it with" - )), - } +/// Why the swarm's closure for an agent cannot be deployed. +enum NoClosure { + /// The builder has recorded no build of the agent. + NoRecord, + /// A build is recorded, but its closure cannot be had: the record could not + /// be read, the builder did not push the closure, or the download failed. + Unpullable(String), } -/// Why a rebuild or create builds instead of downloading the swarm's closure. -struct LocalBuild { - reason: String, - /// The cache served a path no trusted key signed. - signature_refused: bool, -} - -/// Download the closure the swarm built for `name` at `rev`; its store path. -async fn substitute(host: &impl Host, name: &str, rev: &str) -> Result { - let path = host - .cached_closure(name, rev) +/// The store path of the closure the swarm built for `name`, downloaded into +/// the store. The deploy applies the swarm's closure whatever commit it was +/// built from: when that is not the `rev` this hive chose, it is logged. +/// +/// # Errors +/// [`without_closure`]'s refusal, after logging it. +async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result { + pull_closure(host, name, rev, step) .await - .map_err(|e| LocalBuild { - reason: format!("{e:#}"), - signature_refused: false, - })?; - host.realise_closure(name, &path).await.map_err(|e| { - let reason = format!("{e:#}"); - LocalBuild { - signature_refused: reason.contains(hive_priv_sock::SIGNATURE_REFUSED), - reason, - } - })?; - Ok(path) + .map_err(|why| without_closure(name, rev, step, &why)) } -/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), -/// `nixos-container update`, then the post-rebuild bookkeeping tail +async fn pull_closure( + host: &impl Host, + name: &str, + rev: &str, + step: &str, +) -> Result { + let record = host + .closure_record(name) + .await + .map_err(|e| NoClosure::Unpullable(format!("{e:#}")))? + .ok_or(NoClosure::NoRecord)?; + if !record.cached { + return Err(NoClosure::Unpullable( + "the builder did not push the closure".to_owned(), + )); + } + host.realise_closure(name, &record.store_path) + .await + .map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?; + if record.meta_rev != rev { + tracing::warn!( + %name, %rev, built_from = %record.meta_rev, + "{step}: the swarm built {} from internal/meta {}, not the {rev} this hive chose; deploying the swarm's", + record.store_path, record.meta_rev + ); + } + Ok(record.store_path) +} + +/// What a rebuild or create does when there is no closure to deploy: it +/// fails, and nothing changes, so a running container keeps running as it is +/// and a create makes none. The failure is logged at error level with the +/// agent's name. +/// +/// This is the one place that decides what an unpullable closure (the cache +/// unreachable, a TLS error, a failed download) costs. A rebuild never builds +/// locally on the hive. +fn without_closure(name: &str, rev: &str, step: &str, why: &NoClosure) -> anyhow::Error { + let reason = match why { + NoClosure::NoRecord => "the swarm has recorded no build of this agent".to_owned(), + NoClosure::Unpullable(e) => format!("the swarm's closure cannot be pulled: {e}"), + }; + tracing::error!(%name, %rev, "{step}: {reason}; nothing was changed"); + anyhow::anyhow!("{name}: {reason}") +} + +/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), point the +/// container at the swarm's closure, then the post-rebuild bookkeeping tail /// (rev marker, `Rebuilt` event, forge sync, kick, rescan). /// The recovery-start on failure is NOT here — the DAG's tail /// `Reconcile` runs after this node terminal ok *or* fail. @@ -657,7 +638,11 @@ async fn run_swap( let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir)?; let rev = host.recorded_rev(name)?; - let result = host.swap_update(name, &hive, &paths, &rev, id.get()).await; + let toplevel = closure_for(host, name, &rev, "swap").await?; + tracing::info!(%name, %rev, "swap: applying {toplevel}"); + let result = host + .swap_update(name, &hive, &paths, &toplevel, id.get()) + .await; // On success the Ok-only bookkeeping tail (rev marker, forge // sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node, // which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded @@ -719,30 +704,13 @@ async fn run_provision(host: &impl Host, name: &str) -> Result<()> { /// node stays purely "create", not "create + start". Marks the agent as /// running the pins it was created from. /// -/// The swarm's cache is tried first, as in [`run_prebuild`]: when it holds the -/// closure for the resolved commit it is downloaded, and the local evaluation -/// that follows is a no-op build on a path match and builds the evaluated path -/// on a mismatch. Without a cached closure the create builds it locally. -async fn run_create(host: &impl Host, name: &str, id: NodeId) -> Result<()> { +/// The container is created from the swarm's closure for the resolved commit; +/// with none to deploy the create fails and makes no container. +async fn run_create(host: &impl Host, name: &str) -> Result<()> { let build = host.resolve_meta(name).await?; - match substitute(host, name, &build.rev).await { - Ok(path) => { - let evaluated = host.prebuild_toplevel(name, &build.rev, id.get()).await?; - match drift(&path, evaluated.as_deref()) { - None => tracing::info!(%name, rev = %build.rev, "create: substituted {path}"), - Some(note) => tracing::warn!(%name, rev = %build.rev, "create: {note}"), - } - } - Err(fallback) => { - let reason = &fallback.reason; - if fallback.signature_refused { - tracing::error!(%name, rev = %build.rev, "create: built locally: {reason}"); - } else { - tracing::warn!(%name, rev = %build.rev, "create: built locally: {reason}"); - } - } - } - host.create_only(name, &build.rev).await?; + let toplevel = closure_for(host, name, &build.rev, "create").await?; + tracing::info!(%name, rev = %build.rev, "create: substituted {toplevel}"); + host.create_only(name, &toplevel).await?; host.mark_deployed(name, &build)?; Ok(()) } diff --git a/hive-c0re/src/job_queue/exec/tests.rs b/hive-c0re/src/job_queue/exec/tests.rs index 90fb95664..e1b6154d5 100644 --- a/hive-c0re/src/job_queue/exec/tests.rs +++ b/hive-c0re/src/job_queue/exec/tests.rs @@ -11,16 +11,17 @@ use hive_jobq::NodeId; use hive_jobq::scheduler::Scheduler; use super::{ - Host, dispatch, drift, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain, + Host, dispatch, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain, run_emit_rebuilt, run_forge_sweep, run_knowledge_pull, run_meta_sync, run_pause_drain, run_pause_signal, run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping, run_reconcile, run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap, - run_wanted_pull, run_write_dropin, substitute, + run_wanted_pull, run_write_dropin, }; use crate::coordinator::{AgentPaths, Coordinator, HiveEnv}; use crate::job_queue::model::NodeKind; use crate::meta_rev::MetaBuild; use crate::power::Wanted; +use swarm_wire::agent_closures::AgentClosure; const REV: &str = "2222222222222222222222222222222222222222"; const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-alice"; @@ -30,18 +31,12 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system- struct FakeHost { running: bool, rev: Option, - /// The store path the swarm's record names, when it has a pushed build of - /// `REV`; `None` reads as no record. - closure: Option, - /// A failing `realise` says nix refused the path for want of a signature. - refusal: bool, - /// The toplevel this hive's evaluation produces; `None` is `CLOSURE`. - evaluates: Option, + /// What the swarm's builder recorded for the agent; `None` reads as no + /// record. + built: Option, /// Store paths a download put in the store. realised: Mutex>, - /// Store paths a prebuild had to build because the store lacked them. - builds: Mutex>, - /// Names of the calls that fail (`kill`, `realise`, `prebuild`, `swap_update`, + /// Names of the calls that fail (`kill`, `realise`, `closure_record`, `swap_update`, /// `set_paused`, ...). `recorded_build` fails by finding nothing. failing: Vec<&'static str>, /// The sweeps and pulls never finish, so only a deadline ends them. @@ -56,6 +51,7 @@ impl FakeHost { Self { running: true, rev: Some(REV.to_owned()), + built: Some(closure(REV, true)), ..Self::default() } } @@ -64,12 +60,19 @@ impl FakeHost { Self { running: false, rev: Some(REV.to_owned()), + built: Some(closure(REV, true)), ..Self::default() } } - fn with_closure(mut self) -> Self { - self.closure = Some(CLOSURE.to_owned()); + /// The builder recorded `meta_rev` for the agent, pushed or not. + fn built(mut self, meta_rev: &str, cached: bool) -> Self { + self.built = Some(closure(meta_rev, cached)); + self + } + + fn without_record(mut self) -> Self { + self.built = None; self } @@ -98,10 +101,6 @@ impl FakeHost { fn calls(&self) -> Vec { self.calls.lock().unwrap().clone() } - - fn builds(&self) -> Vec { - self.builds.lock().unwrap().clone() - } } impl Host for FakeHost { @@ -125,54 +124,29 @@ impl Host for FakeHost { .ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded")) } - async fn cached_closure(&self, name: &str, rev: &str) -> anyhow::Result { - self.record(format!("cached_closure {name} {rev}")); - self.closure - .clone() - .ok_or_else(|| anyhow::anyhow!("no-record: no build is recorded for the agent")) + async fn closure_record(&self, name: &str) -> anyhow::Result> { + self.step("closure_record", format!("closure_record {name}"))?; + Ok(self.built.clone()) } async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> { self.record(format!("realise {name} {store_path}")); if self.fails("realise") { - if self.refusal { - anyhow::bail!( - "not available: {}: lacks a signature", - hive_priv_sock::SIGNATURE_REFUSED - ); - } anyhow::bail!("not available: Couldn't connect to server"); } self.realised.lock().unwrap().push(store_path.to_owned()); Ok(()) } - async fn prebuild_toplevel( - &self, - name: &str, - rev: &str, - node_id: u64, - ) -> anyhow::Result> { - self.record(format!("prebuild {name} {rev} node={node_id}")); - if self.fails("prebuild") { - anyhow::bail!("prebuild failed"); - } - let path = self.evaluates.clone().unwrap_or_else(|| CLOSURE.to_owned()); - if !self.realised.lock().unwrap().contains(&path) { - self.builds.lock().unwrap().push(path.clone()); - } - Ok(Some(path)) - } - async fn swap_update( &self, name: &str, _hive: &HiveEnv, _paths: &AgentPaths, - rev: &str, + toplevel: &str, node_id: u64, ) -> anyhow::Result<()> { - self.record(format!("swap_update {name} {rev} node={node_id}")); + self.record(format!("swap_update {name} {toplevel} node={node_id}")); if self.fails("swap_update") { anyhow::bail!("nixos-container update failed"); } @@ -231,8 +205,8 @@ impl Host for FakeHost { self.step("provision", format!("provision {name}")) } - async fn create_only(&self, name: &str, rev: &str) -> anyhow::Result<()> { - self.step("create_only", format!("create_only {name} {rev}")) + async fn create_only(&self, name: &str, toplevel: &str) -> anyhow::Result<()> { + self.step("create_only", format!("create_only {name} {toplevel}")) } async fn write_dropins( @@ -287,6 +261,17 @@ impl Host for FakeHost { } } +/// The builder's record of a closure built from `internal/meta` at `meta_rev`. +fn closure(meta_rev: &str, cached: bool) -> AgentClosure { + AgentClosure { + meta_rev: meta_rev.to_owned(), + store_path: CLOSURE.to_owned(), + system: "x86_64-linux".to_owned(), + built_unix: 1, + cached, + } +} + fn build() -> MetaBuild { MetaBuild { rev: REV.to_owned(), @@ -323,188 +308,185 @@ fn node_id() -> NodeId { // --- Prebuild --------------------------------------------------------------- +const OTHER_REV: &str = "3333333333333333333333333333333333333333"; + +/// What a block logged, level and text: the message and each field. +#[derive(Clone, Default)] +struct Logged(Arc>>); + +impl tracing::Subscriber for Logged { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + use std::fmt::Write as _; + + let mut text = String::new(); + event.record( + &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| { + let _ = write!(text, "{}={value:?} ", field.name()); + }, + ); + self.0 + .lock() + .unwrap() + .push((*event.metadata().level(), text)); + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + +impl Logged { + fn at(&self, level: tracing::Level) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .filter(|(l, _)| *l == level) + .map(|(_, text)| text.clone()) + .collect() + } +} + +/// `fut`'s output and what it logged. The test runtime is single-threaded, so +/// the subscriber set here sees every event the future emits. +async fn logging(fut: impl std::future::Future) -> (T, Logged) { + let logged = Logged::default(); + let _guard = tracing::subscriber::set_default(logged.clone()); + let out = fut.await; + (out, logged) +} + +/// Every way the swarm's closure for `alice` cannot be had, with the text the +/// error log must carry and the host calls that must not happen. +fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] { + [ + ("no record", FakeHost::up().without_record(), "no build"), + ( + "not pushed", + FakeHost::up().built(REV, false), + "did not push", + ), + ( + "download fails", + FakeHost::up().failing("realise"), + "Couldn't connect", + ), + ( + "record unreadable", + FakeHost::up().failing("closure_record"), + "closure_record failed", + ), + ] +} + #[tokio::test] -async fn prebuild_warms_the_recorded_rev_when_the_container_is_up() { +async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() { let host = FakeHost::up(); - let id = node_id(); - run_prebuild(&host, "alice", id).await.expect("prebuild ok"); + let (result, log) = logging(run_prebuild(&host, "alice")).await; + result.expect("prebuild ok"); assert_eq!( host.calls(), [ "is_running alice".to_owned(), "recorded_rev alice".to_owned(), - format!("cached_closure alice {REV}"), - format!("prebuild alice {REV} node={}", id.get()), - ] - ); -} - -/// The download is followed by the local build either way. When this hive -/// evaluates the path the swarm built, the build finds it in the store. -#[tokio::test] -async fn a_downloaded_closure_the_hive_also_evaluates_is_found_not_built() { - let host = FakeHost::up().with_closure(); - let id = node_id(); - run_prebuild(&host, "alice", id).await.expect("prebuild ok"); - assert_eq!( - host.calls(), - [ - "is_running alice".to_owned(), - "recorded_rev alice".to_owned(), - format!("cached_closure alice {REV}"), + "closure_record alice".to_owned(), format!("realise alice {CLOSURE}"), - format!("prebuild alice {REV} node={}", id.get()), ] ); - assert_eq!(host.builds(), Vec::::new()); -} - -/// When it evaluates another path, the build happens in `Prebuild`, after the -/// check that the agent is up and before `Swap` could stop it. -#[tokio::test] -async fn a_downloaded_closure_the_hive_does_not_evaluate_is_built_before_the_swap() { - let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice"; - let host = FakeHost { - evaluates: Some(other.to_owned()), - ..FakeHost::up().with_closure() - }; - let id = node_id(); - run_prebuild(&host, "alice", id).await.expect("prebuild ok"); - let calls = host.calls(); - assert_eq!(calls[0], "is_running alice", "{calls:?}"); - assert_eq!( - calls.last().map(String::as_str), - Some(format!("prebuild alice {REV} node={}", id.get()).as_str()), - "{calls:?}" - ); - assert_eq!(host.builds(), [other.to_owned()]); -} - -#[test] -fn only_a_different_path_is_drift() { - assert_eq!(drift(CLOSURE, Some(CLOSURE)), None); - let note = drift(CLOSURE, Some("/nix/store/other")).expect("drift"); + let info = log.at(tracing::Level::INFO); assert!( - note.contains(CLOSURE) && note.contains("/nix/store/other"), - "{note}" + info.iter() + .any(|l| l.contains("prebuild: substituted") && l.contains(CLOSURE)), + "{info:?}" ); - assert!(drift(CLOSURE, None).expect("no path").contains(CLOSURE)); + assert!(log.at(tracing::Level::WARN).is_empty()); + assert!(log.at(tracing::Level::ERROR).is_empty()); } +/// No closure to deploy fails the node before `StopForUpdate`, so the running +/// container is untouched, and the log names the agent and why. #[tokio::test] -async fn a_failed_download_builds_locally() { - let host = FakeHost::up().with_closure().failing("realise"); - let id = node_id(); - run_prebuild(&host, "alice", id) - .await - .expect("the build covers it"); - let calls = host.calls(); - assert_eq!(calls[3], format!("realise alice {CLOSURE}"), "{calls:?}"); - assert_eq!( - calls[4], - format!("prebuild alice {REV} node={}", id.get()), - "{calls:?}" - ); - assert_eq!(calls.len(), 5, "{calls:?}"); +async fn a_rebuild_without_a_closure_fails_the_prebuild_and_logs_an_error() { + for (case, host, why) in without_a_closure() { + let (result, log) = logging(run_prebuild(&host, "alice")).await; + let err = result.expect_err(case); + assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}"); + let errors = log.at(tracing::Level::ERROR); + assert_eq!(errors.len(), 1, "{case}: {errors:?}"); + assert!( + errors[0].contains("alice") && errors[0].contains(why), + "{case}: {errors:?}" + ); + let calls = host.calls(); + assert!( + !calls.iter().any(|c| c.starts_with("kill")), + "{case}: {calls:?}" + ); + assert!( + !calls + .iter() + .any(|c| c.starts_with("swap_update") || c.starts_with("create_only")), + "{case}: {calls:?}" + ); + } } -/// The fallback is the build, so a build that fails still fails the node. +/// The builder's record is not for the commit this hive chose: its closure is +/// deployed and the difference is a warning. #[tokio::test] -async fn a_failed_download_and_a_failed_build_fail_the_node() { - let host = FakeHost::up() - .with_closure() - .failing("realise") - .failing("prebuild"); - let err = run_prebuild(&host, "alice", node_id()) - .await - .expect_err("nothing produced the closure"); - assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}"); +async fn a_closure_built_from_another_commit_is_deployed_with_a_warning() { + let host = FakeHost::up().built(OTHER_REV, true); + let (result, log) = logging(run_prebuild(&host, "alice")).await; + result.expect("drift is not a refusal"); + assert!( + host.calls().contains(&format!("realise alice {CLOSURE}")), + "{:?}", + host.calls() + ); + let warnings = log.at(tracing::Level::WARN); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!( + [REV, OTHER_REV, CLOSURE] + .iter() + .all(|part| warnings[0].contains(part)), + "{warnings:?}" + ); + assert!(log.at(tracing::Level::ERROR).is_empty()); } #[tokio::test] async fn a_stopped_container_looks_up_nothing() { - let host = FakeHost::down().with_closure(); - run_prebuild(&host, "alice", node_id()) - .await - .expect("skipped"); - assert_eq!(host.calls(), ["is_running alice"]); -} - -/// The error log is for a path the cache served unsigned, not for a cache that -/// is down or a missing record. -#[tokio::test] -async fn only_a_signature_refusal_is_flagged() { - let down = substitute( - &FakeHost::up().with_closure().failing("realise"), - "alice", - REV, - ) - .await - .expect_err("down"); - assert!(!down.signature_refused, "{}", down.reason); - assert!(down.reason.contains("Couldn't connect"), "{}", down.reason); - - let no_record = substitute(&FakeHost::up(), "alice", REV) - .await - .expect_err("no record"); - assert!(!no_record.signature_refused); - assert!( - no_record.reason.starts_with("no-record"), - "{}", - no_record.reason - ); - - let refused = FakeHost { - refusal: true, - ..FakeHost::up().with_closure().failing("realise") - }; - let refused = substitute(&refused, "alice", REV) - .await - .expect_err("refused"); - assert!(refused.signature_refused, "{}", refused.reason); - - let ok = substitute(&FakeHost::up().with_closure(), "alice", REV).await; - assert_eq!(ok.ok().as_deref(), Some(CLOSURE)); -} - -#[tokio::test] -async fn prebuild_is_skipped_when_the_container_is_down() { let host = FakeHost::down(); - run_prebuild(&host, "alice", node_id()) - .await - .expect("a skipped prebuild is ok"); + run_prebuild(&host, "alice").await.expect("skipped"); assert_eq!(host.calls(), ["is_running alice"]); } #[tokio::test] -async fn prebuild_without_a_recorded_rev_fails_before_building() { +async fn prebuild_without_a_recorded_rev_fails_before_looking_up_a_closure() { let host = FakeHost { rev: None, ..FakeHost::up() }; - let err = run_prebuild(&host, "alice", node_id()) + let err = run_prebuild(&host, "alice") .await - .expect_err("nothing to build from"); + .expect_err("nothing to deploy"); assert!( format!("{err:#}").contains("no internal/meta commit"), "{err:#}" ); - assert!(!host.calls().iter().any(|c| c.starts_with("prebuild"))); -} - -#[tokio::test] -async fn prebuild_failure_fails_the_node() { - let host = FakeHost::up().failing("prebuild"); - let err = run_prebuild(&host, "alice", node_id()) - .await - .expect_err("build failure propagates"); - assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}"); + assert!(!host.calls().iter().any(|c| c.starts_with("closure_record"))); } // --- Swap ------------------------------------------------------------------- #[tokio::test] -async fn swap_updates_at_the_recorded_rev() { +async fn swap_applies_the_swarms_closure() { let (_dir, coord) = coordinator(); let host = FakeHost::down(); let id = node_id(); @@ -513,11 +495,50 @@ async fn swap_updates_at_the_recorded_rev() { host.calls(), [ "recorded_rev alice".to_owned(), - format!("swap_update alice {REV} node={}", id.get()), + "closure_record alice".to_owned(), + format!("realise alice {CLOSURE}"), + format!("swap_update alice {CLOSURE} node={}", id.get()), ] ); } +/// A swap with no closure to deploy never updates: the container stays as it +/// was, and the log names the agent and why. +#[tokio::test] +async fn a_swap_without_a_closure_never_updates_and_logs_an_error() { + for (case, host, why) in without_a_closure() { + let (_dir, coord) = coordinator(); + let (result, log) = logging(run_swap(&host, &coord, "alice", node_id())).await; + result.expect_err(case); + let errors = log.at(tracing::Level::ERROR); + assert_eq!(errors.len(), 1, "{case}: {errors:?}"); + assert!( + errors[0].contains("alice") && errors[0].contains(why), + "{case}: {errors:?}" + ); + assert!( + !host.calls().iter().any(|c| c.starts_with("swap_update")), + "{case}: {:?}", + host.calls() + ); + } +} + +#[tokio::test] +async fn swap_applies_a_closure_built_from_another_commit_with_a_warning() { + let (_dir, coord) = coordinator(); + let host = FakeHost::down().built(OTHER_REV, true); + let id = node_id(); + let (result, log) = logging(run_swap(&host, &coord, "alice", id)).await; + result.expect("drift is not a refusal"); + assert_eq!( + host.calls().last(), + Some(&format!("swap_update alice {CLOSURE} node={}", id.get())) + ); + assert_eq!(log.at(tracing::Level::WARN).len(), 1); + assert!(log.at(tracing::Level::ERROR).is_empty()); +} + /// A failed update fails the node and nothing else: the swap never restarts or /// stops the container itself — recovery is the DAG's tail `Reconcile`. #[tokio::test] @@ -529,8 +550,8 @@ async fn a_failed_swap_reports_the_error_and_touches_no_container() { .expect_err("update failure propagates"); assert!(format!("{err:#}").contains("update failed"), "{err:#}"); let calls = host.calls(); - assert_eq!(calls.len(), 2, "{calls:?}"); - assert!(calls[1].starts_with("swap_update alice"), "{calls:?}"); + assert_eq!(calls.len(), 4, "{calls:?}"); + assert!(calls[3].starts_with("swap_update alice"), "{calls:?}"); } #[tokio::test] @@ -824,84 +845,73 @@ async fn provision_and_write_dropin_forward_and_propagate_failure() { } #[tokio::test] -async fn create_without_a_record_builds_locally_at_the_resolved_rev() { +async fn create_downloads_the_swarms_closure_and_creates_from_it() { let host = FakeHost::up(); - run_create(&host, "alice", node_id()).await.expect("ok"); + let (result, log) = logging(run_create(&host, "alice")).await; + result.expect("ok"); assert_eq!( host.calls(), [ "resolve_meta alice".to_owned(), - format!("cached_closure alice {REV}"), - format!("create_only alice {REV}"), - format!("mark_deployed alice {REV}"), - ] - ); -} - -#[tokio::test] -async fn create_downloads_a_cached_closure_and_builds_nothing() { - let host = FakeHost::up().with_closure(); - let id = node_id(); - run_create(&host, "alice", id).await.expect("ok"); - assert_eq!( - host.calls(), - [ - "resolve_meta alice".to_owned(), - format!("cached_closure alice {REV}"), + "closure_record alice".to_owned(), format!("realise alice {CLOSURE}"), - format!("prebuild alice {REV} node={}", id.get()), - format!("create_only alice {REV}"), + format!("create_only alice {CLOSURE}"), format!("mark_deployed alice {REV}"), ] ); - assert_eq!(host.builds(), Vec::::new()); -} - -#[tokio::test] -async fn create_builds_locally_when_the_download_fails() { - let host = FakeHost::up().with_closure().failing("realise"); - run_create(&host, "alice", node_id()).await.expect("ok"); - assert_eq!( - host.calls(), - [ - "resolve_meta alice".to_owned(), - format!("cached_closure alice {REV}"), - format!("realise alice {CLOSURE}"), - format!("create_only alice {REV}"), - format!("mark_deployed alice {REV}"), - ] + assert!( + log.at(tracing::Level::INFO) + .iter() + .any(|l| l.contains("create: substituted") && l.contains(CLOSURE)) ); + assert!(log.at(tracing::Level::WARN).is_empty()); +} + +/// A create with no closure to deploy is refused: no container is made, none is +/// marked deployed, and the log names the agent and why. +#[tokio::test] +async fn a_create_without_a_closure_is_refused_and_logs_an_error() { + for (case, host, why) in without_a_closure() { + let (result, log) = logging(run_create(&host, "alice")).await; + let err = result.expect_err(case); + assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}"); + let errors = log.at(tracing::Level::ERROR); + assert_eq!(errors.len(), 1, "{case}: {errors:?}"); + assert!( + errors[0].contains("alice") && errors[0].contains(why), + "{case}: {errors:?}" + ); + let calls = host.calls(); + assert!( + !calls + .iter() + .any(|c| c.starts_with("create_only") || c.starts_with("mark_deployed")), + "{case}: {calls:?}" + ); + } } #[tokio::test] -async fn create_builds_the_evaluated_path_when_it_differs_from_the_download() { - let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice"; - let host = FakeHost { - evaluates: Some(other.to_owned()), - ..FakeHost::up().with_closure() - }; - run_create(&host, "alice", node_id()).await.expect("ok"); - assert_eq!(host.builds(), [other.to_owned()]); +async fn create_uses_a_closure_built_from_another_commit_with_a_warning() { + let host = FakeHost::up().built(OTHER_REV, true); + let (result, log) = logging(run_create(&host, "alice")).await; + result.expect("drift is not a refusal"); let calls = host.calls(); assert!( - calls - .iter() - .any(|c| c == &format!("create_only alice {REV}")), + calls.contains(&format!("create_only alice {CLOSURE}")), "{calls:?}" ); + assert_eq!(log.at(tracing::Level::WARN).len(), 1); + assert!(log.at(tracing::Level::ERROR).is_empty()); } #[tokio::test] async fn a_failed_create_is_never_marked_deployed() { let host = FakeHost::up().failing("create_only"); - run_create(&host, "alice", node_id()) - .await - .expect_err("create fails"); - assert_eq!(host.calls().len(), 3); + run_create(&host, "alice").await.expect_err("create fails"); + assert_eq!(host.calls().len(), 4); let host = FakeHost::up().failing("resolve_meta"); - run_create(&host, "alice", node_id()) - .await - .expect_err("resolve fails"); + run_create(&host, "alice").await.expect_err("resolve fails"); assert_eq!(host.calls(), ["resolve_meta alice"]); } @@ -1165,11 +1175,19 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() { ), ( alice(|agent| NodeKind::Prebuild { agent }), - &["is_running alice", "recorded_rev alice", "prebuild alice"], + &[ + "is_running alice", + "recorded_rev alice", + "closure_record alice", + ], ), ( alice(|agent| NodeKind::Swap { agent }), - &["recorded_rev alice", "swap_update alice"], + &[ + "recorded_rev alice", + "closure_record alice", + "swap_update alice", + ], ), ( alice(|agent| NodeKind::RebuildBookkeeping { agent }), @@ -1183,6 +1201,7 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() { alice(|agent| NodeKind::Create { agent }), &[ "resolve_meta alice", + "closure_record alice", "create_only alice", "mark_deployed alice", ], diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 69a9cead0..8eb0fe0dd 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -32,9 +32,9 @@ pub enum NodeKind { /// The rebuild's preamble: runtime-dir prep and the `internal/meta` /// commit the build reads. MetaSync { agent: String }, - /// Out-of-band toplevel build while the container keeps serving. + /// Download of the swarm's closure while the container keeps serving. Prebuild { agent: String }, - /// `nixos-container update` profile-swap. + /// Profile-swap onto the swarm's closure. Swap { agent: String }, /// The post-`Swap` bookkeeping tail as a first-class node. RebuildBookkeeping { agent: String }, diff --git a/hive-c0re/src/job_queue/templates.rs b/hive-c0re/src/job_queue/templates.rs index f1817bbe0..7e4bef860 100644 --- a/hive-c0re/src/job_queue/templates.rs +++ b/hive-c0re/src/job_queue/templates.rs @@ -190,7 +190,7 @@ fn rebuild_subtree<'a>( .needs(Resource::Agent(a())) .after_ok(meta_sync); - // Siblings under the brace: the build and the quiesce chain run + // Siblings under the brace: the download and the quiesce chain run // concurrently, borrowing the brace's grants rather than declaring their // own. Declaring the lease on both would make them mutually exclusive — // it is single-unit — which is exactly what this shape exists to avoid. @@ -200,8 +200,8 @@ fn rebuild_subtree<'a>( let drain = graceful.then(|| quiesce(builder, agent, agent_window)); - // The container goes down here, not earlier: `AfterOk` the build so a - // failed build never stops a healthy container, and `AfterOk` the drain so + // The container goes down here, not earlier: `AfterOk` the download so a + // missing closure never stops a healthy container, and `AfterOk` the drain so // the agent has checkpointed. let mut stop_for_update = builder .node(NodeKind::StopForUpdate { agent: a() }) diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index beb680609..15633a7cc 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -234,9 +234,9 @@ pub async fn provision_container(name: &str, paths: &AgentPaths) -> Result<()> { /// The `nixos-container create` proper — the job queue's `Create` node. /// Assumes `provision_container` already registered the agent. -/// `meta_rev` is the `internal/meta` commit to build from (`crate::meta_rev`). -pub async fn create_only(name: &str, meta_rev: &str) -> Result<()> { - priv_run_inner("create", name, Some(meta_rev), None).await +/// `toplevel` is the store path of the system closure to create it from. +pub async fn create_only(name: &str, toplevel: &str) -> Result<()> { + priv_run_inner("create", name, Some(toplevel), None).await } /// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on @@ -285,20 +285,20 @@ pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> } /// Profile-swap for an existing, stopped container: re-apply the -/// drop-ins, then `nixos-container update`. The job queue's `Swap` -/// node. Requires the container stopped (the queue's `StopForUpdate` -/// upstream); does NOT start it — the DAG's tail `Reconcile` owns -/// bringing the agent back to its wanted power state. `meta_rev` is as for +/// drop-ins, then point the container at the closure `toplevel`. The job +/// queue's `Swap` node. Requires the container stopped (the queue's +/// `StopForUpdate` upstream); does NOT start it — the DAG's tail `Reconcile` owns +/// bringing the agent back to its wanted power state. `toplevel` is as for /// [`create_only`]. pub async fn swap_update( name: &str, hive: &HiveEnv, paths: &AgentPaths, - meta_rev: &str, + toplevel: &str, node_id: Option, ) -> Result<()> { write_dropins(name, hive, paths).await?; - priv_run_inner("update", name, Some(meta_rev), node_id).await + priv_run_inner("update", name, Some(toplevel), node_id).await } /// Build the `AgentSpec` list for [`crate::agent_set::refresh`] from @@ -720,195 +720,6 @@ fn confirm_gone_after_failed_destroy( } } -/// Wall-clock bound on [`prebuild_toplevel`]'s `nix build`. Four times CI's -/// observed cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`), -/// which compiles this workspace's crates, the part of an agent toplevel no -/// binary cache holds. A stalled download is bounded by nix's own -/// `stalled-download-timeout`; this covers the rest, such as a wedged -/// nix-daemon holding the rebuild's build slot forever. -const PREBUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1); - -/// Wait for `child` until `limit`. `None` means it was still running, and its -/// whole process group was `SIGKILL`ed and the child reaped, so a grandchild -/// holding the daemon connection dies with it. The child must have been -/// spawned with `process_group(0)`. -async fn wait_bounded( - child: &mut tokio::process::Child, - limit: std::time::Duration, -) -> std::io::Result> { - // Group leader, so its pid is the pgid. Read before waiting: a reaped - // child has no id. - let pgid = child.id().and_then(|pid| i32::try_from(pid).ok()); - if let Ok(status) = tokio::time::timeout(limit, child.wait()).await { - return status.map(Some); - } - if let Some(pgid) = pgid { - // SAFETY: plain `kill(2)`; `-pgid` targets the child's own process - // group, which `process_group(0)` made it lead. - unsafe { - libc::kill(-pgid, libc::SIGKILL); - } - } - child.wait().await?; - Ok(None) -} - -/// Log and record a prebuild's stdout; the last non-empty line is the -/// toplevel `nix build --print-out-paths` printed. -async fn pump_prebuild_stdout( - stdout: tokio::process::ChildStdout, - cmdline: String, - logs: Option>, - log_id: Option, -) -> Option { - use tokio::io::{AsyncBufReadExt, BufReader}; - let mut lines = BufReader::new(stdout).lines(); - let mut out_path = None; - while let Ok(Some(line)) = lines.next_line().await { - tracing::info!(target: "nix-prebuild", cmdline = %cmdline, "{line}"); - if !line.trim().is_empty() { - out_path = Some(line.trim().to_owned()); - } - if let (Some(h), Some(id)) = (&logs, log_id) { - h.append_stdout(id, &line); - } - } - out_path -} - -/// Pre-build `system.build.toplevel` against `meta#` so the -/// subsequent `nixos-container update` finds the result cached and -/// skips straight to the profile-swap. Store-warming only — container -/// is untouched. See `docs/scheduler/coordinator.md::Rebuild path` for why -/// the prebuild happens before stop, and `docs/scheduler/coordinator.md::Prebuild -/// attr path` for why the explicit nixosConfigurations attr is required. -/// -/// `node_id` is the queue node this build belongs to, when there is one — -/// it is stored on the `build_logs` row so the dashboard can find the log -/// from the node. Pass `None` for builds that run outside the queue. -/// -/// Returns the toplevel `nix build` printed, `None` if it printed nothing. -pub async fn prebuild_toplevel( - name: &str, - flake_ref: &str, - node_id: Option, -) -> Result> { - use tokio::io::{AsyncBufReadExt, BufReader}; - // Split `#` so we can re-emit with the explicit - // `nixosConfigurations.` segment. The flake_ref shape is - // constructed by the caller and always contains exactly one - // `#`; `split_once` returning None here would be a programmer - // error we'd want to surface loudly rather than paper over. - let (flake_root, fragment) = flake_ref - .split_once('#') - .with_context(|| format!("flake_ref {flake_ref:?} missing '#' fragment"))?; - // Sanity-check the fragment matches the agent name we were - // passed — guards against future calls that pass a divergent - // pair (no current callsite does, but the pair is redundant - // and worth checking once). - if fragment != name { - anyhow::bail!("prebuild_toplevel: flake_ref fragment '{fragment}' ≠ agent name '{name}'"); - } - let attr = format!("{flake_root}#nixosConfigurations.{name}.config.system.build.toplevel"); - let args = vec![ - "--extra-experimental-features", - "nix-command flakes", - "build", - "--no-link", - "--print-out-paths", - &attr, - ]; - let cmdline = format!("nix {}", args.join(" ")); - tracing::info!(%name, %cmdline, "prebuild: warming system toplevel"); - - // Open a build_logs row for this attempt (best-effort — None when - // the global handle hasn't been installed, e.g. early startup - // or standalone tests). Lines pumped from stdout/stderr append - // into the row; `finish` lands the terminal status before we bail. - let logs = crate::build_logs::global(); - let log_id = logs.as_ref().and_then(|h| { - h.start(name, "prebuild", &cmdline, node_id) - .map_err(|e| { - tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)"); - }) - .ok() - }); - - let mut child = Command::new("nix") - .args(&args) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .process_group(0) - .spawn() - .with_context(|| format!("spawn {cmdline}"))?; - - let stdout = child.stdout.take().expect("piped stdout"); - let stderr = child.stderr.take().expect("piped stderr"); - - let pump_stdout = tokio::spawn(pump_prebuild_stdout( - stdout, - cmdline.clone(), - logs.clone(), - log_id, - )); - - let stderr_cmdline = cmdline.clone(); - let stderr_logs = logs.clone(); - let pump_stderr = tokio::spawn(async move { - let mut lines = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = lines.next_line().await { - if is_actionable_nix_line(&line) { - tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}"); - } else { - tracing::info!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}"); - } - if let (Some(h), Some(id)) = (&stderr_logs, log_id) { - h.append_stderr(id, &line); - } - } - }); - - let Some(status) = wait_bounded(&mut child, PREBUILD_TIMEOUT) - .await - .with_context(|| format!("wait {cmdline}"))? - else { - // A process that left the group can still hold the pipes open. - pump_stdout.abort(); - pump_stderr.abort(); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.finish(id, crate::build_logs::BuildStatus::Fail); - } - let secs = PREBUILD_TIMEOUT.as_secs(); - match log_id { - Some(id) => { - bail!("prebuild {cmdline} timed out after {secs}s; killed it; see build log #{id}") - } - None => bail!("prebuild {cmdline} timed out after {secs}s; killed it"), - } - }; - let out_path = pump_stdout.await.ok().flatten(); - let _ = pump_stderr.await; - - let ok = status.success(); - if let (Some(h), Some(id)) = (&logs, log_id) { - h.finish( - id, - if ok { - crate::build_logs::BuildStatus::Ok - } else { - crate::build_logs::BuildStatus::Fail - }, - ); - } - if !ok { - match log_id { - Some(id) => bail!("prebuild {cmdline} failed ({status}); see build log #{id}"), - None => bail!("prebuild {cmdline} failed ({status})"), - } - } - Ok(out_path) -} - pub async fn list() -> Result> { let stdout = crate::priv_client::list_containers().await?; Ok(stdout @@ -990,11 +801,11 @@ fn make_log_callback( } } -/// A `create` or `update` asked for without an `internal/meta` commit. The -/// public entry points take the commit, so this guards the string dispatch +/// A `create` or `update` asked for without a closure. The +/// public entry points take the closure, so this guards the string dispatch /// below only. -fn missing_meta_rev(kind: &str, name: &str) -> anyhow::Error { - anyhow::anyhow!("{name}: {kind} needs an internal/meta commit to build from") +fn missing_toplevel(kind: &str, name: &str) -> anyhow::Error { + anyhow::anyhow!("{name}: {kind} needs a closure to apply") } /// Execute a container operation via hive-priv and integrate with @@ -1016,7 +827,7 @@ async fn priv_run(kind: &str, name: &str) -> Result<()> { async fn priv_run_inner( kind: &str, name: &str, - meta_rev: Option<&str>, + toplevel: Option<&str>, node_id: Option, ) -> Result<()> { let container = container_name(name); @@ -1034,27 +845,27 @@ async fn priv_run_inner( // For long-running ops use the streaming protocol so build_logs // receives lines in real time rather than as a batch at completion. let result: Result<()> = match kind { - "create" => match meta_rev { - Some(rev) => { + "create" => match toplevel { + Some(toplevel) => { crate::priv_client::create_container_streaming( name, - rev, + toplevel, make_log_callback(logs.clone(), log_id, cmdline.clone()), ) .await } - None => Err(missing_meta_rev(kind, name)), + None => Err(missing_toplevel(kind, name)), }, - "update" => match meta_rev { - Some(rev) => { + "update" => match toplevel { + Some(toplevel) => { crate::priv_client::update_container_streaming( name, - rev, + toplevel, make_log_callback(logs.clone(), log_id, cmdline.clone()), ) .await } - None => Err(missing_meta_rev(kind, name)), + None => Err(missing_toplevel(kind, name)), }, "start" => crate::priv_client::start_container(name).await, "stop" => crate::priv_client::stop_container(name).await, diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs index 9a7ec33af..1124da337 100644 --- a/hive-c0re/src/lifecycle/tests.rs +++ b/hive-c0re/src/lifecycle/tests.rs @@ -349,68 +349,6 @@ fn agent_names_propagates_an_unreadable_list() { assert!(format!("{err:#}").contains("hive-priv"), "{err:#}"); } -fn is_dead(pid: &str) -> bool { - // A killed process that is not our child stays a zombie until its own - // parent reaps it, so a zombie counts as dead. - match std::fs::read_to_string(format!("/proc/{pid}/stat")) { - Err(_) => true, - Ok(stat) => stat - .rsplit_once(") ") - .is_some_and(|(_, rest)| rest.starts_with('Z')), - } -} - -/// A child still running at the limit comes back `None`, and the kill reaches -/// its whole process group: the backgrounded grandchild dies too, not just the -/// shell that started it. -#[tokio::test] -async fn a_prebuild_past_its_limit_is_killed_with_its_group() { - use std::time::{Duration, Instant}; - let dir = tempfile::tempdir().expect("tempdir"); - let pidfile = dir.path().join("grandchild"); - let mut child = Command::new("sh") - .arg("-c") - .arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display())) - .process_group(0) - .spawn() - .expect("spawn sh"); - - let started = Instant::now(); - let status = wait_bounded(&mut child, Duration::from_secs(2)) - .await - .expect("wait_bounded"); - assert!(status.is_none(), "must time out, got {status:?}"); - assert!( - started.elapsed() < Duration::from_mins(1), - "the limit did not bound the wait" - ); - - let pid = std::fs::read_to_string(&pidfile).expect("grandchild pid"); - let pid = pid.trim(); - let deadline = Instant::now() + Duration::from_secs(5); - while !is_dead(pid) && Instant::now() < deadline { - tokio::time::sleep(Duration::from_millis(50)).await; - } - assert!(is_dead(pid), "grandchild {pid} outlived the timeout"); -} - -/// The control: a child that exits inside its limit comes back with its -/// status. -#[tokio::test] -async fn a_prebuild_inside_its_limit_returns_its_status() { - let mut child = Command::new("sh") - .arg("-c") - .arg("exit 3") - .process_group(0) - .spawn() - .expect("spawn sh"); - let status = wait_bounded(&mut child, std::time::Duration::from_mins(1)) - .await - .expect("wait_bounded") - .expect("a fast child must not time out"); - assert_eq!(status.code(), Some(3)); -} - /// The setup path creates the agent's `work/` dir beside `state/`, so the /// bind source exists before nspawn starts the container. #[test] diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index ff32c5313..c8568b9da 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -115,7 +115,7 @@ enum Cmd { /// option. #[arg(long)] model_prices: Option, - /// Override: number of concurrent nix-heavy job-queue nodes + /// Override: number of concurrent closure-fetching job-queue nodes /// (prebuild / profile-swap / create / meta lock). Set via the /// `services.hyperhive.deploy.hive-controller.buildSlots` NixOS option. #[arg(long)] diff --git a/hive-c0re/src/meta_rev.rs b/hive-c0re/src/meta_rev.rs index 0d692cbe8..92af161ff 100644 --- a/hive-c0re/src/meta_rev.rs +++ b/hive-c0re/src/meta_rev.rs @@ -1,27 +1,25 @@ -//! The `internal/meta` commit a hive builds an agent from. +//! The `internal/meta` commit a hive deploys an agent from. //! //! `internal/meta` is the swarm's meta flake, committed by swarm-controller; -//! a hive has no meta flake of its own. A rebuild's `MetaSync` node resolves -//! the head ([`resolve`]) and records it ([`record`]); `Prebuild` and `Swap` -//! read the record ([`recorded`]), so both build the same flake. A create -//! resolves and uses the commit directly. When the head does not pin the -//! agent's current config, the build fails rather than falling back: the -//! swarm re-pins before it asks a hive to deploy. +//! a hive has no meta flake of its own and builds nothing from it. A rebuild's +//! `MetaSync` node resolves the head ([`resolve`]) and records it ([`record`]); +//! `Prebuild` and `Swap` read the record ([`recorded`]) for the commit the +//! swarm's closure is expected to come from. A create resolves and uses the +//! commit directly. When the head does not pin the agent's current config, the +//! deploy fails rather than falling back: the swarm re-pins before it asks a +//! hive to deploy. //! -//! The record also names the pins a successful build deploys. [`mark_deployed`] +//! The record also names the pins a successful deploy applies. [`mark_deployed`] //! copies them to the agent's markers, which the boot reconcile //! ([`crate::auto_update`]) and the dashboard's update chip read. use anyhow::{Context, Result, bail}; -use hive_priv_sock::{meta_flake_url, valid_meta_rev}; +use hive_priv_sock::valid_meta_rev; use serde::{Deserialize, Serialize}; use swarm_queue_client::agent_archetype::ArchetypeRef; /// What the forge says about `internal/meta` and an agent's config repo. pub(crate) trait MetaSource { - /// Whether nix's own `git` can authenticate to the forge: it reads as this - /// hive's account, from the credential file the source publishes. - fn nix_can_fetch(&self) -> bool; /// The default branch's head commit of `internal/meta`; `None` when the /// repo is absent or has no commit. async fn meta_head(&self) -> Result>; @@ -42,15 +40,13 @@ pub(crate) struct MetaBuild { pub config: Option, } -/// The `internal/meta` commit to build `agent` from. +/// The `internal/meta` commit to deploy `agent` from. /// -/// `internal/meta` is private, so a hive whose nix cannot authenticate -/// ([`MetaSource::nix_can_fetch`]) cannot build. The head is taken only when -/// its `flake.nix` builds the agent the way it is wanted now: from -/// `archetype` when that is set, otherwise from the config commit the forge's -/// `main` is at. A head that is behind (the swarm's update has not run, or -/// failed), lacks the agent or lists it under another archetype would build an -/// older module than the one being deployed. +/// The head is taken only when its `flake.nix` builds the agent the way it is +/// wanted now: from `archetype` when that is set, otherwise from the config +/// commit the forge's `main` is at. A head that is behind (the swarm's update +/// has not run, or failed), lacks the agent or lists it under another archetype +/// would build an older module than the one being deployed. /// /// # Errors /// No usable head, a head that does not pin the agent's current config, or a @@ -60,9 +56,6 @@ pub(crate) async fn choose( agent: &str, archetype: Option<&ArchetypeRef>, ) -> Result { - if !source.nix_can_fetch() { - bail!("{agent}: this hive has no forge credential for nix to fetch internal/meta with"); - } let Some(rev) = source.meta_head().await? else { bail!("{agent}: the swarm has not published internal/meta yet"); }; @@ -174,7 +167,7 @@ fn recorded_in(dir: &std::path::Path, agent: &str) -> Option { } /// The record of `agent`'s build, or the error naming why a `Prebuild` or -/// `Swap` has nothing to build from. +/// `Swap` has no commit to expect. /// /// # Errors /// No valid record: its `MetaSync` did not run, or ran in an older hive-c0re. @@ -219,21 +212,11 @@ fn mark_deployed_in( /// Whether an agent's `stored` marker says it runs the hyperhive commit `pin` /// names. With no `pin`, because the hive can't read `internal/meta`, nothing -/// is stale: a rebuild would have no flake to build from either. +/// is stale: a rebuild would have no commit to deploy either. pub(crate) fn marker_is_fresh(pin: Option<&str>, stored: Option<&str>) -> bool { pin.is_none_or(|pin| stored == Some(pin)) } -/// The `#` ref [`crate::lifecycle::prebuild_toplevel`] builds -/// for `rev`: the same flake hive-priv builds, from the same forge URL. -pub(crate) fn flake_ref(agent: &str, rev: &str) -> String { - flake_ref_at(crate::forge::forge_http_base(), agent, rev) -} - -fn flake_ref_at(forge_base: &str, agent: &str, rev: &str) -> String { - format!("{}#{agent}", meta_flake_url(forge_base, rev)) -} - #[cfg(test)] mod tests { use super::*; @@ -249,13 +232,9 @@ mod tests { head: Option<&'static str>, flake: String, config_head: Option<&'static str>, - fetchable: bool, } impl MetaSource for Forge { - fn nix_can_fetch(&self) -> bool { - self.fetchable - } async fn meta_head(&self) -> Result> { Ok(self.head.map(str::to_owned)) } @@ -313,7 +292,6 @@ mod tests { head: Some(META), flake: flake(), config_head, - fetchable: true, } } @@ -369,9 +347,6 @@ mod tests { async fn an_archetype_agent_is_taken_when_the_head_lists_that_archetype() { struct Archetypes(Forge); impl MetaSource for Archetypes { - fn nix_can_fetch(&self) -> bool { - self.0.nix_can_fetch() - } async fn meta_head(&self) -> Result> { self.0.meta_head().await } @@ -422,19 +397,6 @@ mod tests { assert_unpinned(&choose(&source, "rev", None).await.unwrap_err(), "rev"); } - /// A hive without the credential nix's git reads cannot build from the - /// repo even though its own token reads it; the first test is the control. - #[tokio::test] - async fn a_hive_whose_nix_cannot_authenticate_cannot_build() { - let mut source = forge(Some(ARGUS_HEAD)); - source.fetchable = false; - let err = choose(&source, "argus", None).await.unwrap_err(); - assert!( - format!("{err:#}").contains("no forge credential"), - "{err:#}" - ); - } - #[tokio::test] async fn no_repo_or_no_commit_is_an_error_naming_the_agent() { let mut source = forge(Some(ARGUS_HEAD)); @@ -568,12 +530,4 @@ mod tests { assert!(marker_is_fresh(None, Some(STALE)), "no pin to compare with"); assert!(marker_is_fresh(None, None)); } - - #[test] - fn the_prebuild_ref_is_the_flake_hive_priv_builds() { - assert_eq!( - flake_ref_at("http://forge.t.local", "argus", META), - format!("git+http://forge.t.local/internal/meta.git?rev={META}#argus") - ); - } } diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index 35c31c537..48741238e 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -235,14 +235,14 @@ pub async fn kill_container(name: &str) -> Result<()> { /// appending lines to `build_logs` or otherwise capturing the output. pub async fn update_container_streaming( name: &str, - meta_rev: &str, + toplevel: &str, on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { ok(call_streaming( &PrivRequest::UpdateContainer { name: name.to_owned(), stream: true, - meta_rev: meta_rev.to_owned(), + toplevel: toplevel.to_owned(), }, on_line, ) @@ -253,14 +253,14 @@ pub async fn update_container_streaming( /// arrive. Returns `Ok(())` on success. pub async fn create_container_streaming( name: &str, - meta_rev: &str, + toplevel: &str, on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { ok(call_streaming( &PrivRequest::CreateContainer { name: name.to_owned(), stream: true, - meta_rev: meta_rev.to_owned(), + toplevel: toplevel.to_owned(), }, on_line, ) diff --git a/hive-c0re/src/stores/build_logs.rs b/hive-c0re/src/stores/build_logs.rs index 6af05a713..203ed4b6e 100644 --- a/hive-c0re/src/stores/build_logs.rs +++ b/hive-c0re/src/stores/build_logs.rs @@ -16,7 +16,7 @@ use utoipa::ToSchema; use crate::db::Migration; /// Process-singleton handle, set once at coordinator startup. Lets -/// the `lifecycle` module's `run` / `prebuild_toplevel` access the +/// the `lifecycle` module's `run` access the /// writer without threading an `Arc` through every /// `pub async fn` entry point in the lifecycle surface — there are /// 10+ callsites and the handle is the same `Arc` everywhere diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 623fe4702..fce863f99 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -101,10 +101,7 @@ impl std::str::FromStr for InfraContainer { } } -/// Whether `rev` is a git commit id: 40 lowercase hex digits. The only part of -/// the `internal/meta` flake ref that crosses the wire (`meta_rev` on -/// `PrivRequest::UpdateContainer` and `CreateContainer`), so `hive-priv` checks -/// it before it builds a ref. +/// Whether `rev` is a git commit id: 40 lowercase hex digits. #[must_use] pub fn valid_meta_rev(rev: &str) -> bool { rev.len() == 40 && rev.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) @@ -113,7 +110,8 @@ pub fn valid_meta_rev(rev: &str) -> bool { /// Whether `path` is a store path in the shape nix prints one: `/nix/store/`, /// 32 lowercase letters or digits, `-`, then a name of the characters store /// names allow. The path crosses the wire in [`PrivRequest::RealiseClosure`], -/// so `hive-priv` checks it before it hands it to nix. +/// [`PrivRequest::UpdateContainer`] and [`PrivRequest::CreateContainer`], so +/// `hive-priv` checks it before it hands it to nix. #[must_use] pub fn valid_closure_path(path: &str) -> bool { let Some(rest) = path.strip_prefix("/nix/store/") else { @@ -137,20 +135,6 @@ pub fn valid_closure_path(path: &str) -> bool { /// cache that is down. pub const SIGNATURE_REFUSED: &str = "no signature by the swarm cache's key"; -/// The flake the swarm publishes at `internal/meta`, at `rev`: -/// `git+/internal/meta.git?rev=`. `forge_base` is the forge's -/// `http` origin, which the hive's git credential helper answers for. -/// -/// One definition for `hive-c0re`'s prebuild and `hive-priv`'s build, so both -/// name the same flake. -#[must_use] -pub fn meta_flake_url(forge_base: &str, rev: &str) -> String { - format!( - "git+{}/internal/meta.git?rev={rev}", - forge_base.trim_end_matches('/') - ) -} - /// Root of per-agent state directories on the host. /// Subdirectory layout: `//state/`. /// Used by `WriteAgentStateFile` to derive the write path so the @@ -278,31 +262,28 @@ pub enum PrivRequest { /// in the container. nixos-container has no kill verb. KillContainer { name: String }, - /// `nixos-container update --flake ` - /// The flake ref is derived from `name` and `meta_rev` by hive-priv: the - /// swarm's `internal/meta` at that commit (the forge URL is hive-priv's - /// own and never on the wire). + /// Point `name`'s container profile at `toplevel`, then reload the unit if + /// it runs. hive-priv never builds: `toplevel` is a closure already in the + /// store (see `RealiseClosure`). /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. UpdateContainer { name: String, #[serde(default)] stream: bool, - /// The commit of `internal/meta` to build from, checked by - /// `valid_meta_rev`. - meta_rev: String, + /// The agent's system closure, checked by `valid_closure_path`. + toplevel: String, }, - /// `nixos-container create --flake ` - /// The flake ref is derived from `name` (and `meta_rev`) as for - /// `UpdateContainer`. + /// `nixos-container create --system-path ` + /// `toplevel` is as for `UpdateContainer`. /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. CreateContainer { name: String, #[serde(default)] stream: bool, - meta_rev: String, + toplevel: String, }, /// `nix-store --realise ` from the swarm's binary cache, never @@ -718,7 +699,7 @@ pub enum PrivEvent { #[cfg(test)] mod tests { - use super::{InfraContainer, PrivRequest, meta_flake_url, valid_closure_path, valid_meta_rev}; + use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev}; #[test] fn infra_control_names_no_c0re_or_swarm_service() { @@ -765,15 +746,6 @@ mod tests { } } - #[test] - fn the_meta_flake_url_names_internal_meta_at_the_rev() { - let rev = "a".repeat(40); - assert_eq!( - meta_flake_url("http://forge.t.local/", &rev), - format!("git+http://forge.t.local/internal/meta.git?rev={rev}") - ); - } - #[test] fn only_a_store_path_is_a_closure_path() { let hash = "0123456789abcdfghijklmnpqrsvwxyz"; @@ -814,9 +786,9 @@ mod tests { assert!(serde_json::from_str::(without).is_err()); } - /// `meta_rev` is required: there is no local flake to fall back to. + /// `toplevel` is required: hive-priv has no flake to build one from. #[test] - fn a_request_without_meta_rev_is_refused() { + fn a_request_without_a_toplevel_is_refused() { for op in ["update_container", "create_container"] { let without = format!(r#"{{"op":"{op}","name":"atlas"}}"#); assert!( @@ -824,8 +796,8 @@ mod tests { "{op}" ); let with = format!( - r#"{{"op":"{op}","name":"atlas","meta_rev":"{}"}}"#, - "b".repeat(40) + r#"{{"op":"{op}","name":"atlas","toplevel":"/nix/store/{}-nixos-system-atlas"}}"#, + "b".repeat(32) ); assert!(serde_json::from_str::(&with).is_ok(), "{op}"); } diff --git a/hive-priv/README.md b/hive-priv/README.md index 45f2b27b9..108646096 100644 --- a/hive-priv/README.md +++ b/hive-priv/README.md @@ -42,60 +42,22 @@ Each module's `//!` header lists its ops. ## Implementation notes -### Container toplevel builds (create/update) +### Container toplevels (create/update) -`container_flake_action` (in `src/container.rs`) builds -`nixosConfigurations..config.system.build.toplevel` itself -(`nix_build_toplevel`) and passes the resolved store path to -`nixos-container create`/`update` via `--system-path`, for both verbs. +`container_toplevel_action` (in `src/container.rs`) applies a closure that is +already in the store; hive-priv builds nothing. hive-c0re names the closure +(`toplevel` on `CreateContainer` / `UpdateContainer`), downloaded beforehand +with `RealiseClosure`, and hive-priv accepts it only in the shape of a store +path. -**Why not let `nixos-container` build it (the old `create`-only -behaviour, `update` used its own `--flake` path)**: `nixos-container`'s -own `buildFlake()` — invoked whenever `--system-path` isn't passed — -builds to a _hardcoded relative path_. `$systemPath` is only ever -assigned from the CLI flag or from `buildFlake()`'s own result, so with -no flag it stays `undef` and `"$systemPath.tmp"` interpolates to the -bare string `.tmp` in whatever the caller's cwd happens to be. -`buildFlake()` itself takes no lock at all: `create` wraps its _whole -action_ in an exclusive `flock` before calling it, but `update` used to -call it with no lock whatsoever — so `create`'s lock never protected -against a concurrent `update` clobbering the same `.tmp`. hive-priv never -sets a per-call cwd, so with `services.hyperhive.deploy.hive-controller.buildSlots > 1`, -two concurrent calls (any mix of `create`/`update`) could share that one -`.tmp`: one's `readlink(".tmp")` resolving to the _other's_ build -output, handing an agent's container the wrong agent's closure — the -"agent container gets closure of other agent" bug. +`create` passes the path to `nixos-container create --system-path`, so +`nixos-container`'s own `buildFlake()` never runs. Without the flag, it builds +to a hardcoded relative `.tmp` out-link: `$systemPath` stays `undef` and +`"$systemPath.tmp"` interpolates to the bare string `.tmp` in the caller's cwd. +With `services.hyperhive.deploy.hive-controller.buildSlots > 1`, two concurrent +calls could share that one `.tmp`, and one agent's container got another +agent's closure. -Building the toplevel here and passing the resolved store path via -`--system-path` for _every_ call means `buildFlake()` never runs at all, -for either verb — no shared `.tmp` left to race on, no locking invariant -of a script we don't own to keep track of. `--no-link` avoids a -competing out-link race of our own; we only need the store path, not a -GC root (it's safe from collection for as long as it takes -`nixos-container` to register it against the container's own profile, -same window every other `--print-out-paths` consumer already relies on). - -**Forwards stderr live, captures stdout silently — deliberately not -symmetric.** This build is the multi-minute phase of a `create`/ -`update`, and it used to run _inside_ `nixos-container`'s own `--flake` -invocation, which streams every line to the caller in real time. -Buffering it instead (`Command::output()`, as this function first -shipped) regressed that: nothing on the wire — dashboard or -`journalctl -f` alike — until the whole build finishes, then everything -at once. So both pipes are drained concurrently (needed to avoid -deadlocking if either pipe fills while the other is being read), but -only stderr — where nix's own progress goes — is forwarded live, same -shape as `container_run_streaming`. stdout is different: -`--print-out-paths` writes _only_ the final store path there, once, at -the end — forwarding it the same way would risk interleaving a progress -line into the value this function hands back as `--system-path`, trading -a closure-mixup bug for a corrupted-argument one. So stdout lines are -accumulated silently and only consulted after the exit status is known -to be success — and even then, exactly one non-empty, trimmed line is -required (`nix build --print-out-paths` prints one line _per output_, -not one line total; `config.system.build.toplevel` is single-output -today, but a bare whole-buffer `.trim()` would silently hand a -multi-line string to `--system-path` the day that ever changes, and a -bare untrimmed/unfiltered `.lines()` turns a lone `"\n"` into a bogus -empty-string "path" — a wrong line count, or a blank/whitespace one, -`bail!`s instead). +`update` points the container's profile at the path with `nix-env --set` and +reloads the unit if it runs, because `nixos-container update` dies on its +version probe before it honors `--system-path`. diff --git a/hive-priv/src/closure.rs b/hive-priv/src/closure.rs index 5c1bbc3a3..d54e4e008 100644 --- a/hive-priv/src/closure.rs +++ b/hive-priv/src/closure.rs @@ -170,7 +170,7 @@ async fn realise_with( let cache = Cache::read(url, dir, owner)?; let mut cmd = tokio::process::Command::new("nix-store"); cmd.args(realise_args(store_path, &cache)); - match run_bounded(cmd, REALISE_TIMEOUT, None) + match run_bounded(cmd, REALISE_TIMEOUT) .await .context("running nix-store")? { @@ -342,8 +342,8 @@ mod tests { assert!(Cache::read(Some(URL), &dir.join("nowhere"), me).is_err()); } - /// Missing facts reach the caller as `not available`, which its prebuild - /// answers by building. + /// Missing facts reach the caller as `not available`, which a rebuild + /// reports as a closure that cannot be pulled. #[tokio::test] async fn without_the_root_files_the_answer_is_not_available() { let nowhere = scratch().join("nowhere"); diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs index fc25a6c75..50ac3737c 100644 --- a/hive-priv/src/container.rs +++ b/hive-priv/src/container.rs @@ -3,12 +3,11 @@ //! and `ListContainers`. Every op takes a logical agent name and reaches only //! the `h-`-prefixed container [`container_system_name`] derives from it. //! -//! `CreateContainer` / `UpdateContainer` build the toplevel here -//! ([`nix_build_toplevel`]) and stream progress to the caller as -//! [`PrivEvent::Line`]s when asked to. +//! `CreateContainer` / `UpdateContainer` apply a closure already in the store +//! and stream progress to the caller as [`PrivEvent::Line`]s when asked to. -use anyhow::{Context as _, Result, anyhow, bail}; -use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine, meta_flake_url, valid_meta_rev}; +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine, valid_closure_path}; use tokio::io::{AsyncWriteExt, BufReader}; use tokio::net::unix::OwnedWriteHalf; use tokio::process::Command; @@ -72,52 +71,43 @@ pub(crate) async fn destroy_container(name: &str) -> Result<(String, String)> { container_run(&["destroy", &container_system_name(name)]).await } -/// Shared body for `CreateContainer` / `UpdateContainer`: validate the -/// name, build the toplevel ourselves, and apply it (streaming line -/// events to `writer` when `stream` is set). +/// `toplevel` as a store path, or the error saying it is not one. The path +/// crosses the socket from hive-c0re, which is not root. +fn toplevel_to_apply(toplevel: &str) -> Result<&str> { + if !valid_closure_path(toplevel) { + bail!("{toplevel:?} is not a store path"); + } + Ok(toplevel) +} + +/// Shared body for `CreateContainer` / `UpdateContainer`: validate the name +/// and the closure, and apply it (streaming line events to `writer` when +/// `stream` is set). Nothing is built here: `toplevel` is a closure already in +/// the store, which `RealiseClosure` put there. /// -/// **Both verbs build explicitly now — not just `update`.** The first cut -/// of this fix only rewrote `update`, reasoning that `create` was already -/// safe: it wraps its whole action in an exclusive `flock` before calling -/// `nixos-container`'s own `buildFlake()`, and once `update` stopped -/// writing to `buildFlake()`'s shared `.tmp` out-link, concurrent -/// `create`s were the only remaining writers — mutually excluded by that -/// lock. True, but it leaves `create`'s safety resting on an internal -/// implementation detail of a script we don't own (its current locking -/// behavior, which could change upstream without notice) instead of on -/// something we control. Building here for both verbs removes `buildFlake()` -/// from the picture entirely — there's no shared `.tmp` left to race on, -/// so there's nothing left to reason about staying in sync with. +/// `create` passes `--system-path` so `nixos-container`'s own `buildFlake()` +/// never runs; it builds to a relative `.tmp` out-link that concurrent calls +/// share, which handed an agent another agent's closure. /// -/// **`update` no longer calls `nixos-container update` at all.** That -/// action's own version-compat probe runs unconditionally before -/// `--system-path` is ever honored, dying on every agent's update. -/// [`swap_container_profile`] replicates exactly what `update`'s own -/// action does *past* that probe (confirmed against its source): `nix-env -/// --set` the per-container profile, then `systemctl reload` if the -/// container is running. `create` is untouched — it isn't the failing -/// verb. -pub(crate) async fn container_flake_action( +/// `update` does not call `nixos-container update` at all: that action's +/// version-compat probe runs before `--system-path` is honored and dies on every +/// agent. [`swap_container_profile`] does what the action does past that probe: +/// `nix-env --set` the per-container profile, then `systemctl reload` if the +/// container is running. +pub(crate) async fn container_toplevel_action( verb: &str, name: &str, - meta_rev: &str, + toplevel: &str, stream: bool, writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { validate_agent_name(name)?; - // The build is the multi-minute phase of this operation — give it the - // same live-line treatment `container_run_streaming` gives - // `nixos-container` itself when the caller asked for it. Without - // this, moving the build out of the streamed `nixos-container` call - // (which is the whole point of this fix) would silently regress every - // UI that shows build progress: nothing until the longest phase - // finishes, then everything at once. - let toplevel = nix_build_toplevel(name, meta_rev, stream.then_some(&mut *writer)).await?; + let toplevel = toplevel_to_apply(toplevel)?; let system_name = container_system_name(name); if verb == "update" { - return swap_container_profile(&system_name, &toplevel, stream, writer).await; + return swap_container_profile(&system_name, toplevel, stream, writer).await; } - let args = [verb, &system_name, "--system-path", &toplevel]; + let args = [verb, &system_name, "--system-path", toplevel]; if stream { container_run_streaming(&args, writer).await } else { @@ -126,7 +116,7 @@ pub(crate) async fn container_flake_action( } /// Apply a prebuilt toplevel to an existing container directly, without -/// going through `nixos-container update` (see `container_flake_action`'s +/// going through `nixos-container update` (see `container_toplevel_action`'s /// doc comment for why). Mirrors `nixos-container.pl`'s own `update` /// action verbatim, past its version probe: point the per-container `nix- /// env` profile at the new toplevel, then reload the container unit *if* @@ -209,46 +199,10 @@ async fn log_and_forward(stdout: &str, stderr: &str, stream: bool, writer: &mut } } -/// The environment variable naming the forge's `http` origin, which -/// `internal/meta` is fetched from. Set by the unit; never taken from a -/// request. -pub(crate) const FORGE_URL_ENV: &str = "HIVE_PRIV_FORGE_URL"; - -/// The flake agent toplevels are built from: `internal/meta` at `rev`. The URL -/// is `forge_base`, hive-priv's own; only the rev comes from the request, and -/// it is checked here. -fn meta_flake(rev: &str, forge_base: Option<&str>) -> Result { - if !valid_meta_rev(rev) { - bail!("meta_rev {rev:?} is not a 40-digit lowercase hex commit id"); - } - let base = forge_base.filter(|b| !b.is_empty()).with_context(|| { - format!("{FORGE_URL_ENV} is unset, so there is no internal/meta to build from") - })?; - Ok(meta_flake_url(base, rev)) -} - -/// The explicit `nixosConfigurations..config.system.build.toplevel` -/// flake attr path — same construction `hive-c0re`'s own -/// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so -/// the exact string shape is unit-tested without needing to run `nix`. -fn toplevel_attr(flake: &str, name: &str) -> String { - format!("{flake}#nixosConfigurations.{name}.config.system.build.toplevel") -} - -/// Wall-clock bound on [`nix_build_toplevel`]. Four times CI's observed -/// cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`), which -/// compiles this workspace's crates, the part of an agent toplevel no binary -/// cache holds; the rebuild path has usually warmed this exact attr already -/// (`hive-c0re`'s `prebuild_toplevel`). A stalled download is bounded by nix's -/// own `stalled-download-timeout`; this covers the rest, such as a wedged -/// nix-daemon holding the caller's build slot forever. -const NIX_BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1); - /// How a [`run_bounded`] child ended. pub(crate) enum BoundedRun { Exited { status: std::process::ExitStatus, - stdout: String, stderr: String, }, /// Still running at the limit; its whole process group was killed. @@ -256,8 +210,7 @@ pub(crate) enum BoundedRun { } /// Run `cmd` to completion or until `limit`, draining both pipes: stdout is -/// captured silently, stderr is logged and, when `writer` is set, forwarded -/// live as [`PrivEvent::Line`]s. +/// discarded, stderr is kept and also logged as it arrives. /// /// The child leads its own process group and a timeout `SIGKILL`s the whole /// group, so a grandchild still holding the pipes (or the daemon connection) @@ -265,7 +218,6 @@ pub(crate) enum BoundedRun { pub(crate) async fn run_bounded( mut cmd: Command, limit: std::time::Duration, - mut writer: Option<&mut OwnedWriteHalf>, ) -> std::io::Result { use tokio::io::AsyncBufReadExt as _; @@ -282,13 +234,10 @@ pub(crate) async fn run_bounded( let run = async { let mut stdout_lines = BufReader::new(stdout).lines(); let mut stderr_lines = BufReader::new(stderr).lines(); - let mut stdout_buf = String::new(); let mut stderr_buf = String::new(); - // ⚠️ Both pipes are drained concurrently even though only one is - // streamed: reading stderr alone would let stdout fill its pipe - // buffer and deadlock the child on a build with enough stdout output - // to fill it. + // ⚠️ Both pipes are drained concurrently: reading one alone would let + // the other fill its pipe buffer and deadlock the child. // // ⚠️ Each stream's EOF is tracked separately rather than breaking on // the first `None`: `next_line()` on a closed stream returns @@ -300,27 +249,18 @@ pub(crate) async fn run_bounded( tokio::select! { line = stdout_lines.next_line(), if !stdout_done => { match line { - // Captured, never streamed — this is the store path. - Ok(Some(l)) => { - stdout_buf.push_str(&l); - stdout_buf.push('\n'); - } + Ok(Some(_)) => {} Ok(None) => stdout_done = true, Err(e) => { - tracing::warn!(error = %e, "nix build stdout read error"); + tracing::warn!(error = %e, "child stdout read error"); stdout_done = true; } } } line = stderr_lines.next_line(), if !stderr_done => { match line { - // Streamed as it arrives, so the dashboard and - // `journalctl -f` show build progress live. Ok(Some(l)) => { - tracing::info!(target: "nix-build-toplevel", "{l}"); - if let Some(w) = writer.as_deref_mut() { - write_line_event(w, PrivStream::Stderr, &l).await; - } + tracing::info!(target: "nix-store", "{l}"); if !stderr_buf.is_empty() { stderr_buf.push('\n'); } @@ -328,7 +268,7 @@ pub(crate) async fn run_bounded( } Ok(None) => stderr_done = true, Err(e) => { - tracing::warn!(error = %e, "nix build stderr read error"); + tracing::warn!(error = %e, "child stderr read error"); stderr_done = true; } } @@ -338,7 +278,6 @@ pub(crate) async fn run_bounded( let status = child.wait().await?; Ok::<_, std::io::Error>(BoundedRun::Exited { status, - stdout: stdout_buf, stderr: stderr_buf, }) }; @@ -356,90 +295,6 @@ pub(crate) async fn run_bounded( Ok(BoundedRun::TimedOut) } -/// Build `nixosConfigurations..config.system.build.toplevel` and -/// return the resulting store path, so `create`/`update` can hand -/// `nixos-container` an explicit `--system-path` instead of letting its -/// own `buildFlake()` build to a racy shared out-link. See "Container -/// toplevel builds" in this crate's README for the full story — the -/// concurrency bug this closes (the "agent container gets closure of -/// other agent" mystery bug) and why stdout/stderr are drained -/// concurrently but handled asymmetrically (stderr streamed live, -/// stdout captured and required to be exactly one line). -/// -/// `writer` is `None` for the non-streaming call shape (`stream: false`); -/// stderr still logs to journald either way, just without the -/// `PrivEvent::Line` forwarding. Bounded by [`NIX_BUILD_TIMEOUT`]. -async fn nix_build_toplevel( - name: &str, - meta_rev: &str, - writer: Option<&mut OwnedWriteHalf>, -) -> Result { - let flake = meta_flake(meta_rev, std::env::var(FORGE_URL_ENV).ok().as_deref())?; - let attr = toplevel_attr(&flake, name); - let mut cmd = Command::new("nix"); - cmd.args([ - "--extra-experimental-features", - "nix-command flakes", - "build", - "--no-link", - "--print-out-paths", - &attr, - ]); - let (status, stdout_buf, stderr_buf) = match run_bounded(cmd, NIX_BUILD_TIMEOUT, writer) - .await - .with_context(|| format!("run nix build {attr}"))? - { - BoundedRun::Exited { - status, - stdout, - stderr, - } => (status, stdout, stderr), - BoundedRun::TimedOut => bail!( - "nix build {attr} timed out after {}s; killed it", - NIX_BUILD_TIMEOUT.as_secs() - ), - }; - - // ⚠️ Success is decided by the exit status, not by "we parsed a - // path" — a build can print to stdout and still fail. - if !status.success() { - bail!( - "nix build {attr} failed ({status}): {}", - stderr_buf.lines().last().unwrap_or("").trim() - ); - } - single_output_path(&stdout_buf) - .map(str::to_owned) - .map_err(|count| { - anyhow!("nix build {attr} produced {count} output path(s), expected exactly 1: {stdout_buf:?}") - }) -} - -/// Parse `nix build --print-out-paths`' stdout down to the single output -/// path this function's caller expects. `--print-out-paths` prints one -/// line *per output*, not one line total (`nix build --no-link -/// --print-out-paths nixpkgs#openssl` prints two: `…-bin`, `…-man`); -/// `config.system.build.toplevel` is single-output today, so this is one -/// line in practice — but a bare whole-buffer `.trim()` would silently -/// hand a multi-line string on to `--system-path` the day that ever -/// changes, the same corrupted-argument failure this function exists to -/// avoid. Trims and drops empty lines *before* counting, so a lone -/// `"\n"` (or trailing whitespace on the real line) can't be mistaken -/// for a present-but-blank path — see the unit tests below for the exact -/// table this closes. `Err` carries the surviving line count, for the -/// caller's error message. -fn single_output_path(stdout: &str) -> Result<&str, usize> { - let lines: Vec<&str> = stdout - .lines() - .map(str::trim) - .filter(|l| !l.is_empty()) - .collect(); - match lines[..] { - [path] => Ok(path), - _ => Err(lines.len()), - } -} - /// Invoke `nixos-container` with the given args, log output to journald. pub(crate) async fn container_run(args: &[&str]) -> Result<(String, String)> { let out = Command::new("nixos-container") @@ -709,85 +564,37 @@ async fn container_run_streaming( } Ok((String::new(), String::new())) } + #[cfg(test)] mod tests { - use super::{ - BoundedRun, FORGE_URL_ENV, meta_flake, run_bounded, single_output_path, toplevel_attr, - }; + use super::{BoundedRun, run_bounded, toplevel_to_apply}; use crate::shared::test_support::scratch; - /// Pins the exact attr path we hand to `nix build` — it has to match - /// `hive-c0re`'s own `lifecycle::prebuild_toplevel` construction, since - /// that step's whole point is warming the store for this later build. - /// A drifted attr path defeats the cache-warming silently — no error, - /// just a slower `update`. The flake is `internal/meta` at the rev, and its - /// URL is the unit's, never the request's. - #[test] - fn a_meta_rev_builds_from_internal_meta_at_the_units_forge() { - let rev = "c".repeat(40); - let forge = Some("http://forge.t.local"); - assert_eq!( - toplevel_attr(&meta_flake(&rev, forge).unwrap(), "atlas"), - format!( - "git+http://forge.t.local/internal/meta.git?rev={rev}\ - #nixosConfigurations.atlas.config.system.build.toplevel" - ) - ); - } + const TOPLEVEL: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; - /// Nothing but a commit id gets into the ref, whatever the request says. + /// A store path passes through unchanged; anything else, whatever the + /// request says, never reaches `nix-env` or `nixos-container`. #[test] - fn a_meta_rev_that_is_not_a_commit_id_is_refused() { - let forge = Some("http://forge.t.local"); + fn only_a_store_path_is_applied() { + assert_eq!(toplevel_to_apply(TOPLEVEL).unwrap(), TOPLEVEL); for bad in [ - "main", "", + "main", "../../x", - &format!("{}#evil", "a".repeat(34)), - &format!("{}&ref=x", "a".repeat(34)), + "/etc/passwd", + "/nix/store/x", + &format!("{TOPLEVEL}/bin"), + &format!("{TOPLEVEL};id"), + &"c".repeat(40), ] { - let err = meta_flake(bad, forge).unwrap_err(); + let err = toplevel_to_apply(bad).unwrap_err(); assert!( - format!("{err:#}").contains("not a 40-digit"), + format!("{err:#}").contains("not a store path"), "{bad}: {err:#}" ); } } - #[test] - fn a_meta_rev_without_a_forge_in_the_unit_is_refused() { - let rev = "c".repeat(40); - for forge in [None, Some("")] { - let err = meta_flake(&rev, forge).unwrap_err(); - assert!(format!("{err:#}").contains(FORGE_URL_ENV), "{err:#}"); - } - } - - /// `--print-out-paths`' exact per-shape table (measured against real - /// `rustc` semantics, not just reasoned about) — a lone `"\n"` and a - /// trailing-space path are the two shapes a bare `.lines().collect()` - /// gets wrong, both catchable only by trimming + dropping empties - /// before counting rather than after. - #[test] - fn single_output_path_rejects_blank_and_trims_whitespace() { - assert_eq!(single_output_path(""), Err(0)); - assert_eq!(single_output_path("\n"), Err(0)); - assert_eq!(single_output_path("\n\n"), Err(0)); - assert_eq!(single_output_path("/nix/store/abc\n"), Ok("/nix/store/abc")); - assert_eq!( - single_output_path("/nix/store/abc \n"), - Ok("/nix/store/abc") - ); - assert_eq!( - single_output_path("/nix/store/abc\r\n"), - Ok("/nix/store/abc") - ); - assert_eq!( - single_output_path("/nix/store/abc\n/nix/store/def\n"), - Err(2) - ); - } - /// Whether `pid` is gone, a zombie (`Z`), or being reaped (`X`): none of /// these runs again, and a process passes through them in that order, /// so once this is true it stays true. @@ -817,7 +624,7 @@ mod tests { /// reaches its whole process group: the backgrounded grandchild dies too, /// not just the shell that started it. #[tokio::test] - async fn a_build_past_its_limit_is_killed_with_its_group() { + async fn a_child_past_its_limit_is_killed_with_its_group() { use std::time::{Duration, Instant}; let dir = scratch(); let pidfile = dir.join("grandchild"); @@ -826,9 +633,7 @@ mod tests { .arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display())); let started = Instant::now(); - let run = run_bounded(cmd, Duration::from_secs(2), None) - .await - .unwrap(); + let run = run_bounded(cmd, Duration::from_secs(2)).await.unwrap(); assert!(matches!(run, BoundedRun::TimedOut), "must time out"); assert!( started.elapsed() < Duration::from_mins(1), @@ -846,23 +651,18 @@ mod tests { } /// The control: a child that exits inside its limit is `Exited`, with its - /// status and both streams kept apart. + /// status and its stderr. #[tokio::test] - async fn a_build_inside_its_limit_returns_its_output() { + async fn a_child_inside_its_limit_returns_its_status_and_stderr() { let mut cmd = tokio::process::Command::new("sh"); cmd.arg("-c") .arg("echo progress >&2; echo /nix/store/abc-toplevel"); - match run_bounded(cmd, std::time::Duration::from_mins(1), None) + match run_bounded(cmd, std::time::Duration::from_mins(1)) .await .unwrap() { - BoundedRun::Exited { - status, - stdout, - stderr, - } => { + BoundedRun::Exited { status, stderr } => { assert!(status.success()); - assert_eq!(stdout, "/nix/store/abc-toplevel\n"); assert_eq!(stderr, "progress"); } BoundedRun::TimedOut => panic!("a fast child must not time out"), diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 48ca9f7b2..10971f6aa 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -366,14 +366,14 @@ async fn exec( PrivRequest::UpdateContainer { ref name, stream, - ref meta_rev, - } => container::container_flake_action("update", name, meta_rev, stream, writer).await, + ref toplevel, + } => container::container_toplevel_action("update", name, toplevel, stream, writer).await, PrivRequest::CreateContainer { ref name, stream, - ref meta_rev, - } => container::container_flake_action("create", name, meta_rev, stream, writer).await, + ref toplevel, + } => container::container_toplevel_action("create", name, toplevel, stream, writer).await, PrivRequest::RealiseClosure { ref agent, diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 4de6106ef..6cd79d13c 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -41,13 +41,12 @@ let }; # Privsep splits ownership across users, so git/libgit2's dubious- - # ownership guard trips on legitimate cross-user reads: hive-priv (root) - # fetches the hive-core-owned meta/applied repos via nix, and hive-c0re - # (hive-core) fetches the agent-owned proposed-config repos. Both - # processes are trusted and can already read the files; this gitconfig - # only satisfies the ownership guard. libgit2 honours the literal `*` - # (mid-path globs aren't supported, so per-agent repos can't be listed); - # in practice these processes only ever touch hyperhive's own repos. + # ownership guard trips on legitimate cross-user reads: hive-c0re + # (hive-core) fetches the agent-owned proposed-config repos. The process is + # trusted and can already read the files; this gitconfig only satisfies the + # ownership guard. libgit2 honours the literal `*` (mid-path globs aren't + # supported, so per-agent repos can't be listed); in practice this process + # only ever touches hyperhive's own repos. # # The forge sections (`forgeGit.config`) answer for the `git+http` forge # fetches nix makes: the agent config inputs and `internal/meta`. hive-c0re's diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 8f395357d..bad8fc369 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -41,11 +41,9 @@ let ); in { - # nix (the prebuild `nix build`, flake-check, and meta eval) writes - # its fetcher/eval cache under $HOME/.cache. As a system user + # git writes its config and caches under $HOME. As a system user # hive-core has no home, so HOME defaults to the unwritable - # /var/empty and Lix fails to initialise its cache. Point HOME at - # the writable StateDirectory. + # /var/empty. Point HOME at the writable StateDirectory. HOME = "/var/lib/hyperhive"; HYPERHIVE_GIT = "${pkgs.git}/bin/git"; # Never let git block on an interactive credential prompt. hive-core is a diff --git a/nix/host-modules/hive-c0re/options.nix b/nix/host-modules/hive-c0re/options.nix index 5d2eefc01..75fdfe0e8 100644 --- a/nix/host-modules/hive-c0re/options.nix +++ b/nix/host-modules/hive-c0re/options.nix @@ -290,11 +290,11 @@ default = 1; example = 2; description = '' - Number of nix-heavy job-queue nodes (container prebuilds, + Number of closure-fetching job-queue nodes (container prebuilds, profile swaps, first-spawn creates, meta lock bumps) hive-c0re - runs concurrently. The default of 1 serializes all heavy nix - work; raise it on hosts with the cores/RAM to build several - agent toplevels at once. Per-agent correctness is independent + runs concurrently. The default of 1 serializes all of them; + raise it on hosts with the bandwidth and disk to take several + agents at once. Per-agent correctness is independent of this count — each agent's container-affecting operations are serialized by its lifecycle lease regardless. ''; diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index 5f865f570..3975e9266 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -9,22 +9,6 @@ }: let controllerCfg = config.services.hyperhive.deploy.hive-controller; - - forgeGit = import ./lib/hive-forge-git.nix { - inherit pkgs; - forgeDomain = config.services.hyperhive.swarm.forge.domain; - }; - - # Same gitconfig as the c0re unit (see ./hive-c0re) apart from its stalled- - # transfer limit: hive-priv (root) runs nix, which fetches the hive-core-owned - # meta/applied repos (libgit2 refuses cross-user reads without - # safe.directory) and, from a fresh HOME, the forge inputs of `internal/meta` - # (the forge sections). - safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" '' - [safe] - directory = * - ${forgeGit.config} - ''; in { config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable { @@ -77,41 +61,25 @@ in # findutils already in the default unit PATH. Without `nixos-container` # on PATH every container op fails ENOENT, which `build_all` silently # swallows into an empty list ("no managed containers"). - # - # `nix` itself shells out by bare name too: `git` whenever it has to - # fetch/re-resolve a git-source flake input (an agent.nix with a - # `git+https://…` input, or a stale flake.lock whose node URL no longer - # matches the flake's declared input → nix re-resolves at eval), and - # `ssh` to dispatch to remote builders (`nix.buildMachines` / - # `ssh-ng://`). Without these on PATH `nixos-container update` dies with - # `executing "git": No such file or directory` / `Could not find - # executable 'ssh'` — the agent build fails before it starts. path = [ pkgs.nixos-container pkgs.nix # nix, nix-env, nix-instantiate — create + update - pkgs.gitMinimal # git — nix fetches/re-resolves git-source flake inputs - forgeGit.credentialHelper # git-credential-hive-forge — the forge's `git+http` inputs - pkgs.openssh # ssh — nix dispatches builds to remote builders pkgs.util-linux # umount (nsenter is hardcoded in the script) pkgs.e2fsprogs # chattr pkgs.btrfs-progs # btrfs subvolume create/delete — Ensure/DeleteAgentSubvolume ]; environment = { - # `nixos-container update/create` runs `nix`, which writes its - # fetcher/eval cache under $HOME/.cache. With ProtectHome and no - # explicit HOME this lands on the unwritable /var/empty and Lix - # errors out. Point HOME at the StateDirectory below (persistent, - # so the cache survives across rebuilds). + # nix (`nix-env`, `nix-store`) writes its cache under $HOME/.cache. + # With ProtectHome and no explicit HOME this lands on the unwritable + # /var/empty and Lix errors out. Point HOME at the StateDirectory + # below (persistent, so the cache survives across restarts). HOME = "/var/lib/hive-priv"; # hive-priv runs as root. Root nix defaults to store=auto which # resolves to the LOCAL store — bypassing the host daemon, its # remote builders, and prebuilt derivation outputs. Force daemon - # routing so nixos-container update and the nix prebuild see the - # same store and substituters as every other build context. + # routing so the closure `RealiseClosure` downloads is in the same + # store the container is then pointed at. NIX_REMOTE = "daemon"; - # The forge origin `internal/meta` is built from when a request names - # one of its commits. Fixed here; no request chooses it. - HIVE_PRIV_FORGE_URL = "http://${config.services.hyperhive.swarm.forge.domain}"; }; serviceConfig = { ExecStart = "${controllerCfg.package}/bin/hive-priv"; @@ -159,11 +127,6 @@ in # Point the working directory at the writable StateDirectory so nix # drops its temp symlink there instead. WorkingDirectory = "/var/lib/hive-priv"; - # nix (run here as root for `nixos-container update --flake - # /var/lib/hyperhive/meta#`) fetches the hive-core-owned - # meta/applied repos; libgit2 refuses them without safe.directory. - # See safeDirGitconfig above. - ExecStartPre = "+-${pkgs.coreutils}/bin/cp ${safeDirGitconfig} /var/lib/hive-priv/.gitconfig"; # Kernel surfaces neither hive-priv nor its children use: no # personality(2), no real-time scheduling, no /dev/kmsg or # /proc/kmsg, no explicit module loading (kernel-initiated autoload diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index bcdb300f5..3c6bb9a50 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -234,13 +234,12 @@ let && env shared.controllerAlone == { }; } { - # nix fetches the forge inputs with `git`, so the credential helper and - # the https-to-http rewrite are git config in both processes that build: - # hive-c0re (prebuilds) and hive-priv (builds as root, from its own - # HOME). The controller's unit is the control that the lookup tells a - # unit with the helper from one without; the URL hive-priv builds - # `internal/meta` from is the one hive-c0re reads the forge at. - name = "hive-c0re and hive-priv answer the forge's git fetches and rewrite its https inputs to http"; + # hive-c0re carries the credential helper and the https-to-http rewrite as + # git config. hive-priv builds nothing and fetches nothing from the forge, + # so it carries neither and has no forge URL. The hive-c0re arm is the + # control that the lookups find a helper and a forge URL where there is + # one, so the absences on hive-priv and the controller are real. + name = "hive-c0re answers the forge's git fetches and rewrites its https inputs to http; hive-priv has neither"; ok = let forge = allLocal.services.hyperhive.swarm.forge.domain; @@ -256,7 +255,7 @@ let env = unit: allLocal.systemd.services.${unit}.environment; in helperOn "hive-c0re" - && helperOn "hive-priv" + && !(helperOn "hive-priv") # The helper answers from the file hive-c0re publishes the hive's own # token to. The first arm is the control for the second: the lookup # finds the helper's text, so the absent `core` strings are really absent. @@ -267,8 +266,8 @@ let && !(helperOn "swarm-controller") && lib.hasInfix "[credential \"http://${forge}\"]\nhelper =\nhelper = hive-forge" git.config && lib.hasInfix "[url \"http://${forge}/\"]\ninsteadOf = https://${forge}/" git.config - && (env "hive-priv").HIVE_PRIV_FORGE_URL == "http://${forge}" - && (env "hive-priv").HIVE_PRIV_FORGE_URL == (env "hive-c0re").HIVE_FORGE_URL; + && (env "hive-c0re") ? HIVE_FORGE_URL + && !((env "hive-priv") ? HIVE_PRIV_FORGE_URL); } { # `hyperhive_rev` defaults to none on the controller, so a settings JSON From 6b800600e26575f5ee48fb9af32818d1e8f0f899 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 22:36:58 +0100 Subject: [PATCH 2/3] bind a container apply to the closure realised for the agent, and record the deployed pins on drift hive-priv applies only the path its last RealiseClosure returned for that agent. On drift the swap and create record the pins of the commit the closure was built from. --- docs/agent-lifecycle/config-changes.md | 3 +- docs/swarm/builder.md | 10 +- docs/trust-boundary/security.md | 2 +- hive-c0re/src/job_queue/exec.rs | 79 +++++++++++++--- hive-c0re/src/job_queue/exec/tests.rs | 60 +++++++++++- hive-c0re/src/meta_rev.rs | 75 ++++++++++++++- hive-priv-sock/src/lib.rs | 4 +- hive-priv/README.md | 10 +- hive-priv/src/closure.rs | 124 ++++++++++++++++++++++++- hive-priv/src/container.rs | 42 +-------- 10 files changed, 338 insertions(+), 71 deletions(-) diff --git a/docs/agent-lifecycle/config-changes.md b/docs/agent-lifecycle/config-changes.md index 85b231112..72f134378 100644 --- a/docs/agent-lifecycle/config-changes.md +++ b/docs/agent-lifecycle/config-changes.md @@ -172,7 +172,8 @@ the swarm builder built the agent from the updated repo (the writes it from the builder's reply) and have `hive-priv` download the closure it names from the swarm cache (`RealiseClosure`). `Swap` and `Create` then give `hive-priv` the store path (`toplevel` on `UpdateContainer` and - `CreateContainer`), which accepts only a store path. On the wire the + `CreateContainer`), which accepts only the path its last `RealiseClosure` for + that agent returned. On the wire the `hive_priv_sock::PrivRequest` variants carry a required field: `{"op":"update_container","name":"atlas","stream":true,"toplevel":"/nix/store/-nixos-system-atlas"}`. The three cases when the swarm's closure isn't the one this hive expects diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index f75fb1095..34b384ecb 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -492,11 +492,11 @@ same and creates the container from that path. A rebuild or create has three outcomes, by what the lookup and the download find: -| Found | What happens | -| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| a record that the builder pushed, for the commit this hive chose | The hive downloads the closure and deploys it. The hive logs `prebuild: substituted ` (`create: substituted ` for a create) at info level. | -| a record that the builder pushed, for another commit (the hive and the builder disagree) | The hive downloads the builder's closure and deploys it, with a warning that names the agent, both commits and the path. The hive evaluates nothing. | -| no record, or one the hive can't use (not pushed, queue or cache unreachable, download or signature failed) | The rebuild or create fails with an error that names the agent and the reason, logged at error level, and nothing changes. A running container keeps running unchanged, because `Prebuild` fails before `StopForUpdate`. A create makes no container. | +| Found | What happens | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| a record that the builder pushed, for the commit this hive chose | The hive downloads the closure and deploys it. The hive logs `prebuild: substituted ` (`create: substituted ` for a create) at info level. | +| a record that the builder pushed, for another commit (the hive and the builder disagree) | The hive downloads the builder's closure and deploys it, with a warning that names the agent, both commits and the path. The hive evaluates nothing, and records the pins of the commit the builder built the closure from, so the agent's markers say what runs. | +| no record, or one the hive can't use (not pushed, queue or cache unreachable, download or signature failed) | The rebuild or create fails with an error that names the agent and the reason, logged at error level, and nothing changes. A running container keeps running unchanged, because `Prebuild` fails before `StopForUpdate`. A create makes no container. | The hive never falls back to a local build. The third row has one owner in the code, `without_closure` in `hive-c0re/src/job_queue/exec.rs`, so a change to what diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index 65c4f5738..ebe65029f 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -256,7 +256,7 @@ known operations; there is no arbitrary command pass-through: | ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `StartContainer` / `StopContainer` | `nixos-container start/stop ` | | `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | -| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must be a store path (`/nix/store/<32 chars>-`), and hive-priv builds nothing | +| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must be the store path the last successful `RealiseClosure` returned for that agent (in memory, per agent), so hive-c0re can't name a path never realised for the agent; hive-priv builds nothing | | `RealiseClosure` | `nix-store --realise --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-` and the agent name valid. It downloads and never builds | | `DestroyContainer` | `nixos-container destroy ` | | `ListContainers` | `nixos-container list` | diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index cbfce208e..2de5dd9d5 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -48,6 +48,8 @@ pub(super) trait Host: Sync { /// The record the swarm's builder wrote for the agent; `None` when it wrote /// none. async fn closure_record(&self, name: &str) -> Result>; + /// The pins `internal/meta` at the commit `rev` holds for the agent. + async fn pins_at(&self, name: &str, rev: &str) -> Result; /// Download `store_path` from the swarm's cache into the store, never /// building it. async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>; @@ -125,6 +127,10 @@ impl Host for RealHost { crate::attic::closure::record(name).await } + async fn pins_at(&self, name: &str, rev: &str) -> Result { + crate::meta_rev::resolve_at(name, rev).await + } + async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> { crate::priv_client::realise_closure(name, store_path).await } @@ -548,8 +554,8 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> { async fn run_prebuild(host: &impl Host, name: &str) -> Result<()> { if host.is_running(name).await { let rev = host.recorded_rev(name)?; - let path = closure_for(host, name, &rev, "prebuild").await?; - tracing::info!(%name, %rev, "prebuild: substituted {path}"); + let closure = closure_for(host, name, &rev, "prebuild").await?; + tracing::info!(%name, %rev, "prebuild: substituted {}", closure.path); } Ok(()) } @@ -563,13 +569,21 @@ enum NoClosure { Unpullable(String), } -/// The store path of the closure the swarm built for `name`, downloaded into -/// the store. The deploy applies the swarm's closure whatever commit it was -/// built from: when that is not the `rev` this hive chose, it is logged. +/// The closure the swarm built for an agent, in the store. +struct Closure { + path: String, + /// The `internal/meta` commit the builder built it from. + meta_rev: String, +} + +/// The closure the swarm built for `name`, downloaded into the store. The +/// deploy applies the swarm's closure whatever commit it was built from: when +/// that is not the `rev` this hive chose, it is logged, and the caller records +/// the pins of the commit actually deployed ([`deployed_pins`]). /// /// # Errors /// [`without_closure`]'s refusal, after logging it. -async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result { +async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result { pull_closure(host, name, rev, step) .await .map_err(|why| without_closure(name, rev, step, &why)) @@ -580,7 +594,7 @@ async fn pull_closure( name: &str, rev: &str, step: &str, -) -> Result { +) -> Result { let record = host .closure_record(name) .await @@ -601,7 +615,33 @@ async fn pull_closure( record.store_path, record.meta_rev ); } - Ok(record.store_path) + Ok(Closure { + path: record.store_path, + meta_rev: record.meta_rev, + }) +} + +/// The pins to record for a deploy of `closure` when this hive chose the commit +/// `rev`: those of the commit the closure was built from, so the markers say +/// what runs. `None` when they are the same commit. +/// +/// # Errors +/// The pins of the closure's commit could not be read. The caller fails before +/// applying anything. +async fn deployed_pins( + host: &impl Host, + name: &str, + closure: &Closure, + rev: &str, +) -> Result> { + if closure.meta_rev == rev { + return Ok(None); + } + let pins = host + .pins_at(name, &closure.meta_rev) + .await + .with_context(|| format!("{name}: reading the pins of the commit the swarm built from"))?; + Ok(Some(pins)) } /// What a rebuild or create does when there is no closure to deploy: it @@ -638,11 +678,19 @@ async fn run_swap( let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir)?; let rev = host.recorded_rev(name)?; - let toplevel = closure_for(host, name, &rev, "swap").await?; - tracing::info!(%name, %rev, "swap: applying {toplevel}"); + let closure = closure_for(host, name, &rev, "swap").await?; + let deployed = deployed_pins(host, name, &closure, &rev).await?; + tracing::info!(%name, %rev, "swap: applying {}", closure.path); let result = host - .swap_update(name, &hive, &paths, &toplevel, id.get()) + .swap_update(name, &hive, &paths, &closure.path, id.get()) .await; + // The bookkeeping node marks what the record holds, so it holds what was + // applied. + if result.is_ok() + && let Some(deployed) = &deployed + { + host.record_meta(name, deployed)?; + } // On success the Ok-only bookkeeping tail (rev marker, forge // sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node, // which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded @@ -708,9 +756,12 @@ async fn run_provision(host: &impl Host, name: &str) -> Result<()> { /// with none to deploy the create fails and makes no container. async fn run_create(host: &impl Host, name: &str) -> Result<()> { let build = host.resolve_meta(name).await?; - let toplevel = closure_for(host, name, &build.rev, "create").await?; - tracing::info!(%name, rev = %build.rev, "create: substituted {toplevel}"); - host.create_only(name, &toplevel).await?; + let closure = closure_for(host, name, &build.rev, "create").await?; + let build = deployed_pins(host, name, &closure, &build.rev) + .await? + .unwrap_or(build); + tracing::info!(%name, rev = %build.rev, "create: substituted {}", closure.path); + host.create_only(name, &closure.path).await?; host.mark_deployed(name, &build)?; Ok(()) } diff --git a/hive-c0re/src/job_queue/exec/tests.rs b/hive-c0re/src/job_queue/exec/tests.rs index e1b6154d5..6d4e29203 100644 --- a/hive-c0re/src/job_queue/exec/tests.rs +++ b/hive-c0re/src/job_queue/exec/tests.rs @@ -129,6 +129,15 @@ impl Host for FakeHost { Ok(self.built.clone()) } + async fn pins_at(&self, name: &str, rev: &str) -> anyhow::Result { + self.step("pins_at", format!("pins_at {name} {rev}"))?; + Ok(MetaBuild { + rev: rev.to_owned(), + hyperhive: "dh".to_owned(), + config: Some("dc".to_owned()), + }) + } + async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> { self.record(format!("realise {name} {store_path}")); if self.fails("realise") { @@ -531,14 +540,46 @@ async fn swap_applies_a_closure_built_from_another_commit_with_a_warning() { let id = node_id(); let (result, log) = logging(run_swap(&host, &coord, "alice", id)).await; result.expect("drift is not a refusal"); + // The pins of the commit the closure came from are read before the apply + // and recorded after it, in place of the ones this hive chose; the + // bookkeeping node then marks those. assert_eq!( - host.calls().last(), - Some(&format!("swap_update alice {CLOSURE} node={}", id.get())) + host.calls(), + [ + "recorded_rev alice".to_owned(), + "closure_record alice".to_owned(), + format!("realise alice {CLOSURE}"), + format!("pins_at alice {OTHER_REV}"), + format!("swap_update alice {CLOSURE} node={}", id.get()), + format!("record_meta alice {OTHER_REV}"), + ] ); assert_eq!(log.at(tracing::Level::WARN).len(), 1); assert!(log.at(tracing::Level::ERROR).is_empty()); } +/// Pins that can't be read, or an apply that fails, leave the record alone. +#[tokio::test] +async fn a_drifted_swap_records_pins_only_for_an_apply_that_happened() { + for failing in ["pins_at", "swap_update"] { + let (_dir, coord) = coordinator(); + let host = FakeHost::down().built(OTHER_REV, true).failing(failing); + run_swap(&host, &coord, "alice", node_id()) + .await + .expect_err(failing); + let calls = host.calls(); + assert!( + !calls.iter().any(|c| c.starts_with("record_meta")), + "{failing}: {calls:?}" + ); + assert_eq!( + calls.iter().any(|c| c.starts_with("swap_update")), + failing == "swap_update", + "{failing}: {calls:?}" + ); + } +} + /// A failed update fails the node and nothing else: the swap never restarts or /// stops the container itself — recovery is the DAG's tail `Reconcile`. #[tokio::test] @@ -897,9 +938,18 @@ async fn create_uses_a_closure_built_from_another_commit_with_a_warning() { let (result, log) = logging(run_create(&host, "alice")).await; result.expect("drift is not a refusal"); let calls = host.calls(); - assert!( - calls.contains(&format!("create_only alice {CLOSURE}")), - "{calls:?}" + // Created from the swarm's closure, and marked as the commit it was built + // from, not the one this hive chose. + assert_eq!( + calls, + [ + "resolve_meta alice".to_owned(), + "closure_record alice".to_owned(), + format!("realise alice {CLOSURE}"), + format!("pins_at alice {OTHER_REV}"), + format!("create_only alice {CLOSURE}"), + format!("mark_deployed alice {OTHER_REV}"), + ] ); assert_eq!(log.at(tracing::Level::WARN).len(), 1); assert!(log.at(tracing::Level::ERROR).is_empty()); diff --git a/hive-c0re/src/meta_rev.rs b/hive-c0re/src/meta_rev.rs index 92af161ff..1573a1525 100644 --- a/hive-c0re/src/meta_rev.rs +++ b/hive-c0re/src/meta_rev.rs @@ -130,7 +130,58 @@ pub(crate) async fn resolve(agent: &str) -> Result { Ok(build) } -/// Record `build` as what `agent`'s rebuild builds from. The record is in +/// The pins `internal/meta` at `rev` holds for `agent`: the hyperhive commit and, +/// for a custom agent (`archetype` false), its config commit. This is what a +/// deploy of the swarm's closure for `rev` applies, which can differ from the +/// head [`choose`] took. +/// +/// # Errors +/// `rev` is not a commit id, the forge read failed, or the flake pins no +/// hyperhive commit or, for a custom agent, no config commit. +pub(crate) async fn pins_at( + source: &impl MetaSource, + agent: &str, + archetype: bool, + rev: &str, +) -> Result { + if !valid_meta_rev(rev) { + bail!("{agent}: {rev:?} is not an internal/meta commit id"); + } + let flake = source.flake_at(rev).await?; + let Some(hyperhive) = hive_meta_flake::pinned_rev(&flake, "hyperhive") else { + bail!("{agent}: internal/meta at {rev} pins no hyperhive commit"); + }; + let config = if archetype { + None + } else { + let Some(config) = hive_meta_flake::pinned_rev(&flake, &format!("agent-{agent}")) else { + bail!("{agent}: internal/meta at {rev} pins no config commit for it"); + }; + Some(config.to_owned()) + }; + Ok(MetaBuild { + rev: rev.to_owned(), + hyperhive: hyperhive.to_owned(), + config, + }) +} + +/// [`pins_at`] against the forge this hive reads. +/// +/// # Errors +/// As [`pins_at`], or this hive holds no forge token or cannot read the +/// agent's archetype. +pub(crate) async fn resolve_at(agent: &str, rev: &str) -> Result { + let archetype = crate::archetypes::of(agent) + .with_context(|| format!("{agent}: cannot read the agent's archetype"))?; + let Some(source) = crate::forge::internal_meta_source().await else { + bail!("{agent}: this hive holds no forge token to read internal/meta with"); + }; + pins_at(&source, agent, archetype.is_some(), rev).await +} + +/// Record `build` as the commit `agent`'s rebuild expects the swarm's closure +/// from, and the pins it deploys. The record is in /// hive-c0re's own state ([`crate::paths::meta_rev_dir`]), where the agent /// cannot write. /// @@ -397,6 +448,28 @@ mod tests { assert_unpinned(&choose(&source, "rev", None).await.unwrap_err(), "rev"); } + #[tokio::test] + async fn the_pins_at_a_commit_are_what_its_flake_holds() { + let source = forge(Some(ARGUS_HEAD)); + let argus = pins_at(&source, "argus", false, META).await.unwrap(); + assert_eq!( + argus, + MetaBuild { + rev: META.to_owned(), + hyperhive: HYPERHIVE.to_owned(), + config: Some(ARGUS_HEAD.to_owned()), + } + ); + // An archetype agent has no config pin of its own. + let rev = pins_at(&source, "rev", true, META).await.unwrap(); + assert_eq!((rev.hyperhive.as_str(), rev.config), (HYPERHIVE, None)); + // A custom agent the flake doesn't list, and a rev that is no commit id. + let err = pins_at(&source, "nobody", false, META).await.unwrap_err(); + assert!(format!("{err:#}").contains("no config commit"), "{err:#}"); + let err = pins_at(&source, "argus", false, "main").await.unwrap_err(); + assert!(format!("{err:#}").contains("not an internal/meta commit")); + } + #[tokio::test] async fn no_repo_or_no_commit_is_an_error_naming_the_agent() { let mut source = forge(Some(ARGUS_HEAD)); diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index fce863f99..069567094 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -263,8 +263,8 @@ pub enum PrivRequest { KillContainer { name: String }, /// Point `name`'s container profile at `toplevel`, then reload the unit if - /// it runs. hive-priv never builds: `toplevel` is a closure already in the - /// store (see `RealiseClosure`). + /// it runs. hive-priv never builds: `toplevel` must be the path the last + /// successful `RealiseClosure` for `name` returned, and it refuses any other. /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. UpdateContainer { diff --git a/hive-priv/README.md b/hive-priv/README.md index 108646096..9d3282231 100644 --- a/hive-priv/README.md +++ b/hive-priv/README.md @@ -47,8 +47,14 @@ Each module's `//!` header lists its ops. `container_toplevel_action` (in `src/container.rs`) applies a closure that is already in the store; hive-priv builds nothing. hive-c0re names the closure (`toplevel` on `CreateContainer` / `UpdateContainer`), downloaded beforehand -with `RealiseClosure`, and hive-priv accepts it only in the shape of a store -path. +with `RealiseClosure`. hive-priv remembers, per agent, the path its last +successful `RealiseClosure` returned (in memory), and applies only that path to +that agent. It refuses a path never realised for the agent, another agent's +and the host's own, and logs an error naming the agent and both paths. A +restart between the download and the apply empties the record, so the apply +fails and the job retries. hive-priv does not re-check the signature at apply +time: `RealiseClosure` enforces it, and the binding limits an apply to what that +call stored. `create` passes the path to `nixos-container create --system-path`, so `nixos-container`'s own `buildFlake()` never runs. Without the flag, it builds diff --git a/hive-priv/src/closure.rs b/hive-priv/src/closure.rs index d54e4e008..49f759b5b 100644 --- a/hive-priv/src/closure.rs +++ b/hive-priv/src/closure.rs @@ -143,14 +143,78 @@ fn reason(stderr: &str) -> String { /// out of the store: no cache or key on this host, nix failing or timing out. pub(crate) async fn realise_closure(agent: &str, store_path: &str) -> Result<(String, String)> { let url = std::env::var(CACHE_URL_ENV).ok(); - realise_with( + let done = realise_with( url.as_deref(), Path::new(ROOT_FACTS_DIR), 0, agent, store_path, ) - .await + .await?; + if let Ok(mut realised) = REALISED.lock() { + remember(&mut realised, agent, store_path); + } + Ok(done) +} + +/// The store path each agent's last successful `RealiseClosure` put in the +/// store. `CreateContainer` / `UpdateContainer` apply only that path for that +/// agent. hive-priv is a long-running process, so the entry outlives the +/// calls between the download and the apply; a restart in between empties it +/// and the apply is refused. +static REALISED: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +fn remember(realised: &mut Vec<(String, String)>, agent: &str, store_path: &str) { + realised.retain(|(a, _)| a != agent); + realised.push((agent.to_owned(), store_path.to_owned())); +} + +/// The path recorded for `agent`. +fn lookup<'a>(realised: &'a [(String, String)], agent: &str) -> Option<&'a str> { + realised + .iter() + .find(|(a, _)| a == agent) + .map(|(_, p)| p.as_str()) +} + +/// `toplevel` when it is a store path and the one `agent`'s last +/// `RealiseClosure` returned (`realised`); else the refusal, which is also +/// logged at error level with the agent and both paths. +/// +/// The request comes from hive-c0re, which is not root. A signed path is not +/// enough: it could be another agent's closure or the host's own. +/// +/// # Errors +/// `toplevel` is not a store path, or is not `realised`. +pub(crate) fn applicable<'a>( + agent: &str, + toplevel: &'a str, + realised: Option<&str>, +) -> Result<&'a str> { + if !valid_closure_path(toplevel) { + bail!("{toplevel:?} is not a store path"); + } + if realised == Some(toplevel) { + return Ok(toplevel); + } + let realised = realised.unwrap_or("none"); + tracing::error!( + %agent, requested = %toplevel, %realised, + "refusing to apply a closure that is not the one realised for the agent" + ); + bail!("{agent}: {toplevel} is not the closure realised for this agent ({realised})") +} + +/// [`applicable`] against what `RealiseClosure` recorded for `agent`. +/// +/// # Errors +/// As [`applicable`]. +pub(crate) fn applicable_for<'a>(agent: &str, toplevel: &'a str) -> Result<&'a str> { + let realised = REALISED + .lock() + .ok() + .and_then(|r| lookup(&r, agent).map(str::to_owned)); + applicable(agent, toplevel, realised.as_deref()) } /// [`realise_closure`] with the cache's address and the key's directory and @@ -216,6 +280,62 @@ mod tests { } } + const OTHER: &str = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-iris"; + + #[test] + fn the_path_realised_for_the_agent_is_applied() { + assert_eq!(applicable("atlas", PATH, Some(PATH)).unwrap(), PATH); + } + + /// Another agent's realised path, a valid-shaped path never realised, and + /// a restart that emptied the record are all refused, naming the agent + /// and both paths. + #[test] + fn any_other_path_is_refused() { + for (case, realised) in [ + ("another path was realised for the agent", Some(OTHER)), + ("nothing was realised for the agent", None), + ] { + let err = applicable("atlas", PATH, realised).unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("atlas") + && msg.contains(PATH) + && msg.contains(realised.unwrap_or("none")), + "{case}: {msg}" + ); + } + } + + #[test] + fn only_a_store_path_is_applied_even_when_it_was_recorded() { + for bad in [ + "", + "main", + "/etc/passwd", + "/nix/store/x", + &format!("{PATH}/bin"), + ] { + let err = applicable("atlas", bad, Some(bad)).unwrap_err(); + assert!(format!("{err:#}").contains("not a store path"), "{bad}"); + } + } + + /// Each agent keeps its own entry, and a newer download replaces the + /// older one. + #[test] + fn a_record_is_per_agent_and_the_latest_wins() { + let mut realised = Vec::new(); + remember(&mut realised, "atlas", PATH); + remember(&mut realised, "iris", OTHER); + assert_eq!(lookup(&realised, "atlas"), Some(PATH)); + assert_eq!(lookup(&realised, "iris"), Some(OTHER)); + remember(&mut realised, "atlas", OTHER); + assert_eq!(lookup(&realised, "atlas"), Some(OTHER)); + assert_eq!(lookup(&realised, "nobody"), None); + assert_eq!(realised.len(), 2); + } + fn option<'a>(args: &'a [String], name: &str) -> Option<&'a str> { args.windows(3) .find(|w| w[0] == "--option" && w[1] == name) diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs index 50ac3737c..34e470d1c 100644 --- a/hive-priv/src/container.rs +++ b/hive-priv/src/container.rs @@ -7,7 +7,7 @@ //! and stream progress to the caller as [`PrivEvent::Line`]s when asked to. use anyhow::{Context as _, Result, bail}; -use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine, valid_closure_path}; +use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine}; use tokio::io::{AsyncWriteExt, BufReader}; use tokio::net::unix::OwnedWriteHalf; use tokio::process::Command; @@ -71,17 +71,8 @@ pub(crate) async fn destroy_container(name: &str) -> Result<(String, String)> { container_run(&["destroy", &container_system_name(name)]).await } -/// `toplevel` as a store path, or the error saying it is not one. The path -/// crosses the socket from hive-c0re, which is not root. -fn toplevel_to_apply(toplevel: &str) -> Result<&str> { - if !valid_closure_path(toplevel) { - bail!("{toplevel:?} is not a store path"); - } - Ok(toplevel) -} - /// Shared body for `CreateContainer` / `UpdateContainer`: validate the name -/// and the closure, and apply it (streaming line events to `writer` when +/// and that `toplevel` is the closure realised for the agent, and apply it (streaming line events to `writer` when /// `stream` is set). Nothing is built here: `toplevel` is a closure already in /// the store, which `RealiseClosure` put there. /// @@ -102,7 +93,7 @@ pub(crate) async fn container_toplevel_action( writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { validate_agent_name(name)?; - let toplevel = toplevel_to_apply(toplevel)?; + let toplevel = crate::closure::applicable_for(name, toplevel)?; let system_name = container_system_name(name); if verb == "update" { return swap_container_profile(&system_name, toplevel, stream, writer).await; @@ -567,34 +558,9 @@ async fn container_run_streaming( #[cfg(test)] mod tests { - use super::{BoundedRun, run_bounded, toplevel_to_apply}; + use super::{BoundedRun, run_bounded}; use crate::shared::test_support::scratch; - const TOPLEVEL: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; - - /// A store path passes through unchanged; anything else, whatever the - /// request says, never reaches `nix-env` or `nixos-container`. - #[test] - fn only_a_store_path_is_applied() { - assert_eq!(toplevel_to_apply(TOPLEVEL).unwrap(), TOPLEVEL); - for bad in [ - "", - "main", - "../../x", - "/etc/passwd", - "/nix/store/x", - &format!("{TOPLEVEL}/bin"), - &format!("{TOPLEVEL};id"), - &"c".repeat(40), - ] { - let err = toplevel_to_apply(bad).unwrap_err(); - assert!( - format!("{err:#}").contains("not a store path"), - "{bad}: {err:#}" - ); - } - } - /// Whether `pid` is gone, a zombie (`Z`), or being reaped (`X`): none of /// these runs again, and a process passes through them in that order, /// so once this is true it stays true. From 221c3f35a14a74ade3eeb0e7f0a4fb11a2ff6c2a Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 22:55:31 +0100 Subject: [PATCH 3/3] remove the hive-c0re forge-credential chain nothing reads any more With no nix build on the hive, the boot publish of the forge token for nix has no reader. Removes the ForgeSweep node, the token file, the git credential helper and hive-c0re's forge gitconfig sections. --- docs/integrations/forge.md | 13 +-- docs/scheduler/coordinator.md | 1 - docs/swarm/credentials-reference.md | 2 +- docs/trust-boundary/security.md | 56 +++++------ hive-c0re/src/forge/mod.rs | 14 +-- hive-c0re/src/forge/reader.rs | 111 --------------------- hive-c0re/src/job_queue/exec.rs | 17 ---- hive-c0re/src/job_queue/exec/tests.rs | 20 +--- hive-c0re/src/job_queue/model.rs | 5 +- hive-c0re/src/lifecycle/git.rs | 7 +- hive-c0re/src/main.rs | 4 - hive-c0re/src/paths.rs | 7 -- hive-c0re/src/stats/warnings.rs | 5 +- hive-c0re/src/workers/auto_update.rs | 7 +- nix/host-modules/hive-c0re/default.nix | 17 +--- nix/host-modules/hive-c0re/environment.nix | 4 +- nix/host-modules/lib/hive-forge-git.nix | 45 --------- nix/module-eval/core-toggle.nix | 40 +++----- 18 files changed, 67 insertions(+), 308 deletions(-) delete mode 100644 nix/host-modules/lib/hive-forge-git.nix diff --git a/docs/integrations/forge.md b/docs/integrations/forge.md index 472fd6cc6..f39dbaea4 100644 --- a/docs/integrations/forge.md +++ b/docs/integrations/forge.md @@ -69,14 +69,11 @@ forge container: - seeding `proposed` from `agent-configs/` on first deploy - cloning and pulling `/knowledge` -The builds read as the hive's account too. nix fetches the config inputs and -`internal/meta` with `git`, and `git` gets its credential from -`git-credential-hive-forge`, which reads `/run/hyperhive/forge-hive-token`. -hive-c0re writes that file (account on the first line, token on the second) -from the hive's store token at each boot and each `internal/meta` read, when its -content differs. A hive whose store holds no token has no such file, fetches -nothing private through nix and fails its builds: they have no other flake than -`internal/meta`. This also holds on a hive with no forge container. +The hive builds nothing, so no nix process reads the forge as the hive's +account. hive-c0re hands the token to each `git` command it runs +(`http.extraHeader`) and keeps no copy on disk. A hive whose store holds no token +reads nothing private from the forge. This also holds on a hive with no forge +container. Roll a change to any of this out store host first (its granter writes the controller's grant on `swarm/hives/+/forge/token`), then the controller host, diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 7b5b3dff4..2966f16f7 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -72,7 +72,6 @@ Cheap — no build slot: | `SetWanted` | write the durable power intent (`wanted = Up`/`Offline`) as the head node of a power-op DAG. Takes the agent lease even though it's a store write, so the intent write and the tail `Reconcile` are atomic per-agent — two racing power ops can't clobber each other's intent before either reconciles | | `EmitRebuilt` | tail of a rebuild — emit the agent's `Rebuilt` manager event (ok/fail per outcome, nothing on cancel). One node per agent _and_ per outcome | | `WriteDropin` | `set_nspawn_flags` + `set_resource_limits` + daemon-reload | -| `ForgeSweep` | one-shot boot-time publish of the hive's forge read token for nix (`forge::ensure_all`) as a first-class node, so it shows as real work on the dashboard instead of running invisibly in a bare `tokio::spawn`. Agentless | | `KnowledgePull` | `/knowledge` pull (`knowledge::pull`): at boot (commits that landed while `hive-c0re` was down), on the swarm knowledge-changed event, and hourly as a fallback. Holds `Resource::KnowledgeTree` (capacity 1), so two pulls never overlap on the working tree; each trigger queues its own pass, which waits for the resource if one is already live. Agentless | | `WantedPull` | pull of the agent set the swarm controller declares for this hive (`wanted::pull`), converging the agents it names. Runs at boot, waiting up to a minute in total for the queue connection, and again after every attach of the wanted-state watch (which retries with a backoff when the attach fails or the stream ends); the deploy event (`swarm_status`) and the watch are the fast path, this repairs a missed one. A first deploy already queued or running for an agent isn't queued again. Agentless | diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index 7dd943f55..c75ee094f 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -156,7 +156,7 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. | the cache's public key and pull token, for hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); the cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand, and a hive that never got them has `hive-priv` answer `not available`. | | the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | | the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | -| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity, and copies it (account on the first line, token on the second) to `/run/hyperhive/forge-hive-token` (tmpfs, `0600` `hive-core`) for nix's `git`, which `hive-priv` also runs. Each `internal/meta` read and each boot rewrites the copy when it differs; where they differ after a rotation, a build that starts before the next read fails to authenticate, and a retry reads the new token | +| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | | squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | | the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index ebe65029f..fa131ec34 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -252,34 +252,34 @@ for every operation that genuinely requires root. **Narrow interface** — `PrivRequest` variants map 1:1 to specific known operations; there is no arbitrary command pass-through: -| Operation | What it runs | -| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `StartContainer` / `StopContainer` | `nixos-container start/stop ` | -| `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | -| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must be the store path the last successful `RealiseClosure` returned for that agent (in memory, per agent), so hive-c0re can't name a path never realised for the agent; hive-priv builds nothing | -| `RealiseClosure` | `nix-store --realise --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-` and the agent name valid. It downloads and never builds | -| `DestroyContainer` | `nixos-container destroy ` | -| `ListContainers` | `nixos-container list` | -| `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | -| `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list, network isolation vars, and `--ephemeral` after the root's salvage). hive-priv first copies a stopped container's root to the agent's `work/`, then renames it aside (it deletes nothing), following no link the container could have planted | -| `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else | -| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs ` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names | -| `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy, and the GC root of the hive-priv build it named | -| `DaemonReload` | `systemctl daemon-reload` | -| `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | -| `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | -| `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | -| `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | -| `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | -| `DeleteAgentSubvolume` | `btrfs subvolume delete /` — purge path only, never a plain destroy | -| `EnsureBtrfsQuota` | `btrfs quota enable `. Operator opt-in — enabling forces a full rescan | -| `ReadSubvolumeUsage` | `btrfs qgroup show -f --raw /` | -| `SetSubvolumeQuota` | `btrfs qgroup limit /` | -| `MeasureAgentDisk` | `du -s -x -b -c -P` over `//state`, `//work` and `/var/lib/nixos-containers/h-`, killed after 10 min. hive-priv derives the paths from the name; the request carries none | -| `SnapshotAgentSubvolume` | `btrfs subvolume snapshot -r ` — the frozen point-in-time copy a migration streams from | -| `DeleteAgentSnapshot` | `btrfs subvolume delete ` | -| `SendAgentSnapshotToFile` | `btrfs send [-p ] ` into a bare filename under `MIGRATE_STAGING_ROOT`; refuses to overwrite an existing export | -| `SendAgentSnapshotToFd` | `btrfs send [-p ] ` into a **file descriptor passed with the request** (`SCM_RIGHTS`). hive-c0re connects to the peer and hands over the connected socket, so hive-priv never learns an address or protocol. hive-priv requires exactly one descriptor here, and refuses one that arrives alongside any other operation | +| Operation | What it runs | +| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `StartContainer` / `StopContainer` | `nixos-container start/stop ` | +| `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | +| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must be the store path the last successful `RealiseClosure` returned for that agent (in memory, per agent), so hive-c0re can't name a path never realised for the agent; hive-priv builds nothing. The binding doesn't mean hive-c0re can't choose the closure: it can still ask hive-priv to download any swarm-signed path for any agent and then apply it. The swarm key and the cache bound that to a cross-assignment risk (one agent gets another's closure), not an arbitrary-path one | +| `RealiseClosure` | `nix-store --realise --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-` and the agent name valid. It downloads and never builds | +| `DestroyContainer` | `nixos-container destroy ` | +| `ListContainers` | `nixos-container list` | +| `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | +| `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list, network isolation vars, and `--ephemeral` after the root's salvage). hive-priv first copies a stopped container's root to the agent's `work/`, then renames it aside (it deletes nothing), following no link the container could have planted | +| `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else | +| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs ` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names | +| `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy, and the GC root of the hive-priv build it named | +| `DaemonReload` | `systemctl daemon-reload` | +| `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | +| `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | +| `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | +| `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | +| `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | +| `DeleteAgentSubvolume` | `btrfs subvolume delete /` — purge path only, never a plain destroy | +| `EnsureBtrfsQuota` | `btrfs quota enable `. Operator opt-in — enabling forces a full rescan | +| `ReadSubvolumeUsage` | `btrfs qgroup show -f --raw /` | +| `SetSubvolumeQuota` | `btrfs qgroup limit /` | +| `MeasureAgentDisk` | `du -s -x -b -c -P` over `//state`, `//work` and `/var/lib/nixos-containers/h-`, killed after 10 min. hive-priv derives the paths from the name; the request carries none | +| `SnapshotAgentSubvolume` | `btrfs subvolume snapshot -r ` — the frozen point-in-time copy a migration streams from | +| `DeleteAgentSnapshot` | `btrfs subvolume delete ` | +| `SendAgentSnapshotToFile` | `btrfs send [-p ] ` into a bare filename under `MIGRATE_STAGING_ROOT`; refuses to overwrite an existing export | +| `SendAgentSnapshotToFd` | `btrfs send [-p ] ` into a **file descriptor passed with the request** (`SCM_RIGHTS`). hive-c0re connects to the peer and hands over the connected socket, so hive-priv never learns an address or protocol. hive-priv requires exactly one descriptor here, and refuses one that arrives alongside any other operation | **Closures from the swarm cache** — `RealiseClosure` is how a hive's rebuild takes the closure the swarm built instead of building it. Root-owned sources diff --git a/hive-c0re/src/forge/mod.rs b/hive-c0re/src/forge/mod.rs index a65c9fd87..1a063e98f 100644 --- a/hive-c0re/src/forge/mod.rs +++ b/hive-c0re/src/forge/mod.rs @@ -1,5 +1,5 @@ //! The hive's forge access: reads of the forge go through [`reader_token`] on -//! every hive, and [`ensure_all`] publishes that token for nix's `git`. The +//! every hive. The //! swarm-wide orgs, teams and repos, and each agent's read grant on //! `internal/docs`, are swarm-controller's. Full design: //! `docs/integrations/forge.md`. @@ -10,7 +10,7 @@ mod reconcile; mod repos; pub(crate) use internal_meta::internal_meta_source; -pub use reader::{publish_nix_credential, reader_token}; +pub use reader::reader_token; pub use reconcile::{fetch_forge_main, reconcile_config_apply, reconcile_config_status}; pub use repos::{clone_config_into_proposed, fetch_config_main_into_applied}; @@ -145,16 +145,6 @@ where .map_err(|_| anyhow::anyhow!("{op}: no answer from the forge within {API_TIMEOUT:?}")) } -/// Publish this hive's forge read token for nix's `git` at startup, so a build -/// that starts before any `internal/meta` read still has a credential. Every -/// hive runs it: the token is the hive's own, in the store, whether or not a -/// forge container is local. -pub async fn ensure_all() { - if publish_nix_credential().await { - tracing::debug!("forge: nix's forge credential is current"); - } -} - #[cfg(test)] mod tests { use super::{API_TIMEOUT, bounded, git_url_with_base}; diff --git a/hive-c0re/src/forge/reader.rs b/hive-c0re/src/forge/reader.rs index 149ea67f0..0bc243fac 100644 --- a/hive-c0re/src/forge/reader.rs +++ b/hive-c0re/src/forge/reader.rs @@ -1,14 +1,6 @@ //! The credential this hive reads the forge with: its own read-only account, //! whose token `swarm-controller` mints into the store at //! `swarm/hives//forge/token`. -//! -//! nix's `git` cannot reach the store, so [`ReaderToken::publish_for_nix`] -//! copies the token to a runtime file that `git-credential-hive-forge` -//! (`nix/host-modules/lib/hive-forge-git.nix`) reads. - -use std::path::Path; - -use anyhow::Result; use super::basic_auth_header; @@ -39,30 +31,6 @@ impl ReaderToken { pub(crate) fn api(&self) -> anyhow::Result { super::api(&self.value) } - - /// Copy this token to the file nix's `git` authenticates from - /// ([`crate::paths::FORGE_HIVE_TOKEN`]). Whether the file now holds it; a - /// failed write is logged and leaves the file as it was. - pub(crate) fn publish_for_nix(&self) -> bool { - let path = Path::new(crate::paths::FORGE_HIVE_TOKEN); - match self.write_credential(path) { - Ok(()) => true, - Err(e) => { - tracing::warn!(path = %path.display(), error = ?e, "forge: could not write the nix forge credential"); - false - } - } - } - - /// Login on the first line, token on the second. Written only when the - /// content differs, so a build's resolve is not a disk write. - fn write_credential(&self, path: &Path) -> Result<()> { - let content = format!("{}\n{}\n", self.login, self.value); - if std::fs::read(path).is_ok_and(|have| have == content.as_bytes()) { - return Ok(()); - } - crate::secret_file::write_secret_0600(path, content.as_bytes()) - } } /// The token to read the forge with, or `None` when the store holds none for @@ -77,16 +45,6 @@ pub async fn reader_token() -> Option { token } -/// Publish this hive's read token for nix; see [`ReaderToken::publish_for_nix`]. -/// Whether the file holds a token. -pub async fn publish_nix_credential() -> bool { - let Some(token) = reader_token().await else { - tracing::debug!("forge: no hive read token in the store; nix has no forge credential"); - return false; - }; - token.publish_for_nix() -} - /// This hive's token from the store, under the hive's own identity. /// /// `None`, never an error, for every way it can come up empty — no hive name, @@ -131,8 +89,6 @@ async fn stored_reader_token() -> Option { #[cfg(test)] mod tests { use super::*; - use std::os::unix::fs::PermissionsExt as _; - fn stored() -> ReaderToken { ReaderToken { login: "hive-alpha".to_owned(), @@ -140,73 +96,6 @@ mod tests { } } - fn mode(path: &Path) -> u32 { - std::fs::metadata(path).expect("stat").permissions().mode() & 0o777 - } - - #[test] - fn the_nix_credential_is_the_hives_login_then_its_token_at_0600() { - let dir = tempfile::tempdir().expect("tempdir"); - let path = dir.path().join("forge-hive-token"); - stored().write_credential(&path).expect("written"); - assert_eq!( - std::fs::read_to_string(&path).expect("read"), - "hive-alpha\nfrom-the-store\n" - ); - assert_eq!(mode(&path), 0o600); - } - - #[test] - fn a_rotated_token_replaces_the_nix_credential() { - let dir = tempfile::tempdir().expect("tempdir"); - let path = dir.path().join("forge-hive-token"); - stored().write_credential(&path).expect("first"); - let rotated = ReaderToken { - login: "hive-alpha".to_owned(), - value: "rotated".to_owned(), - }; - rotated.write_credential(&path).expect("second"); - assert_eq!( - std::fs::read_to_string(&path).expect("read"), - "hive-alpha\nrotated\n" - ); - assert_eq!(mode(&path), 0o600); - } - - #[test] - fn an_unchanged_credential_is_not_rewritten() { - let dir = tempfile::tempdir().expect("tempdir"); - let path = dir.path().join("forge-hive-token"); - stored().write_credential(&path).expect("first"); - // A rewrite goes through a sibling temp file and a rename, which - // gives the file a new inode. - let before = std::fs::metadata(&path).expect("stat"); - stored().write_credential(&path).expect("second"); - let after = std::fs::metadata(&path).expect("stat"); - assert_eq!( - std::os::unix::fs::MetadataExt::ino(&before), - std::os::unix::fs::MetadataExt::ino(&after) - ); - } - - /// `hive-forge-git.nix`'s credential helper names this path as a literal. - #[test] - fn the_nix_credential_lives_on_the_runtime_tmpfs_where_the_helper_reads_it() { - assert_eq!( - crate::paths::FORGE_HIVE_TOKEN, - "/run/hyperhive/forge-hive-token" - ); - assert!(crate::paths::FORGE_HIVE_TOKEN.starts_with(crate::paths::RUNTIME_ROOT)); - } - - #[test] - fn a_path_that_cannot_be_written_is_an_error_not_a_panic() { - let dir = tempfile::tempdir().expect("tempdir"); - let file = dir.path().join("a-file"); - std::fs::write(&file, "").expect("file"); - assert!(stored().write_credential(&file.join("sub")).is_err()); - } - #[test] fn the_header_names_the_tokens_own_account() { use base64::Engine as _; diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 2de5dd9d5..39f0994b2 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -99,8 +99,6 @@ pub(super) trait Host: Sync { async fn delete_state_subvolume(&self, name: &str) -> Result<()>; /// The persistent directories a purge removes. fn purge_dirs(&self, name: &str) -> Vec; - /// Publish the hive's forge credential for nix. - async fn forge_ensure_all(&self); /// One `/knowledge` pull. async fn knowledge_pull(&self, coord: &Arc) -> Result<()>; /// One swarm wanted-state pull. @@ -228,10 +226,6 @@ impl Host for RealHost { .collect() } - async fn forge_ensure_all(&self) { - crate::forge::ensure_all().await; - } - async fn knowledge_pull(&self, coord: &Arc) -> Result<()> { crate::workers::knowledge::pull(coord).await } @@ -332,23 +326,12 @@ async fn dispatch( // `Finishing` so the nodes under it start. What they declare stays held // until their whole subtree settles. NodeKind::AgentWindow { .. } => Ok(()), - NodeKind::ForgeSweep => run_forge_sweep(host).await, NodeKind::KnowledgePull => run_knowledge_pull(host, coord, KNOWLEDGE_PULL_DEADLINE).await, NodeKind::WantedPull => run_wanted_pull(host, coord).await, }; (builder, result) } -/// Boot-time publish of the hive's forge credential for nix, as a DAG node — -/// see [`NodeKind::ForgeSweep`]. `forge::ensure_all` logs its own outcome and -/// is best-effort (a build re-publishes the credential before it fetches), so -/// this wrapper has nothing left to report; it exists to make the step a -/// visible unit of work. -async fn run_forge_sweep(host: &impl Host) -> Result<()> { - host.forge_ensure_all().await; - Ok(()) -} - /// `/knowledge` pull as a DAG node — see [`NodeKind::KnowledgePull`]. Every /// pass reports here, whoever submitted it: as the node's own error on the /// dashboard, and into the debounced warning. diff --git a/hive-c0re/src/job_queue/exec/tests.rs b/hive-c0re/src/job_queue/exec/tests.rs index 6d4e29203..a95f947e3 100644 --- a/hive-c0re/src/job_queue/exec/tests.rs +++ b/hive-c0re/src/job_queue/exec/tests.rs @@ -12,9 +12,9 @@ use hive_jobq::scheduler::Scheduler; use super::{ Host, dispatch, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain, - run_emit_rebuilt, run_forge_sweep, run_knowledge_pull, run_meta_sync, run_pause_drain, - run_pause_signal, run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping, - run_reconcile, run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap, + run_emit_rebuilt, run_knowledge_pull, run_meta_sync, run_pause_drain, run_pause_signal, + run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping, run_reconcile, + run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap, run_wanted_pull, run_write_dropin, }; use crate::coordinator::{AgentPaths, Coordinator, HiveEnv}; @@ -254,10 +254,6 @@ impl Host for FakeHost { self.purge.clone() } - async fn forge_ensure_all(&self) { - self.record("forge_ensure_all".to_owned()); - } - async fn knowledge_pull(&self, _coord: &Arc) -> anyhow::Result<()> { if self.hang { std::future::pending::<()>().await; @@ -1131,13 +1127,6 @@ async fn emit_rebuilt_reports_success_and_failure_under_one_key() { // --- Sweeps and pulls --------------------------------------------------------- -#[tokio::test] -async fn forge_sweep_runs_once_and_never_fails() { - let host = FakeHost::up(); - run_forge_sweep(&host).await.expect("ok"); - assert_eq!(host.calls(), ["forge_ensure_all"]); -} - #[tokio::test] async fn knowledge_pull_passes_the_result_through_and_times_out() { let (_dir, coord) = coordinator(); @@ -1305,7 +1294,6 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() { async fn dispatch_routes_the_agentless_and_hostless_kinds() { let host = FakeHost::up(); for kind in [ - NodeKind::ForgeSweep, NodeKind::WantedPull, NodeKind::KnowledgePull, alice(|agent| NodeKind::AgentWindow { agent }), @@ -1325,7 +1313,7 @@ async fn dispatch_routes_the_agentless_and_hostless_kinds() { ); } let calls = host.calls(); - for want in ["forge_ensure_all", "wanted_pull", "knowledge_pull"] { + for want in ["wanted_pull", "knowledge_pull"] { assert!(calls.iter().any(|c| c == want), "{want} in {calls:?}"); } } diff --git a/hive-c0re/src/job_queue/model.rs b/hive-c0re/src/job_queue/model.rs index 8eb0fe0dd..6c3e67e5b 100644 --- a/hive-c0re/src/job_queue/model.rs +++ b/hive-c0re/src/job_queue/model.rs @@ -96,9 +96,6 @@ pub enum NodeKind { /// Write the agent's durable power intent (`wanted = Up` when `up`, /// else `Offline`) as the head node of a power-op DAG. SetWanted { agent: String, up: bool }, - /// One-shot boot-time publish of the hive's forge read token for nix - /// (`forge::ensure_all`). Agentless. - ForgeSweep, /// `/knowledge` pull (`knowledge::pull`): at boot, on the swarm /// knowledge-changed event, and hourly. Agentless. KnowledgePull, @@ -165,7 +162,7 @@ impl NodeKind { | NodeKind::AgentWindow { agent } | NodeKind::EmitRebuilt { agent, .. } | NodeKind::SetWanted { agent, .. } => agent, - NodeKind::ForgeSweep | NodeKind::KnowledgePull | NodeKind::WantedPull => "", + NodeKind::KnowledgePull | NodeKind::WantedPull => "", } } diff --git a/hive-c0re/src/lifecycle/git.rs b/hive-c0re/src/lifecycle/git.rs index 7d97cfefc..c2cbb406a 100644 --- a/hive-c0re/src/lifecycle/git.rs +++ b/hive-c0re/src/lifecycle/git.rs @@ -10,9 +10,8 @@ use anyhow::{Context, Result, bail}; use tokio::io::{AsyncRead, AsyncReadExt}; use tokio::process::Command; -/// Deadline for a child that talks to a remote (forge fetch/push, `nix flake -/// update`/`lock`): a healthy one takes seconds, so ten minutes only ever -/// fires on a hang. +/// Deadline for a child that talks to a remote (forge fetch/push): a healthy +/// one takes seconds, so ten minutes only ever fires on a hang. pub const FETCH_DEADLINE: Duration = Duration::from_mins(10); /// Deadline for a git child that touches only the local repo. @@ -21,7 +20,7 @@ pub const GIT_LOCAL_DEADLINE: Duration = Duration::from_mins(2); /// Run `cmd` to completion and capture its output, failing once `deadline` /// passes. `what` names the command in the error. The child leads its own /// process group and the whole group is `SIGKILL`ed on timeout, so a -/// grandchild (a `git` or credential helper under `nix`) dies with it instead +/// grandchild (a `git` or credential helper it spawned) dies with it instead /// of outliving the lock. pub async fn run_with_deadline( cmd: &mut Command, diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index c8568b9da..231900f94 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -259,10 +259,6 @@ async fn cmd_serve( tracing::warn!(error = ?e, "auto-update task failed"); } }); - // Forge credential publish: a `NodeKind::ForgeSweep` DAG node (see - // `workers::auto_update::submit_startup_sweep_nodes`), submitted - // unconditionally on every boot — moved off a bare `tokio::spawn` so it - // shows as real work on the dashboard. // Knowledge periodic pull: hourly fallback in case a knowledge event from // the swarm is missed (e.g. hive-c0re was down when it was sent). Sleeps // before its first pull: the boot `NodeKind::KnowledgePull` DAG node diff --git a/hive-c0re/src/paths.rs b/hive-c0re/src/paths.rs index 246619a83..11aed6b1a 100644 --- a/hive-c0re/src/paths.rs +++ b/hive-c0re/src/paths.rs @@ -196,13 +196,6 @@ pub fn gateway_agents_conf() -> PathBuf { gateway_dir().join("agents.conf") } -/// `forge-hive-token` — this hive's read-only forge token (login on the first -/// line, token on the second), copied from the swarm secret store for nix's -/// `git`. Tmpfs: re-published on every boot. -// nix: read by `git-credential-hive-forge`; the same literal is in -// `nix/host-modules/lib/hive-forge-git.nix` — must match. -pub const FORGE_HIVE_TOKEN: &str = "/run/hyperhive/forge-hive-token"; - /// `attic/` — the swarm cache's pull credentials for the probe: `netrc`, `substituter` /// and `public-key`, each `0600`, copied from the swarm secret store. Tmpfs: /// re-published on every boot. diff --git a/hive-c0re/src/stats/warnings.rs b/hive-c0re/src/stats/warnings.rs index dca7cf8f9..066f331b1 100644 --- a/hive-c0re/src/stats/warnings.rs +++ b/hive-c0re/src/stats/warnings.rs @@ -172,9 +172,8 @@ pub fn snapshot() -> Vec { /// Raise a warning for a one-shot boot-time step that has no periodic /// retry to observe a later success and clear the warning via -/// [`WarningGuard`] drop — e.g. a step inside `forge::ensure_all()`, -/// which `tokio::spawn`s once at hive-c0re startup and never runs again -/// this process. Intentionally **leaks** the guard for the life of the +/// [`WarningGuard`] drop — e.g. a step that runs once at hive-c0re startup +/// and never again this process. Intentionally **leaks** the guard for the life of the /// process: the warning clears the next time hive-c0re restarts (a fresh /// process starts with an empty registry) and re-runs the step, which is /// exactly when a config/environment fix would take effect anyway. diff --git a/hive-c0re/src/workers/auto_update.rs b/hive-c0re/src/workers/auto_update.rs index 43cd54a52..0807b7f59 100644 --- a/hive-c0re/src/workers/auto_update.rs +++ b/hive-c0re/src/workers/auto_update.rs @@ -196,13 +196,13 @@ fn boot_action(wanted: Option, fresh: bool, running: bool) } } -/// Submit the boot-time forge/knowledge/wanted-state sweeps as DAG -/// nodes — `ForgeSweep`, `KnowledgePull`, `WantedPull`. Unlike +/// Submit the boot-time knowledge and wanted-state sweeps as DAG +/// nodes — `KnowledgePull`, `WantedPull`. Unlike /// [`submit_boot_tree`] this runs on **every** boot, quiet or not: these /// aren't config-drift work, they're startup housekeeping that always needs /// to happen, and the point of moving them here is exactly so they show up /// as real work on the dashboard instead of an invisible `tokio::spawn` that -/// only surfaces on failure. Three independent, build-slot- and lease-exempt +/// only surfaces on failure. Independent, build-slot- and lease-exempt /// roots — no dependency edges between them, matching the existing /// `Reconcile`-root pattern in [`boot_nodes`]. `KnowledgePull` holds its /// sweep's own resource, so it queues behind any pass of the same sweep @@ -211,7 +211,6 @@ fn submit_startup_sweep_nodes(coord: &Arc) { use crate::job_queue::NodeKind; if let Err(e) = coord.job_queue.insert_job(|b| { - let _ = b.node(NodeKind::ForgeSweep); let _ = crate::job_queue::templates::knowledge_pull(b); let _ = b.node(NodeKind::WantedPull); Vec::new() diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 6cd79d13c..7441493f6 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -35,26 +35,18 @@ let gatewayCfg = config.services.hyperhive.gateway; }; - forgeGit = import ../lib/hive-forge-git.nix { - inherit pkgs; - forgeDomain = config.services.hyperhive.swarm.forge.domain; - }; - # Privsep splits ownership across users, so git/libgit2's dubious- # ownership guard trips on legitimate cross-user reads: hive-c0re # (hive-core) fetches the agent-owned proposed-config repos. The process is # trusted and can already read the files; this gitconfig only satisfies the # ownership guard. libgit2 honours the literal `*` (mid-path globs aren't # supported, so per-agent repos can't be listed); in practice this process - # only ever touches hyperhive's own repos. - # - # The forge sections (`forgeGit.config`) answer for the `git+http` forge - # fetches nix makes: the agent config inputs and `internal/meta`. hive-c0re's - # own forge reads use `forge::reader_token()` instead. + # only ever touches hyperhive's own repos. Its forge reads authenticate with + # `forge::reader_token()`, handed to git per command, so no credential helper + # is configured. safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" '' [safe] directory = * - ${forgeGit.config} [http] # Abort a stalled fetch instead of blocking startup indefinitely. If a # git http transfer from the forge (e.g. filling an applied mirror) @@ -251,9 +243,6 @@ in after = [ "hive-c0re.socket" ]; path = [ pkgs.git - # `git-credential-hive-forge` on PATH so git finds it for git+http - # fetches from the forge (helper = hive-forge). - forgeGit.credentialHelper "/run/current-system/sw" ]; environment = import ./environment.nix { inherit lib config pkgs; }; diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index bad8fc369..1d4976b32 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -47,8 +47,8 @@ in HOME = "/var/lib/hyperhive"; HYPERHIVE_GIT = "${pkgs.git}/bin/git"; # Never let git block on an interactive credential prompt. hive-core is a - # TTY-less system user, so a prompt (e.g. the forge credential helper - # returns nothing because the forge isn't reachable yet on cold boot) + # TTY-less system user, so a prompt (e.g. git asking for credentials + # because the forge isn't reachable yet on cold boot) # would hang forever and freeze the daemon during startup. With this set, # git fails fast instead of prompting. Paired with the git http # low-speed abort in `safeDirGitconfig` (bounds a stalled transfer) and diff --git a/nix/host-modules/lib/hive-forge-git.nix b/nix/host-modules/lib/hive-forge-git.nix deleted file mode 100644 index d85d31f1a..000000000 --- a/nix/host-modules/lib/hive-forge-git.nix +++ /dev/null @@ -1,45 +0,0 @@ -# Git settings for the host processes that hand nix a forge URL: hive-c0re -# (prebuilds as `hive-core`) and hive-priv (builds as root, with a HOME of its -# own), each through its own `$HOME/.gitconfig`. nix runs `git` for a `git+…` -# input, so git's config is the only credential and URL plumbing it has. -# -# - `credential`: the forge host answers through `git-credential-hive-forge`, -# which reads the hive's own forge account and token from the file hive-c0re -# publishes. An empty `helper =` first resets the list, so no /etc/gitconfig -# helper (e.g. `oauth`, which waits on a browser) runs ahead of ours. -# - `url`: the swarm's `internal/meta` lock names its inputs by the forge's -# `https` URL, which only the builder's credential helper answers for. They -# are fetched over the `http` URL the helper above is scoped to. -# -# Pure function — NOT a NixOS module (don't add it to the host-modules -# aggregator). Call it from a module's `let`: -# -# forgeGit = import ./lib/hive-forge-git.nix { inherit pkgs; forgeDomain = …; }; -# # then put `forgeGit.credentialHelper` on the unit's `path` and -# # `forgeGit.config` into its gitconfig. -{ pkgs, forgeDomain }: -{ - # Answers only `get`. The file (`/run/hyperhive/forge-hive-token`, hive-c0re's - # `paths::FORGE_HIVE_TOKEN`) holds the account on its first line and the - # token on its second; it is read on every invocation, so it never holds a - # stale copy and survives token rotation. Prints nothing where the file is - # absent or incomplete. - credentialHelper = pkgs.writeShellScriptBin "git-credential-hive-forge" '' - [ "''${1:-}" = "get" ] || exit 0 - [ -r /run/hyperhive/forge-hive-token ] || exit 0 - { read -r login; read -r token; } < /run/hyperhive/forge-hive-token || true - [ -n "$login" ] && [ -n "$token" ] || exit 0 - printf 'username=%s\n' "$login" - printf 'password=%s\n' "$token" - ''; - - config = '' - [credential] - helper = - [credential "http://${forgeDomain}"] - helper = - helper = hive-forge - [url "http://${forgeDomain}/"] - insteadOf = https://${forgeDomain}/ - ''; -} diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 3c6bb9a50..745260b98 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -234,39 +234,25 @@ let && env shared.controllerAlone == { }; } { - # hive-c0re carries the credential helper and the https-to-http rewrite as - # git config. hive-priv builds nothing and fetches nothing from the forge, - # so it carries neither and has no forge URL. The hive-c0re arm is the - # control that the lookups find a helper and a forge URL where there is - # one, so the absences on hive-priv and the controller are real. - name = "hive-c0re answers the forge's git fetches and rewrites its https inputs to http; hive-priv has neither"; + # No host process hands nix a forge URL any more, so none carries a git + # credential helper for the forge. The hive-c0re arm is the control: its + # `git` and its forge URL are found by the same lookups, so the absences + # are real. + name = "no host unit carries the forge credential helper, and hive-priv has no forge URL"; ok = let - forge = allLocal.services.hyperhive.swarm.forge.domain; - git = import ../host-modules/lib/hive-forge-git.nix { - inherit pkgs; - forgeDomain = forge; - }; - helperOn = + pathNames = unit: - lib.any ( - p: builtins.isAttrs p && (p.name or "") == "git-credential-hive-forge" - ) allLocal.systemd.services.${unit}.path; + map (p: if builtins.isAttrs p then p.name or "" else "") allLocal.systemd.services.${unit}.path; env = unit: allLocal.systemd.services.${unit}.environment; in - helperOn "hive-c0re" - && !(helperOn "hive-priv") - # The helper answers from the file hive-c0re publishes the hive's own - # token to. The first arm is the control for the second: the lookup - # finds the helper's text, so the absent `core` strings are really absent. - && lib.hasInfix "/run/hyperhive/forge-hive-token" git.credentialHelper.text - && !(lib.hasInfix "forge-core-token" git.credentialHelper.text) - && !(lib.hasInfix "username=core" git.credentialHelper.text) - && !(lib.hasInfix "username = core" git.config) - && !(helperOn "swarm-controller") - && lib.hasInfix "[credential \"http://${forge}\"]\nhelper =\nhelper = hive-forge" git.config - && lib.hasInfix "[url \"http://${forge}/\"]\ninsteadOf = https://${forge}/" git.config + lib.any (lib.hasPrefix "git-2") (pathNames "hive-c0re") && (env "hive-c0re") ? HIVE_FORGE_URL + && lib.all (unit: !(builtins.elem "git-credential-hive-forge" (pathNames unit))) [ + "hive-c0re" + "hive-priv" + "swarm-controller" + ] && !((env "hive-priv") ? HIVE_PRIV_FORGE_URL); } {