From c16cf1756ff7235b01d0cc0f7c4d4f9f54658f48 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 02:02:29 +0100 Subject: [PATCH] download an agent's cached closure in prebuild, building locally when that fails --- docs/swarm/builder.md | 51 +++- docs/swarm/credentials-reference.md | 2 +- docs/trust-boundary/security.md | 89 ++++-- hive-c0re/src/attic/closure.rs | 62 ++++ hive-c0re/src/attic/mod.rs | 6 +- hive-c0re/src/attic/probe.rs | 44 ++- hive-c0re/src/attic/reader.rs | 3 +- hive-c0re/src/job_queue/exec.rs | 97 ++++++- hive-c0re/src/job_queue/exec/tests.rs | 194 ++++++++++++- hive-c0re/src/lifecycle/mod.rs | 52 +++- hive-c0re/src/paths.rs | 2 +- hive-c0re/src/priv_client.rs | 13 + hive-priv-sock/src/lib.rs | 81 +++++- hive-priv/src/closure.rs | 401 ++++++++++++++++++++++++++ hive-priv/src/container.rs | 4 +- hive-priv/src/main.rs | 12 +- 16 files changed, 1032 insertions(+), 81 deletions(-) create mode 100644 hive-c0re/src/attic/closure.rs create mode 100644 hive-priv/src/closure.rs diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index 6d76b5a7..459f8982 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -202,9 +202,10 @@ refuses a request that waited 30 seconds for it. It runs `nix build` of `git+https:///internal/meta.git?rev=`, with no input overridden: the committed `flake.lock` decides every input. Like the hive's build, it uses that attribute, so a green build here means the flake evaluates and builds for -the builder's platform. The hive doesn't reuse the result yet: it still builds -locally, from whatever head it reads at deploy time, which can be newer than -the one the swarm built. +the builder's platform. A hive [downloads the closure instead of building it](#what-hives-do-with-the-cache) +when the record is for the commit it builds from; otherwise it builds locally, +from whatever head it reads at deploy time, which can be newer than the one the +swarm built. The node ends `done` or `failed`. Every failure skips the deploy and shows on the Jobs page as a failed `build_meta` node, and the timer builds it again: @@ -250,8 +251,7 @@ platform and the time. Only the controller writes the bucket, and the builder holds no grant on it. A failed build writes nothing and leaves the previous key. The record's `cached` field says whether the builder pushed the closure to [the binary cache](#binary-cache). Each hive reads its own agents' keys to -[report whether it can pull them](#what-hives-do-with-the-cache); no deploy -uses a record yet. The closure stays unrooted in the builder host's store. +[report whether it can pull them and download them in a rebuild](#what-hives-do-with-the-cache). The closure stays unrooted in the builder host's store. The controller gives the `builder` forge account read on `internal/meta` when it updates the flake, and again before each build, since the repo is private. A @@ -437,6 +437,47 @@ live attic. A `bad-signature` on every agent in the first pass after a deploy means check attic's fingerprint format; it says nothing about an attacker. A `fetch-failed` on every agent points at the URL, the token or the network. +### Rebuilds download the closure + +A rebuild's `Prebuild` node, which runs while the agent is still up, tries the +cache before it builds. It reads the agent's record in `agent-closures` and +goes on only when the record is for the `internal/meta` commit `MetaSync` +recorded for this rebuild and says the builder pushed the closure. It then +asks `hive-priv` to realise the record's store path (`RealiseClosure`). `hive-priv` runs +`nix-store --realise` with `--max-jobs 0`, so nix can't build the path, and with +the host's configured cache address, and the public key and netrc from the +root-owned `/run/hyperhive-priv/attic/`, as per-call settings. The settings replace the host's `substituters` and +`trusted-public-keys` and set `require-sigs true`: the swarm cache is the only +source, and nix stores the path only when the cache's key signed it. The host's +nix configuration stays as it was. + +The deploy log says which way it went: + +- `prebuild: substituted `: the closure is in the store, and the local build found it there and had nothing to build. +- `prebuild: the downloaded closure is not the one this hive evaluates, ; built the latter locally`, at warn level: the builder and the hive evaluate different closures for the agent, which is worth a look. The download was no use and the rebuild built locally, with the agent still up. +- `prebuild: built locally: `: the reason is the first check that failed, + such as `no-record`, `rev-mismatch`, `not-cached`, or what `hive-priv` answered + (`not available: …`, for a cache that's down or has no credentials). The rebuild builds as before. +- The same line at error level, with `no signature by the swarm cache's key` in + the reason: nix refused the path because no trusted key signed it. The rebuild builds locally + and `hive-priv` logs the refusal as well. + +After a download, `Prebuild` still runs the local `nix build` of the agent's toplevel, +while the agent is up. It's an evaluation and no build when the hive evaluates the path the +swarm built, and a full build, as without a cache, when it doesn't, so `Swap` never meets a +build it didn't expect. `Swap` still evaluates and builds in `hive-priv`, and finds the path in +the store. A stopped agent skips `Prebuild`, so it still builds in `Swap`. + +Mixed versions: + +| hive | controller or record | what happens | +| ---- | -------------------- | ---------------------------------------------------------------------------------- | +| old | any | builds as before | +| new | old, or no record | builds locally and logs why | +| new | cache down | the download fails, the hive builds locally and logs why | +| new | signature mismatch | nix refuses the path, the hive builds locally and logs an error | +| new | old `hive-priv` | the verb is unknown to it, so the call fails and the hive builds locally, as above | + ## Credentials | What | Where | Who reads it | diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index 650fad93..c516f69f 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -1,4 +1,4 @@ -| this hive's cache pull facts | `swarm-controller`, into the secret store at `swarm/hives//attic/{pull-token,url,cache,public-key}`, copied every five minutes from `swarm/services/attic/` | `hive-c0re` reads them under the hive's own store identity and writes `/run/hyperhive/attic/netrc` (the pull token), `substituter` and `public-key` (tmpfs, each `0600` `hive-core`). Nothing uses the files yet. | +| this hive's cache pull facts | `swarm-controller`, into the secret store at `swarm/hives//attic/{pull-token,url,cache,public-key}`, copied every five minutes from `swarm/services/attic/` | `hive-c0re` reads them under the hive's own store identity and writes `/run/hyperhive/attic/netrc` (the pull token), `substituter` and `public-key` (tmpfs, each `0600` `hive-core`). Only the probe reads them; `hive-priv` doesn't. | # Credentials: the per-secret reference diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index 993c2618..e44ec02d 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -247,33 +247,68 @@ for every operation that genuinely requires root. **Narrow interface** — `PrivRequest` variants map 1:1 to specific known operations; there is no arbitrary command pass-through: -| Operation | What it runs | -| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `StartContainer` / `StopContainer` | `nixos-container start/stop ` | -| `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | -| `CreateContainer` / `UpdateContainer` | `nixos-container create/update --flake ` | -| `DestroyContainer` | `nixos-container destroy ` | -| `ListContainers` | `nixos-container list` | -| `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | -| `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list + network isolation vars) | -| `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else | -| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container | -| `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy | -| `DaemonReload` | `systemctl daemon-reload` | -| `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | -| `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | -| `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | -| `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | -| `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | -| `DeleteAgentSubvolume` | `btrfs subvolume delete /` — purge path only, never a plain destroy | -| `EnsureBtrfsQuota` | `btrfs quota enable `. Operator opt-in — enabling forces a full rescan | -| `ReadSubvolumeUsage` | `btrfs qgroup show -f --raw /` | -| `SetSubvolumeQuota` | `btrfs qgroup limit /` | -| `MeasureAgentDisk` | `du -s -x -b -c -P` over `//state`, `//work` and `/var/lib/nixos-containers/h-`, killed after 10 min. hive-priv derives the paths from the name; the request carries none | -| `SnapshotAgentSubvolume` | `btrfs subvolume snapshot -r ` — the frozen point-in-time copy a migration streams from | -| `DeleteAgentSnapshot` | `btrfs subvolume delete ` | -| `SendAgentSnapshotToFile` | `btrfs send [-p ] ` into a bare filename under `MIGRATE_STAGING_ROOT`; refuses to overwrite an existing export | -| `SendAgentSnapshotToFd` | `btrfs send [-p ] ` into a **file descriptor passed with the request** (`SCM_RIGHTS`). hive-c0re connects to the peer and hands over the connected socket, so hive-priv never learns an address or protocol. hive-priv requires exactly one descriptor here, and refuses one that arrives alongside any other operation | +| Operation | What it runs | +| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `StartContainer` / `StopContainer` | `nixos-container start/stop ` | +| `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | +| `CreateContainer` / `UpdateContainer` | `nixos-container create/update --flake ` | +| `RealiseClosure` | `nix-store --realise --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-` and the agent name valid. It downloads and never builds | +| `DestroyContainer` | `nixos-container destroy ` | +| `ListContainers` | `nixos-container list` | +| `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | +| `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list + network isolation vars) | +| `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else | +| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container | +| `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy | +| `DaemonReload` | `systemctl daemon-reload` | +| `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | +| `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | +| `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | +| `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | +| `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | +| `DeleteAgentSubvolume` | `btrfs subvolume delete /` — purge path only, never a plain destroy | +| `EnsureBtrfsQuota` | `btrfs quota enable `. Operator opt-in — enabling forces a full rescan | +| `ReadSubvolumeUsage` | `btrfs qgroup show -f --raw /` | +| `SetSubvolumeQuota` | `btrfs qgroup limit /` | +| `MeasureAgentDisk` | `du -s -x -b -c -P` over `//state`, `//work` and `/var/lib/nixos-containers/h-`, killed after 10 min. hive-priv derives the paths from the name; the request carries none | +| `SnapshotAgentSubvolume` | `btrfs subvolume snapshot -r ` — the frozen point-in-time copy a migration streams from | +| `DeleteAgentSnapshot` | `btrfs subvolume delete ` | +| `SendAgentSnapshotToFile` | `btrfs send [-p ] ` into a bare filename under `MIGRATE_STAGING_ROOT`; refuses to overwrite an existing export | +| `SendAgentSnapshotToFd` | `btrfs send [-p ] ` into a **file descriptor passed with the request** (`SCM_RIGHTS`). hive-c0re connects to the peer and hands over the connected socket, so hive-priv never learns an address or protocol. hive-priv requires exactly one descriptor here, and refuses one that arrives alongside any other operation | + +**Closures from the swarm cache** — `RealiseClosure` is how a hive's rebuild +takes the closure the swarm built instead of building it. Root-owned sources +alone decide what root accepts: + +- The cache signs. The swarm's cache (attic) holds the signing key and signs + each narinfo it serves; the private key never leaves it. The builder pushes + with a push token and holds no signing key. +- The address is the host's configuration: + `services.hyperhive.swarm.attic.domain` and `cache` become + `HIVE_PRIV_CACHE_URL` in the `hive-priv` unit. No request and no file chooses + it, and `hive-priv` refuses an address that isn't `https`. +- The key and the pull token come from the secret store. The root unit + `hive-priv-cache-facts` reads the hive's own copies under the hive's own store + identity and writes them to `/run/hyperhive-priv/attic/` (directory `0700`, + files `0600`, root). `hive-priv` reads nothing from `/run/hyperhive/attic/`, + which `hive-c0re` writes for the probe, and it refuses a file in the root + directory that isn't a plain file owned by root or that others can read, and + the directory under the same rule. +- Nix verifies. For that one call `hive-priv` sets `substituters` and + `trusted-public-keys` to the swarm cache and its key, and `require-sigs true`. + A path without a signature by that key isn't stored; `hive-priv` answers + `not available` and logs the refusal at error level. `--max-jobs 0` means no + answer, whatever the cache sends, starts a build as root. +- A hive realises only paths whose signature checks. The request carries an + agent name and a store path, both validated, and no address, key or file; any + other field never reaches the verb. + +What a compromised `hive-c0re` can do with this verb: ask for any store path +that has the shape of one and that the swarm cache signed, which fills the +store with paths the swarm vouches for. It can't make root accept a path from +a cache or under a key of its choosing, because it writes none of the places +those come from. The trust anchor is the cache's signing key and the store that +hands out its public key, as for every other credential. **Container allowlist** — `hive-priv` validates every request against an allowlist before any operation: the allowlist accepts only names diff --git a/hive-c0re/src/attic/closure.rs b/hive-c0re/src/attic/closure.rs new file mode 100644 index 00000000..f8e2c577 --- /dev/null +++ b/hive-c0re/src/attic/closure.rs @@ -0,0 +1,62 @@ +//! The closure the swarm built for an agent at a given `internal/meta` commit, +//! as a rebuild's `Prebuild` downloads it instead of building. + +use anyhow::{Result, anyhow}; +use swarm_wire::agent_closures::AgentClosure; + +use super::probe::{self, Unpullable}; + +/// The store path of the closure `agent`'s record names, when the record is a +/// build of `rev` that the builder pushed to the cache. +/// +/// # Errors +/// No record, one for another commit, or one the builder did not push; or the +/// queue cannot be reached. The message is what a deploy log says. +pub async fn path_for(agent: &str, rev: &str) -> Result { + let client = crate::swarm_queue::client() + .await + .ok_or_else(|| anyhow!("this hive has no connection to the swarm queue"))?; + let record = probe::read_record(&client, agent).await?; + path_in(record.as_ref(), rev) +} + +fn path_in(record: Option<&AgentClosure>, rev: &str) -> Result { + probe::current_record(record, Some(rev)) + .map(|record| record.store_path.clone()) + .map_err(|Unpullable { reason, detail }| anyhow!("{}: {detail}", reason.label())) +} + +#[cfg(test)] +mod tests { + use super::*; + + const REV: &str = "a1b2c3"; + const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; + + fn record(rev: &str, cached: bool) -> AgentClosure { + AgentClosure { + meta_rev: rev.to_owned(), + store_path: PATH.to_owned(), + system: "x86_64-linux".to_owned(), + built_unix: 1, + cached, + } + } + + #[test] + fn a_pushed_build_of_the_rev_names_its_path() { + assert_eq!(path_in(Some(&record(REV, true)), REV).expect("path"), PATH); + } + + #[test] + fn anything_else_names_why_not() { + for (record, why) in [ + (None, "no-record"), + (Some(record("other", true)), "rev-mismatch"), + (Some(record(REV, false)), "not-cached"), + ] { + let err = path_in(record.as_ref(), REV).expect_err(why); + assert!(format!("{err}").starts_with(why), "{err}"); + } + } +} diff --git a/hive-c0re/src/attic/mod.rs b/hive-c0re/src/attic/mod.rs index c29214ec..48db932a 100644 --- a/hive-c0re/src/attic/mod.rs +++ b/hive-c0re/src/attic/mod.rs @@ -1,7 +1,9 @@ //! The swarm's binary cache, as this hive sees it: the pull credentials it is -//! given ([`reader`]), the narinfo check ([`narinfo`]), and the probe that -//! reports per agent whether the latest closure could be pulled ([`probe`]). +//! given ([`reader`]), the narinfo check ([`narinfo`]), the probe that reports +//! per agent whether the latest closure could be pulled ([`probe`]), and the +//! lookup a rebuild uses to download that closure ([`closure`]). +pub mod closure; pub mod narinfo; pub mod probe; pub mod reader; diff --git a/hive-c0re/src/attic/probe.rs b/hive-c0re/src/attic/probe.rs index 5c39e76f..46f4f98d 100644 --- a/hive-c0re/src/attic/probe.rs +++ b/hive-c0re/src/attic/probe.rs @@ -3,7 +3,8 @@ //! //! Every [`PASS_INTERVAL`] each agent's record in `agent-closures` is looked up //! and its narinfo fetched with the pull token and checked against the cache's -//! public key. Nothing is downloaded and nothing is deployed from it. +//! public key. Nothing is downloaded here; a rebuild downloads through +//! [`super::closure`]. //! //! The verdict is the first of these that fails, in this order: //! no credentials (`no-credentials`), no record (`no-record`), a record built @@ -82,20 +83,12 @@ impl Cache for Http { } } -/// Whether the closure `record` names can be pulled with `facts`. `head` is the -/// `internal/meta` commit a deploy would build from. -pub(crate) async fn judge( - facts: Option<&Facts>, - record: Option<&AgentClosure>, +/// `record` when it is the latest build of the `internal/meta` commit `head` +/// and the builder pushed it to the cache. +pub(crate) fn current_record<'a>( + record: Option<&'a AgentClosure>, head: Option<&str>, - cache: &impl Cache, -) -> Result<(), Unpullable> { - let facts = facts.ok_or_else(|| { - no( - Reason::NoCredentials, - "the swarm has given this hive no cache facts", - ) - })?; +) -> Result<&'a AgentClosure, Unpullable> { let record = record.ok_or_else(|| no(Reason::NoRecord, "no build is recorded for the agent"))?; if head != Some(record.meta_rev.as_str()) { @@ -114,6 +107,24 @@ pub(crate) async fn judge( "the builder did not push the closure", )); } + Ok(record) +} + +/// Whether the closure `record` names can be pulled with `facts`. `head` is the +/// `internal/meta` commit a deploy would build from. +pub(crate) async fn judge( + facts: Option<&Facts>, + record: Option<&AgentClosure>, + head: Option<&str>, + cache: &impl Cache, +) -> Result<(), Unpullable> { + let facts = facts.ok_or_else(|| { + no( + Reason::NoCredentials, + "the swarm has given this hive no cache facts", + ) + })?; + let record = current_record(record, head)?; let text = match cache.narinfo(facts, &record.store_path).await { Ok(Some(text)) => text, Ok(None) => return Err(no(Reason::NotCached, "the cache does not hold the narinfo")), @@ -195,7 +206,10 @@ async fn pass(client: &async_nats::Client) -> Result<()> { Ok(()) } -async fn read_record(client: &async_nats::Client, agent: &str) -> Result> { +pub(crate) async fn read_record( + client: &async_nats::Client, + agent: &str, +) -> Result> { match agent_closures::read(client, agent) .await .map_err(|e| anyhow::anyhow!(e)) diff --git a/hive-c0re/src/attic/reader.rs b/hive-c0re/src/attic/reader.rs index 6c64a73f..7ce8977f 100644 --- a/hive-c0re/src/attic/reader.rs +++ b/hive-c0re/src/attic/reader.rs @@ -3,7 +3,8 @@ //! ([`swarm_secret_client::attic`]), read here under the hive's own identity. //! //! [`Facts::publish`] puts them in `/run/hyperhive/attic/` as a netrc, the -//! substituter URL and the public key, where nix and the root helper read them. +//! substituter URL and the public key, which the pull probe reads. The root helper +//! reads its own copies, kept by a root unit. use std::path::Path; diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 5de97e56..9257c642 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -44,8 +44,21 @@ pub(super) trait Host: Sync { async fn kill(&self, name: &str) -> Result<()>; /// The `internal/meta` commit `MetaSync` recorded for the agent. fn recorded_rev(&self, name: &str) -> Result; + /// The store path of the closure the swarm built for the agent at the + /// `internal/meta` commit `rev` and pushed to its cache. The error says why + /// there is none. + async fn cached_closure(&self, name: &str, rev: &str) -> Result; + /// Download `store_path` from the swarm's cache into the store, never + /// building it. + async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>; /// Warm the agent's toplevel build from the `internal/meta` commit `rev`. - async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> Result<()>; + /// The toplevel it printed, when it printed one. + async fn prebuild_toplevel( + &self, + name: &str, + rev: &str, + node_id: u64, + ) -> Result>; /// Re-apply drop-ins, then `nixos-container update` at `rev`. async fn swap_update( &self, @@ -117,7 +130,20 @@ impl Host for RealHost { Ok(crate::meta_rev::recorded_or_err(name)?.rev) } - async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> Result<()> { + async fn cached_closure(&self, name: &str, rev: &str) -> Result { + crate::attic::closure::path_for(name, rev).await + } + + async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> { + crate::priv_client::realise_closure(name, store_path).await + } + + async fn prebuild_toplevel( + &self, + name: &str, + rev: &str, + node_id: u64, + ) -> Result> { let flake_ref = crate::meta_rev::flake_ref(name, rev); crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(node_id)).await } @@ -595,6 +621,12 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> { /// container is already down: its only purpose is to shrink the swap's /// downtime window, so a stopped agent (no uptime to preserve) doesn't /// pay the double eval — `Swap` builds inline instead. +/// +/// The swarm's cache is tried first: when the swarm built this agent at the +/// recorded commit and pushed the closure, it is downloaded. The local build +/// runs either way. After a download it is an evaluation and a no-op build when +/// this hive evaluates the path the swarm built, and builds it, with the agent +/// still up, when it doesn't; `Swap` never meets a build it didn't expect. async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> { // Warm the toplevel build only when the container is up — the whole // point of prebuild is to shrink the swap's downtime window. A @@ -602,11 +634,70 @@ async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> { // eval and let the downstream `Swap` build inline. if host.is_running(name).await { let rev = host.recorded_rev(name)?; - host.prebuild_toplevel(name, &rev, id.get()).await?; + let downloaded = match substitute(host, name, &rev).await { + Ok(path) => Some(path), + Err(fallback) => { + let reason = &fallback.reason; + if fallback.signature_refused { + tracing::error!(%name, %rev, "prebuild: built locally: {reason}"); + } else { + tracing::warn!(%name, %rev, "prebuild: built locally: {reason}"); + } + None + } + }; + let evaluated = host.prebuild_toplevel(name, &rev, id.get()).await?; + if let Some(path) = downloaded { + match drift(&path, evaluated.as_deref()) { + None => tracing::info!(%name, %rev, "prebuild: substituted {path}"), + Some(note) => tracing::warn!(%name, %rev, "prebuild: {note}"), + } + } } Ok(()) } +/// What to log when the closure downloaded for a rebuild isn't the one this +/// hive's evaluation produced, which means the builder and the hive disagree +/// about the agent's flake. `None` when they agree. +fn drift(downloaded: &str, evaluated: Option<&str>) -> Option { + match evaluated { + Some(path) if path == downloaded => None, + Some(path) => Some(format!( + "the downloaded closure {downloaded} is not the one this hive evaluates, {path}; built the latter locally" + )), + None => Some(format!( + "downloaded {downloaded}, but the local build printed no path to compare it with" + )), + } +} + +/// Why a rebuild builds instead of downloading the swarm's closure. +struct LocalBuild { + reason: String, + /// The cache served a path no trusted key signed. + signature_refused: bool, +} + +/// Download the closure the swarm built for `name` at `rev`; its store path. +async fn substitute(host: &impl Host, name: &str, rev: &str) -> Result { + let path = host + .cached_closure(name, rev) + .await + .map_err(|e| LocalBuild { + reason: format!("{e:#}"), + signature_refused: false, + })?; + host.realise_closure(name, &path).await.map_err(|e| { + let reason = format!("{e:#}"); + LocalBuild { + signature_refused: reason.contains(hive_priv_sock::SIGNATURE_REFUSED), + reason, + } + })?; + Ok(path) +} + /// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), /// `nixos-container update`, then the post-rebuild bookkeeping tail /// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan). diff --git a/hive-c0re/src/job_queue/exec/tests.rs b/hive-c0re/src/job_queue/exec/tests.rs index 49ccba7f..12b67f27 100644 --- a/hive-c0re/src/job_queue/exec/tests.rs +++ b/hive-c0re/src/job_queue/exec/tests.rs @@ -11,12 +11,12 @@ use hive_jobq::NodeId; use hive_jobq::scheduler::Scheduler; use super::{ - Host, dispatch, matrix_sweep_warning, run_create, run_destroy_bookkeeping, + Host, dispatch, drift, matrix_sweep_warning, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain, run_emit_rebuilt, run_forge_sweep, run_knowledge_pull, run_matrix_sweep, run_meta_sync, run_pause_drain, run_pause_signal, run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping, run_reconcile, run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap, run_wanted_pull, - run_write_dropin, + run_write_dropin, substitute, }; use crate::coordinator::{AgentPaths, Coordinator, HiveEnv}; use crate::job_queue::model::NodeKind; @@ -24,14 +24,27 @@ use crate::meta_rev::MetaBuild; use crate::power::Wanted; const REV: &str = "2222222222222222222222222222222222222222"; +const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-alice"; /// A [`Host`] that records every call as one string and fails on request. #[derive(Default)] struct FakeHost { running: bool, rev: Option, - /// Names of the calls that fail (`kill`, `swap_update`, ...). `recorded_build` fails by - /// finding nothing and `matrix_ensure_all` by reporting false. + /// The store path the swarm's record names, when it has a pushed build of + /// `REV`; `None` reads as no record. + closure: Option, + /// A failing `realise` says nix refused the path for want of a signature. + refusal: bool, + /// The toplevel this hive's evaluation produces; `None` is `CLOSURE`. + evaluates: Option, + /// Store paths a download put in the store. + realised: Mutex>, + /// Store paths a prebuild had to build because the store lacked them. + builds: Mutex>, + /// Names of the calls that fail (`kill`, `realise`, `prebuild`, `swap_update`, + /// `set_paused`, ...). `recorded_build` fails by finding nothing and + /// `matrix_ensure_all` by reporting false. failing: Vec<&'static str>, /// The sweeps and pulls never finish, so only a deadline ends them. hang: bool, @@ -57,6 +70,11 @@ impl FakeHost { } } + fn with_closure(mut self) -> Self { + self.closure = Some(CLOSURE.to_owned()); + self + } + fn failing(mut self, call: &'static str) -> Self { self.failing.push(call); self @@ -82,6 +100,10 @@ impl FakeHost { fn calls(&self) -> Vec { self.calls.lock().unwrap().clone() } + + fn builds(&self) -> Vec { + self.builds.lock().unwrap().clone() + } } impl Host for FakeHost { @@ -105,12 +127,43 @@ impl Host for FakeHost { .ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded")) } - async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> anyhow::Result<()> { + async fn cached_closure(&self, name: &str, rev: &str) -> anyhow::Result { + self.record(format!("cached_closure {name} {rev}")); + self.closure + .clone() + .ok_or_else(|| anyhow::anyhow!("no-record: no build is recorded for the agent")) + } + + async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> { + self.record(format!("realise {name} {store_path}")); + if self.fails("realise") { + if self.refusal { + anyhow::bail!( + "not available: {}: lacks a signature", + hive_priv_sock::SIGNATURE_REFUSED + ); + } + anyhow::bail!("not available: Couldn't connect to server"); + } + self.realised.lock().unwrap().push(store_path.to_owned()); + Ok(()) + } + + async fn prebuild_toplevel( + &self, + name: &str, + rev: &str, + node_id: u64, + ) -> anyhow::Result> { self.record(format!("prebuild {name} {rev} node={node_id}")); if self.fails("prebuild") { anyhow::bail!("prebuild failed"); } - Ok(()) + let path = self.evaluates.clone().unwrap_or_else(|| CLOSURE.to_owned()); + if !self.realised.lock().unwrap().contains(&path) { + self.builds.lock().unwrap().push(path.clone()); + } + Ok(Some(path)) } async fn swap_update( @@ -290,11 +343,140 @@ async fn prebuild_warms_the_recorded_rev_when_the_container_is_up() { [ "is_running alice".to_owned(), "recorded_rev alice".to_owned(), + format!("cached_closure alice {REV}"), format!("prebuild alice {REV} node={}", id.get()), ] ); } +/// The download is followed by the local build either way. When this hive +/// evaluates the path the swarm built, the build finds it in the store. +#[tokio::test] +async fn a_downloaded_closure_the_hive_also_evaluates_is_found_not_built() { + let host = FakeHost::up().with_closure(); + let id = node_id(); + run_prebuild(&host, "alice", id).await.expect("prebuild ok"); + assert_eq!( + host.calls(), + [ + "is_running alice".to_owned(), + "recorded_rev alice".to_owned(), + format!("cached_closure alice {REV}"), + format!("realise alice {CLOSURE}"), + format!("prebuild alice {REV} node={}", id.get()), + ] + ); + assert_eq!(host.builds(), Vec::::new()); +} + +/// When it evaluates another path, the build happens in `Prebuild`, after the +/// check that the agent is up and before `Swap` could stop it. +#[tokio::test] +async fn a_downloaded_closure_the_hive_does_not_evaluate_is_built_before_the_swap() { + let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice"; + let host = FakeHost { + evaluates: Some(other.to_owned()), + ..FakeHost::up().with_closure() + }; + let id = node_id(); + run_prebuild(&host, "alice", id).await.expect("prebuild ok"); + let calls = host.calls(); + assert_eq!(calls[0], "is_running alice", "{calls:?}"); + assert_eq!( + calls.last().map(String::as_str), + Some(format!("prebuild alice {REV} node={}", id.get()).as_str()), + "{calls:?}" + ); + assert_eq!(host.builds(), [other.to_owned()]); +} + +#[test] +fn only_a_different_path_is_drift() { + assert_eq!(drift(CLOSURE, Some(CLOSURE)), None); + let note = drift(CLOSURE, Some("/nix/store/other")).expect("drift"); + assert!( + note.contains(CLOSURE) && note.contains("/nix/store/other"), + "{note}" + ); + assert!(drift(CLOSURE, None).expect("no path").contains(CLOSURE)); +} + +#[tokio::test] +async fn a_failed_download_builds_locally() { + let host = FakeHost::up().with_closure().failing("realise"); + let id = node_id(); + run_prebuild(&host, "alice", id) + .await + .expect("the build covers it"); + let calls = host.calls(); + assert_eq!(calls[3], format!("realise alice {CLOSURE}"), "{calls:?}"); + assert_eq!( + calls[4], + format!("prebuild alice {REV} node={}", id.get()), + "{calls:?}" + ); + assert_eq!(calls.len(), 5, "{calls:?}"); +} + +/// The fallback is the build, so a build that fails still fails the node. +#[tokio::test] +async fn a_failed_download_and_a_failed_build_fail_the_node() { + let host = FakeHost::up() + .with_closure() + .failing("realise") + .failing("prebuild"); + let err = run_prebuild(&host, "alice", node_id()) + .await + .expect_err("nothing produced the closure"); + assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}"); +} + +#[tokio::test] +async fn a_stopped_container_looks_up_nothing() { + let host = FakeHost::down().with_closure(); + run_prebuild(&host, "alice", node_id()) + .await + .expect("skipped"); + assert_eq!(host.calls(), ["is_running alice"]); +} + +/// The error log is for a path the cache served unsigned, not for a cache that +/// is down or a missing record. +#[tokio::test] +async fn only_a_signature_refusal_is_flagged() { + let down = substitute( + &FakeHost::up().with_closure().failing("realise"), + "alice", + REV, + ) + .await + .expect_err("down"); + assert!(!down.signature_refused, "{}", down.reason); + assert!(down.reason.contains("Couldn't connect"), "{}", down.reason); + + let no_record = substitute(&FakeHost::up(), "alice", REV) + .await + .expect_err("no record"); + assert!(!no_record.signature_refused); + assert!( + no_record.reason.starts_with("no-record"), + "{}", + no_record.reason + ); + + let refused = FakeHost { + refusal: true, + ..FakeHost::up().with_closure().failing("realise") + }; + let refused = substitute(&refused, "alice", REV) + .await + .expect_err("refused"); + assert!(refused.signature_refused, "{}", refused.reason); + + let ok = substitute(&FakeHost::up().with_closure(), "alice", REV).await; + assert_eq!(ok.ok().as_deref(), Some(CLOSURE)); +} + #[tokio::test] async fn prebuild_is_skipped_when_the_container_is_down() { let host = FakeHost::down(); diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index 82fee621..fbe481f1 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -751,6 +751,29 @@ async fn wait_bounded( Ok(None) } +/// Log and record a prebuild's stdout; the last non-empty line is the +/// toplevel `nix build --print-out-paths` printed. +async fn pump_prebuild_stdout( + stdout: tokio::process::ChildStdout, + cmdline: String, + logs: Option>, + log_id: Option, +) -> Option { + use tokio::io::{AsyncBufReadExt, BufReader}; + let mut lines = BufReader::new(stdout).lines(); + let mut out_path = None; + while let Ok(Some(line)) = lines.next_line().await { + tracing::info!(target: "nix-prebuild", cmdline = %cmdline, "{line}"); + if !line.trim().is_empty() { + out_path = Some(line.trim().to_owned()); + } + if let (Some(h), Some(id)) = (&logs, log_id) { + h.append_stdout(id, &line); + } + } + out_path +} + /// Pre-build `system.build.toplevel` against `meta#` so the /// subsequent `nixos-container update` finds the result cached and /// skips straight to the profile-swap. Store-warming only — container @@ -761,7 +784,13 @@ async fn wait_bounded( /// `node_id` is the queue node this build belongs to, when there is one — /// it is stored on the `build_logs` row so the dashboard can find the log /// from the node. Pass `None` for builds that run outside the queue. -pub async fn prebuild_toplevel(name: &str, flake_ref: &str, node_id: Option) -> Result<()> { +/// +/// Returns the toplevel `nix build` printed, `None` if it printed nothing. +pub async fn prebuild_toplevel( + name: &str, + flake_ref: &str, + node_id: Option, +) -> Result> { use tokio::io::{AsyncBufReadExt, BufReader}; // Split `#` so we can re-emit with the explicit // `nixosConfigurations.` segment. The flake_ref shape is @@ -814,17 +843,12 @@ pub async fn prebuild_toplevel(name: &str, flake_ref: &str, node_id: Option let stdout = child.stdout.take().expect("piped stdout"); let stderr = child.stderr.take().expect("piped stderr"); - let stdout_cmdline = cmdline.clone(); - let stdout_logs = logs.clone(); - let pump_stdout = tokio::spawn(async move { - let mut lines = BufReader::new(stdout).lines(); - while let Ok(Some(line)) = lines.next_line().await { - tracing::info!(target: "nix-prebuild", cmdline = %stdout_cmdline, "{line}"); - if let (Some(h), Some(id)) = (&stdout_logs, log_id) { - h.append_stdout(id, &line); - } - } - }); + let pump_stdout = tokio::spawn(pump_prebuild_stdout( + stdout, + cmdline.clone(), + logs.clone(), + log_id, + )); let stderr_cmdline = cmdline.clone(); let stderr_logs = logs.clone(); @@ -860,7 +884,7 @@ pub async fn prebuild_toplevel(name: &str, flake_ref: &str, node_id: Option None => bail!("prebuild {cmdline} timed out after {secs}s; killed it"), } }; - let _ = pump_stdout.await; + let out_path = pump_stdout.await.ok().flatten(); let _ = pump_stderr.await; let ok = status.success(); @@ -880,7 +904,7 @@ pub async fn prebuild_toplevel(name: &str, flake_ref: &str, node_id: Option None => bail!("prebuild {cmdline} failed ({status})"), } } - Ok(()) + Ok(out_path) } pub async fn list() -> Result> { diff --git a/hive-c0re/src/paths.rs b/hive-c0re/src/paths.rs index 732ef9fd..bcc5b140 100644 --- a/hive-c0re/src/paths.rs +++ b/hive-c0re/src/paths.rs @@ -231,7 +231,7 @@ pub fn gateway_agents_conf() -> PathBuf { // `nix/host-modules/lib/hive-forge-git.nix` — must match. pub const FORGE_HIVE_TOKEN: &str = "/run/hyperhive/forge-hive-token"; -/// `attic/` — the swarm cache's pull credentials for nix: `netrc`, `substituter` +/// `attic/` — the swarm cache's pull credentials for the probe: `netrc`, `substituter` /// and `public-key`, each `0600`, copied from the swarm secret store. Tmpfs: /// re-published on every boot. pub const ATTIC_DIR: &str = "/run/hyperhive/attic"; diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index 96eaa318..35c31c53 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -267,6 +267,19 @@ pub async fn create_container_streaming( .await?) } +/// See [`PrivRequest::RealiseClosure`]. A path hive-priv could not download +/// reads as an error whose message starts `not available: `. +/// +/// # Errors +/// The call failed, or hive-priv answered `not available`. +pub async fn realise_closure(agent: &str, store_path: &str) -> Result<()> { + ok(call(&PrivRequest::RealiseClosure { + agent: agent.to_owned(), + store_path: store_path.to_owned(), + }) + .await?) +} + pub async fn destroy_container(name: &str) -> Result<()> { ok(call(&PrivRequest::DestroyContainer { name: name.to_owned(), diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 3b64f6f2..4b02dfc7 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -110,6 +110,33 @@ pub fn valid_meta_rev(rev: &str) -> bool { rev.len() == 40 && rev.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) } +/// Whether `path` is a store path in the shape nix prints one: `/nix/store/`, +/// 32 lowercase letters or digits, `-`, then a name of the characters store +/// names allow. The path crosses the wire in [`PrivRequest::RealiseClosure`], +/// so `hive-priv` checks it before it hands it to nix. +#[must_use] +pub fn valid_closure_path(path: &str) -> bool { + let Some(rest) = path.strip_prefix("/nix/store/") else { + return false; + }; + let Some((hash, name)) = rest.split_once('-') else { + return false; + }; + hash.len() == 32 + && hash + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) + && !name.is_empty() + && name.bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.' | b'_' | b'?' | b'=') + }) +} + +/// What a failed [`PrivRequest::RealiseClosure`] says when nix refused a path +/// because no trusted key signed it, so a caller can tell that apart from a +/// cache that is down. +pub const SIGNATURE_REFUSED: &str = "no signature by the swarm cache's key"; + /// The flake the swarm publishes at `internal/meta`, at `rev`: /// `git+/internal/meta.git?rev=`. `forge_base` is the forge's /// `http` origin, which the hive's git credential helper answers for. @@ -270,6 +297,18 @@ pub enum PrivRequest { meta_rev: String, }, + /// `nix-store --realise ` from the swarm's binary cache, never + /// building: the closure the swarm built for `agent`, downloaded into the + /// store ahead of the update that applies it. + /// + /// The request names no cache and no key: hive-priv takes the cache's + /// address from its own unit's environment and the key and the pull token + /// from files only root can write, and nix accepts the path only with a + /// signature by that key. `store_path` is checked by + /// [`valid_closure_path`]. A path nix could not substitute answers with an + /// error that starts `not available: `. + RealiseClosure { agent: String, store_path: String }, + /// `nixos-container destroy ` DestroyContainer { name: String }, @@ -661,7 +700,7 @@ pub enum PrivEvent { #[cfg(test)] mod tests { - use super::{InfraContainer, PrivRequest, meta_flake_url, valid_meta_rev}; + use super::{InfraContainer, PrivRequest, meta_flake_url, valid_closure_path, valid_meta_rev}; #[test] fn infra_control_names_no_c0re_or_swarm_service() { @@ -717,6 +756,46 @@ mod tests { ); } + #[test] + fn only_a_store_path_is_a_closure_path() { + let hash = "0123456789abcdfghijklmnpqrsvwxyz"; + for good in [ + format!("/nix/store/{hash}-nixos-system-atlas-26.05.20261026.718f670"), + format!("/nix/store/{hash}-a"), + ] { + assert!(valid_closure_path(&good), "{good}"); + } + for bad in [ + String::new(), + "/nix/store/".to_owned(), + format!("/nix/store/{hash}"), + format!("/nix/store/{hash}-"), + format!("/nix/store/{}-x", &hash[1..]), + format!("/nix/store/{hash}a-x"), + format!("/nix/store/{}-x", hash.to_uppercase()), + format!("/tmp/{hash}-x"), + format!("/nix/store/{hash}-x/bin/sh"), + format!("/nix/store/{hash}-../x"), + format!("/nix/store/{hash}-x y"), + format!("/nix/store/{hash}-x\n"), + format!("/nix/store/../../etc/{hash}-x"), + format!("/nix/store/{hash}-x;id"), + ] { + assert!(!valid_closure_path(&bad), "{bad:?}"); + } + } + + #[test] + fn a_closure_request_names_an_agent_and_a_path() { + let line = r#"{"op":"realise_closure","agent":"atlas","store_path":"/nix/store/x"}"#; + assert!(matches!( + serde_json::from_str::(line), + Ok(PrivRequest::RealiseClosure { .. }) + )); + let without = r#"{"op":"realise_closure","agent":"atlas"}"#; + assert!(serde_json::from_str::(without).is_err()); + } + /// `meta_rev` is required: there is no local flake to fall back to. #[test] fn a_request_without_meta_rev_is_refused() { diff --git a/hive-priv/src/closure.rs b/hive-priv/src/closure.rs new file mode 100644 index 00000000..5c1bbc3a --- /dev/null +++ b/hive-priv/src/closure.rs @@ -0,0 +1,401 @@ +//! `RealiseClosure`: download the closure the swarm built for an agent from +//! the swarm's binary cache into the store, without ever building. +//! +//! What this accepts is decided by root-owned sources alone: the cache's +//! address is the host's configuration, handed to the unit in +//! [`CACHE_URL_ENV`], and the cache's public key and the pull token's netrc are +//! files a root unit keeps in [`ROOT_FACTS_DIR`]. Nothing in the request, and +//! nothing hive-c0re (unprivileged) writes, chooses a cache or a key. They go to +//! nix's daemon as per-call settings, which the daemon applies for a trusted +//! client such as this one (root), so the host's nix configuration is untouched. + +use std::os::unix::fs::MetadataExt as _; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{SIGNATURE_REFUSED, valid_closure_path}; + +use crate::container::{BoundedRun, run_bounded}; +use crate::shared::validate_agent_name; + +/// The unit's environment variable naming the cache: `https:///` +/// from the host's `services.hyperhive.swarm.attic` options. +pub(crate) const CACHE_URL_ENV: &str = "HIVE_PRIV_CACHE_URL"; + +/// Where the root unit `hive-priv-cache-facts` keeps `public-key` and `netrc`. +/// Not `/run/hyperhive/attic`, which hive-c0re writes. +const ROOT_FACTS_DIR: &str = "/run/hyperhive-priv/attic"; + +/// Wall-clock bound on one download. A stalled transfer is bounded by nix's +/// own `stalled-download-timeout`; this covers the rest. +const REALISE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(30); + +/// The cache this host trusts. +#[derive(Debug, PartialEq, Eq)] +struct Cache { + substituter: String, + public_key: String, + netrc: PathBuf, +} + +/// Whether `path` is a regular file or directory owned by `owner` that nobody +/// else can read or write. +fn private_to(path: &Path, owner: u32, directory: bool) -> Result<()> { + let meta = std::fs::symlink_metadata(path) + .with_context(|| format!("{} is missing", path.display()))?; + let kind_ok = if directory { + meta.is_dir() + } else { + meta.is_file() + }; + if !kind_ok { + bail!( + "{} is not a plain {}", + path.display(), + if directory { "directory" } else { "file" } + ); + } + if meta.uid() != owner { + bail!("{} is not owned by uid {owner}", path.display()); + } + if meta.mode() & 0o077 != 0 { + bail!("{} is readable or writable by others", path.display()); + } + Ok(()) +} + +impl Cache { + /// The cache at `url`, trusting the key in `dir`. + /// + /// # Errors + /// No `url`, one that is not a plain `https` address (the token would go + /// out in clear), a `dir` or file that is missing, not owned by `owner` or + /// open to others, or a key that is not `:`. + fn read(url: Option<&str>, dir: &Path, owner: u32) -> Result { + let url = url + .map(str::trim) + .filter(|u| !u.is_empty()) + .context("this host names no cache")?; + if !url.starts_with("https://") || url.contains(char::is_whitespace) { + bail!("the cache address is not an https URL"); + } + private_to(dir, owner, true)?; + let key_file = dir.join("public-key"); + private_to(&key_file, owner, false)?; + let netrc = dir.join("netrc"); + private_to(&netrc, owner, false)?; + let key = std::fs::read_to_string(&key_file) + .with_context(|| format!("reading {}", key_file.display()))?; + let key = key.trim(); + if key.contains(char::is_whitespace) + || key + .split_once(':') + .is_none_or(|(name, key)| name.is_empty() || key.is_empty()) + { + bail!("the cache's public key is not :"); + } + Ok(Self { + substituter: url.to_owned(), + public_key: key.to_owned(), + netrc, + }) + } +} + +/// The `nix-store` arguments that fetch `store_path` and nothing else. +/// +/// - `--max-jobs 0` forbids a local build, so a path the cache lacks fails. +/// - `substituters` and `trusted-public-keys` are replaced, not extended: the +/// swarm cache is the only source and its key the only one that counts. +/// - `require-sigs true` makes nix refuse a path no trusted key signed. +fn realise_args(store_path: &str, cache: &Cache) -> Vec { + let mut args: Vec = ["--realise", store_path, "--max-jobs", "0"] + .map(str::to_owned) + .into(); + let netrc = cache.netrc.display().to_string(); + for (name, value) in [ + ("substituters", cache.substituter.as_str()), + ("trusted-public-keys", cache.public_key.as_str()), + ("require-sigs", "true"), + ("netrc-file", netrc.as_str()), + ] { + args.extend(["--option".to_owned(), name.to_owned(), value.to_owned()]); + } + args +} + +/// Why nix did not realise the path, from its last stderr line. A refusal for +/// want of a signature is named, so the caller can tell it from a cache that +/// is down. +fn reason(stderr: &str) -> String { + let last = stderr.lines().last().unwrap_or("").trim(); + if last.to_lowercase().contains("signature") { + format!("{SIGNATURE_REFUSED}: {last}") + } else { + last.to_owned() + } +} + +/// `RealiseClosure`. +/// +/// # Errors +/// Bad arguments, or `not available: ` for anything that left the path +/// out of the store: no cache or key on this host, nix failing or timing out. +pub(crate) async fn realise_closure(agent: &str, store_path: &str) -> Result<(String, String)> { + let url = std::env::var(CACHE_URL_ENV).ok(); + realise_with( + url.as_deref(), + Path::new(ROOT_FACTS_DIR), + 0, + agent, + store_path, + ) + .await +} + +/// [`realise_closure`] with the cache's address and the key's directory and +/// owner named, so a test can stand in for the host. +async fn realise_with( + url: Option<&str>, + dir: &Path, + owner: u32, + agent: &str, + store_path: &str, +) -> Result<(String, String)> { + validate_agent_name(agent)?; + if !valid_closure_path(store_path) { + bail!("{store_path:?} is not a store path"); + } + let outcome = async { + let cache = Cache::read(url, dir, owner)?; + let mut cmd = tokio::process::Command::new("nix-store"); + cmd.args(realise_args(store_path, &cache)); + match run_bounded(cmd, REALISE_TIMEOUT, None) + .await + .context("running nix-store")? + { + BoundedRun::Exited { status, .. } if status.success() => Ok(()), + BoundedRun::Exited { stderr, .. } => bail!("{}", reason(&stderr)), + BoundedRun::TimedOut => bail!( + "nix-store timed out after {}s; killed it", + REALISE_TIMEOUT.as_secs() + ), + } + } + .await; + match outcome { + Ok(()) => { + tracing::info!(%agent, %store_path, "substituted the agent's closure"); + Ok(("substituted".to_owned(), String::new())) + } + Err(e) => { + let reason = format!("{e:#}"); + if reason.contains(SIGNATURE_REFUSED) { + tracing::error!(%agent, %store_path, %reason, "the cache's closure was refused"); + } + bail!("not available: {reason}") + } + } +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt as _; + + use super::*; + use crate::shared::test_support::scratch; + + const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; + const URL: &str = "https://cache.example/swarm"; + + fn cache() -> Cache { + Cache { + substituter: URL.to_owned(), + public_key: "swarm:AAAA".to_owned(), + netrc: PathBuf::from("/run/hyperhive-priv/attic/netrc"), + } + } + + fn option<'a>(args: &'a [String], name: &str) -> Option<&'a str> { + args.windows(3) + .find(|w| w[0] == "--option" && w[1] == name) + .map(|w| w[2].as_str()) + } + + #[test] + fn the_command_never_builds() { + let args = realise_args(PATH, &cache()); + assert_eq!(args[..2], ["--realise", PATH]); + let jobs = args.iter().position(|a| a == "--max-jobs").expect("flag"); + assert_eq!(args[jobs + 1], "0"); + } + + #[test] + fn only_the_swarm_cache_and_its_key_count_and_signatures_are_required() { + let args = realise_args(PATH, &cache()); + assert_eq!(option(&args, "substituters"), Some(URL)); + assert_eq!(option(&args, "trusted-public-keys"), Some("swarm:AAAA")); + assert_eq!(option(&args, "require-sigs"), Some("true")); + assert_eq!( + option(&args, "netrc-file"), + Some("/run/hyperhive-priv/attic/netrc") + ); + // The settings that would add to the defaults are not used. + assert!(!args.iter().any(|a| a.starts_with("extra-")), "{args:?}"); + } + + /// The uid the test runs as, which stands in for root. + fn me() -> u32 { + std::fs::metadata(scratch()).expect("stat").uid() + } + + fn put(path: &Path, text: &str, mode: u32) { + std::fs::write(path, text).expect("write"); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).expect("chmod"); + } + + /// A facts directory as the root unit leaves it. + fn facts(key: &str) -> PathBuf { + let dir = scratch(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("chmod"); + put(&dir.join("public-key"), &format!("{key}\n"), 0o600); + put( + &dir.join("netrc"), + "machine cache.example password t\n", + 0o600, + ); + dir + } + + #[test] + fn the_cache_is_the_hosts_address_and_the_root_files_key() { + let dir = facts("swarm:AAAA"); + let cache = Cache::read(Some(URL), &dir, me()).expect("read"); + assert_eq!(cache.substituter, URL); + assert_eq!(cache.public_key, "swarm:AAAA"); + assert_eq!(cache.netrc, dir.join("netrc")); + } + + /// A `substituter` or `public-key` file beside the facts, as hive-c0re + /// writes them, decides nothing: the address is the argument and the key is + /// the root file's. + #[test] + fn the_address_is_never_read_from_a_file_and_the_key_only_from_the_root_files() { + let dir = facts("swarm:ROOT"); + put( + &dir.join("substituter"), + "https://evil.example/cache\n", + 0o600, + ); + let cache = Cache::read(Some(URL), &dir, me()).expect("read"); + assert_eq!(cache.substituter, URL); + assert_eq!(cache.public_key, "swarm:ROOT"); + // The directory hive-c0re writes is not the one this reads. + assert!(!ROOT_FACTS_DIR.starts_with("/run/hyperhive/")); + assert_eq!(ROOT_FACTS_DIR, "/run/hyperhive-priv/attic"); + // And without an address from the host there is no cache, whatever the + // directory holds. + for none in [None, Some(""), Some(" ")] { + assert!(Cache::read(none, &dir, me()).is_err(), "{none:?}"); + } + } + + #[test] + fn a_file_that_is_not_the_roots_or_is_open_to_others_is_refused() { + let me = me(); + // Control: the facts as the unit leaves them are accepted. + assert!(Cache::read(Some(URL), &facts("swarm:AAAA"), me).is_ok()); + // Another owner than the one trusted. + assert!(Cache::read(Some(URL), &facts("swarm:AAAA"), me + 1).is_err()); + // The key or the netrc readable by others, or the directory open. + for (name, mode) in [("public-key", 0o644), ("netrc", 0o644), ("netrc", 0o660)] { + let dir = facts("swarm:AAAA"); + std::fs::set_permissions(dir.join(name), std::fs::Permissions::from_mode(mode)) + .expect("chmod"); + assert!(Cache::read(Some(URL), &dir, me).is_err(), "{name} {mode:o}"); + } + let dir = facts("swarm:AAAA"); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + assert!(Cache::read(Some(URL), &dir, me).is_err()); + // A symlink in place of the key. + let dir = facts("swarm:AAAA"); + std::fs::remove_file(dir.join("public-key")).expect("rm"); + std::os::unix::fs::symlink(dir.join("netrc"), dir.join("public-key")).expect("link"); + assert!(Cache::read(Some(URL), &dir, me).is_err()); + } + + #[test] + fn missing_files_and_unusable_values_are_refused() { + let me = me(); + for url in ["http://cache.example/swarm", "https://cache.example/a b"] { + assert!( + Cache::read(Some(url), &facts("swarm:AAAA"), me).is_err(), + "{url}" + ); + } + for key in ["nokeyname", "swarm: AAAA", ":AAAA", "swarm:"] { + assert!(Cache::read(Some(URL), &facts(key), me).is_err(), "{key:?}"); + } + let dir = facts("swarm:AAAA"); + std::fs::remove_file(dir.join("netrc")).expect("rm"); + assert!(Cache::read(Some(URL), &dir, me).is_err()); + assert!(Cache::read(Some(URL), &dir.join("nowhere"), me).is_err()); + } + + /// Missing facts reach the caller as `not available`, which its prebuild + /// answers by building. + #[tokio::test] + async fn without_the_root_files_the_answer_is_not_available() { + let nowhere = scratch().join("nowhere"); + let err = realise_with(Some(URL), &nowhere, 0, "atlas", PATH) + .await + .expect_err("no facts"); + assert!(format!("{err:#}").starts_with("not available: "), "{err:#}"); + let err = realise_with(None, &facts("swarm:AAAA"), me(), "atlas", PATH) + .await + .expect_err("no address"); + assert!(format!("{err:#}").starts_with("not available: "), "{err:#}"); + } + + /// The request carries an agent and a path and nothing that chooses a + /// cache: fields beyond those are dropped when it is read. + #[test] + fn the_request_cannot_carry_an_address_or_a_key() { + let line = r#"{"op":"realise_closure","agent":"atlas","store_path":"/nix/store/x", + "substituter":"https://evil.example/c","public_key":"evil:AAAA","netrc":"/tmp/x"}"#; + let hive_priv_sock::PrivRequest::RealiseClosure { agent, store_path } = + serde_json::from_str(line).expect("reads") + else { + panic!("not a RealiseClosure"); + }; + assert_eq!( + (agent.as_str(), store_path.as_str()), + ("atlas", "/nix/store/x") + ); + } + + #[test] + fn a_signature_refusal_is_named_and_another_failure_is_not() { + let refused = + reason("error: cannot add path 'x' because it lacks a signature by a trusted key"); + assert!(refused.starts_with(SIGNATURE_REFUSED), "{refused}"); + let down = + reason("warning: x\nerror: unable to download 'https://c/x.narinfo': Couldn't connect"); + assert!(!down.contains(SIGNATURE_REFUSED), "{down}"); + assert!(down.contains("Couldn't connect"), "{down}"); + } + + #[tokio::test] + async fn bad_arguments_are_refused_before_anything_runs() { + for (agent, path) in [ + ("Bad Agent", PATH), + ("", PATH), + ("atlas", "/nix/store/x"), + ("atlas", "/etc/passwd"), + ("atlas", &format!("{PATH}/bin/sh")), + ] { + let err = realise_closure(agent, path).await.expect_err("refused"); + assert!(!format!("{err:#}").contains("not available"), "{err:#}"); + } + } +} diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs index a8172652..fc25a6c7 100644 --- a/hive-priv/src/container.rs +++ b/hive-priv/src/container.rs @@ -245,7 +245,7 @@ fn toplevel_attr(flake: &str, name: &str) -> String { const NIX_BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1); /// How a [`run_bounded`] child ended. -enum BoundedRun { +pub(crate) enum BoundedRun { Exited { status: std::process::ExitStatus, stdout: String, @@ -262,7 +262,7 @@ enum BoundedRun { /// The child leads its own process group and a timeout `SIGKILL`s the whole /// group, so a grandchild still holding the pipes (or the daemon connection) /// dies with it instead of outliving the error. -async fn run_bounded( +pub(crate) async fn run_bounded( mut cmd: Command, limit: std::time::Duration, mut writer: Option<&mut OwnedWriteHalf>, diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 1f1d379f..c6afa11b 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -20,10 +20,11 @@ //! dispatch table. Each op family's handlers and validation live in their //! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags, //! systemd drop-ins), [`storage`] (btrfs subvolumes, quotas, snapshots, disk -//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles) and -//! [`infra`] (gateway nginx, infra containers). Helpers more than -//! one family uses are in [`shared`]. +//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles), [`infra`] (gateway nginx, +//! infra containers) and [`closure`] (downloads from the swarm cache). Helpers +//! more than one family uses are in [`shared`]. +mod closure; mod container; mod container_config; mod files; @@ -361,6 +362,11 @@ async fn exec( ref meta_rev, } => container::container_flake_action("create", name, meta_rev, stream, writer).await, + PrivRequest::RealiseClosure { + ref agent, + ref store_path, + } => closure::realise_closure(agent, store_path).await, + PrivRequest::DestroyContainer { ref name } => container::destroy_container(name).await, PrivRequest::ListContainers => container::container_run(&["list"]).await,