From 5485151ba66449a11d1d236c3fa1c24639863fbb Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 3 Nov 2026 00:30:55 +0100 Subject: [PATCH] add a per-agent nix builds mode, host or none, carried like privateUsers --- Cargo.lock | 3 + docs/process/gotchas.md | 5 +- docs/trust-boundary/security.md | 42 ++++++- hive-c0re/src/attic/closure.rs | 1 + hive-c0re/src/attic/probe.rs | 1 + hive-c0re/src/job_queue/exec/tests.rs | 1 + hive-c0re/src/lifecycle/host_config.rs | 67 ++++++++++- hive-c0re/src/priv_client.rs | 3 + hive-priv-sock/Cargo.toml | 2 + hive-priv-sock/src/lib.rs | 49 ++++++++ hive-priv/Cargo.toml | 1 + hive-priv/src/container_config.rs | 146 ++++++++++++++++++++++-- hive-priv/src/main.rs | 2 + hive-types/README.md | 4 + hive-types/src/lib.rs | 4 +- hive-types/src/nix_builds.rs | 58 ++++++++++ nix/agent-modules/default.nix | 30 +++++ nix/module-eval/agent-user.nix | 29 ++++- nix/vm-tests/agent-ephemeral-rootfs.nix | 38 ++++-- swarm-builder/Cargo.toml | 2 + swarm-builder/src/main.rs | 15 ++- swarm-builder/src/nix_build.rs | 100 ++++++++++++++-- swarm-builder/src/push.rs | 16 +++ swarm-controller/src/cache_miss.rs | 16 ++- swarm-controller/src/meta_build.rs | 17 ++- swarm-wire/src/agent_closures.rs | 12 +- swarm-wire/src/builder.rs | 12 +- 27 files changed, 618 insertions(+), 58 deletions(-) create mode 100644 hive-types/src/nix_builds.rs diff --git a/Cargo.lock b/Cargo.lock index 77480d01..6cbf8792 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2003,6 +2003,7 @@ version = "0.1.0" dependencies = [ "anyhow", "hive-priv-sock", + "hive-types", "libc", "mockall", "rustix", @@ -2016,6 +2017,7 @@ dependencies = [ name = "hive-priv-sock" version = "0.1.0" dependencies = [ + "hive-types", "serde", "serde_json", ] @@ -4929,6 +4931,7 @@ dependencies = [ "async-nats", "futures-util", "hive-log", + "hive-types", "mockall", "opentelemetry", "opentelemetry-otlp", diff --git a/docs/process/gotchas.md b/docs/process/gotchas.md index f4eba9c4..5fdcb766 100644 --- a/docs/process/gotchas.md +++ b/docs/process/gotchas.md @@ -306,7 +306,10 @@ nix build "git+/internal/meta.git?rev=#nixosConfigurations.argus.con ### Containerized nix-daemon needs `sandbox-fallback = true` -Agent containers bind-mount the host's nix-daemon socket. nspawn +nixos-containers' start script bind-mounts the host's nix-daemon socket +directory into every container (hyperhive adds no bind for it; an agent +with `services.hyperhive.agent.nixBuilds = "none"` gets it shadowed, see +`docs/trust-boundary/security.md`). nspawn containers don't get user-namespaces by default, so `nix build` invocations _inside_ the container can't set up the build sandbox and fail outright if the host daemon's diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index d770665b..467cc5fa 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -174,15 +174,53 @@ before building `agent_notes_dir(name)`, so a `..` component can't traverse. ### Background -Agent containers bind-mount the host's `nix-daemon` socket. The host daemon may +nixos-containers' start script bind-mounts the host's `nix-daemon` socket +directory (`/nix/var/nix/daemon-socket`) into every container, hyperhive's +agents included; hyperhive adds no bind for it. The host daemon may have `sandbox-fallback = false` (strict NixOS defaults), which causes `nix build` inside nspawn containers to fail — containers lack kernel user namespaces, so nix can't set up its build sandbox. the agent modules set `sandbox-fallback = true` so that builds fall back to unsandboxed execution rather than failing outright. +### Per-agent nix builds mode + +`services.hyperhive.agent.nixBuilds` picks where an agent's nix builds run. It +has two values, and defaults to `host`: + +- **`host`**: builds go to the host daemon through the socket above. The daemon + forks them in its own cgroup, so they run **outside the agent container's + CPU and memory caps**, which sit on `container-.service`. +- **`none`**: the container has no route to the host daemon. hive-priv adds + `--tmpfs=/nix/var/nix/daemon-socket` to the container's nspawn flags. nspawn + mounts its custom mounts sorted by destination, then by kind (a bind before + a tmpfs), so the empty tmpfs lands on top of the start script's bind + whatever order the flags are in, and the container finds no `socket` in the + directory. `NIX_REMOTE=daemon` stays, so a build fails with `cannot connect +to socket at '/nix/var/nix/daemon-socket/socket'`, which names the cause. The + agent's `/nix/store` and its read-only nix database stay bound, so store + paths already there stay readable; only a realisation of a new path, or any + `nix` command that needs the daemon, fails. + +The mode reaches the container as `privateUsers` does: the swarm builder reads +the option from the evaluated config into the agent's closure record +(`nix_builds`), hive-c0re reads the record when it writes the container's +flags and sends the mode to hive-priv, and the flags apply at the container's +next start. + +- A record or builder from before the field reads `host`. +- A mode this build doesn't know (a record written by a newer swarm) reads as + `none`, never `host`: an agent that asked for a mode its hive can't give gets + no builds, not builds outside its limits. +- A record hive-c0re can't read, or none, sends no mode, and hive-priv keeps + what the container's last write chose. +- A hive-c0re or hive-priv from before the option ignores it, so the agent keeps + host builds until you upgrade both. + +`none` cuts the daemon only. It leaves the agent's egress alone. + ### Threat model -Unsandboxed nix builds run as `nixbld` users (non-root, typically UIDs 30001-30010). +Unsandboxed nix builds run as `nixbld` users (non-root; `nrBuildUsers` is at least 32, so UIDs 30001 and up). Without sandbox isolation, a build derivation's builder script has read access to any file in the container that the nixbld user can read. diff --git a/hive-c0re/src/attic/closure.rs b/hive-c0re/src/attic/closure.rs index ebcc1396..b39a574e 100644 --- a/hive-c0re/src/attic/closure.rs +++ b/hive-c0re/src/attic/closure.rs @@ -174,6 +174,7 @@ mod tests { built_unix: 1, cached, private_users: false, + nix_builds: hive_types::NixBuilds::Host, } } diff --git a/hive-c0re/src/attic/probe.rs b/hive-c0re/src/attic/probe.rs index d66e2b03..66c2c1a6 100644 --- a/hive-c0re/src/attic/probe.rs +++ b/hive-c0re/src/attic/probe.rs @@ -246,6 +246,7 @@ mod tests { built_unix: 1, cached: true, private_users: false, + nix_builds: hive_types::NixBuilds::Host, } } diff --git a/hive-c0re/src/job_queue/exec/tests.rs b/hive-c0re/src/job_queue/exec/tests.rs index 56ca798c..ccc6106d 100644 --- a/hive-c0re/src/job_queue/exec/tests.rs +++ b/hive-c0re/src/job_queue/exec/tests.rs @@ -275,6 +275,7 @@ fn closure(cached: bool) -> AgentClosure { built_unix: 1, cached, private_users: false, + nix_builds: hive_types::NixBuilds::Host, } } diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index 14541315..ef19adcc 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use hive_priv_sock::BindMount; -use hive_types::Permission; +use hive_types::{NixBuilds, Permission}; use crate::coordinator::{AgentPaths, HiveEnv}; @@ -451,9 +451,11 @@ async fn set_nspawn_flags( }); } - // Whether the agent's config sets `privateUsers`, as the swarm builder - // recorded it. Read from the record, never from the agent's nix. - let private_users = private_users_of(&crate::attic::closure::record(agent_name).await); + // What the agent's config sets for `privateUsers` and `nixBuilds`, as the + // swarm builder recorded it. Read from the record, never from the agent's nix. + let record = crate::attic::closure::record(agent_name).await; + let private_users = private_users_of(&record); + let nix_builds = nix_builds_of(&record); // Web-socket subdir: bind-mount `/run/hive-agent//` into the // container so the harness can bind `web.sock` there and the host-side @@ -487,6 +489,7 @@ async fn set_nspawn_flags( &load_creds, super::hive_facts(hive), private_users, + nix_builds, ) .await } @@ -529,6 +532,21 @@ fn private_users_of( } } +/// What hive-priv is told about the agent's nix builds: the record's mode, or +/// `None` (leave the agent as the last write set it) when there is no record or +/// it cannot be read, so a swarm outage never turns an agent's host builds on. +fn nix_builds_of( + record: &Result>, +) -> Option { + match record { + Ok(record) => record.as_ref().map(|r| r.nix_builds), + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "closure record unreadable; the agent's nix builds are left as they are"); + None + } + } +} + /// The binds every agent container gets, before its own state, config and /// role-specific dirs. fn base_binds(runtime_dir: &Path, claude_dir: &Path, claude_mount: String) -> Vec { @@ -559,8 +577,16 @@ mod tests { stage_limits, }; use crate::coordinator::HiveEnv; + use hive_types::NixBuilds; fn record_with(private_users: bool) -> swarm_wire::agent_closures::AgentClosure { + record_with_builds(private_users, NixBuilds::Host) + } + + fn record_with_builds( + private_users: bool, + nix_builds: NixBuilds, + ) -> swarm_wire::agent_closures::AgentClosure { swarm_wire::agent_closures::AgentClosure { meta_rev: "a".repeat(40), store_path: "/nix/store/x".to_owned(), @@ -568,6 +594,7 @@ mod tests { built_unix: 1, cached: true, private_users, + nix_builds, } } @@ -609,6 +636,38 @@ mod tests { assert_eq!(super::private_users_of(&Ok(Some(old))), Some(false)); } + /// The record's mode is what hive-priv is told, both ways. No record and an + /// unreadable one say nothing, so a swarm outage changes no agent; a record + /// from before the field reads as host builds. + #[test] + fn the_records_nix_builds_is_what_the_start_path_passes() { + for mode in [NixBuilds::Host, NixBuilds::Disabled] { + assert_eq!( + super::nix_builds_of(&Ok(Some(record_with_builds(false, mode)))), + Some(mode) + ); + } + assert_eq!(super::nix_builds_of(&Ok(None)), None); + assert_eq!(super::nix_builds_of(&Err(anyhow::anyhow!("down"))), None); + let old = serde_json::json!({ + "meta_rev": "a", "store_path": "/nix/store/x", + "system": "s", "built_unix": 1, + }); + let old = serde_json::from_value(old).expect("decodes"); + assert_eq!(super::nix_builds_of(&Ok(Some(old))), Some(NixBuilds::Host)); + // A mode a newer swarm wrote, which this build does not know, decodes + // and builds nothing. + let newer = serde_json::json!({ + "meta_rev": "a", "store_path": "/nix/store/x", + "system": "s", "built_unix": 1, "nix_builds": "container", + }); + let newer = serde_json::from_value(newer).expect("decodes"); + assert_eq!( + super::nix_builds_of(&Ok(Some(newer))), + Some(NixBuilds::Disabled) + ); + } + /// No agent container gets a cross-agent scratch dir; `/knowledge` is the /// control that the lookup finds a bind that is there. #[test] diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index a67d9f23..431a55b9 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -11,6 +11,7 @@ use hive_priv_sock::{ BindMount, CredentialMount, HiveFacts, InfraAction, InfraContainer, MEASURE_AGENT_DISK_TIMEOUT, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream, }; +use hive_types::NixBuilds; use std::os::fd::{AsRawFd as _, OwnedFd, RawFd}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Interest}; @@ -320,6 +321,7 @@ pub async fn write_nspawn_flags( load_credentials: &[CredentialMount], hive_facts: HiveFacts, private_users: Option, + nix_builds: Option, ) -> Result<()> { ok(call(&PrivRequest::WriteNspawnFlags { container: container.to_owned(), @@ -328,6 +330,7 @@ pub async fn write_nspawn_flags( load_credentials: load_credentials.to_vec(), hive_facts: Box::new(hive_facts), private_users, + nix_builds, }) .await?) } diff --git a/hive-priv-sock/Cargo.toml b/hive-priv-sock/Cargo.toml index ee48717a..ad4f7f93 100644 --- a/hive-priv-sock/Cargo.toml +++ b/hive-priv-sock/Cargo.toml @@ -8,6 +8,8 @@ readme = "README.md" workspace = true [dependencies] +# `NixBuilds`, shared with the closure record. +hive-types.workspace = true serde.workspace = true [dev-dependencies] diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index b50d8500..67c6c9ed 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -10,6 +10,7 @@ //! audit. No server/client implementation lives here, only the wire //! contract (mirrors `hive-host-sock`'s split for the host admin socket). +use hive_types::NixBuilds; use serde::{Deserialize, Serialize}; /// Default socket path for the privileged helper. @@ -350,6 +351,16 @@ pub enum PrivRequest { /// agent as it is: namespaced if hive-priv moved its data, else not. #[serde(default)] private_users: Option, + /// Whether the container may reach the host's nix-daemon, as the + /// swarm builder recorded the agent's `services.hyperhive.agent.nixBuilds`. + /// `Some(NixBuilds::Disabled)` has hive-priv shadow the daemon socket + /// directory with an empty tmpfs. `None`, which is what a hive-c0re + /// without the field or without a readable record sends, keeps the + /// agent as the last write left it, so a swarm outage never turns an + /// agent's host builds on. An agent never written before gets the + /// host daemon. + #[serde(default)] + nix_builds: Option, }, /// Create the agent's socket dir `/run/hive-agent/` as `0751 @@ -718,6 +729,8 @@ pub enum PrivEvent { #[cfg(test)] mod tests { + use hive_types::NixBuilds; + use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev}; /// A request from a hive-c0re that predates the field decodes with the @@ -746,6 +759,42 @@ mod tests { assert_eq!(private_users, want); } } + /// The nix-builds mode is optional on the wire: a request from a hive-c0re + /// that predates the field reads as `None`, the field round-trips, and a + /// mode this build does not know reads as the one that builds nothing. + #[test] + fn nix_builds_is_optional_on_the_wire() { + let request = |extra: Option<(&str, serde_json::Value)>| { + let mut wire = serde_json::json!({ + "op": "write_nspawn_flags", + "container": "h-iris", + "binds": [], + "isolation": {"bridge": "hive0", "gateway_ip": "10.42.0.1"}, + }); + if let Some((key, value)) = extra { + wire[key] = value; + } + let PrivRequest::WriteNspawnFlags { nix_builds, .. } = + serde_json::from_value(wire).expect("decodes") + else { + panic!("variant"); + }; + nix_builds + }; + assert_eq!(request(None), None); + for (spelling, want) in [ + ("host", NixBuilds::Host), + ("none", NixBuilds::Disabled), + ("container", NixBuilds::Disabled), + ] { + assert_eq!( + request(Some(("nix_builds", spelling.into()))), + Some(want), + "{spelling}" + ); + } + } + #[test] fn infra_control_names_no_c0re_or_swarm_service() { // Neither the daemon nor the swarm's forge and matrix have a diff --git a/hive-priv/Cargo.toml b/hive-priv/Cargo.toml index abcfe7bd..61803861 100644 --- a/hive-priv/Cargo.toml +++ b/hive-priv/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] anyhow.workspace = true hive-priv-sock.workspace = true +hive-types.workspace = true libc.workspace = true rustix.workspace = true serde_json.workspace = true diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs index c86675f4..95b8f9fd 100644 --- a/hive-priv/src/container_config.rs +++ b/hive-priv/src/container_config.rs @@ -14,6 +14,7 @@ use anyhow::{Context as _, Result, bail}; use hive_priv_sock::{ AGENT_PREFIX, AGENT_RUNTIME_ROOT, BindMount, CredentialMount, HiveFacts, NetworkIsolation, }; +use hive_types::NixBuilds; use tokio::process::Command; use crate::shared::{StagedFile, open_dir, validate_credential_name, validate_name_chars}; @@ -27,6 +28,7 @@ pub(crate) fn handle_write_nspawn_flags( load_credentials: &[CredentialMount], hive_facts: &HiveFacts, private_users: Option, + nix_builds: Option, ) -> Result<(String, String)> { let agent = validate_container_system_name(container)?; for bind in binds { @@ -60,8 +62,11 @@ pub(crate) fn handle_write_nspawn_flags( isolation, load_credentials, &hive_facts, - ephemeral, - userns, + StartMode { + ephemeral, + userns, + nix_builds, + }, )?; Ok((String::new(), String::new())) } @@ -79,6 +84,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking( load_credentials: Vec, hive_facts: HiveFacts, private_users: Option, + nix_builds: Option, ) -> Result<(String, String)> { tokio::task::spawn_blocking(move || { handle_write_nspawn_flags( @@ -88,6 +94,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking( &load_credentials, &hive_facts, private_users, + nix_builds, ) }) .await @@ -358,6 +365,18 @@ fn git_overlay_flags(binds: &[BindMount]) -> Vec { .collect() } +/// What a start of the container asks of nspawn besides its binds, as +/// [`write_nspawn_flags`] writes it. +#[derive(Clone, Copy)] +struct StartMode { + /// `--ephemeral`, once [`crate::rootfs`] has made the root fit for it. + ephemeral: bool, + /// The container runs in its own user namespace. + userns: bool, + /// `None` keeps the choice the conf's last write made. + nix_builds: Option, +} + /// Update `/etc/nixos-containers/.conf`: strip old network vars /// (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`), /// write the current network-isolation settings, then append @@ -370,11 +389,15 @@ fn write_nspawn_flags( isolation: &NetworkIsolation, load_credentials: &[CredentialMount], hive_facts: &str, - ephemeral: bool, - userns: bool, + mode: StartMode, ) -> Result<()> { use std::fmt::Write as _; use std::os::unix::fs::MetadataExt as _; + let StartMode { + ephemeral, + userns, + nix_builds, + } = mode; let path = format!("/etc/nixos-containers/{container}.conf"); let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?; let lines: Vec<&str> = original @@ -406,7 +429,9 @@ fn write_nspawn_flags( out.push_str("LOCAL_ADDRESS6=\n"); let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge); } - let flags_joined = nspawn_flags(binds, load_credentials, ephemeral, userns).join(" "); + let nix_builds = chosen_nix_builds(nix_builds, &original); + let flags_joined = + nspawn_flags(binds, load_credentials, ephemeral, userns, nix_builds).join(" "); let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\""); // Keeps the owner and mode `nixos-container` created the file with; this // helper only rewrites its lines. @@ -457,6 +482,38 @@ fn private_users_line(userns: bool) -> &'static str { if userns { "PRIVATE_USERS=pick\n" } else { "" } } +/// An empty tmpfs over `/nix/var/nix/daemon-socket`, the directory nixos-containers +/// always binds the host's nix-daemon socket through, so the container finds no +/// socket to connect to. nspawn mounts its custom mounts sorted by destination +/// and then by kind, and a tmpfs sorts after a bind, so this lands on top of the +/// start script's bind whatever order the flags are in. +const NIX_DAEMON_SHADOW_FLAG: &str = "--tmpfs=/nix/var/nix/daemon-socket"; + +/// What the previous write of the conf chose: [`NixBuilds::Disabled`] when its +/// `EXTRA_NSPAWN_FLAGS` line carries [`NIX_DAEMON_SHADOW_FLAG`], else +/// [`NixBuilds::Host`], which is also what a conf never written gets. +fn previous_nix_builds(conf: &str) -> NixBuilds { + let shadowed = conf + .lines() + .filter_map(|line| line.trim_start().strip_prefix("EXTRA_NSPAWN_FLAGS=")) + .any(|flags| { + flags + .split(|c: char| c.is_whitespace() || c == '"') + .any(|flag| flag == NIX_DAEMON_SHADOW_FLAG) + }); + if shadowed { + NixBuilds::Disabled + } else { + NixBuilds::Host + } +} + +/// The mode a write uses: the one asked for, else what the conf's last write +/// chose. +fn chosen_nix_builds(requested: Option, conf: &str) -> NixBuilds { + requested.unwrap_or_else(|| previous_nix_builds(conf)) +} + /// The flags `EXTRA_NSPAWN_FLAGS` carries, in the order nspawn needs them. /// /// `--ephemeral` runs the container on a snapshot of its root and drops the @@ -464,6 +521,8 @@ fn private_users_line(userns: bool) -> &'static str { /// the flag is asked for. The git overlays and credentials follow the binds /// they refer to. /// +/// `nix_builds` set to [`NixBuilds::Disabled`] adds [`NIX_DAEMON_SHADOW_FLAG`]. +/// /// `userns` mounts every bind `owneridmap` and idmaps the root instead of /// chowning it, so a different `pick` range never walks the tree. fn nspawn_flags( @@ -471,6 +530,7 @@ fn nspawn_flags( load_credentials: &[CredentialMount], ephemeral: bool, userns: bool, + nix_builds: NixBuilds, ) -> Vec { let mut flags: Vec = ephemeral .then(|| "--ephemeral".to_owned()) @@ -485,6 +545,9 @@ fn nspawn_flags( format!("{flag}={}:{}{idmap}", b.host_path, b.container_path) })); flags.extend(git_overlay_flags(binds)); + if nix_builds == NixBuilds::Disabled { + flags.push(NIX_DAEMON_SHADOW_FLAG.to_owned()); + } // Credential forwarding: nspawn loads each host secret into the // container's credential store under ``; inner units inherit it // via `LoadCredential=`. Validated (name charset + bind-path @@ -653,8 +716,9 @@ pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: & #[cfg(test)] mod tests { use super::{ - BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags, - hive_priv_exe, is_owned_var, limits_dropin_body, nspawn_flags, private_users_line, + BindMount, HiveFacts, NIX_DAEMON_SHADOW_FLAG, NetworkIsolation, NixBuilds, + chosen_nix_builds, git_overlay_flags, handle_write_nspawn_flags, hive_priv_exe, + is_owned_var, limits_dropin_body, nspawn_flags, previous_nix_builds, private_users_line, publish_file, remove_service_dropin, render_hive_facts, rootfs_check_command, validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits, }; @@ -1105,10 +1169,10 @@ mod tests { #[test] fn ephemeral_is_passed_only_when_asked() { let binds = [bind("/agents/atlas/work")]; - let with = nspawn_flags(&binds, &[], true, false); + let with = nspawn_flags(&binds, &[], true, false, NixBuilds::Host); assert_eq!(with[0], "--ephemeral", "{with:?}"); assert!(with.iter().any(|f| f.starts_with("--bind-ro=")), "{with:?}"); - let without = nspawn_flags(&binds, &[], false, false); + let without = nspawn_flags(&binds, &[], false, false, NixBuilds::Host); assert!(!without.iter().any(|f| f == "--ephemeral"), "{without:?}"); assert_eq!(without.len() + 1, with.len()); } @@ -1119,7 +1183,7 @@ mod tests { #[test] fn binds_are_owneridmap_only_for_a_namespaced_agent() { let binds = [bind("/knowledge"), bind("/agents/a/config")]; - let plain = nspawn_flags(&binds, &[], false, false); + let plain = nspawn_flags(&binds, &[], false, false, NixBuilds::Host); assert!( plain.iter().any(|f| f.starts_with("--bind-ro=")), "{plain:?}" @@ -1130,7 +1194,7 @@ mod tests { .all(|f| !f.contains("idmap") && !f.contains("ownership")), "{plain:?}" ); - let userns = nspawn_flags(&binds, &[], false, true); + let userns = nspawn_flags(&binds, &[], false, true, NixBuilds::Host); assert!( userns.contains(&"--private-users-ownership=map".to_owned()), "{userns:?}" @@ -1143,6 +1207,63 @@ mod tests { ); } + /// `Disabled` puts the tmpfs over the daemon socket directory after the + /// binds; `Host` leaves it out. The binds are the control that the lookup + /// finds what is there, and the flag is the whole of the difference. + #[test] + fn nix_builds_none_shadows_the_daemon_socket_dir() { + let binds = [bind("/agents/atlas/work")]; + let host = nspawn_flags(&binds, &[], false, false, NixBuilds::Host); + let none = nspawn_flags(&binds, &[], false, false, NixBuilds::Disabled); + assert!(host.iter().any(|f| f.starts_with("--bind-ro=")), "{host:?}"); + assert!( + !host.iter().any(|f| f == NIX_DAEMON_SHADOW_FLAG), + "{host:?}" + ); + assert_eq!( + none.last().map(String::as_str), + Some(NIX_DAEMON_SHADOW_FLAG) + ); + assert_eq!(none.len(), host.len() + 1); + assert_eq!(NIX_DAEMON_SHADOW_FLAG, "--tmpfs=/nix/var/nix/daemon-socket"); + } + + /// An unspecified mode keeps what the conf's last write chose, whatever + /// else the line carries; a conf with no such line is a host-builds agent. + #[test] + fn an_unspecified_nix_builds_mode_keeps_the_last_writes_choice() { + let binds = [bind("/knowledge")]; + for want in [NixBuilds::Host, NixBuilds::Disabled] { + let flags = nspawn_flags(&binds, &[], true, true, want).join(" "); + let conf = format!("AUTO_START=1\nEXTRA_NSPAWN_FLAGS=\"{flags}\"\n"); + assert_eq!(previous_nix_builds(&conf), want, "{conf}"); + } + assert_eq!(previous_nix_builds(""), NixBuilds::Host); + // A mode asked for wins over the conf; none asked for keeps it. + let shadowed = format!("EXTRA_NSPAWN_FLAGS=\"{NIX_DAEMON_SHADOW_FLAG}\"\n"); + assert_eq!(chosen_nix_builds(None, &shadowed), NixBuilds::Disabled); + assert_eq!(chosen_nix_builds(None, ""), NixBuilds::Host); + assert_eq!( + chosen_nix_builds(Some(NixBuilds::Host), &shadowed), + NixBuilds::Host + ); + assert_eq!( + chosen_nix_builds(Some(NixBuilds::Disabled), ""), + NixBuilds::Disabled + ); + // The flag has to be a flag of its own, on the flags line. + assert_eq!( + previous_nix_builds(&format!( + "EXTRA_NSPAWN_FLAGS=\"{NIX_DAEMON_SHADOW_FLAG}/x\"\n" + )), + NixBuilds::Host + ); + assert_eq!( + previous_nix_builds(&format!("OTHER=\"{NIX_DAEMON_SHADOW_FLAG}\"\n")), + NixBuilds::Host + ); + } + /// `PRIVATE_USERS=` is dropped from the conf whatever the agent now is, so /// leaving the list cannot leave the line behind. `AUTO_START=` is the /// control: a line that is not ours stays. @@ -1166,7 +1287,8 @@ mod tests { let facts = HiveFacts::default(); for name in ["hive-forge", "hive-matrix"] { let errs = [ - handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None).unwrap_err(), + handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None, None) + .unwrap_err(), write_resource_limits(name, "4G", "200%", None, None).unwrap_err(), remove_service_dropin(name).unwrap_err(), ]; diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index faf1c845..585d6ba9 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -414,6 +414,7 @@ async fn exec( load_credentials, hive_facts, private_users, + nix_builds, } => { container_config::handle_write_nspawn_flags_blocking( container, @@ -422,6 +423,7 @@ async fn exec( load_credentials, *hive_facts, private_users, + nix_builds, ) .await } diff --git a/hive-types/README.md b/hive-types/README.md index 34514019..93d585d8 100644 --- a/hive-types/README.md +++ b/hive-types/README.md @@ -10,6 +10,10 @@ Foundational shared **newtypes** for the hyperhive workspace. A zero-dependency - **`Permission`** — the set of named permissions an agent may hold, with its stored `snake_case` names and the MCP and built-in tools each grants. +- **`NixBuilds`** — where an agent's nix builds run (`host` or `none`), as + `services.hyperhive.agent.nixBuilds` sets it. It rides the swarm closure record, + the builder's reply and the `hive-priv` wire; a spelling it doesn't know reads + as `none`. - **`Ident`** — a validated agent-name newtype. Typing agent-name fields as `Ident` (rather than bare `String`) gets serde-validated parsing **at the socket boundary for free**: a malformed name is rejected at deserialize time diff --git a/hive-types/src/lib.rs b/hive-types/src/lib.rs index bc7d84dc..fc184260 100644 --- a/hive-types/src/lib.rs +++ b/hive-types/src/lib.rs @@ -1,4 +1,4 @@ -//! Foundational shared types for the hyperhive workspace: [`Ident`], [`Permission`], +//! Foundational shared types for the hyperhive workspace: [`Ident`], [`Permission`], [`NixBuilds`], //! the [`term_msg`] row markers and the [`review`] rule. //! //! A zero-dependency (bar `serde`) leaf crate so every wire-type crate @@ -7,8 +7,10 @@ //! and get serde-validated parsing at the socket boundary for free — with //! no cross-crate coupling and without growing `hive-sh4re`. +mod nix_builds; mod permission; +pub use nix_builds::NixBuilds; pub use permission::{Permission, deserialize_known}; pub mod review; diff --git a/hive-types/src/nix_builds.rs b/hive-types/src/nix_builds.rs new file mode 100644 index 00000000..b6b94737 --- /dev/null +++ b/hive-types/src/nix_builds.rs @@ -0,0 +1,58 @@ +//! How an agent's nix builds run: [`NixBuilds`]. +//! +//! Lives in this leaf crate because the same value is read by the swarm +//! builder's reply, the closure record, `hive-c0re` and the `hive-priv` wire. + +use serde::{Deserialize, Serialize}; + +/// Where an agent's nix builds run, as `services.hyperhive.agent.nixBuilds` +/// sets it. Stored by its option spelling. +/// +/// A spelling this build does not know reads as [`NixBuilds::Disabled`], so a +/// record written by a newer swarm never fails to decode. An agent that asked +/// for a mode this build lacks therefore gets no builds, never builds on the +/// host daemon outside its limits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum NixBuilds { + /// Builds go to the host's nix-daemon, which runs them outside the agent + /// container's CPU and memory caps. The default. + #[default] + #[serde(rename = "host")] + Host, + /// The container cannot reach the host's nix-daemon, so it builds nothing. + #[serde(rename = "none", other)] + Disabled, +} + +#[cfg(test)] +mod tests { + use super::NixBuilds; + + #[test] + fn the_spellings_are_pinned() { + for (value, wire) in [(NixBuilds::Host, "host"), (NixBuilds::Disabled, "none")] { + assert_eq!(serde_json::to_value(value).expect("encodes"), wire); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).expect("decodes"), + value + ); + } + } + + #[test] + fn the_default_is_the_host_daemon() { + assert_eq!(NixBuilds::default(), NixBuilds::Host); + } + + /// A mode a newer swarm writes decodes, and as the one that builds nothing. + #[test] + fn an_unknown_spelling_reads_as_disabled() { + for unknown in ["container", "", "HOST"] { + assert_eq!( + serde_json::from_value::(serde_json::json!(unknown)).expect("decodes"), + NixBuilds::Disabled, + "{unknown:?}" + ); + } + } +} diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 668fbf7a..6b1dafce 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -135,6 +135,32 @@ in ''; }; + options.services.hyperhive.agent.nixBuilds = lib.mkOption { + type = lib.types.enum [ + "host" + "none" + ]; + default = "host"; + example = "none"; + description = '' + Where this agent's nix builds run. + + - `host` (default): builds go through the host's nix-daemon socket, + which every agent container gets bound in by default. The daemon runs + them outside this container, so the container's CPU and memory caps do + not apply to a build. + - `none`: the container cannot reach the host's nix-daemon, so + `nix build`, `nix run`, `nix shell` and the like fail for anything not + already in the store, with a "cannot connect to socket" error. The + agent's own system closure stays readable. + + The swarm builder reads the option from the evaluated config into the + agent's closure record, and the hive applies it when it next starts the + container, so a running agent keeps its mode until restarted. A host + that does not know the option leaves the agent on `host`. + ''; + }; + config = { assertions = [ # Guard the inputs-routed-as-output pattern: the agent flake.nix is @@ -235,6 +261,10 @@ in # Non-root nix clients already default to the daemon socket, so this # is a no-op for them; it only matters for root services that would # otherwise silently use the local store. + # + # Stays set under `nixBuilds = "none"`: the missing socket then fails a + # build with an error that names it, where the local store would fail on a + # read-only filesystem. systemd.globalEnvironment.NIX_REMOTE = "daemon"; # `claude-code` is unfree. Each per-agent container's nixosConfiguration diff --git a/nix/module-eval/agent-user.nix b/nix/module-eval/agent-user.nix index dbbc6cc8..47a2024f 100644 --- a/nix/module-eval/agent-user.nix +++ b/nix/module-eval/agent-user.nix @@ -22,11 +22,20 @@ let ; # A named agent, so the rendered script carries a name no default supplies. - migrate = - (agentWith { services.hyperhive.agent.user.name = "iris"; }) - .system.activationScripts.hive-agent-user-migrate.text; + iris = agentWith { services.hyperhive.agent.user.name = "iris"; }; + migrate = iris.system.activationScripts.hive-agent-user-migrate.text; # The tmpfiles rules of the same named agent. - rules = (agentWith { services.hyperhive.agent.user.name = "iris"; }).systemd.tmpfiles.rules; + rules = iris.systemd.tmpfiles.rules; + # The same agent with its nix builds switched off, and one asking for a mode + # the option does not have. + noBuilds = agentWith { + services.hyperhive.agent.user.name = "iris"; + services.hyperhive.agent.nixBuilds = "none"; + }; + unknownBuilds = agentWith { + services.hyperhive.agent.user.name = "iris"; + services.hyperhive.agent.nixBuilds = "container"; + }; # `privateUsers` with the given pins; the agent name is the one the cases # above use, so only the pins differ. withUsers = @@ -113,6 +122,18 @@ let && lib.elem "L /home/iris/.local/state/nix - - - - /nix/var/nix/profiles/per-user/iris" rules && lib.elem "L /home/iris/.nix-profile - - - - /home/iris/.local/state/nix/profiles/profile" rules; } + { + # The option defaults to host builds, accepts `none`, and refuses a mode + # it does not declare. The default and `none` arms are each other's + # control; `none` leaves the in-container routing alone, because the + # error an agent gets then comes from the missing socket. + name = "nixBuilds defaults to host, takes none and refuses a mode it does not declare"; + ok = + iris.services.hyperhive.agent.nixBuilds == "host" + && noBuilds.services.hyperhive.agent.nixBuilds == "none" + && noBuilds.systemd.globalEnvironment.NIX_REMOTE == "daemon" + && !(builtins.tryEval unknownBuilds.services.hyperhive.agent.nixBuilds).success; + } ]; in runGroup "agent-user" cases diff --git a/nix/vm-tests/agent-ephemeral-rootfs.nix b/nix/vm-tests/agent-ephemeral-rootfs.nix index a49d7676..7be6e5f5 100644 --- a/nix/vm-tests/agent-ephemeral-rootfs.nix +++ b/nix/vm-tests/agent-ephemeral-rootfs.nix @@ -16,15 +16,15 @@ # - a link planted at `root` in the template refuses the next start through # the limits drop-in's `ExecStartPre=` check (whose build has a GC root), # leaves the link's target untouched, and the start goes through once the -# link is gone. -# +# link is gone; # - an agent whose config sets `services.hyperhive.agent.user.privateUsers` # (hive-c0re sends the flag from the closure record; the test sends it) -# starts in its own -# user namespace with state it already had, owned by uid 1000 on disk: the -# data moves once to `dataOwner`, the agent user reads and writes it as -# itself, the other agent's conf (no flag sent) is untouched, and a second -# `WriteNspawnFlags` moves nothing. +# starts in its own user namespace with state it already had, owned by uid +# 1000 on disk: the data moves once to `dataOwner`, the agent user reads and +# writes it as itself, the other agent's conf (no flag sent) is untouched, +# and a second `WriteNspawnFlags` moves nothing; +# - an agent set to `services.hyperhive.agent.nixBuilds = "none"` has no +# nix-daemon socket and cannot build; a request naming no mode keeps that. # # The agent's journal forwarder itself (an OTLP push to the hive's collector) is # not booted here; the module-eval suite covers its wiring. @@ -277,6 +277,9 @@ pkgs.testers.runNixOSTest { assert host.succeed(f"cat {WORK}/kept.txt").strip() == "kept" as_agent("hello") as_agent("tree --version") + # The default mode: the host daemon's socket is there and answers. + assert "socket" in in_ct("ls /nix/var/nix/daemon-socket").split() + as_agent(f"nix-store --store daemon -q --hash {HELLO}") with subtest("the switch ran once"): salvages = [d for d in salvage_dirs() if d.startswith("salvaged-rootfs-")] @@ -335,10 +338,14 @@ pkgs.testers.runNixOSTest { "binds": NS_BINDS, "isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"}, "private_users": True, + "nix_builds": "none", } priv_request(host, write) ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf") assert "PRIVATE_USERS=pick" in ns_conf, ns_conf + assert "--tmpfs=/nix/var/nix/daemon-socket" in ns_conf, ns_conf + # The other agent's conf has no such flag: the control for the line above. + assert "daemon-socket" not in conf(), conf() assert "--private-users-ownership=map" in ns_conf, ns_conf # Every bind, the read-only config included, is mounted owneridmap. assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf @@ -379,12 +386,27 @@ pkgs.testers.runNixOSTest { host.succeed("stat -c %u:%g ${nsSocketDir}/web.sock | grep -x ${dataOwner}:${dataOwner}") host.succeed(f"nixos-container run {NS} -- cat /proc/self/uid_map | grep -v ' 4294967295$'") + def no_daemon(): + # The start script's bind is under the tmpfs: no socket in the + # directory, and a build cannot connect. The store path is still + # readable, and the first agent's directory (checked above) has the + # socket where this one has none. + host.fail(f"nixos-container run {NS} -- test -S /nix/var/nix/daemon-socket/socket") + assert as_ns("ls -A /nix/var/nix/daemon-socket") == "" + host.fail(f"nixos-container run {NS} -- runuser -l {NS_AGENT} -c 'nix-store --store daemon -q --hash {HELLO}'") + as_ns(f"test -r {HELLO}/bin/hello") + no_daemon() + # The move runs once: a file a host process leaves owned by 1000 after it # is not chowned by the next request, and the agent restarts. host.succeed(f"nixos-container stop {NS}") host.succeed(f"echo late > {NS_DIR}/state/late.txt && chown 1000:1000 {NS_DIR}/state/late.txt") - priv_request(host, write) + # A request that names no nix-builds mode, as one with no readable + # record is, keeps the choice the last write made. + priv_request(host, {k: v for k, v in write.items() if k != "nix_builds"}) host.succeed(f"stat -c %u:%g {NS_DIR}/state/late.txt | grep -x 1000:1000") + assert "--tmpfs=/nix/var/nix/daemon-socket" in host.succeed(f"cat /etc/nixos-containers/{NS}.conf") boot_ns() + no_daemon() ''; } diff --git a/swarm-builder/Cargo.toml b/swarm-builder/Cargo.toml index 3487af8f..f1cd1212 100644 --- a/swarm-builder/Cargo.toml +++ b/swarm-builder/Cargo.toml @@ -14,6 +14,8 @@ async-nats.workspace = true # StreamExt::next on the subscription. futures-util.workspace = true hive-log.workspace = true +# `NixBuilds`, the type of the reply's `nix_builds`. +hive-types.workspace = true # The build metrics, exported over OTLP like the review classifier's. opentelemetry.workspace = true opentelemetry-otlp.workspace = true diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index 7dda6326..f12e123f 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -620,6 +620,7 @@ async fn handle_meta( system, cached, private_users, + nix_builds, } => tracing::info!( agent = %job.agent, meta_rev = %job.meta_rev, @@ -627,6 +628,7 @@ async fn handle_meta( %system, cached, private_users, + ?nix_builds, "agent meta build built" ), FromMetaReply::Failed { error_tail } => tracing::warn!( @@ -859,6 +861,7 @@ mod tests { system: "s".to_owned(), cached: false, private_users: false, + nix_builds: hive_types::NixBuilds::Host, } }, ) @@ -881,6 +884,7 @@ mod tests { system: "s".to_owned(), cached: false, private_users: false, + nix_builds: hive_types::NixBuilds::Host, } }, ) @@ -1462,11 +1466,12 @@ mod tests { "system": "x86_64-linux", "cached": false, "private_users": true, + "nix_builds": "host", }) ); - assert_eq!(fake.lines("tokens"), ["forge-token"; 2]); + assert_eq!(fake.lines("tokens"), ["forge-token"; 3]); let calls = fake.calls(); - assert_eq!(calls.len(), 2, "{calls:?}"); + assert_eq!(calls.len(), 3, "{calls:?}"); assert!( calls[0].contains(&format!( "git+https://forge.example/internal/meta.git?rev={REV}#nixosConfigurations.atlas.config.system.build.toplevel" @@ -1477,6 +1482,10 @@ mod tests { calls[1].contains("services.hyperhive.agent.user.privateUsers"), "{calls:?}" ); + assert!( + calls[2].contains("services.hyperhive.agent.nixBuilds"), + "{calls:?}" + ); } #[tokio::test] @@ -1582,7 +1591,7 @@ mod tests { ]; serve_meta(fake.builder(), env, msgs).await; assert_eq!(next(&mut sent).await.0, "_INBOX.2"); - assert_eq!(fake.calls().len(), 2, "one build and its option read"); + assert_eq!(fake.calls().len(), 3, "one build and its two option reads"); } #[tokio::test(start_paused = true)] diff --git a/swarm-builder/src/nix_build.rs b/swarm-builder/src/nix_build.rs index 9ce91431..f7c2b4df 100644 --- a/swarm-builder/src/nix_build.rs +++ b/swarm-builder/src/nix_build.rs @@ -18,6 +18,7 @@ use std::path::PathBuf; use std::process::Stdio; use std::time::Duration; +use hive_types::NixBuilds; use swarm_wire::builder::{ BuildAgent, BuildFromMeta, FromMetaReply, LockMeta, LockReply, META_REPO_PATH, Reply, }; @@ -93,6 +94,7 @@ impl Builder { system: self.system.clone(), cached: false, private_users: self.private_users(job, token).await, + nix_builds: self.nix_builds(job, token).await, }, Reply::Failed { error_tail } => FromMetaReply::Failed { error_tail }, }, @@ -182,28 +184,61 @@ impl Builder { ) } - /// Whether the agent's config sets `services.hyperhive.agent.user.privateUsers`, - /// read by `nix eval` of the flake the build just evaluated, in this - /// sandbox. Anything but a clean `true` is false: an agent whose module - /// predates the option, a failed eval and a timeout all keep the agent out - /// of a user namespace. - async fn private_users(&self, job: &BuildFromMeta, token: &str) -> bool { + /// `nix eval --json` of `option` in the agent's evaluated config, run in this + /// sandbox against the flake the build just evaluated. `None` when the eval + /// fails or times out, and logs why along with what the caller falls back to. + async fn eval_option( + &self, + job: &BuildFromMeta, + token: &str, + option: &str, + fallback: &str, + ) -> Option> { let mut cmd = self.nix_command(token); cmd.args(EVAL_FLAGS) - .arg(format!("{}.{PRIVATE_USERS_OPTION}", self.meta_config(job))); + .arg(format!("{}.{option}", self.meta_config(job))); match run_raw(cmd, self.deadline).await { - Ok((status, out, _)) if status.success() => parse_flag(&out), + Ok((status, out, _)) if status.success() => Some(out), Ok((status, _, err)) => { - tracing::info!(agent = %job.agent, %status, error = %tail(&String::from_utf8_lossy(&err), status), "privateUsers not read; the agent stays out of a user namespace"); - false + tracing::info!(agent = %job.agent, %status, error = %tail(&String::from_utf8_lossy(&err), status), "{option} not read; {fallback}"); + None } Err(why) => { - tracing::warn!(agent = %job.agent, %why, "privateUsers not read; the agent stays out of a user namespace"); - false + tracing::warn!(agent = %job.agent, %why, "{option} not read; {fallback}"); + None } } } + /// Whether the agent's config sets `services.hyperhive.agent.user.privateUsers`. + /// Anything but a clean `true` is false: an agent whose module predates the + /// option, a failed eval and a timeout all keep the agent out of a user + /// namespace. + async fn private_users(&self, job: &BuildFromMeta, token: &str) -> bool { + self.eval_option( + job, + token, + PRIVATE_USERS_OPTION, + "the agent stays out of a user namespace", + ) + .await + .is_some_and(|out| parse_flag(&out)) + } + + /// The agent's `services.hyperhive.agent.nixBuilds`. An agent whose module + /// predates the option, a failed eval and a timeout all read as + /// [`NixBuilds::Host`], which is what every agent did before the option. + async fn nix_builds(&self, job: &BuildFromMeta, token: &str) -> NixBuilds { + self.eval_option( + job, + token, + NIX_BUILDS_OPTION, + "the agent builds through the host daemon", + ) + .await + .map_or_else(NixBuilds::default, |out| parse_nix_builds(&out)) + } + /// The config commit as a flake reference, fetched through the PR's head /// ref. fn agent_url(&self, job: &BuildAgent) -> String { @@ -242,11 +277,20 @@ const EVAL_FLAGS: [&str; 7] = [ /// The agent option the controller records, relative to the evaluated config. const PRIVATE_USERS_OPTION: &str = "services.hyperhive.agent.user.privateUsers"; +/// The agent option holding where its nix builds run. +const NIX_BUILDS_OPTION: &str = "services.hyperhive.agent.nixBuilds"; + /// `nix eval --json` of a bool: true only for exactly `true`. fn parse_flag(out: &[u8]) -> bool { out.trim_ascii() == b"true" } +/// `nix eval --json` of a [`NixBuilds`]: the spelling, or the default when the +/// output is not a string. +fn parse_nix_builds(out: &[u8]) -> NixBuilds { + serde_json::from_slice(out).unwrap_or_default() +} + /// `nix flake lock` of the flake in the working directory. The flake's config /// is not trusted, and nothing is built. const LOCK_ARGS: &[&str] = &["flake", "lock", "--option", "accept-flake-config", "false"]; @@ -560,6 +604,37 @@ mod tests { } } + /// The reply carries the option's spelling. `host` is the control for + /// `none`; a mode this build does not know reads as `none`, and anything + /// that is no spelling at all (a failed eval, an older module) as `host`. + #[tokio::test] + async fn a_meta_build_reports_the_agents_nix_builds_option() { + for (i, (answer, want)) in [ + ("echo '\"host\"'", NixBuilds::Host), + ("echo '\"none\"'", NixBuilds::Disabled), + ("echo '\"container\"'", NixBuilds::Disabled), + ("echo null", NixBuilds::Host), + ("echo '\"none\"'; exit 1", NixBuilds::Host), + ("echo error: attribute missing >&2; exit 1", NixBuilds::Host), + ] + .into_iter() + .enumerate() + { + let nix = fake_nix( + &format!("nix-builds-{i}"), + &format!( + "for a in \"$@\"; do last=$a; done; case \"$last\" in *nixBuilds) {answer};; *) if [ \"$1\" = eval ]; then echo false; else echo /nix/store/x; fi;; esac\n" + ), + ); + let b = Builder { nix, ..builder() }; + let reply = b.build_from_meta(&meta_job(), "t").await; + assert!( + matches!(&reply, FromMetaReply::Built { nix_builds, .. } if *nix_builds == want), + "{answer}: {reply:?}" + ); + } + } + #[test] fn the_option_is_read_from_the_evaluated_config_not_the_toplevel() { assert_eq!( @@ -610,6 +685,7 @@ mod tests { system: "x86_64-linux".to_owned(), cached: false, private_users: false, + nix_builds: NixBuilds::Host, } ); } diff --git a/swarm-builder/src/push.rs b/swarm-builder/src/push.rs index 29becee3..a0d0c2dd 100644 --- a/swarm-builder/src/push.rs +++ b/swarm-builder/src/push.rs @@ -211,6 +211,7 @@ pub async fn cache_built( out_path, system, private_users, + nix_builds, .. }, ) = (cache, &reply) @@ -238,6 +239,7 @@ pub async fn cache_built( system: system.clone(), cached: true, private_users: *private_users, + nix_builds: *nix_builds, } } Err(reason) => { @@ -252,6 +254,8 @@ mod tests { use std::os::unix::fs::PermissionsExt as _; use std::sync::atomic::{AtomicBool, Ordering}; + use hive_types::NixBuilds; + use super::*; const TOKEN: &str = "eyJ0b2tlbiI6InNlY3JldCJ9.c2lnbmVk"; @@ -263,6 +267,7 @@ mod tests { system: "x86_64-linux".to_owned(), cached: false, private_users: false, + nix_builds: NixBuilds::Disabled, } } @@ -294,6 +299,17 @@ mod tests { let cache = cache_with("true"); let reply = cache_built(built(), Some(&cache), token(), PUSH_DEADLINE, "atlas").await; assert!(cached(&reply), "{reply:?}"); + // The rewritten reply keeps what the evaluated config said. + assert!( + matches!( + reply, + FromMetaReply::Built { + nix_builds: NixBuilds::Disabled, + .. + } + ), + "{reply:?}" + ); } #[tokio::test] diff --git a/swarm-controller/src/cache_miss.rs b/swarm-controller/src/cache_miss.rs index f28efef4..89191cf0 100644 --- a/swarm-controller/src/cache_miss.rs +++ b/swarm-controller/src/cache_miss.rs @@ -288,13 +288,14 @@ async fn rebuild(world: &impl World, agent: &str, stored: Stored) -> Outcome { return Outcome::Unavailable; } }; - let (out_path, system, cached, private_users) = match reply { + let (out_path, system, cached, private_users, nix_builds) = match reply { FromMetaReply::Built { out_path, system, cached, private_users, - } => (out_path, system, cached, private_users), + nix_builds, + } => (out_path, system, cached, private_users, nix_builds), FromMetaReply::Failed { error_tail } => { tracing::warn!(%agent, %meta_rev, %error_tail, "cache miss: the rebuild failed"); return Outcome::Failed; @@ -323,6 +324,7 @@ async fn rebuild(world: &impl World, agent: &str, stored: Stored) -> Outcome { built_unix: unix_now(), cached: true, private_users, + nix_builds, }; match replace_if_current(world, agent, &now).await { Ok(true) => Outcome::PathChanged, @@ -540,6 +542,8 @@ mod tests { use std::collections::BTreeSet; use std::sync::Mutex; + use hive_types::NixBuilds; + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader}; use tracing_subscriber::layer::SubscriberExt as _; @@ -561,6 +565,7 @@ mod tests { built_unix: 7, cached, private_users: false, + nix_builds: NixBuilds::Host, } } @@ -648,6 +653,7 @@ mod tests { system: "x86_64-linux".to_owned(), cached, private_users: false, + nix_builds: NixBuilds::Host, } } @@ -893,9 +899,9 @@ mod tests { } /// The record a rebuild at another path writes carries what the builder - /// answered about the user namespace, not what the old record held. + /// answered about the user namespace and nix builds, not what the old record held. #[tokio::test] - async fn the_replacement_record_carries_the_rebuilds_private_users_answer() { + async fn the_replacement_record_carries_the_rebuilds_user_namespace_and_nix_builds_answers() { let other = path("iris-other"); let FromMetaReply::Built { out_path, @@ -911,12 +917,14 @@ mod tests { system, cached, private_users: true, + nix_builds: NixBuilds::Disabled, }; let world = Fake::missing(&["iris"]).answering("iris", answer); pass_at(&world, &mut Backoff::default(), Instant::now()).await; let writes = world.writes(); assert_eq!(writes.len(), 1); assert!(writes[0].1.private_users); + assert_eq!(writes[0].1.nix_builds, NixBuilds::Disabled); } #[tokio::test] async fn another_path_leaves_a_record_that_moved_on() { diff --git a/swarm-controller/src/meta_build.rs b/swarm-controller/src/meta_build.rs index a6d09c08..cd3de123 100644 --- a/swarm-controller/src/meta_build.rs +++ b/swarm-controller/src/meta_build.rs @@ -371,6 +371,7 @@ pub(crate) async fn run( system, cached, private_users, + nix_builds, }) => { let built_rev = meta_rev.clone(); let closure = AgentClosure { @@ -380,6 +381,7 @@ pub(crate) async fn run( built_unix: unix_now(), cached, private_users, + nix_builds, }; match queue.record(&agent, &closure).await { Ok(()) => { @@ -398,6 +400,8 @@ pub(crate) async fn run( mod tests { use std::sync::{Arc, Mutex}; + use hive_types::NixBuilds; + use crate::builder::runs::tests::FakeDoc; use super::*; @@ -488,15 +492,16 @@ mod tests { } fn built_cached(cached: bool) -> Self { - Self::built_with(cached, false) + Self::built_with(cached, false, NixBuilds::Host) } - fn built_with(cached: bool, private_users: bool) -> Self { + fn built_with(cached: bool, private_users: bool, nix_builds: NixBuilds) -> Self { Self::answering(Ok(FromMetaReply::Built { out_path: "/nix/store/x-toplevel".to_owned(), system: "x86_64-linux".to_owned(), cached, private_users, + nix_builds, })) } } @@ -627,9 +632,9 @@ mod tests { /// The record says what the builder said about the agent's user namespace, /// both ways; a builder that said nothing leaves it off. #[tokio::test] - async fn the_record_carries_the_builders_private_users_answer() { - for private_users in [true, false] { - let queue = FakeQueue::built_with(true, private_users); + async fn the_record_carries_the_builders_user_namespace_and_nix_builds_answers() { + for (private_users, nix_builds) in [(true, NixBuilds::Disabled), (false, NixBuilds::Host)] { + let queue = FakeQueue::built_with(true, private_users, nix_builds); let outcome = run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await; assert_eq!(outcome, Outcome::Done); let recorded = queue.recorded.lock().unwrap(); @@ -637,6 +642,7 @@ mod tests { panic!("one record: {recorded:?}") }; assert_eq!(closure.private_users, private_users); + assert_eq!(closure.nix_builds, nix_builds); } } @@ -922,6 +928,7 @@ mod tests { built_unix: 1, cached: false, private_users: true, + nix_builds: NixBuilds::Disabled, }; client.record("atlas", &closure).await.expect("written"); closure.store_path = "/nix/store/b".to_owned(); diff --git a/swarm-wire/src/agent_closures.rs b/swarm-wire/src/agent_closures.rs index 0e86f66e..305b109a 100644 --- a/swarm-wire/src/agent_closures.rs +++ b/swarm-wire/src/agent_closures.rs @@ -9,6 +9,8 @@ //! Keyed by the agent alone, like [`crate::agent_limits`]: an agent can move //! between hives, and its closure goes with it. +use hive_types::NixBuilds; + /// The KV bucket closures are recorded in, keyed by agent name. pub const BUCKET: &str = "agent-closures"; @@ -42,6 +44,11 @@ pub struct AgentClosure { /// false, which keeps the agent out of one. #[serde(default)] pub private_users: bool, + /// The agent's `services.hyperhive.agent.nixBuilds`: where its nix builds + /// run. A record written before the field existed reads as + /// [`NixBuilds::Host`], which is what every agent did before it. + #[serde(default)] + pub nix_builds: NixBuilds, } #[cfg(test)] @@ -63,6 +70,7 @@ mod tests { built_unix: 7, cached: true, private_users: true, + nix_builds: NixBuilds::Disabled, }; let wire = serde_json::json!({ "meta_rev": "abc", @@ -71,6 +79,7 @@ mod tests { "built_unix": 7, "cached": true, "private_users": true, + "nix_builds": "none", }); assert_eq!(serde_json::to_value(&value).expect("encodes"), wire); assert_eq!( @@ -80,7 +89,7 @@ mod tests { } #[test] - fn a_record_without_the_fields_reads_as_not_cached_and_not_namespaced() { + fn a_record_without_the_fields_reads_as_not_cached_not_namespaced_and_host_builds() { let old = serde_json::json!({ "meta_rev": "abc", "store_path": "/nix/store/x", @@ -90,5 +99,6 @@ mod tests { let record = serde_json::from_value::(old).expect("decodes"); assert!(!record.cached); assert!(!record.private_users); + assert_eq!(record.nix_builds, NixBuilds::Host); } } diff --git a/swarm-wire/src/builder.rs b/swarm-wire/src/builder.rs index 649eaf0d..c9041697 100644 --- a/swarm-wire/src/builder.rs +++ b/swarm-wire/src/builder.rs @@ -7,6 +7,7 @@ use std::time::Duration; +use hive_types::NixBuilds; use serde::{Deserialize, Serialize}; /// The subject every build goes to. The controller is its one publisher and @@ -154,6 +155,11 @@ pub enum FromMetaReply { /// predates the field answers without it, which reads as false. #[serde(default)] private_users: bool, + /// The agent's `services.hyperhive.agent.nixBuilds`. A builder that + /// predates the field answers without it, which reads as + /// [`NixBuilds::Host`]. + #[serde(default)] + nix_builds: NixBuilds, }, /// The build did not succeed: the last lines `nix` printed, or why it /// stopped (it timed out, or could not start). The controller holds the @@ -268,6 +274,7 @@ mod tests { system: "x86_64-linux".to_owned(), cached: true, private_users: true, + nix_builds: NixBuilds::Disabled, }, json!({ "state": "built", @@ -275,6 +282,7 @@ mod tests { "system": "x86_64-linux", "cached": true, "private_users": true, + "nix_builds": "none", }), ), ( @@ -299,7 +307,8 @@ mod tests { } #[test] - fn a_built_reply_from_a_builder_without_the_fields_reads_as_not_cached_and_not_namespaced() { + fn a_built_reply_from_a_builder_without_the_fields_reads_as_not_cached_not_namespaced_and_host_builds() + { let old = json!({"state": "built", "out_path": "/nix/store/x", "system": "s"}); assert_eq!( serde_json::from_value::(old).expect("decodes"), @@ -308,6 +317,7 @@ mod tests { system: "s".to_owned(), cached: false, private_users: false, + nix_builds: NixBuilds::Host, } ); }