mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

add a per-agent nix builds mode, host or none, carried like privateUsers

Browse source
This commit is contained in:
atlas 2026-11-03 00:30:55 +01:00 • committed by mara
commit 5485151ba6
3 changed files with 618 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

15
swarm-builder/src/main.rs
View file

@ -620,6 +620,7 @@ async fn handle_meta(
system,
cached,
private_users,
nix_builds,
} => tracing::info!(
agent = %job.agent,
meta_rev = %job.meta_rev,
@ -627,6 +628,7 @@ async fn handle_meta(
%system,
cached,
private_users,
?nix_builds,
"agent meta build built"
),
FromMetaReply::Failed { error_tail } => tracing::warn!(
@ -859,6 +861,7 @@ mod tests {
system: "s".to_owned(),
cached: false,
private_users: false,
nix_builds: hive_types::NixBuilds::Host,
}
},
)
@ -881,6 +884,7 @@ mod tests {
system: "s".to_owned(),
cached: false,
private_users: false,
nix_builds: hive_types::NixBuilds::Host,
}
},
)
@ -1462,11 +1466,12 @@ mod tests {
"system": "x86_64-linux",
"cached": false,
"private_users": true,
"nix_builds": "host",
})
);
assert_eq!(fake.lines("tokens"), ["forge-token"; 2]);
assert_eq!(fake.lines("tokens"), ["forge-token"; 3]);
let calls = fake.calls();
assert_eq!(calls.len(), 2, "{calls:?}");
assert_eq!(calls.len(), 3, "{calls:?}");
assert!(
calls[0].contains(&format!(
"git+https://forge.example/internal/meta.git?rev={REV}#nixosConfigurations.atlas.config.system.build.toplevel"
@ -1477,6 +1482,10 @@ mod tests {
calls[1].contains("services.hyperhive.agent.user.privateUsers"),
"{calls:?}"
);
assert!(
calls[2].contains("services.hyperhive.agent.nixBuilds"),
"{calls:?}"
);
}
#[tokio::test]
@ -1582,7 +1591,7 @@ mod tests {
];
serve_meta(fake.builder(), env, msgs).await;
assert_eq!(next(&mut sent).await.0, "_INBOX.2");
assert_eq!(fake.calls().len(), 2, "one build and its option read");
assert_eq!(fake.calls().len(), 3, "one build and its two option reads");
}
#[tokio::test(start_paused = true)]

100
swarm-builder/src/nix_build.rs
View file

@ -18,6 +18,7 @@ use std::path::PathBuf;
use std::process::Stdio;
use std::time::Duration;
use hive_types::NixBuilds;
use swarm_wire::builder::{
BuildAgent, BuildFromMeta, FromMetaReply, LockMeta, LockReply, META_REPO_PATH, Reply,
};
@ -93,6 +94,7 @@ impl Builder {
system: self.system.clone(),
cached: false,
private_users: self.private_users(job, token).await,
nix_builds: self.nix_builds(job, token).await,
},
Reply::Failed { error_tail } => FromMetaReply::Failed { error_tail },
},
@ -182,28 +184,61 @@ impl Builder {
)
}
/// Whether the agent's config sets `services.hyperhive.agent.user.privateUsers`,
/// read by `nix eval` of the flake the build just evaluated, in this
/// sandbox. Anything but a clean `true` is false: an agent whose module
/// predates the option, a failed eval and a timeout all keep the agent out
/// of a user namespace.
async fn private_users(&self, job: &BuildFromMeta, token: &str) -> bool {
/// `nix eval --json` of `option` in the agent's evaluated config, run in this
/// sandbox against the flake the build just evaluated. `None` when the eval
/// fails or times out, and logs why along with what the caller falls back to.
async fn eval_option(
&self,
job: &BuildFromMeta,
token: &str,
option: &str,
fallback: &str,
) -> Option<Vec<u8>> {
let mut cmd = self.nix_command(token);
cmd.args(EVAL_FLAGS)
.arg(format!("{}.{PRIVATE_USERS_OPTION}", self.meta_config(job)));
.arg(format!("{}.{option}", self.meta_config(job)));
match run_raw(cmd, self.deadline).await {
Ok((status, out, _)) if status.success() => parse_flag(&out),
Ok((status, out, _)) if status.success() => Some(out),
Ok((status, _, err)) => {
tracing::info!(agent = %job.agent, %status, error = %tail(&String::from_utf8_lossy(&err), status), "privateUsers not read; the agent stays out of a user namespace");
false
tracing::info!(agent = %job.agent, %status, error = %tail(&String::from_utf8_lossy(&err), status), "{option} not read; {fallback}");
None
}
Err(why) => {
tracing::warn!(agent = %job.agent, %why, "privateUsers not read; the agent stays out of a user namespace");
false
tracing::warn!(agent = %job.agent, %why, "{option} not read; {fallback}");
None
}
}
}
/// Whether the agent's config sets `services.hyperhive.agent.user.privateUsers`.
/// Anything but a clean `true` is false: an agent whose module predates the
/// option, a failed eval and a timeout all keep the agent out of a user
/// namespace.
async fn private_users(&self, job: &BuildFromMeta, token: &str) -> bool {
self.eval_option(
job,
token,
PRIVATE_USERS_OPTION,
"the agent stays out of a user namespace",
)
.await
.is_some_and(|out| parse_flag(&out))
}
/// The agent's `services.hyperhive.agent.nixBuilds`. An agent whose module
/// predates the option, a failed eval and a timeout all read as
/// [`NixBuilds::Host`], which is what every agent did before the option.
async fn nix_builds(&self, job: &BuildFromMeta, token: &str) -> NixBuilds {
self.eval_option(
job,
token,
NIX_BUILDS_OPTION,
"the agent builds through the host daemon",
)
.await
.map_or_else(NixBuilds::default, |out| parse_nix_builds(&out))
}
/// The config commit as a flake reference, fetched through the PR's head
/// ref.
fn agent_url(&self, job: &BuildAgent) -> String {
@ -242,11 +277,20 @@ const EVAL_FLAGS: [&str; 7] = [
/// The agent option the controller records, relative to the evaluated config.
const PRIVATE_USERS_OPTION: &str = "services.hyperhive.agent.user.privateUsers";
/// The agent option holding where its nix builds run.
const NIX_BUILDS_OPTION: &str = "services.hyperhive.agent.nixBuilds";
/// `nix eval --json` of a bool: true only for exactly `true`.
fn parse_flag(out: &[u8]) -> bool {
out.trim_ascii() == b"true"
}
/// `nix eval --json` of a [`NixBuilds`]: the spelling, or the default when the
/// output is not a string.
fn parse_nix_builds(out: &[u8]) -> NixBuilds {
serde_json::from_slice(out).unwrap_or_default()
}
/// `nix flake lock` of the flake in the working directory. The flake's config
/// is not trusted, and nothing is built.
const LOCK_ARGS: &[&str] = &["flake", "lock", "--option", "accept-flake-config", "false"];
@ -560,6 +604,37 @@ mod tests {
}
}
/// The reply carries the option's spelling. `host` is the control for
/// `none`; a mode this build does not know reads as `none`, and anything
/// that is no spelling at all (a failed eval, an older module) as `host`.
#[tokio::test]
async fn a_meta_build_reports_the_agents_nix_builds_option() {
for (i, (answer, want)) in [
("echo '\"host\"'", NixBuilds::Host),
("echo '\"none\"'", NixBuilds::Disabled),
("echo '\"container\"'", NixBuilds::Disabled),
("echo null", NixBuilds::Host),
("echo '\"none\"'; exit 1", NixBuilds::Host),
("echo error: attribute missing >&2; exit 1", NixBuilds::Host),
]
.into_iter()
.enumerate()
{
let nix = fake_nix(
&format!("nix-builds-{i}"),
&format!(
"for a in \"$@\"; do last=$a; done; case \"$last\" in *nixBuilds) {answer};; *) if [ \"$1\" = eval ]; then echo false; else echo /nix/store/x; fi;; esac\n"
),
);
let b = Builder { nix, ..builder() };
let reply = b.build_from_meta(&meta_job(), "t").await;
assert!(
matches!(&reply, FromMetaReply::Built { nix_builds, .. } if *nix_builds == want),
"{answer}: {reply:?}"
);
}
}
#[test]
fn the_option_is_read_from_the_evaluated_config_not_the_toplevel() {
assert_eq!(
@ -610,6 +685,7 @@ mod tests {
system: "x86_64-linux".to_owned(),
cached: false,
private_users: false,
nix_builds: NixBuilds::Host,
}
);
}

16
swarm-builder/src/push.rs
View file

@ -211,6 +211,7 @@ pub async fn cache_built(
out_path,
system,
private_users,
nix_builds,
..
},
) = (cache, &reply)
@ -238,6 +239,7 @@ pub async fn cache_built(
system: system.clone(),
cached: true,
private_users: *private_users,
nix_builds: *nix_builds,
}
}
Err(reason) => {
@ -252,6 +254,8 @@ mod tests {
use std::os::unix::fs::PermissionsExt as _;
use std::sync::atomic::{AtomicBool, Ordering};
use hive_types::NixBuilds;
use super::*;
const TOKEN: &str = "eyJ0b2tlbiI6InNlY3JldCJ9.c2lnbmVk";
@ -263,6 +267,7 @@ mod tests {
system: "x86_64-linux".to_owned(),
cached: false,
private_users: false,
nix_builds: NixBuilds::Disabled,
}
}
@ -294,6 +299,17 @@ mod tests {
let cache = cache_with("true");
let reply = cache_built(built(), Some(&cache), token(), PUSH_DEADLINE, "atlas").await;
assert!(cached(&reply), "{reply:?}");
// The rewritten reply keeps what the evaluated config said.
assert!(
matches!(
reply,
FromMetaReply::Built {
nix_builds: NixBuilds::Disabled,
..
}
),
"{reply:?}"
);
}
#[tokio::test]