mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

add a per-agent nix builds mode, host or none, carried like privateUsers

Browse source
This commit is contained in:
atlas 2026-11-03 00:30:55 +01:00 • committed by mara
commit 5485151ba6
2 changed files with 618 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

146
hive-priv/src/container_config.rs
View file

@ -14,6 +14,7 @@ use anyhow::{Context as _, Result, bail};
use hive_priv_sock::{
AGENT_PREFIX, AGENT_RUNTIME_ROOT, BindMount, CredentialMount, HiveFacts, NetworkIsolation,
};
use hive_types::NixBuilds;
use tokio::process::Command;
use crate::shared::{StagedFile, open_dir, validate_credential_name, validate_name_chars};
@ -27,6 +28,7 @@ pub(crate) fn handle_write_nspawn_flags(
load_credentials: &[CredentialMount],
hive_facts: &HiveFacts,
private_users: Option<bool>,
nix_builds: Option<NixBuilds>,
) -> Result<(String, String)> {
let agent = validate_container_system_name(container)?;
for bind in binds {
@ -60,8 +62,11 @@ pub(crate) fn handle_write_nspawn_flags(
isolation,
load_credentials,
&hive_facts,
ephemeral,
userns,
StartMode {
ephemeral,
userns,
nix_builds,
},
)?;
Ok((String::new(), String::new()))
}
@ -79,6 +84,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking(
load_credentials: Vec<CredentialMount>,
hive_facts: HiveFacts,
private_users: Option<bool>,
nix_builds: Option<NixBuilds>,
) -> Result<(String, String)> {
tokio::task::spawn_blocking(move || {
handle_write_nspawn_flags(
@ -88,6 +94,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking(
&load_credentials,
&hive_facts,
private_users,
nix_builds,
)
})
.await
@ -358,6 +365,18 @@ fn git_overlay_flags(binds: &[BindMount]) -> Vec<String> {
.collect()
}
/// What a start of the container asks of nspawn besides its binds, as
/// [`write_nspawn_flags`] writes it.
#[derive(Clone, Copy)]
struct StartMode {
/// `--ephemeral`, once [`crate::rootfs`] has made the root fit for it.
ephemeral: bool,
/// The container runs in its own user namespace.
userns: bool,
/// `None` keeps the choice the conf's last write made.
nix_builds: Option<NixBuilds>,
}
/// Update `/etc/nixos-containers/<container>.conf`: strip old network vars
/// (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`),
/// write the current network-isolation settings, then append
@ -370,11 +389,15 @@ fn write_nspawn_flags(
isolation: &NetworkIsolation,
load_credentials: &[CredentialMount],
hive_facts: &str,
ephemeral: bool,
userns: bool,
mode: StartMode,
) -> Result<()> {
use std::fmt::Write as _;
use std::os::unix::fs::MetadataExt as _;
let StartMode {
ephemeral,
userns,
nix_builds,
} = mode;
let path = format!("/etc/nixos-containers/{container}.conf");
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
let lines: Vec<&str> = original
@ -406,7 +429,9 @@ fn write_nspawn_flags(
out.push_str("LOCAL_ADDRESS6=\n");
let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge);
}
let flags_joined = nspawn_flags(binds, load_credentials, ephemeral, userns).join(" ");
let nix_builds = chosen_nix_builds(nix_builds, &original);
let flags_joined =
nspawn_flags(binds, load_credentials, ephemeral, userns, nix_builds).join(" ");
let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
// Keeps the owner and mode `nixos-container` created the file with; this
// helper only rewrites its lines.
@ -457,6 +482,38 @@ fn private_users_line(userns: bool) -> &'static str {
if userns { "PRIVATE_USERS=pick\n" } else { "" }
}
/// An empty tmpfs over `/nix/var/nix/daemon-socket`, the directory nixos-containers
/// always binds the host's nix-daemon socket through, so the container finds no
/// socket to connect to. nspawn mounts its custom mounts sorted by destination
/// and then by kind, and a tmpfs sorts after a bind, so this lands on top of the
/// start script's bind whatever order the flags are in.
const NIX_DAEMON_SHADOW_FLAG: &str = "--tmpfs=/nix/var/nix/daemon-socket";
/// What the previous write of the conf chose: [`NixBuilds::Disabled`] when its
/// `EXTRA_NSPAWN_FLAGS` line carries [`NIX_DAEMON_SHADOW_FLAG`], else
/// [`NixBuilds::Host`], which is also what a conf never written gets.
fn previous_nix_builds(conf: &str) -> NixBuilds {
let shadowed = conf
.lines()
.filter_map(|line| line.trim_start().strip_prefix("EXTRA_NSPAWN_FLAGS="))
.any(|flags| {
flags
.split(|c: char| c.is_whitespace() || c == '"')
.any(|flag| flag == NIX_DAEMON_SHADOW_FLAG)
});
if shadowed {
NixBuilds::Disabled
} else {
NixBuilds::Host
}
}
/// The mode a write uses: the one asked for, else what the conf's last write
/// chose.
fn chosen_nix_builds(requested: Option<NixBuilds>, conf: &str) -> NixBuilds {
requested.unwrap_or_else(|| previous_nix_builds(conf))
}
/// The flags `EXTRA_NSPAWN_FLAGS` carries, in the order nspawn needs them.
///
/// `--ephemeral` runs the container on a snapshot of its root and drops the
@ -464,6 +521,8 @@ fn private_users_line(userns: bool) -> &'static str {
/// the flag is asked for. The git overlays and credentials follow the binds
/// they refer to.
///
/// `nix_builds` set to [`NixBuilds::Disabled`] adds [`NIX_DAEMON_SHADOW_FLAG`].
///
/// `userns` mounts every bind `owneridmap` and idmaps the root instead of
/// chowning it, so a different `pick` range never walks the tree.
fn nspawn_flags(
@ -471,6 +530,7 @@ fn nspawn_flags(
load_credentials: &[CredentialMount],
ephemeral: bool,
userns: bool,
nix_builds: NixBuilds,
) -> Vec<String> {
let mut flags: Vec<String> = ephemeral
.then(|| "--ephemeral".to_owned())
@ -485,6 +545,9 @@ fn nspawn_flags(
format!("{flag}={}:{}{idmap}", b.host_path, b.container_path)
}));
flags.extend(git_overlay_flags(binds));
if nix_builds == NixBuilds::Disabled {
flags.push(NIX_DAEMON_SHADOW_FLAG.to_owned());
}
// Credential forwarding: nspawn loads each host secret into the
// container's credential store under `<name>`; inner units inherit it
// via `LoadCredential=<name>`. Validated (name charset + bind-path
@ -653,8 +716,9 @@ pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &
#[cfg(test)]
mod tests {
use super::{
BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags,
hive_priv_exe, is_owned_var, limits_dropin_body, nspawn_flags, private_users_line,
BindMount, HiveFacts, NIX_DAEMON_SHADOW_FLAG, NetworkIsolation, NixBuilds,
chosen_nix_builds, git_overlay_flags, handle_write_nspawn_flags, hive_priv_exe,
is_owned_var, limits_dropin_body, nspawn_flags, previous_nix_builds, private_users_line,
publish_file, remove_service_dropin, render_hive_facts, rootfs_check_command,
validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits,
};
@ -1105,10 +1169,10 @@ mod tests {
#[test]
fn ephemeral_is_passed_only_when_asked() {
let binds = [bind("/agents/atlas/work")];
let with = nspawn_flags(&binds, &[], true, false);
let with = nspawn_flags(&binds, &[], true, false, NixBuilds::Host);
assert_eq!(with[0], "--ephemeral", "{with:?}");
assert!(with.iter().any(|f| f.starts_with("--bind-ro=")), "{with:?}");
let without = nspawn_flags(&binds, &[], false, false);
let without = nspawn_flags(&binds, &[], false, false, NixBuilds::Host);
assert!(!without.iter().any(|f| f == "--ephemeral"), "{without:?}");
assert_eq!(without.len() + 1, with.len());
}
@ -1119,7 +1183,7 @@ mod tests {
#[test]
fn binds_are_owneridmap_only_for_a_namespaced_agent() {
let binds = [bind("/knowledge"), bind("/agents/a/config")];
let plain = nspawn_flags(&binds, &[], false, false);
let plain = nspawn_flags(&binds, &[], false, false, NixBuilds::Host);
assert!(
plain.iter().any(|f| f.starts_with("--bind-ro=")),
"{plain:?}"
@ -1130,7 +1194,7 @@ mod tests {
.all(|f| !f.contains("idmap") && !f.contains("ownership")),
"{plain:?}"
);
let userns = nspawn_flags(&binds, &[], false, true);
let userns = nspawn_flags(&binds, &[], false, true, NixBuilds::Host);
assert!(
userns.contains(&"--private-users-ownership=map".to_owned()),
"{userns:?}"
@ -1143,6 +1207,63 @@ mod tests {
);
}
/// `Disabled` puts the tmpfs over the daemon socket directory after the
/// binds; `Host` leaves it out. The binds are the control that the lookup
/// finds what is there, and the flag is the whole of the difference.
#[test]
fn nix_builds_none_shadows_the_daemon_socket_dir() {
let binds = [bind("/agents/atlas/work")];
let host = nspawn_flags(&binds, &[], false, false, NixBuilds::Host);
let none = nspawn_flags(&binds, &[], false, false, NixBuilds::Disabled);
assert!(host.iter().any(|f| f.starts_with("--bind-ro=")), "{host:?}");
assert!(
!host.iter().any(|f| f == NIX_DAEMON_SHADOW_FLAG),
"{host:?}"
);
assert_eq!(
none.last().map(String::as_str),
Some(NIX_DAEMON_SHADOW_FLAG)
);
assert_eq!(none.len(), host.len() + 1);
assert_eq!(NIX_DAEMON_SHADOW_FLAG, "--tmpfs=/nix/var/nix/daemon-socket");
}
/// An unspecified mode keeps what the conf's last write chose, whatever
/// else the line carries; a conf with no such line is a host-builds agent.
#[test]
fn an_unspecified_nix_builds_mode_keeps_the_last_writes_choice() {
let binds = [bind("/knowledge")];
for want in [NixBuilds::Host, NixBuilds::Disabled] {
let flags = nspawn_flags(&binds, &[], true, true, want).join(" ");
let conf = format!("AUTO_START=1\nEXTRA_NSPAWN_FLAGS=\"{flags}\"\n");
assert_eq!(previous_nix_builds(&conf), want, "{conf}");
}
assert_eq!(previous_nix_builds(""), NixBuilds::Host);
// A mode asked for wins over the conf; none asked for keeps it.
let shadowed = format!("EXTRA_NSPAWN_FLAGS=\"{NIX_DAEMON_SHADOW_FLAG}\"\n");
assert_eq!(chosen_nix_builds(None, &shadowed), NixBuilds::Disabled);
assert_eq!(chosen_nix_builds(None, ""), NixBuilds::Host);
assert_eq!(
chosen_nix_builds(Some(NixBuilds::Host), &shadowed),
NixBuilds::Host
);
assert_eq!(
chosen_nix_builds(Some(NixBuilds::Disabled), ""),
NixBuilds::Disabled
);
// The flag has to be a flag of its own, on the flags line.
assert_eq!(
previous_nix_builds(&format!(
"EXTRA_NSPAWN_FLAGS=\"{NIX_DAEMON_SHADOW_FLAG}/x\"\n"
)),
NixBuilds::Host
);
assert_eq!(
previous_nix_builds(&format!("OTHER=\"{NIX_DAEMON_SHADOW_FLAG}\"\n")),
NixBuilds::Host
);
}
/// `PRIVATE_USERS=` is dropped from the conf whatever the agent now is, so
/// leaving the list cannot leave the line behind. `AUTO_START=` is the
/// control: a line that is not ours stays.
@ -1166,7 +1287,8 @@ mod tests {
let facts = HiveFacts::default();
for name in ["hive-forge", "hive-matrix"] {
let errs = [
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None).unwrap_err(),
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None, None)
.unwrap_err(),
write_resource_limits(name, "4G", "200%", None, None).unwrap_err(),
remove_service_dropin(name).unwrap_err(),
];

2
hive-priv/src/main.rs
View file

@ -414,6 +414,7 @@ async fn exec(
load_credentials,
hive_facts,
private_users,
nix_builds,
} => {
container_config::handle_write_nspawn_flags_blocking(
container,
@ -422,6 +423,7 @@ async fn exec(
load_credentials,
*hive_facts,
private_users,
nix_builds,
)
.await
}