Watch
0
0
Fork
You've already forked hyperhive
0

read privateUsers from the closure record at start, and move the data owner option under hive-controller

This commit is contained in:
atlas 2026-11-01 16:18:34 +01:00 • committed by mara
commit f5a609911c
10 changed files with 208 additions and 97 deletions

View file

@ -451,6 +451,10 @@ async fn set_nspawn_flags(
}); });
} }
// Whether the agent's config sets `privateUsers`, as the swarm builder
// recorded it. Read from the record, never from the agent's nix.
let private_users = private_users_of(&crate::attic::closure::record(agent_name).await);
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the // Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
// container so the harness can bind `web.sock` there and the host-side // container so the harness can bind `web.sock` there and the host-side
// gateway sees it. Subdir bind (not socket file) keeps the inode // gateway sees it. Subdir bind (not socket file) keeps the inode
@ -460,7 +464,7 @@ async fn set_nspawn_flags(
// and leaves an existing one alone; the container's activation hands it // and leaves an existing one alone; the container's activation hands it
// to the agent user. Don't chown or chmod it here. // to the agent user. Don't chown or chmod it here.
let socket_dir = crate::agent_sockets::agent_dir_for(agent_name); let socket_dir = crate::agent_sockets::agent_dir_for(agent_name);
crate::priv_client::ensure_agent_socket_dir(agent_name).await?; crate::priv_client::ensure_agent_socket_dir(agent_name, private_users).await?;
binds.push(BindMount { binds.push(BindMount {
host_path: socket_dir.to_string_lossy().into_owned(), host_path: socket_dir.to_string_lossy().into_owned(),
container_path: socket_dir.to_string_lossy().into_owned(), container_path: socket_dir.to_string_lossy().into_owned(),
@ -486,10 +490,26 @@ async fn set_nspawn_flags(
isolation, isolation,
&load_creds, &load_creds,
super::hive_facts(hive), super::hive_facts(hive),
private_users,
) )
.await .await
} }
/// What hive-priv is told about the agent's user namespace: the record's flag,
/// or `None` (leave the agent as it is) when there is no record or it cannot be
/// read, so a swarm outage never turns a namespaced agent off or on.
fn private_users_of(
record: &Result<Option<swarm_wire::agent_closures::AgentClosure>>,
) -> Option<bool> {
match record {
Ok(record) => record.as_ref().map(|r| r.private_users),
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "closure record unreadable; the agent's user namespace is left as it is");
None
}
}
}
/// The binds every agent container gets, before its own state, config and /// The binds every agent container gets, before its own state, config and
/// role-specific dirs. /// role-specific dirs.
fn base_binds(runtime_dir: &Path, claude_dir: &Path, claude_mount: String) -> Vec<BindMount> { fn base_binds(runtime_dir: &Path, claude_dir: &Path, claude_mount: String) -> Vec<BindMount> {
@ -521,6 +541,40 @@ mod tests {
}; };
use crate::coordinator::HiveEnv; use crate::coordinator::HiveEnv;
fn record_with(private_users: bool) -> swarm_wire::agent_closures::AgentClosure {
swarm_wire::agent_closures::AgentClosure {
meta_rev: "a".repeat(40),
store_path: "/nix/store/x".to_owned(),
system: "x86_64-linux".to_owned(),
built_unix: 1,
cached: true,
private_users,
}
}
/// The flag in the record is what hive-priv is told, both ways. No record
/// and an unreadable one say nothing, so a swarm outage changes no agent's
/// namespace; a record from before the field reads false.
#[test]
fn the_records_flag_is_what_the_start_path_passes() {
assert_eq!(
super::private_users_of(&Ok(Some(record_with(true)))),
Some(true)
);
assert_eq!(
super::private_users_of(&Ok(Some(record_with(false)))),
Some(false)
);
assert_eq!(super::private_users_of(&Ok(None)), None);
assert_eq!(super::private_users_of(&Err(anyhow::anyhow!("down"))), None);
let old = serde_json::json!({
"meta_rev": "a", "store_path": "/nix/store/x",
"system": "s", "built_unix": 1,
});
let old = serde_json::from_value(old).expect("decodes");
assert_eq!(super::private_users_of(&Ok(Some(old))), Some(false));
}
/// No agent container gets a cross-agent scratch dir; `/knowledge` is the /// No agent container gets a cross-agent scratch dir; `/knowledge` is the
/// control that the lookup finds a bind that is there. /// control that the lookup finds a bind that is there.
#[test] #[test]

View file

@ -306,21 +306,24 @@ pub async fn write_nspawn_flags(
isolation: NetworkIsolation, isolation: NetworkIsolation,
load_credentials: &[CredentialMount], load_credentials: &[CredentialMount],
hive_facts: HiveFacts, hive_facts: HiveFacts,
private_users: Option<bool>,
) -> Result<()> { ) -> Result<()> {
ok(call(&PrivRequest::WriteNspawnFlags { ok(call(&PrivRequest::WriteNspawnFlags {
container: container.to_owned(), container: container.to_owned(),
binds: binds.to_vec(), binds: binds.to_vec(),
isolation, isolation,
load_credentials: load_credentials.to_vec(), load_credentials: load_credentials.to_vec(),
hive_facts, hive_facts: Box::new(hive_facts),
private_users,
}) })
.await?) .await?)
} }
/// See [`PrivRequest::EnsureAgentSocketDir`]. /// See [`PrivRequest::EnsureAgentSocketDir`].
pub async fn ensure_agent_socket_dir(name: &str) -> Result<()> { pub async fn ensure_agent_socket_dir(name: &str, private_users: Option<bool>) -> Result<()> {
ok(call(&PrivRequest::EnsureAgentSocketDir { ok(call(&PrivRequest::EnsureAgentSocketDir {
name: name.to_owned(), name: name.to_owned(),
private_users,
}) })
.await?) .await?)
} }

View file

@ -335,17 +335,31 @@ pub enum PrivRequest {
/// Written to the container's `/etc/hyperhive-hive-facts` on every /// Written to the container's `/etc/hyperhive-hive-facts` on every
/// call, so a fact the hive no longer has is removed, not kept. /// call, so a fact the hive no longer has is removed, not kept.
#[serde(default)] #[serde(default)]
hive_facts: HiveFacts, hive_facts: Box<HiveFacts>,
/// Whether the agent's evaluated config sets
/// `services.hyperhive.agent.user.privateUsers`, as the swarm builder
/// recorded it. `Some(true)` starts the container in its own user
/// namespace; `Some(false)` does not, and hive-priv refuses an agent
/// whose data it already moved. `None`, which is what a hive-c0re
/// without the field or without a readable record sends, keeps the
/// agent as it is: namespaced if hive-priv moved its data, else not.
#[serde(default)]
private_users: Option<bool>,
}, },
/// Create the agent's socket dir `/run/hive-agent/<name>` as `0751 /// Create the agent's socket dir `/run/hive-agent/<name>` as `0751
/// root:root` if it is missing; an existing directory is left as it is. /// root:root` if it is missing; an existing directory is left as it is.
/// It is a bind source, so it has to exist before every container start, /// It is a bind source, so it has to exist before every container start,
/// and `/run` is a tmpfs. The container's own activation hands it to the /// and `/run` is a tmpfs. The container's own activation hands it to the
/// agent user. Called from `lifecycle::set_nspawn_flags`. /// agent user, or, for a namespaced agent, by this request. Called from
/// `lifecycle::set_nspawn_flags`.
EnsureAgentSocketDir { EnsureAgentSocketDir {
/// Logical agent name (validated by `validate_agent_name`). /// Logical agent name (validated by `validate_agent_name`).
name: String, name: String,
/// As for `WriteNspawnFlags`: a namespaced agent's dir goes to the
/// data owner.
#[serde(default)]
private_users: Option<bool>,
}, },
/// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf` /// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`
@ -701,6 +715,32 @@ pub enum PrivEvent {
mod tests { mod tests {
use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev}; use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev};
/// A request from a hive-c0re that predates the field decodes with the
/// agent's namespace left to hive-priv, and the field round-trips.
#[test]
fn private_users_is_optional_on_the_wire() {
let old = serde_json::json!({"op": "ensure_agent_socket_dir", "name": "iris"});
let PrivRequest::EnsureAgentSocketDir {
private_users: None,
..
} = serde_json::from_value(old).expect("decodes")
else {
panic!("no field reads as None");
};
for want in [Some(true), Some(false)] {
let wire = serde_json::to_value(PrivRequest::EnsureAgentSocketDir {
name: "iris".to_owned(),
private_users: want,
})
.expect("encodes");
let PrivRequest::EnsureAgentSocketDir { private_users, .. } =
serde_json::from_value(wire).expect("decodes")
else {
panic!("variant");
};
assert_eq!(private_users, want);
}
}
#[test] #[test]
fn infra_control_names_no_c0re_or_swarm_service() { fn infra_control_names_no_c0re_or_swarm_service() {
// Neither the daemon nor the swarm's forge and matrix have a // Neither the daemon nor the swarm's forge and matrix have a

View file

@ -26,6 +26,7 @@ pub(crate) fn handle_write_nspawn_flags(
isolation: &NetworkIsolation, isolation: &NetworkIsolation,
load_credentials: &[CredentialMount], load_credentials: &[CredentialMount],
hive_facts: &HiveFacts, hive_facts: &HiveFacts,
private_users: Option<bool>,
) -> Result<(String, String)> { ) -> Result<(String, String)> {
let agent = validate_container_system_name(container)?; let agent = validate_container_system_name(container)?;
for bind in binds { for bind in binds {
@ -52,7 +53,7 @@ pub(crate) fn handle_write_nspawn_flags(
}); });
// After the salvage, so the mount points land in the root the start will // After the salvage, so the mount points land in the root the start will
// snapshot. // snapshot.
let userns = crate::userns::activate(container, agent, binds)?; let userns = crate::userns::activate(container, agent, binds, private_users)?;
write_nspawn_flags( write_nspawn_flags(
container, container,
binds, binds,
@ -77,6 +78,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking(
isolation: NetworkIsolation, isolation: NetworkIsolation,
load_credentials: Vec<CredentialMount>, load_credentials: Vec<CredentialMount>,
hive_facts: HiveFacts, hive_facts: HiveFacts,
private_users: Option<bool>,
) -> Result<(String, String)> { ) -> Result<(String, String)> {
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
handle_write_nspawn_flags( handle_write_nspawn_flags(
@ -85,6 +87,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking(
&isolation, &isolation,
&load_credentials, &load_credentials,
&hive_facts, &hive_facts,
private_users,
) )
}) })
.await .await
@ -1163,7 +1166,7 @@ mod tests {
let facts = HiveFacts::default(); let facts = HiveFacts::default();
for name in ["hive-forge", "hive-matrix"] { for name in ["hive-forge", "hive-matrix"] {
let errs = [ let errs = [
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts).unwrap_err(), handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None).unwrap_err(),
write_resource_limits(name, "4G", "200%", None, None).unwrap_err(), write_resource_limits(name, "4G", "200%", None, None).unwrap_err(),
remove_service_dropin(name).unwrap_err(), remove_service_dropin(name).unwrap_err(),
]; ];

View file

@ -153,11 +153,14 @@ pub(crate) fn remove_legacy_tmpfiles() -> Result<()> {
} }
/// `EnsureAgentSocketDir` — create `/run/hive-agent/<name>` if missing. /// `EnsureAgentSocketDir` — create `/run/hive-agent/<name>` if missing.
pub(crate) fn ensure_agent_socket_dir(name: &str) -> Result<(String, String)> { pub(crate) fn ensure_agent_socket_dir(
name: &str,
private_users: Option<bool>,
) -> Result<(String, String)> {
ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?; ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?;
// A namespaced agent's activation cannot chown the dir through the id // A namespaced agent's activation cannot chown the dir through the id
// mapping, so the data owner, which is the agent user inside, gets it here. // mapping, so the data owner, which is the agent user inside, gets it here.
if let Some(owner) = crate::userns::owner_for_agent(name)? { if let Some(owner) = crate::userns::owner_for_agent(name, private_users)? {
let path = Path::new(SOCKET_DIR_ROOT).join(name); let path = Path::new(SOCKET_DIR_ROOT).join(name);
std::os::unix::fs::lchown(&path, Some(owner), Some(owner)) std::os::unix::fs::lchown(&path, Some(owner), Some(owner))
.with_context(|| format!("chown {}", path.display()))?; .with_context(|| format!("chown {}", path.display()))?;

View file

@ -391,18 +391,23 @@ async fn exec(
isolation, isolation,
load_credentials, load_credentials,
hive_facts, hive_facts,
private_users,
} => { } => {
container_config::handle_write_nspawn_flags_blocking( container_config::handle_write_nspawn_flags_blocking(
container, container,
binds, binds,
isolation, isolation,
load_credentials, load_credentials,
hive_facts, *hive_facts,
private_users,
) )
.await .await
} }
PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name), PrivRequest::EnsureAgentSocketDir {
ref name,
private_users,
} => files::ensure_agent_socket_dir(name, private_users),
PrivRequest::WriteResourceLimits { PrivRequest::WriteResourceLimits {
ref container, ref container,

View file

@ -1,11 +1,12 @@
//! Agents that run in their own user namespace. //! Agents that run in their own user namespace.
//! //!
//! The operator lists them in `services.hyperhive.userns.agents` and names the //! An agent's config sets `services.hyperhive.agent.user.privateUsers`; the
//! one host uid:gid that owns their data on disk (`dataOwner`); the unit's //! swarm builder records it, hive-c0re reads the record and sends the flag with
//! environment carries both, so only root decides who is namespaced. A listed //! each start. The host's data owner (`dataOwner`, one uid:gid) comes from this
//! agent starts with `--private-users=pick`, a root idmapped rather than //! unit's environment. A namespaced agent starts with `--private-users=pick`, a
//! chowned, and every bind mounted `owneridmap`: the owner of the bind's //! root idmapped rather than chowned, and every bind mounted `owneridmap`: the
//! target inside the container, the agent user, is the data owner on disk. //! owner of the bind's target inside the container, the agent user, is the data
//! owner on disk.
//! //!
//! Getting there moves the agent's data once. [`activate`] chowns its `state`, //! Getting there moves the agent's data once. [`activate`] chowns its `state`,
//! `harness`, `work` and `claude` dirs to the data owner, never touching //! `harness`, `work` and `claude` dirs to the data owner, never touching
@ -28,7 +29,6 @@ use rustix::io::Errno;
use crate::files::SOCKET_DIR_ROOT; use crate::files::SOCKET_DIR_ROOT;
const AGENTS_ENV: &str = "HIVE_PRIV_USERNS_AGENTS";
const DATA_OWNER_ENV: &str = "HIVE_PRIV_USERNS_DATA_OWNER"; const DATA_OWNER_ENV: &str = "HIVE_PRIV_USERNS_DATA_OWNER";
/// One `<owner> <uid> <gid>` file per agent: the data owner its data was moved /// One `<owner> <uid> <gid>` file per agent: the data owner its data was moved
@ -49,7 +49,7 @@ pub(crate) struct Env<'a> {
pub(crate) socket_dir: &'a Path, pub(crate) socket_dir: &'a Path,
pub(crate) marker_root: &'a Path, pub(crate) marker_root: &'a Path,
pub(crate) rootfs: &'a Path, pub(crate) rootfs: &'a Path,
/// The data owner when the agent is listed, else `None`. /// The data owner when the agent is to be namespaced, else `None`.
pub(crate) owner: Option<u32>, pub(crate) owner: Option<u32>,
pub(crate) stopped: bool, pub(crate) stopped: bool,
/// Lowest uid taken for an agent's own: below it a dir belongs to root or /// Lowest uid taken for an agent's own: below it a dir belongs to root or
@ -78,22 +78,26 @@ fn chown_at(dir: BorrowedFd<'_>, name: &CStr, uid: u32, gid: u32) -> std::io::Re
)?) )?)
} }
/// The data owner for `agent` under the unit's environment: `None` when the /// The data owner when `agent` is to run in a user namespace, else `None`.
/// agent is not listed. ///
/// `requested` is what hive-c0re sent from the closure record. `None` there
/// means it could not say, and the agent stays as it is: namespaced if
/// hive-priv already `moved` its data.
/// ///
/// # Errors /// # Errors
/// The agent is listed and the data owner is missing or not a plain uid. /// The agent is to be namespaced and the data owner is missing or not a plain
/// uid.
pub(crate) fn owner_for( pub(crate) fn owner_for(
agents: Option<&str>, requested: Option<bool>,
moved: bool,
owner: Option<&str>, owner: Option<&str>,
agent: &str, agent: &str,
) -> Result<Option<u32>> { ) -> Result<Option<u32>> {
if !agents.is_some_and(|list| list.split_whitespace().any(|a| a == agent)) { if !requested.unwrap_or(moved) {
return Ok(None); return Ok(None);
} }
let owner = owner.with_context(|| { let owner = owner
format!("{agent} is listed in {AGENTS_ENV}, but {DATA_OWNER_ENV} is not set") .with_context(|| format!("{agent} sets privateUsers, but {DATA_OWNER_ENV} is not set"))?;
})?;
let owner = owner let owner = owner
.trim() .trim()
.parse() .parse()
@ -101,30 +105,37 @@ pub(crate) fn owner_for(
Ok(Some(owner)) Ok(Some(owner))
} }
/// [`owner_for`] under the unit's environment. /// [`owner_for`] under the unit's environment and the agent's marker.
/// ///
/// # Errors /// # Errors
/// As [`owner_for`]. /// As [`owner_for`], or `agent` is not an agent name.
pub(crate) fn owner_for_agent(agent: &str) -> Result<Option<u32>> { pub(crate) fn owner_for_agent(agent: &str, requested: Option<bool>) -> Result<Option<u32>> {
crate::shared::validate_agent_name(agent)?;
owner_for( owner_for(
std::env::var(AGENTS_ENV).ok().as_deref(), requested,
Path::new(MARKER_ROOT).join(agent).exists(),
std::env::var(DATA_OWNER_ENV).ok().as_deref(), std::env::var(DATA_OWNER_ENV).ok().as_deref(),
agent, agent,
) )
} }
/// Make `agent`'s next start of `container` namespaced when it is listed, and /// Make `agent`'s next start of `container` namespaced when its config asks
/// say whether it is. `binds` are the ones the start will mount. /// for it, and say whether it is. `binds` are the ones the start will mount.
/// ///
/// A container that is running, and whose data has not been moved yet, keeps /// A container that is running, and whose data has not been moved yet, keeps
/// its current start: the move waits for a stopped container. /// its current start: the move waits for a stopped container.
/// ///
/// # Errors /// # Errors
/// The environment is inconsistent, the data owner changed since the data /// The environment is inconsistent, the data owner changed since the data
/// moved, the agent was dropped from the list after it moved, or a chown or /// moved, the agent stopped asking for a namespace after it moved, or a chown
/// mkdir fails. /// or mkdir fails.
pub(crate) fn activate(container: &str, agent: &str, binds: &[BindMount]) -> Result<bool> { pub(crate) fn activate(
let owner = owner_for_agent(agent)?; container: &str,
agent: &str,
binds: &[BindMount],
requested: Option<bool>,
) -> Result<bool> {
let owner = owner_for_agent(agent, requested)?;
let rootfs = crate::container_config::container_rootfs(container); let rootfs = crate::container_config::container_rootfs(container);
let stopped = crate::rootfs::unit_is_stopped(container)?; let stopped = crate::rootfs::unit_is_stopped(container)?;
activate_in( activate_in(
@ -153,8 +164,8 @@ pub(crate) fn activate_in(env: &Env<'_>, agent: &str, binds: &[BindMount]) -> Re
let Some(owner) = env.owner else { let Some(owner) = env.owner else {
if moved.is_some() { if moved.is_some() {
bail!( bail!(
"{agent}: its data was moved to the userns data owner and it is no longer \ "{agent}: its data was moved to the userns data owner and its config no longer \
listed; list it again, or chown its dirs back and remove {}", sets privateUsers; set it again, or chown its dirs back and remove {}",
marker.display() marker.display()
); );
} }
@ -470,14 +481,25 @@ mod tests {
} }
} }
/// The flag decides; with no word from the record the marker does. The
/// refusal arms and the `None` arms are each other's controls.
#[test] #[test]
fn only_listed_agents_have_an_owner() { fn the_requested_flag_decides_and_without_it_the_marker_does() {
let owner = |agents, owner| owner_for(agents, owner, "a"); let owner = |requested, moved, env| owner_for(requested, moved, env, "a");
assert_eq!(owner(Some("b a"), Some("60000")).unwrap(), Some(60000)); assert_eq!(
assert_eq!(owner(Some("b c"), Some("60000")).unwrap(), None); owner(Some(true), false, Some("60000")).unwrap(),
assert_eq!(owner(None, None).unwrap(), None); Some(60000)
assert!(owner(Some("a"), None).is_err(), "listed without an owner"); );
assert!(owner(Some("a"), Some("root")).is_err(), "not a uid"); assert_eq!(owner(Some(false), false, Some("60000")).unwrap(), None);
assert_eq!(owner(Some(false), true, Some("60000")).unwrap(), None);
assert_eq!(owner(None, true, Some("60000")).unwrap(), Some(60000));
assert_eq!(owner(None, false, Some("60000")).unwrap(), None);
assert!(owner(Some(true), false, None).is_err(), "no data owner");
assert!(owner(Some(true), false, Some("root")).is_err(), "not a uid");
assert!(
owner(Some(false), false, None).is_ok(),
"not asked, not needed"
);
} }
/// The move chowns the four data dirs and what is in them and nothing /// The move chowns the four data dirs and what is in them and nothing
@ -552,7 +574,7 @@ mod tests {
} }
#[test] #[test]
fn an_unlisted_agent_is_left_alone_and_a_moved_one_is_refused() { fn an_agent_without_the_flag_is_left_alone_and_a_moved_one_is_refused() {
let root = tree(); let root = tree();
let (answer, seen) = run(&root, None, true, "a", &[]); let (answer, seen) = run(&root, None, true, "a", &[]);
assert!(!answer.unwrap() && seen.is_empty()); assert!(!answer.unwrap() && seen.is_empty());

View file

@ -10,7 +10,7 @@
let let
controllerCfg = config.services.hyperhive.deploy.hive-controller; controllerCfg = config.services.hyperhive.deploy.hive-controller;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
userns = config.services.hyperhive.userns; userns = controllerCfg.userns;
dataOwnerProblems = import ./lib/userns-data-owner.nix { inherit lib; }; dataOwnerProblems = import ./lib/userns-data-owner.nix { inherit lib; };
in in
{ {
@ -26,41 +26,20 @@ in
} }
]; ];
options.services.hyperhive.userns = { options.services.hyperhive.deploy.hive-controller.userns.dataOwner = lib.mkOption {
agents = lib.mkOption { type = lib.types.ints.u32;
type = lib.types.listOf lib.types.str; default = 60000;
default = [ ]; description = ''
example = [ "alice" ]; The host uid and gid that own the data of an agent running in its own
description = '' user namespace (the agent sets `services.hyperhive.agent.user.privateUsers`).
Agents that run in their own user namespace (`--private-users=pick`, One number for both: nspawn copies the uid map for gids, so the gid
root idmapped). Every hive bind into such a container is mounted cannot differ. It must be no host user or group, outside the normal and
`owneridmap`, so its data on disk belongs to `dataOwner`, which the system users (below 30000) and the dynamic users (61184 and above), and
agent user is inside. Listing an agent moves its `state`, `harness`, below the range `pick` allocates from (524288 and above), so no
`work` and `claude` dirs to `dataOwner` once, at its next start with container's range covers it. `hive-priv` chowns such an agent's data to
the container stopped. Take the agent off the list only after chowning it once. Changing it after an agent has moved is refused at that agent's
those dirs back and removing `/var/lib/hive-priv/userns/<agent>`; next start.
until then it will not start. '';
The agent's own config must pin `services.hyperhive.agent.user.uid`
and `.gid` to one number: nspawn maps a bind's gids with its uid map.
Keep an agent that holds `ManageRootAgent` off the list, since the
other agents' dirs it mounts are not owned by its data owner.
'';
};
dataOwner = lib.mkOption {
type = lib.types.ints.u32;
default = 60000;
description = ''
The host uid and gid that own a namespaced agent's data on disk. One
number for both: nspawn copies the uid map for gids, so the gid
cannot differ. It must be no host user or group, outside the normal and
system users (below 30000) and the dynamic users (61184 and above), and
below the range `pick` allocates from (524288 and above), so no
container's range covers it. Changing it after an agent has moved is
refused at that agent's next start.
'';
};
}; };
config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable { config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
@ -79,8 +58,8 @@ in
}; };
in in
{ {
assertion = userns.agents == [ ] || problems == [ ]; assertion = problems == [ ];
message = "services.hyperhive.userns.dataOwner: ${lib.concatStringsSep "; " problems}"; message = "services.hyperhive.deploy.hive-controller.userns.dataOwner: ${lib.concatStringsSep "; " problems}";
} }
) )
]; ];
@ -148,9 +127,8 @@ in
# routing so the closure `RealiseClosure` downloads is in the same # routing so the closure `RealiseClosure` downloads is in the same
# store the container is then pointed at. # store the container is then pointed at.
NIX_REMOTE = "daemon"; NIX_REMOTE = "daemon";
# Who is namespaced, and the owner their data moves to: set by the # The owner a namespaced agent's data moves to. Which agents are
# operator in the unit, so only root decides. # namespaced is not set here: hive-c0re sends it with each start.
HIVE_PRIV_USERNS_AGENTS = lib.concatStringsSep " " userns.agents;
HIVE_PRIV_USERNS_DATA_OWNER = toString userns.dataOwner; HIVE_PRIV_USERNS_DATA_OWNER = toString userns.dataOwner;
}; };
serviceConfig = { serviceConfig = {

View file

@ -178,15 +178,15 @@ let
{ {
# Read back from the rendered unit and assertion list: a misspelt # Read back from the rendered unit and assertion list: a misspelt
# variable or a dropped assertion fails here. # variable or a dropped assertion fails here.
name = "hive-priv carries the userns agents and data owner, and the owner assertion passes"; name = "hive-priv carries the userns data owner, no agent list, and the owner assertion passes";
ok = ok =
let let
e = allLocal.systemd.services.hive-priv.environment; e = allLocal.systemd.services.hive-priv.environment;
in in
e.HIVE_PRIV_USERNS_AGENTS == "" !(e ? HIVE_PRIV_USERNS_AGENTS)
&& e.HIVE_PRIV_USERNS_DATA_OWNER == "60000" && e.HIVE_PRIV_USERNS_DATA_OWNER == "60000"
&& lib.any ( && lib.any (
a: lib.hasPrefix "services.hyperhive.userns.dataOwner" a.message && a.assertion a: lib.hasInfix "hive-controller.userns.dataOwner" a.message && a.assertion
) allLocal.assertions; ) allLocal.assertions;
} }
{ {

View file

@ -18,10 +18,12 @@
# leaves the link's target untouched, and the start goes through once the # leaves the link's target untouched, and the start goes through once the
# link is gone. # link is gone.
# #
# - an agent listed in `services.hyperhive.userns.agents` starts in its own # - an agent whose config sets `services.hyperhive.agent.user.privateUsers`
# (hive-c0re sends the flag from the closure record; the test sends it)
# starts in its own
# user namespace with state it already had, owned by uid 1000 on disk: the # user namespace with state it already had, owned by uid 1000 on disk: the
# data moves once to `dataOwner`, the agent user reads and writes it as # data moves once to `dataOwner`, the agent user reads and writes it as
# itself, the unlisted agent's conf is untouched, and a second # itself, the other agent's conf (no flag sent) is untouched, and a second
# `WriteNspawnFlags` moves nothing. # `WriteNspawnFlags` moves nothing.
# #
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is # The agent's journal forwarder itself (an OTLP push to the hive's collector) is
@ -118,7 +120,6 @@ pkgs.testers.runNixOSTest {
prefixLength = 24; prefixLength = 24;
} }
]; ];
services.hyperhive.userns.agents = [ nsAgent ];
systemd.tmpfiles.rules = systemd.tmpfiles.rules =
map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs
# What the namespaced agent has from before: dirs its user owned. # What the namespaced agent has from before: dirs its user owned.
@ -154,6 +155,7 @@ pkgs.testers.runNixOSTest {
name = nsAgent; name = nsAgent;
uid = 1000; uid = 1000;
gid = 1000; gid = 1000;
privateUsers = true;
}; };
}; };
}; };
@ -312,19 +314,20 @@ pkgs.testers.runNixOSTest {
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root") host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
boot() boot()
with subtest("a listed agent starts in its own user namespace with the state it had"): with subtest("an agent whose config sets privateUsers starts in its own user namespace with the state it had"):
NS = "${nsMachine}" NS = "${nsMachine}"
NS_AGENT = "${nsAgent}" NS_AGENT = "${nsAgent}"
NS_DIR = "${nsDir}" NS_DIR = "${nsDir}"
NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)}) NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt") host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000") host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT}) priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT, "private_users": True})
write = { write = {
"op": "write_nspawn_flags", "op": "write_nspawn_flags",
"container": NS, "container": NS,
"binds": NS_BINDS, "binds": NS_BINDS,
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"}, "isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
"private_users": True,
} }
priv_request(host, write) priv_request(host, write)
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf") ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
@ -332,7 +335,7 @@ pkgs.testers.runNixOSTest {
assert "--private-users-ownership=map" in ns_conf, ns_conf assert "--private-users-ownership=map" in ns_conf, ns_conf
# Every bind, the read-only config included, is mounted owneridmap. # Every bind, the read-only config included, is mounted owneridmap.
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
# The unlisted agent's conf has none of it: the control that the lookups # The other agent's conf has none of it: the control that the lookups
# above find these strings where they are. # above find these strings where they are.
assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf() assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf()