read privateUsers from the closure record at start, and move the data owner option under hive-controller
This commit is contained in:
parent
1fd591d28b
commit
f5a609911c
10 changed files with 208 additions and 97 deletions
|
|
@ -451,6 +451,10 @@ async fn set_nspawn_flags(
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Whether the agent's config sets `privateUsers`, as the swarm builder
|
||||||
|
// recorded it. Read from the record, never from the agent's nix.
|
||||||
|
let private_users = private_users_of(&crate::attic::closure::record(agent_name).await);
|
||||||
|
|
||||||
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
|
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
|
||||||
// container so the harness can bind `web.sock` there and the host-side
|
// container so the harness can bind `web.sock` there and the host-side
|
||||||
// gateway sees it. Subdir bind (not socket file) keeps the inode
|
// gateway sees it. Subdir bind (not socket file) keeps the inode
|
||||||
|
|
@ -460,7 +464,7 @@ async fn set_nspawn_flags(
|
||||||
// and leaves an existing one alone; the container's activation hands it
|
// and leaves an existing one alone; the container's activation hands it
|
||||||
// to the agent user. Don't chown or chmod it here.
|
// to the agent user. Don't chown or chmod it here.
|
||||||
let socket_dir = crate::agent_sockets::agent_dir_for(agent_name);
|
let socket_dir = crate::agent_sockets::agent_dir_for(agent_name);
|
||||||
crate::priv_client::ensure_agent_socket_dir(agent_name).await?;
|
crate::priv_client::ensure_agent_socket_dir(agent_name, private_users).await?;
|
||||||
binds.push(BindMount {
|
binds.push(BindMount {
|
||||||
host_path: socket_dir.to_string_lossy().into_owned(),
|
host_path: socket_dir.to_string_lossy().into_owned(),
|
||||||
container_path: socket_dir.to_string_lossy().into_owned(),
|
container_path: socket_dir.to_string_lossy().into_owned(),
|
||||||
|
|
@ -486,10 +490,26 @@ async fn set_nspawn_flags(
|
||||||
isolation,
|
isolation,
|
||||||
&load_creds,
|
&load_creds,
|
||||||
super::hive_facts(hive),
|
super::hive_facts(hive),
|
||||||
|
private_users,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What hive-priv is told about the agent's user namespace: the record's flag,
|
||||||
|
/// or `None` (leave the agent as it is) when there is no record or it cannot be
|
||||||
|
/// read, so a swarm outage never turns a namespaced agent off or on.
|
||||||
|
fn private_users_of(
|
||||||
|
record: &Result<Option<swarm_wire::agent_closures::AgentClosure>>,
|
||||||
|
) -> Option<bool> {
|
||||||
|
match record {
|
||||||
|
Ok(record) => record.as_ref().map(|r| r.private_users),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(error = %format!("{e:#}"), "closure record unreadable; the agent's user namespace is left as it is");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The binds every agent container gets, before its own state, config and
|
/// The binds every agent container gets, before its own state, config and
|
||||||
/// role-specific dirs.
|
/// role-specific dirs.
|
||||||
fn base_binds(runtime_dir: &Path, claude_dir: &Path, claude_mount: String) -> Vec<BindMount> {
|
fn base_binds(runtime_dir: &Path, claude_dir: &Path, claude_mount: String) -> Vec<BindMount> {
|
||||||
|
|
@ -521,6 +541,40 @@ mod tests {
|
||||||
};
|
};
|
||||||
use crate::coordinator::HiveEnv;
|
use crate::coordinator::HiveEnv;
|
||||||
|
|
||||||
|
fn record_with(private_users: bool) -> swarm_wire::agent_closures::AgentClosure {
|
||||||
|
swarm_wire::agent_closures::AgentClosure {
|
||||||
|
meta_rev: "a".repeat(40),
|
||||||
|
store_path: "/nix/store/x".to_owned(),
|
||||||
|
system: "x86_64-linux".to_owned(),
|
||||||
|
built_unix: 1,
|
||||||
|
cached: true,
|
||||||
|
private_users,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The flag in the record is what hive-priv is told, both ways. No record
|
||||||
|
/// and an unreadable one say nothing, so a swarm outage changes no agent's
|
||||||
|
/// namespace; a record from before the field reads false.
|
||||||
|
#[test]
|
||||||
|
fn the_records_flag_is_what_the_start_path_passes() {
|
||||||
|
assert_eq!(
|
||||||
|
super::private_users_of(&Ok(Some(record_with(true)))),
|
||||||
|
Some(true)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
super::private_users_of(&Ok(Some(record_with(false)))),
|
||||||
|
Some(false)
|
||||||
|
);
|
||||||
|
assert_eq!(super::private_users_of(&Ok(None)), None);
|
||||||
|
assert_eq!(super::private_users_of(&Err(anyhow::anyhow!("down"))), None);
|
||||||
|
let old = serde_json::json!({
|
||||||
|
"meta_rev": "a", "store_path": "/nix/store/x",
|
||||||
|
"system": "s", "built_unix": 1,
|
||||||
|
});
|
||||||
|
let old = serde_json::from_value(old).expect("decodes");
|
||||||
|
assert_eq!(super::private_users_of(&Ok(Some(old))), Some(false));
|
||||||
|
}
|
||||||
|
|
||||||
/// No agent container gets a cross-agent scratch dir; `/knowledge` is the
|
/// No agent container gets a cross-agent scratch dir; `/knowledge` is the
|
||||||
/// control that the lookup finds a bind that is there.
|
/// control that the lookup finds a bind that is there.
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -306,21 +306,24 @@ pub async fn write_nspawn_flags(
|
||||||
isolation: NetworkIsolation,
|
isolation: NetworkIsolation,
|
||||||
load_credentials: &[CredentialMount],
|
load_credentials: &[CredentialMount],
|
||||||
hive_facts: HiveFacts,
|
hive_facts: HiveFacts,
|
||||||
|
private_users: Option<bool>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
ok(call(&PrivRequest::WriteNspawnFlags {
|
ok(call(&PrivRequest::WriteNspawnFlags {
|
||||||
container: container.to_owned(),
|
container: container.to_owned(),
|
||||||
binds: binds.to_vec(),
|
binds: binds.to_vec(),
|
||||||
isolation,
|
isolation,
|
||||||
load_credentials: load_credentials.to_vec(),
|
load_credentials: load_credentials.to_vec(),
|
||||||
hive_facts,
|
hive_facts: Box::new(hive_facts),
|
||||||
|
private_users,
|
||||||
})
|
})
|
||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// See [`PrivRequest::EnsureAgentSocketDir`].
|
/// See [`PrivRequest::EnsureAgentSocketDir`].
|
||||||
pub async fn ensure_agent_socket_dir(name: &str) -> Result<()> {
|
pub async fn ensure_agent_socket_dir(name: &str, private_users: Option<bool>) -> Result<()> {
|
||||||
ok(call(&PrivRequest::EnsureAgentSocketDir {
|
ok(call(&PrivRequest::EnsureAgentSocketDir {
|
||||||
name: name.to_owned(),
|
name: name.to_owned(),
|
||||||
|
private_users,
|
||||||
})
|
})
|
||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -335,17 +335,31 @@ pub enum PrivRequest {
|
||||||
/// Written to the container's `/etc/hyperhive-hive-facts` on every
|
/// Written to the container's `/etc/hyperhive-hive-facts` on every
|
||||||
/// call, so a fact the hive no longer has is removed, not kept.
|
/// call, so a fact the hive no longer has is removed, not kept.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
hive_facts: HiveFacts,
|
hive_facts: Box<HiveFacts>,
|
||||||
|
/// Whether the agent's evaluated config sets
|
||||||
|
/// `services.hyperhive.agent.user.privateUsers`, as the swarm builder
|
||||||
|
/// recorded it. `Some(true)` starts the container in its own user
|
||||||
|
/// namespace; `Some(false)` does not, and hive-priv refuses an agent
|
||||||
|
/// whose data it already moved. `None`, which is what a hive-c0re
|
||||||
|
/// without the field or without a readable record sends, keeps the
|
||||||
|
/// agent as it is: namespaced if hive-priv moved its data, else not.
|
||||||
|
#[serde(default)]
|
||||||
|
private_users: Option<bool>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Create the agent's socket dir `/run/hive-agent/<name>` as `0751
|
/// Create the agent's socket dir `/run/hive-agent/<name>` as `0751
|
||||||
/// root:root` if it is missing; an existing directory is left as it is.
|
/// root:root` if it is missing; an existing directory is left as it is.
|
||||||
/// It is a bind source, so it has to exist before every container start,
|
/// It is a bind source, so it has to exist before every container start,
|
||||||
/// and `/run` is a tmpfs. The container's own activation hands it to the
|
/// and `/run` is a tmpfs. The container's own activation hands it to the
|
||||||
/// agent user. Called from `lifecycle::set_nspawn_flags`.
|
/// agent user, or, for a namespaced agent, by this request. Called from
|
||||||
|
/// `lifecycle::set_nspawn_flags`.
|
||||||
EnsureAgentSocketDir {
|
EnsureAgentSocketDir {
|
||||||
/// Logical agent name (validated by `validate_agent_name`).
|
/// Logical agent name (validated by `validate_agent_name`).
|
||||||
name: String,
|
name: String,
|
||||||
|
/// As for `WriteNspawnFlags`: a namespaced agent's dir goes to the
|
||||||
|
/// data owner.
|
||||||
|
#[serde(default)]
|
||||||
|
private_users: Option<bool>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`
|
/// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`
|
||||||
|
|
@ -701,6 +715,32 @@ pub enum PrivEvent {
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev};
|
use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev};
|
||||||
|
|
||||||
|
/// A request from a hive-c0re that predates the field decodes with the
|
||||||
|
/// agent's namespace left to hive-priv, and the field round-trips.
|
||||||
|
#[test]
|
||||||
|
fn private_users_is_optional_on_the_wire() {
|
||||||
|
let old = serde_json::json!({"op": "ensure_agent_socket_dir", "name": "iris"});
|
||||||
|
let PrivRequest::EnsureAgentSocketDir {
|
||||||
|
private_users: None,
|
||||||
|
..
|
||||||
|
} = serde_json::from_value(old).expect("decodes")
|
||||||
|
else {
|
||||||
|
panic!("no field reads as None");
|
||||||
|
};
|
||||||
|
for want in [Some(true), Some(false)] {
|
||||||
|
let wire = serde_json::to_value(PrivRequest::EnsureAgentSocketDir {
|
||||||
|
name: "iris".to_owned(),
|
||||||
|
private_users: want,
|
||||||
|
})
|
||||||
|
.expect("encodes");
|
||||||
|
let PrivRequest::EnsureAgentSocketDir { private_users, .. } =
|
||||||
|
serde_json::from_value(wire).expect("decodes")
|
||||||
|
else {
|
||||||
|
panic!("variant");
|
||||||
|
};
|
||||||
|
assert_eq!(private_users, want);
|
||||||
|
}
|
||||||
|
}
|
||||||
#[test]
|
#[test]
|
||||||
fn infra_control_names_no_c0re_or_swarm_service() {
|
fn infra_control_names_no_c0re_or_swarm_service() {
|
||||||
// Neither the daemon nor the swarm's forge and matrix have a
|
// Neither the daemon nor the swarm's forge and matrix have a
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ pub(crate) fn handle_write_nspawn_flags(
|
||||||
isolation: &NetworkIsolation,
|
isolation: &NetworkIsolation,
|
||||||
load_credentials: &[CredentialMount],
|
load_credentials: &[CredentialMount],
|
||||||
hive_facts: &HiveFacts,
|
hive_facts: &HiveFacts,
|
||||||
|
private_users: Option<bool>,
|
||||||
) -> Result<(String, String)> {
|
) -> Result<(String, String)> {
|
||||||
let agent = validate_container_system_name(container)?;
|
let agent = validate_container_system_name(container)?;
|
||||||
for bind in binds {
|
for bind in binds {
|
||||||
|
|
@ -52,7 +53,7 @@ pub(crate) fn handle_write_nspawn_flags(
|
||||||
});
|
});
|
||||||
// After the salvage, so the mount points land in the root the start will
|
// After the salvage, so the mount points land in the root the start will
|
||||||
// snapshot.
|
// snapshot.
|
||||||
let userns = crate::userns::activate(container, agent, binds)?;
|
let userns = crate::userns::activate(container, agent, binds, private_users)?;
|
||||||
write_nspawn_flags(
|
write_nspawn_flags(
|
||||||
container,
|
container,
|
||||||
binds,
|
binds,
|
||||||
|
|
@ -77,6 +78,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking(
|
||||||
isolation: NetworkIsolation,
|
isolation: NetworkIsolation,
|
||||||
load_credentials: Vec<CredentialMount>,
|
load_credentials: Vec<CredentialMount>,
|
||||||
hive_facts: HiveFacts,
|
hive_facts: HiveFacts,
|
||||||
|
private_users: Option<bool>,
|
||||||
) -> Result<(String, String)> {
|
) -> Result<(String, String)> {
|
||||||
tokio::task::spawn_blocking(move || {
|
tokio::task::spawn_blocking(move || {
|
||||||
handle_write_nspawn_flags(
|
handle_write_nspawn_flags(
|
||||||
|
|
@ -85,6 +87,7 @@ pub(crate) async fn handle_write_nspawn_flags_blocking(
|
||||||
&isolation,
|
&isolation,
|
||||||
&load_credentials,
|
&load_credentials,
|
||||||
&hive_facts,
|
&hive_facts,
|
||||||
|
private_users,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
|
|
@ -1163,7 +1166,7 @@ mod tests {
|
||||||
let facts = HiveFacts::default();
|
let facts = HiveFacts::default();
|
||||||
for name in ["hive-forge", "hive-matrix"] {
|
for name in ["hive-forge", "hive-matrix"] {
|
||||||
let errs = [
|
let errs = [
|
||||||
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts).unwrap_err(),
|
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None).unwrap_err(),
|
||||||
write_resource_limits(name, "4G", "200%", None, None).unwrap_err(),
|
write_resource_limits(name, "4G", "200%", None, None).unwrap_err(),
|
||||||
remove_service_dropin(name).unwrap_err(),
|
remove_service_dropin(name).unwrap_err(),
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -153,11 +153,14 @@ pub(crate) fn remove_legacy_tmpfiles() -> Result<()> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `EnsureAgentSocketDir` — create `/run/hive-agent/<name>` if missing.
|
/// `EnsureAgentSocketDir` — create `/run/hive-agent/<name>` if missing.
|
||||||
pub(crate) fn ensure_agent_socket_dir(name: &str) -> Result<(String, String)> {
|
pub(crate) fn ensure_agent_socket_dir(
|
||||||
|
name: &str,
|
||||||
|
private_users: Option<bool>,
|
||||||
|
) -> Result<(String, String)> {
|
||||||
ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?;
|
ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?;
|
||||||
// A namespaced agent's activation cannot chown the dir through the id
|
// A namespaced agent's activation cannot chown the dir through the id
|
||||||
// mapping, so the data owner, which is the agent user inside, gets it here.
|
// mapping, so the data owner, which is the agent user inside, gets it here.
|
||||||
if let Some(owner) = crate::userns::owner_for_agent(name)? {
|
if let Some(owner) = crate::userns::owner_for_agent(name, private_users)? {
|
||||||
let path = Path::new(SOCKET_DIR_ROOT).join(name);
|
let path = Path::new(SOCKET_DIR_ROOT).join(name);
|
||||||
std::os::unix::fs::lchown(&path, Some(owner), Some(owner))
|
std::os::unix::fs::lchown(&path, Some(owner), Some(owner))
|
||||||
.with_context(|| format!("chown {}", path.display()))?;
|
.with_context(|| format!("chown {}", path.display()))?;
|
||||||
|
|
|
||||||
|
|
@ -391,18 +391,23 @@ async fn exec(
|
||||||
isolation,
|
isolation,
|
||||||
load_credentials,
|
load_credentials,
|
||||||
hive_facts,
|
hive_facts,
|
||||||
|
private_users,
|
||||||
} => {
|
} => {
|
||||||
container_config::handle_write_nspawn_flags_blocking(
|
container_config::handle_write_nspawn_flags_blocking(
|
||||||
container,
|
container,
|
||||||
binds,
|
binds,
|
||||||
isolation,
|
isolation,
|
||||||
load_credentials,
|
load_credentials,
|
||||||
hive_facts,
|
*hive_facts,
|
||||||
|
private_users,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name),
|
PrivRequest::EnsureAgentSocketDir {
|
||||||
|
ref name,
|
||||||
|
private_users,
|
||||||
|
} => files::ensure_agent_socket_dir(name, private_users),
|
||||||
|
|
||||||
PrivRequest::WriteResourceLimits {
|
PrivRequest::WriteResourceLimits {
|
||||||
ref container,
|
ref container,
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,12 @@
|
||||||
//! Agents that run in their own user namespace.
|
//! Agents that run in their own user namespace.
|
||||||
//!
|
//!
|
||||||
//! The operator lists them in `services.hyperhive.userns.agents` and names the
|
//! An agent's config sets `services.hyperhive.agent.user.privateUsers`; the
|
||||||
//! one host uid:gid that owns their data on disk (`dataOwner`); the unit's
|
//! swarm builder records it, hive-c0re reads the record and sends the flag with
|
||||||
//! environment carries both, so only root decides who is namespaced. A listed
|
//! each start. The host's data owner (`dataOwner`, one uid:gid) comes from this
|
||||||
//! agent starts with `--private-users=pick`, a root idmapped rather than
|
//! unit's environment. A namespaced agent starts with `--private-users=pick`, a
|
||||||
//! chowned, and every bind mounted `owneridmap`: the owner of the bind's
|
//! root idmapped rather than chowned, and every bind mounted `owneridmap`: the
|
||||||
//! target inside the container, the agent user, is the data owner on disk.
|
//! owner of the bind's target inside the container, the agent user, is the data
|
||||||
|
//! owner on disk.
|
||||||
//!
|
//!
|
||||||
//! Getting there moves the agent's data once. [`activate`] chowns its `state`,
|
//! Getting there moves the agent's data once. [`activate`] chowns its `state`,
|
||||||
//! `harness`, `work` and `claude` dirs to the data owner, never touching
|
//! `harness`, `work` and `claude` dirs to the data owner, never touching
|
||||||
|
|
@ -28,7 +29,6 @@ use rustix::io::Errno;
|
||||||
|
|
||||||
use crate::files::SOCKET_DIR_ROOT;
|
use crate::files::SOCKET_DIR_ROOT;
|
||||||
|
|
||||||
const AGENTS_ENV: &str = "HIVE_PRIV_USERNS_AGENTS";
|
|
||||||
const DATA_OWNER_ENV: &str = "HIVE_PRIV_USERNS_DATA_OWNER";
|
const DATA_OWNER_ENV: &str = "HIVE_PRIV_USERNS_DATA_OWNER";
|
||||||
|
|
||||||
/// One `<owner> <uid> <gid>` file per agent: the data owner its data was moved
|
/// One `<owner> <uid> <gid>` file per agent: the data owner its data was moved
|
||||||
|
|
@ -49,7 +49,7 @@ pub(crate) struct Env<'a> {
|
||||||
pub(crate) socket_dir: &'a Path,
|
pub(crate) socket_dir: &'a Path,
|
||||||
pub(crate) marker_root: &'a Path,
|
pub(crate) marker_root: &'a Path,
|
||||||
pub(crate) rootfs: &'a Path,
|
pub(crate) rootfs: &'a Path,
|
||||||
/// The data owner when the agent is listed, else `None`.
|
/// The data owner when the agent is to be namespaced, else `None`.
|
||||||
pub(crate) owner: Option<u32>,
|
pub(crate) owner: Option<u32>,
|
||||||
pub(crate) stopped: bool,
|
pub(crate) stopped: bool,
|
||||||
/// Lowest uid taken for an agent's own: below it a dir belongs to root or
|
/// Lowest uid taken for an agent's own: below it a dir belongs to root or
|
||||||
|
|
@ -78,22 +78,26 @@ fn chown_at(dir: BorrowedFd<'_>, name: &CStr, uid: u32, gid: u32) -> std::io::Re
|
||||||
)?)
|
)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The data owner for `agent` under the unit's environment: `None` when the
|
/// The data owner when `agent` is to run in a user namespace, else `None`.
|
||||||
/// agent is not listed.
|
///
|
||||||
|
/// `requested` is what hive-c0re sent from the closure record. `None` there
|
||||||
|
/// means it could not say, and the agent stays as it is: namespaced if
|
||||||
|
/// hive-priv already `moved` its data.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// The agent is listed and the data owner is missing or not a plain uid.
|
/// The agent is to be namespaced and the data owner is missing or not a plain
|
||||||
|
/// uid.
|
||||||
pub(crate) fn owner_for(
|
pub(crate) fn owner_for(
|
||||||
agents: Option<&str>,
|
requested: Option<bool>,
|
||||||
|
moved: bool,
|
||||||
owner: Option<&str>,
|
owner: Option<&str>,
|
||||||
agent: &str,
|
agent: &str,
|
||||||
) -> Result<Option<u32>> {
|
) -> Result<Option<u32>> {
|
||||||
if !agents.is_some_and(|list| list.split_whitespace().any(|a| a == agent)) {
|
if !requested.unwrap_or(moved) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let owner = owner.with_context(|| {
|
let owner = owner
|
||||||
format!("{agent} is listed in {AGENTS_ENV}, but {DATA_OWNER_ENV} is not set")
|
.with_context(|| format!("{agent} sets privateUsers, but {DATA_OWNER_ENV} is not set"))?;
|
||||||
})?;
|
|
||||||
let owner = owner
|
let owner = owner
|
||||||
.trim()
|
.trim()
|
||||||
.parse()
|
.parse()
|
||||||
|
|
@ -101,30 +105,37 @@ pub(crate) fn owner_for(
|
||||||
Ok(Some(owner))
|
Ok(Some(owner))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// [`owner_for`] under the unit's environment.
|
/// [`owner_for`] under the unit's environment and the agent's marker.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// As [`owner_for`].
|
/// As [`owner_for`], or `agent` is not an agent name.
|
||||||
pub(crate) fn owner_for_agent(agent: &str) -> Result<Option<u32>> {
|
pub(crate) fn owner_for_agent(agent: &str, requested: Option<bool>) -> Result<Option<u32>> {
|
||||||
|
crate::shared::validate_agent_name(agent)?;
|
||||||
owner_for(
|
owner_for(
|
||||||
std::env::var(AGENTS_ENV).ok().as_deref(),
|
requested,
|
||||||
|
Path::new(MARKER_ROOT).join(agent).exists(),
|
||||||
std::env::var(DATA_OWNER_ENV).ok().as_deref(),
|
std::env::var(DATA_OWNER_ENV).ok().as_deref(),
|
||||||
agent,
|
agent,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Make `agent`'s next start of `container` namespaced when it is listed, and
|
/// Make `agent`'s next start of `container` namespaced when its config asks
|
||||||
/// say whether it is. `binds` are the ones the start will mount.
|
/// for it, and say whether it is. `binds` are the ones the start will mount.
|
||||||
///
|
///
|
||||||
/// A container that is running, and whose data has not been moved yet, keeps
|
/// A container that is running, and whose data has not been moved yet, keeps
|
||||||
/// its current start: the move waits for a stopped container.
|
/// its current start: the move waits for a stopped container.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// The environment is inconsistent, the data owner changed since the data
|
/// The environment is inconsistent, the data owner changed since the data
|
||||||
/// moved, the agent was dropped from the list after it moved, or a chown or
|
/// moved, the agent stopped asking for a namespace after it moved, or a chown
|
||||||
/// mkdir fails.
|
/// or mkdir fails.
|
||||||
pub(crate) fn activate(container: &str, agent: &str, binds: &[BindMount]) -> Result<bool> {
|
pub(crate) fn activate(
|
||||||
let owner = owner_for_agent(agent)?;
|
container: &str,
|
||||||
|
agent: &str,
|
||||||
|
binds: &[BindMount],
|
||||||
|
requested: Option<bool>,
|
||||||
|
) -> Result<bool> {
|
||||||
|
let owner = owner_for_agent(agent, requested)?;
|
||||||
let rootfs = crate::container_config::container_rootfs(container);
|
let rootfs = crate::container_config::container_rootfs(container);
|
||||||
let stopped = crate::rootfs::unit_is_stopped(container)?;
|
let stopped = crate::rootfs::unit_is_stopped(container)?;
|
||||||
activate_in(
|
activate_in(
|
||||||
|
|
@ -153,8 +164,8 @@ pub(crate) fn activate_in(env: &Env<'_>, agent: &str, binds: &[BindMount]) -> Re
|
||||||
let Some(owner) = env.owner else {
|
let Some(owner) = env.owner else {
|
||||||
if moved.is_some() {
|
if moved.is_some() {
|
||||||
bail!(
|
bail!(
|
||||||
"{agent}: its data was moved to the userns data owner and it is no longer \
|
"{agent}: its data was moved to the userns data owner and its config no longer \
|
||||||
listed; list it again, or chown its dirs back and remove {}",
|
sets privateUsers; set it again, or chown its dirs back and remove {}",
|
||||||
marker.display()
|
marker.display()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -470,14 +481,25 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The flag decides; with no word from the record the marker does. The
|
||||||
|
/// refusal arms and the `None` arms are each other's controls.
|
||||||
#[test]
|
#[test]
|
||||||
fn only_listed_agents_have_an_owner() {
|
fn the_requested_flag_decides_and_without_it_the_marker_does() {
|
||||||
let owner = |agents, owner| owner_for(agents, owner, "a");
|
let owner = |requested, moved, env| owner_for(requested, moved, env, "a");
|
||||||
assert_eq!(owner(Some("b a"), Some("60000")).unwrap(), Some(60000));
|
assert_eq!(
|
||||||
assert_eq!(owner(Some("b c"), Some("60000")).unwrap(), None);
|
owner(Some(true), false, Some("60000")).unwrap(),
|
||||||
assert_eq!(owner(None, None).unwrap(), None);
|
Some(60000)
|
||||||
assert!(owner(Some("a"), None).is_err(), "listed without an owner");
|
);
|
||||||
assert!(owner(Some("a"), Some("root")).is_err(), "not a uid");
|
assert_eq!(owner(Some(false), false, Some("60000")).unwrap(), None);
|
||||||
|
assert_eq!(owner(Some(false), true, Some("60000")).unwrap(), None);
|
||||||
|
assert_eq!(owner(None, true, Some("60000")).unwrap(), Some(60000));
|
||||||
|
assert_eq!(owner(None, false, Some("60000")).unwrap(), None);
|
||||||
|
assert!(owner(Some(true), false, None).is_err(), "no data owner");
|
||||||
|
assert!(owner(Some(true), false, Some("root")).is_err(), "not a uid");
|
||||||
|
assert!(
|
||||||
|
owner(Some(false), false, None).is_ok(),
|
||||||
|
"not asked, not needed"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The move chowns the four data dirs and what is in them and nothing
|
/// The move chowns the four data dirs and what is in them and nothing
|
||||||
|
|
@ -552,7 +574,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_unlisted_agent_is_left_alone_and_a_moved_one_is_refused() {
|
fn an_agent_without_the_flag_is_left_alone_and_a_moved_one_is_refused() {
|
||||||
let root = tree();
|
let root = tree();
|
||||||
let (answer, seen) = run(&root, None, true, "a", &[]);
|
let (answer, seen) = run(&root, None, true, "a", &[]);
|
||||||
assert!(!answer.unwrap() && seen.is_empty());
|
assert!(!answer.unwrap() && seen.is_empty());
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
let
|
let
|
||||||
controllerCfg = config.services.hyperhive.deploy.hive-controller;
|
controllerCfg = config.services.hyperhive.deploy.hive-controller;
|
||||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||||
userns = config.services.hyperhive.userns;
|
userns = controllerCfg.userns;
|
||||||
dataOwnerProblems = import ./lib/userns-data-owner.nix { inherit lib; };
|
dataOwnerProblems = import ./lib/userns-data-owner.nix { inherit lib; };
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
|
|
@ -26,41 +26,20 @@ in
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
options.services.hyperhive.userns = {
|
options.services.hyperhive.deploy.hive-controller.userns.dataOwner = lib.mkOption {
|
||||||
agents = lib.mkOption {
|
type = lib.types.ints.u32;
|
||||||
type = lib.types.listOf lib.types.str;
|
default = 60000;
|
||||||
default = [ ];
|
description = ''
|
||||||
example = [ "alice" ];
|
The host uid and gid that own the data of an agent running in its own
|
||||||
description = ''
|
user namespace (the agent sets `services.hyperhive.agent.user.privateUsers`).
|
||||||
Agents that run in their own user namespace (`--private-users=pick`,
|
One number for both: nspawn copies the uid map for gids, so the gid
|
||||||
root idmapped). Every hive bind into such a container is mounted
|
cannot differ. It must be no host user or group, outside the normal and
|
||||||
`owneridmap`, so its data on disk belongs to `dataOwner`, which the
|
system users (below 30000) and the dynamic users (61184 and above), and
|
||||||
agent user is inside. Listing an agent moves its `state`, `harness`,
|
below the range `pick` allocates from (524288 and above), so no
|
||||||
`work` and `claude` dirs to `dataOwner` once, at its next start with
|
container's range covers it. `hive-priv` chowns such an agent's data to
|
||||||
the container stopped. Take the agent off the list only after chowning
|
it once. Changing it after an agent has moved is refused at that agent's
|
||||||
those dirs back and removing `/var/lib/hive-priv/userns/<agent>`;
|
next start.
|
||||||
until then it will not start.
|
'';
|
||||||
|
|
||||||
The agent's own config must pin `services.hyperhive.agent.user.uid`
|
|
||||||
and `.gid` to one number: nspawn maps a bind's gids with its uid map.
|
|
||||||
Keep an agent that holds `ManageRootAgent` off the list, since the
|
|
||||||
other agents' dirs it mounts are not owned by its data owner.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
dataOwner = lib.mkOption {
|
|
||||||
type = lib.types.ints.u32;
|
|
||||||
default = 60000;
|
|
||||||
description = ''
|
|
||||||
The host uid and gid that own a namespaced agent's data on disk. One
|
|
||||||
number for both: nspawn copies the uid map for gids, so the gid
|
|
||||||
cannot differ. It must be no host user or group, outside the normal and
|
|
||||||
system users (below 30000) and the dynamic users (61184 and above), and
|
|
||||||
below the range `pick` allocates from (524288 and above), so no
|
|
||||||
container's range covers it. Changing it after an agent has moved is
|
|
||||||
refused at that agent's next start.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
|
config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
|
||||||
|
|
@ -79,8 +58,8 @@ in
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
assertion = userns.agents == [ ] || problems == [ ];
|
assertion = problems == [ ];
|
||||||
message = "services.hyperhive.userns.dataOwner: ${lib.concatStringsSep "; " problems}";
|
message = "services.hyperhive.deploy.hive-controller.userns.dataOwner: ${lib.concatStringsSep "; " problems}";
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
];
|
];
|
||||||
|
|
@ -148,9 +127,8 @@ in
|
||||||
# routing so the closure `RealiseClosure` downloads is in the same
|
# routing so the closure `RealiseClosure` downloads is in the same
|
||||||
# store the container is then pointed at.
|
# store the container is then pointed at.
|
||||||
NIX_REMOTE = "daemon";
|
NIX_REMOTE = "daemon";
|
||||||
# Who is namespaced, and the owner their data moves to: set by the
|
# The owner a namespaced agent's data moves to. Which agents are
|
||||||
# operator in the unit, so only root decides.
|
# namespaced is not set here: hive-c0re sends it with each start.
|
||||||
HIVE_PRIV_USERNS_AGENTS = lib.concatStringsSep " " userns.agents;
|
|
||||||
HIVE_PRIV_USERNS_DATA_OWNER = toString userns.dataOwner;
|
HIVE_PRIV_USERNS_DATA_OWNER = toString userns.dataOwner;
|
||||||
};
|
};
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
|
|
|
||||||
|
|
@ -178,15 +178,15 @@ let
|
||||||
{
|
{
|
||||||
# Read back from the rendered unit and assertion list: a misspelt
|
# Read back from the rendered unit and assertion list: a misspelt
|
||||||
# variable or a dropped assertion fails here.
|
# variable or a dropped assertion fails here.
|
||||||
name = "hive-priv carries the userns agents and data owner, and the owner assertion passes";
|
name = "hive-priv carries the userns data owner, no agent list, and the owner assertion passes";
|
||||||
ok =
|
ok =
|
||||||
let
|
let
|
||||||
e = allLocal.systemd.services.hive-priv.environment;
|
e = allLocal.systemd.services.hive-priv.environment;
|
||||||
in
|
in
|
||||||
e.HIVE_PRIV_USERNS_AGENTS == ""
|
!(e ? HIVE_PRIV_USERNS_AGENTS)
|
||||||
&& e.HIVE_PRIV_USERNS_DATA_OWNER == "60000"
|
&& e.HIVE_PRIV_USERNS_DATA_OWNER == "60000"
|
||||||
&& lib.any (
|
&& lib.any (
|
||||||
a: lib.hasPrefix "services.hyperhive.userns.dataOwner" a.message && a.assertion
|
a: lib.hasInfix "hive-controller.userns.dataOwner" a.message && a.assertion
|
||||||
) allLocal.assertions;
|
) allLocal.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -18,10 +18,12 @@
|
||||||
# leaves the link's target untouched, and the start goes through once the
|
# leaves the link's target untouched, and the start goes through once the
|
||||||
# link is gone.
|
# link is gone.
|
||||||
#
|
#
|
||||||
# - an agent listed in `services.hyperhive.userns.agents` starts in its own
|
# - an agent whose config sets `services.hyperhive.agent.user.privateUsers`
|
||||||
|
# (hive-c0re sends the flag from the closure record; the test sends it)
|
||||||
|
# starts in its own
|
||||||
# user namespace with state it already had, owned by uid 1000 on disk: the
|
# user namespace with state it already had, owned by uid 1000 on disk: the
|
||||||
# data moves once to `dataOwner`, the agent user reads and writes it as
|
# data moves once to `dataOwner`, the agent user reads and writes it as
|
||||||
# itself, the unlisted agent's conf is untouched, and a second
|
# itself, the other agent's conf (no flag sent) is untouched, and a second
|
||||||
# `WriteNspawnFlags` moves nothing.
|
# `WriteNspawnFlags` moves nothing.
|
||||||
#
|
#
|
||||||
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
|
# The agent's journal forwarder itself (an OTLP push to the hive's collector) is
|
||||||
|
|
@ -118,7 +120,6 @@ pkgs.testers.runNixOSTest {
|
||||||
prefixLength = 24;
|
prefixLength = 24;
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
services.hyperhive.userns.agents = [ nsAgent ];
|
|
||||||
systemd.tmpfiles.rules =
|
systemd.tmpfiles.rules =
|
||||||
map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs
|
map (dir: "d ${agentDir}/${dir} 0755 root root - -") hostDirs
|
||||||
# What the namespaced agent has from before: dirs its user owned.
|
# What the namespaced agent has from before: dirs its user owned.
|
||||||
|
|
@ -154,6 +155,7 @@ pkgs.testers.runNixOSTest {
|
||||||
name = nsAgent;
|
name = nsAgent;
|
||||||
uid = 1000;
|
uid = 1000;
|
||||||
gid = 1000;
|
gid = 1000;
|
||||||
|
privateUsers = true;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
@ -312,19 +314,20 @@ pkgs.testers.runNixOSTest {
|
||||||
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
|
host.succeed(f"rm {ROOT}/root && mkdir -m 0700 {ROOT}/root")
|
||||||
boot()
|
boot()
|
||||||
|
|
||||||
with subtest("a listed agent starts in its own user namespace with the state it had"):
|
with subtest("an agent whose config sets privateUsers starts in its own user namespace with the state it had"):
|
||||||
NS = "${nsMachine}"
|
NS = "${nsMachine}"
|
||||||
NS_AGENT = "${nsAgent}"
|
NS_AGENT = "${nsAgent}"
|
||||||
NS_DIR = "${nsDir}"
|
NS_DIR = "${nsDir}"
|
||||||
NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
|
NS_BINDS = json.loads(${builtins.toJSON (builtins.toJSON nsBinds)})
|
||||||
host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
|
host.succeed(f"echo before > {NS_DIR}/state/pre.txt && chown 1000:1000 {NS_DIR}/state/pre.txt")
|
||||||
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
|
host.succeed(f"stat -c %u:%g {NS_DIR}/state/pre.txt | grep -x 1000:1000")
|
||||||
priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT})
|
priv_request(host, {"op": "ensure_agent_socket_dir", "name": NS_AGENT, "private_users": True})
|
||||||
write = {
|
write = {
|
||||||
"op": "write_nspawn_flags",
|
"op": "write_nspawn_flags",
|
||||||
"container": NS,
|
"container": NS,
|
||||||
"binds": NS_BINDS,
|
"binds": NS_BINDS,
|
||||||
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
|
"isolation": {"bridge": "${bridge}", "gateway_ip": "${gateway}"},
|
||||||
|
"private_users": True,
|
||||||
}
|
}
|
||||||
priv_request(host, write)
|
priv_request(host, write)
|
||||||
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
|
ns_conf = host.succeed(f"cat /etc/nixos-containers/{NS}.conf")
|
||||||
|
|
@ -332,7 +335,7 @@ pkgs.testers.runNixOSTest {
|
||||||
assert "--private-users-ownership=map" in ns_conf, ns_conf
|
assert "--private-users-ownership=map" in ns_conf, ns_conf
|
||||||
# Every bind, the read-only config included, is mounted owneridmap.
|
# Every bind, the read-only config included, is mounted owneridmap.
|
||||||
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
|
assert ns_conf.count(":owneridmap") == len(NS_BINDS), ns_conf
|
||||||
# The unlisted agent's conf has none of it: the control that the lookups
|
# The other agent's conf has none of it: the control that the lookups
|
||||||
# above find these strings where they are.
|
# above find these strings where they are.
|
||||||
assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf()
|
assert "owneridmap" not in conf() and "PRIVATE_USERS" not in conf(), conf()
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue