HIVE_PRIV_EXE, set by the module to the /run/hive-rootfs-check link, is checked at start and used for the drop-in's ExecStartPre. Without it the running binary's path is used as before, and an unlinked one gets an error that names the cause.
1775 lines
71 KiB
Rust
1775 lines
71 KiB
Rust
//! Container config writes: `WriteNspawnFlags` (the container's
|
|
//! `/etc/nixos-containers/<name>.conf`, plus the bridge-DNS marker and
|
|
//! hive-facts file in its rootfs), `WriteResourceLimits` /
|
|
//! `RemoveServiceDropin` (the `container@<name>.service` drop-in) and
|
|
//! `DaemonReload`.
|
|
//!
|
|
//! Every op that names a container takes its system name and refuses anything
|
|
//! [`validate_container_system_name`] does not accept.
|
|
|
|
use std::ffi::OsStr;
|
|
use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd};
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::OnceLock;
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_priv_sock::{
|
|
AGENT_PREFIX, AGENT_RUNTIME_ROOT, BindMount, CredentialMount, HiveFacts, NetworkIsolation,
|
|
};
|
|
use hive_types::NixBuilds;
|
|
use tokio::process::Command;
|
|
|
|
use crate::shared::{StagedFile, open_dir, validate_credential_name, validate_name_chars};
|
|
|
|
/// `WriteNspawnFlags` — validate the container + every bind path + every
|
|
/// credential entry, then write the container's nspawn flag overrides.
|
|
pub(crate) fn handle_write_nspawn_flags(
|
|
container: &str,
|
|
binds: &[BindMount],
|
|
isolation: &NetworkIsolation,
|
|
load_credentials: &[CredentialMount],
|
|
hive_facts: &HiveFacts,
|
|
private_users: Option<bool>,
|
|
nix_builds: Option<NixBuilds>,
|
|
) -> Result<(String, String)> {
|
|
let agent = validate_container_system_name(container)?;
|
|
validate_bridge(&isolation.bridge)?;
|
|
validate_gateway_ip(&isolation.gateway_ip)?;
|
|
for bind in binds {
|
|
validate_bind_path(&bind.host_path)?;
|
|
validate_bind_path(&bind.container_path)?;
|
|
}
|
|
for cred in load_credentials {
|
|
validate_credential_name(&cred.name)?;
|
|
// Same path rules as binds: the colon ban is essential since
|
|
// `--load-credential=name:path` uses `:` as the name/path separator.
|
|
validate_bind_path(&cred.host_path)?;
|
|
}
|
|
let hive_facts = render_hive_facts(hive_facts)?;
|
|
// A root that cannot be salvaged keeps its persistent start: the flag is
|
|
// only ever passed for a root the salvage has dealt with.
|
|
let ephemeral = crate::rootfs::prepare(container, agent).unwrap_or_else(|e| {
|
|
tracing::error!(
|
|
%container,
|
|
error = %format!("{e:#}"),
|
|
"root salvage failed; the container starts with its persistent root"
|
|
);
|
|
false
|
|
});
|
|
// After the salvage, so the mount points land in the root the start will
|
|
// snapshot.
|
|
let userns = crate::userns::activate(container, agent, binds, private_users)?;
|
|
write_nspawn_flags(
|
|
container,
|
|
binds,
|
|
isolation,
|
|
load_credentials,
|
|
&hive_facts,
|
|
StartMode {
|
|
ephemeral,
|
|
userns,
|
|
nix_builds,
|
|
},
|
|
)?;
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// [`handle_write_nspawn_flags`] on the blocking pool. The root salvage inside
|
|
/// it copies files for as long as the root is big, and would hold a runtime
|
|
/// worker for that time.
|
|
///
|
|
/// # Errors
|
|
/// As [`handle_write_nspawn_flags`], or when the task fails to join.
|
|
pub(crate) async fn handle_write_nspawn_flags_blocking(
|
|
container: String,
|
|
binds: Vec<BindMount>,
|
|
isolation: NetworkIsolation,
|
|
load_credentials: Vec<CredentialMount>,
|
|
hive_facts: HiveFacts,
|
|
private_users: Option<bool>,
|
|
nix_builds: Option<NixBuilds>,
|
|
) -> Result<(String, String)> {
|
|
tokio::task::spawn_blocking(move || {
|
|
handle_write_nspawn_flags(
|
|
&container,
|
|
&binds,
|
|
&isolation,
|
|
&load_credentials,
|
|
&hive_facts,
|
|
private_users,
|
|
nix_builds,
|
|
)
|
|
})
|
|
.await
|
|
.context("write nspawn flags task")?
|
|
}
|
|
|
|
/// Validate a system-level container name (`h-` prefix for all agents,
|
|
/// the manager included) and return the logical agent name after it.
|
|
/// Anything else is refused, the swarm's forge and matrix containers
|
|
/// included: they belong to the host that runs them, not to a hive.
|
|
pub(crate) fn validate_container_system_name(name: &str) -> Result<&str> {
|
|
let Some(suffix) = name.strip_prefix(AGENT_PREFIX) else {
|
|
bail!("container name {name:?} is not managed by hive");
|
|
};
|
|
validate_name_chars(suffix)?;
|
|
Ok(suffix)
|
|
}
|
|
|
|
/// `RemoveServiceDropin` — remove the container service's drop-in dir
|
|
/// if present (idempotent), then the GC root of the build it named.
|
|
pub(crate) fn remove_service_dropin(container: &str) -> Result<(String, String)> {
|
|
validate_container_system_name(container)?;
|
|
let dir = format!("/run/systemd/system/container@{container}.service.d");
|
|
if Path::new(&dir).exists() {
|
|
std::fs::remove_dir_all(&dir).with_context(|| format!("remove {dir}"))?;
|
|
}
|
|
crate::prestart::unpin_binary(container)?;
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// `WriteResourceLimits` — drop the systemd resource settings into the
|
|
/// container service's drop-in dir, together with a
|
|
/// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir.
|
|
///
|
|
/// The drop-in also carries the `ExecStartPre=` that runs
|
|
/// `hive-priv check-rootfs <container>` ([`crate::prestart`]) ahead of
|
|
/// nixpkgs' start script. It names the binary [`hive_priv_exe`] picks: the
|
|
/// configured link to the current build when [`EXE_ENV`] is set, else this
|
|
/// process's own binary. A store path there is rooted against garbage
|
|
/// collection for as long as the drop-in names it
|
|
/// ([`crate::prestart::pin_binary`]); the link needs no root.
|
|
///
|
|
/// Two different kinds of setting land in the same file. `MemoryMax=` /
|
|
/// `CPUQuota=` are hard caps that throttle even on an idle host;
|
|
/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only
|
|
/// decide who yields *under contention*. A weight of `None` means "not
|
|
/// configured" and omits the line, so a hive-c0re built before the weights
|
|
/// existed keeps producing the old two-line drop-in.
|
|
///
|
|
/// The condition causes systemd to *skip* (not *fail*) the unit when the
|
|
/// bind-mount source dir is absent — result is `condition`, which does not
|
|
/// increment the start-limit counter. This is belt-and-braces on top of
|
|
/// hive-c0re creating the dir in its start preamble: if it is missing at
|
|
/// start time, the unit idles rather than restart-looping into
|
|
/// `start-limit-hit`.
|
|
pub(crate) fn write_resource_limits(
|
|
container: &str,
|
|
memory_max: &str,
|
|
cpu_quota: &str,
|
|
cpu_weight: Option<u32>,
|
|
io_weight: Option<u32>,
|
|
memory_high: Option<u64>,
|
|
) -> Result<(String, String)> {
|
|
let logical = validate_container_system_name(container)?;
|
|
validate_memory_max(memory_max)?;
|
|
validate_cpu_quota(cpu_quota)?;
|
|
let runtime_dir = format!("{AGENT_RUNTIME_ROOT}/{logical}");
|
|
let dir = format!("/run/systemd/system/container@{container}.service.d");
|
|
std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?;
|
|
let path = format!("{dir}/hyperhive-limits.conf");
|
|
let exe = hive_priv_exe()?;
|
|
// Rooted before the drop-in names it, so the drop-in never names a build a
|
|
// collection can take.
|
|
crate::prestart::pin_binary(container, Path::new(&exe))?;
|
|
let rootfs_check = rootfs_check_command(&exe, container);
|
|
let content = limits_dropin_body(
|
|
&runtime_dir,
|
|
&rootfs_check,
|
|
memory_max,
|
|
cpu_quota,
|
|
cpu_weight,
|
|
io_weight,
|
|
memory_high,
|
|
);
|
|
// Published whole: a `daemon-reload` between truncate and write would
|
|
// otherwise load the unit with its limits and start guard missing.
|
|
publish_file(Path::new(&path), content.as_bytes(), 0o644, None)?;
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// How long a window the start-limit counts over, and how many starts it
|
|
/// allows inside it.
|
|
///
|
|
/// `container@.service` sets `Restart=on-failure` and **no** start limit, so
|
|
/// systemd's defaults apply: 5 starts per 10s, `RestartSec` 100ms. That
|
|
/// makes the bound depend on *how fast* a container dies — one that fails
|
|
/// instantly trips the limit in under a second, one that takes longer than
|
|
/// ~2s never trips it and restarts forever. Whether an agent gets bounded
|
|
/// is not meant to be a function of its failure speed.
|
|
///
|
|
/// The window has to exceed the worst-case time to burn the burst, or the
|
|
/// counter ages out between attempts and the limit is again unreachable:
|
|
/// `TimeoutStartSec` is 1min, so `BURST` slow failures plus their backoff
|
|
/// can span several minutes. 10min covers that with room.
|
|
///
|
|
/// Giving up is cheap here **because it is not terminal** — hive-c0re's
|
|
/// reconcile sweep retries later, and `reset-failed` (see `StartContainer`)
|
|
/// clears the latch first. That is what makes a tight burst safe.
|
|
const START_LIMIT_INTERVAL_SEC: u32 = 600;
|
|
/// One start plus two retries — the operator's ruling was "retry once or
|
|
/// twice", with the reconcile sweep as the slow path after that.
|
|
const START_LIMIT_BURST: u32 = 3;
|
|
/// Backoff between those retries. The 100ms default is for processes that
|
|
/// respawn instantly; a container that just failed to boot gains nothing
|
|
/// from being retried a tenth of a second later.
|
|
const RESTART_SEC: u32 = 5;
|
|
|
|
/// Render the body of `hyperhive-limits.conf`.
|
|
///
|
|
/// `rootfs_check` is the command line of the pre-start root check, the first
|
|
/// `[Service]` line: it runs before the nixpkgs start script and a non-zero
|
|
/// exit fails the start.
|
|
///
|
|
/// `[Unit]`: the condition is checked at start time — it skips (not fails)
|
|
/// the unit when the MCP socket dir is absent, avoiding restart loops —
|
|
/// plus the bounded start limit (see the constants above; `StartLimit*` are
|
|
/// `[Unit]` settings since systemd 229 and are silently ignored under
|
|
/// `[Service]`).
|
|
/// `[Service]`: the restart backoff, the hard caps, then the relative
|
|
/// weights. A weight of `None` means "not configured" and omits its line
|
|
/// entirely, so a request from a hive-c0re built before the weights
|
|
/// existed — or one whose nix option is `null` — renders no weight lines.
|
|
///
|
|
/// `memory_high` is the byte count of `MemoryHigh=`: the throttle point below
|
|
/// the cap, where the kernel slows the container and reclaims from it. `None`
|
|
/// omits the line.
|
|
///
|
|
/// A finite `MemoryMax=` comes with `MemorySwapMax=0`: at the budget the
|
|
/// kernel kills inside the container instead of paging it out to host swap.
|
|
/// `MemoryMax=infinity` is no budget and leaves swap alone.
|
|
fn limits_dropin_body(
|
|
runtime_dir: &str,
|
|
rootfs_check: &str,
|
|
memory_max: &str,
|
|
cpu_quota: &str,
|
|
cpu_weight: Option<u32>,
|
|
io_weight: Option<u32>,
|
|
memory_high: Option<u64>,
|
|
) -> String {
|
|
// Built as two possibly-empty lines rather than pushed onto the
|
|
// string: `format!` appended to a `String` trips clippy::pedantic's
|
|
// `format_push_string`, and a `write!` would need an unwrap.
|
|
let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n"));
|
|
let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n"));
|
|
let high_line = memory_high.map_or_else(String::new, |b| format!("MemoryHigh={b}\n"));
|
|
let swap_line = if memory_max == "infinity" {
|
|
""
|
|
} else {
|
|
"MemorySwapMax=0\n"
|
|
};
|
|
format!(
|
|
"[Unit]\n\
|
|
ConditionPathIsDirectory={runtime_dir}\n\
|
|
StartLimitIntervalSec={START_LIMIT_INTERVAL_SEC}\n\
|
|
StartLimitBurst={START_LIMIT_BURST}\n\
|
|
\n\
|
|
[Service]\n\
|
|
ExecStartPre={rootfs_check}\n\
|
|
RestartSec={RESTART_SEC}\n\
|
|
MemoryMax={memory_max}\n\
|
|
{high_line}\
|
|
{swap_line}\
|
|
CPUQuota={cpu_quota}\n\
|
|
{cpu_weight_line}{io_weight_line}"
|
|
)
|
|
}
|
|
|
|
/// The environment variable naming the binary the drop-in's `ExecStartPre=`
|
|
/// runs. The NixOS module sets it to a path that outlives garbage collection
|
|
/// and switches (`/run/hive-rootfs-check`, a link to the current package).
|
|
pub(crate) const EXE_ENV: &str = "HIVE_PRIV_EXE";
|
|
|
|
/// The binary [`EXE_ENV`] named at start, once [`init_exe`] has checked it.
|
|
static CONFIGURED_EXE: OnceLock<String> = OnceLock::new();
|
|
|
|
/// Read and check [`EXE_ENV`] once at start, so a missing or unsafe value stops
|
|
/// the daemon instead of failing the first drop-in write. With the variable
|
|
/// unset, [`hive_priv_exe`] resolves the running binary's path at each write.
|
|
pub(crate) fn init_exe() -> Result<()> {
|
|
if let Some(value) = std::env::var_os(EXE_ENV) {
|
|
let exe = configured_exe(&value)?;
|
|
// `init_exe` runs once, so the cell is empty.
|
|
let _ = CONFIGURED_EXE.set(exe);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// `value` of [`EXE_ENV`], when it is safe in a unit file and exists.
|
|
fn configured_exe(value: &OsStr) -> Result<String> {
|
|
let exe = safe_unit_path(value.to_str().unwrap_or_default())
|
|
.with_context(|| format!("{EXE_ENV}={} is not safe in a unit file", value.display()))?;
|
|
if !Path::new(exe).exists() {
|
|
bail!("{EXE_ENV}={exe} does not exist");
|
|
}
|
|
Ok(exe.to_owned())
|
|
}
|
|
|
|
/// `path` when it can be written into a command line as is. systemd expands
|
|
/// `%` and `$` in command lines and splits on spaces, so a path with any of
|
|
/// them is refused rather than escaped; store paths never have them.
|
|
fn safe_unit_path(path: &str) -> Option<&str> {
|
|
(path.starts_with('/')
|
|
&& path
|
|
.bytes()
|
|
.all(|b| b.is_ascii_alphanumeric() || b"/._+-".contains(&b)))
|
|
.then_some(path)
|
|
}
|
|
|
|
/// The binary the drop-in's `ExecStartPre=` runs: the one [`EXE_ENV`] named at
|
|
/// start, else this process's own.
|
|
fn hive_priv_exe() -> Result<String> {
|
|
pick_exe(CONFIGURED_EXE.get(), std::env::current_exe())
|
|
}
|
|
|
|
/// [`hive_priv_exe`] with both sources as parameters; the running binary is
|
|
/// not looked at when `configured` is set.
|
|
fn pick_exe(configured: Option<&String>, current: std::io::Result<PathBuf>) -> Result<String> {
|
|
match configured {
|
|
Some(exe) => Ok(exe.clone()),
|
|
None => running_exe(current),
|
|
}
|
|
}
|
|
|
|
/// The path of the running binary, as [`std::env::current_exe`] reported it.
|
|
/// Linux appends ` (deleted)` once the file is unlinked, which a unit file
|
|
/// can't carry, so that case gets its own error.
|
|
fn running_exe(current: std::io::Result<PathBuf>) -> Result<String> {
|
|
let exe = current.context("resolve the hive-priv binary path")?;
|
|
let shown = exe.display().to_string();
|
|
if shown.ends_with(" (deleted)") {
|
|
bail!("hive-priv's own binary {shown} was unlinked; restart hive-priv.service");
|
|
}
|
|
safe_unit_path(&shown)
|
|
.map(str::to_owned)
|
|
.with_context(|| format!("hive-priv binary path {shown} is not safe in a unit file"))
|
|
}
|
|
|
|
/// The `ExecStartPre=` command of the pre-start root check for `container`:
|
|
/// the binary `exe`, then the sub-command and the name.
|
|
fn rootfs_check_command(exe: &str, container: &str) -> String {
|
|
format!("{exe} {} {container}", crate::prestart::SUBCOMMAND)
|
|
}
|
|
|
|
/// `DaemonReload` — `systemctl daemon-reload` on the host.
|
|
pub(crate) async fn daemon_reload() -> Result<(String, String)> {
|
|
let out = Command::new("systemctl")
|
|
.arg("daemon-reload")
|
|
.output()
|
|
.await
|
|
.context("invoke systemctl daemon-reload")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"systemctl daemon-reload failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Publish `content` at `path` through a [`StagedFile`], at `mode` and, when
|
|
/// given, owned by `owner` (`(uid, gid)`).
|
|
fn publish_file(path: &Path, content: &[u8], mode: u32, owner: Option<(u32, u32)>) -> Result<()> {
|
|
let (Some(dir_path), Some(filename)) =
|
|
(path.parent(), path.file_name().and_then(|n| n.to_str()))
|
|
else {
|
|
bail!(
|
|
"publish_file: {} has no directory or file name",
|
|
path.display()
|
|
);
|
|
};
|
|
let dir = open_dir(dir_path)?;
|
|
let staged = StagedFile::write(&dir, dir_path, filename, content, mode)?;
|
|
if let Some((uid, gid)) = owner {
|
|
std::os::unix::fs::fchown(staged.file(), Some(uid), Some(gid))
|
|
.with_context(|| format!("chown {}", path.display()))?;
|
|
}
|
|
staged.publish()
|
|
}
|
|
|
|
/// Validate a bind-mount path: absolute, non-empty, and free of whitespace,
|
|
/// control bytes, `\`, `$`, backtick, `"` and `:`.
|
|
///
|
|
/// The path lands inside `EXTRA_NSPAWN_FLAGS="..."` in the container conf,
|
|
/// and the nixos-containers start script expands that variable unquoted: a
|
|
/// space or tab starts another nspawn argument, `$` and backtick are read by
|
|
/// the shell, `\` and `"` break the quoting, and a newline or NUL ends the
|
|
/// line. The colon separates the two sides of `--bind=SRC:DST`, so a path
|
|
/// carrying one silently becomes a *different mount* than the caller asked
|
|
/// for.
|
|
fn validate_bind_path(path: &str) -> Result<()> {
|
|
if path.is_empty()
|
|
|| !path.starts_with('/')
|
|
|| path
|
|
.bytes()
|
|
.any(|b| b <= b' ' || b == 0x7f || matches!(b, b'"' | b':' | b'\\' | b'$' | b'`'))
|
|
{
|
|
bail!(
|
|
"invalid bind path {path:?}: must be an absolute path with no whitespace, control bytes, colons, backslashes, dollar signs, backticks or double-quotes"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Validate `NetworkIsolation::bridge`, written as `HOST_BRIDGE=` in the
|
|
/// container conf: an interface name of ASCII alphanumerics and `-_.`, at
|
|
/// most 15 bytes (`IFNAMSIZ` less the NUL).
|
|
fn validate_bridge(bridge: &str) -> Result<()> {
|
|
if bridge.is_empty()
|
|
|| bridge.len() > 15
|
|
|| !bridge
|
|
.bytes()
|
|
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.'))
|
|
{
|
|
bail!("invalid bridge {bridge:?}: must be 1-15 ascii letters, digits, '-', '_' or '.'");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Validate `NetworkIsolation::gateway_ip`, written as `HOST_ADDRESS=` in the
|
|
/// container conf and as the bridge-DNS marker: an IPv4 address literal.
|
|
fn validate_gateway_ip(gateway_ip: &str) -> Result<()> {
|
|
if gateway_ip.parse::<std::net::Ipv4Addr>().is_err() {
|
|
bail!("invalid gateway_ip {gateway_ip:?}: must be an IPv4 address literal");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Digits, optionally followed by `.` and more digits.
|
|
fn is_decimal(value: &str) -> bool {
|
|
let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit());
|
|
match value.split_once('.') {
|
|
Some((int, frac)) => digits(int) && digits(frac),
|
|
None => digits(value),
|
|
}
|
|
}
|
|
|
|
/// Validate `memory_max`, written as `MemoryMax=` in the limits drop-in:
|
|
/// `infinity`, a decimal percentage, or a decimal byte size with an optional
|
|
/// `B`/`K`/`M`/`G`/`T`/`P`/`E` suffix. The shape only: the range is
|
|
/// systemd's to judge.
|
|
fn validate_memory_max(value: &str) -> Result<()> {
|
|
let size = value
|
|
.strip_suffix(['B', 'K', 'M', 'G', 'T', 'P', 'E'])
|
|
.unwrap_or(value);
|
|
if value == "infinity" || value.strip_suffix('%').is_some_and(is_decimal) || is_decimal(size) {
|
|
return Ok(());
|
|
}
|
|
bail!(
|
|
"invalid memory_max {value:?}: must be \"infinity\", a percentage such as \"50%\", or a size such as \"8G\""
|
|
)
|
|
}
|
|
|
|
/// Validate `cpu_quota`, written as `CPUQuota=` in the limits drop-in: a
|
|
/// decimal percentage. The shape only, as [`validate_memory_max`].
|
|
fn validate_cpu_quota(value: &str) -> Result<()> {
|
|
if value.strip_suffix('%').is_some_and(is_decimal) {
|
|
return Ok(());
|
|
}
|
|
bail!("invalid cpu_quota {value:?}: must be a percentage such as \"200%\"")
|
|
}
|
|
|
|
/// `--tmpfs=/knowledge/.git`, hiding the shared docs repo's metadata from
|
|
/// inside the container.
|
|
///
|
|
/// **The agent is given a working tree, never a repository.** `/knowledge` is
|
|
/// the hive's shared docs, whose `.git/config` has held a credential the
|
|
/// host-side worker embedded, and whose `.git` carries every branch and the
|
|
/// full history of a document whose *currently deployed* value is the only
|
|
/// thing a reader may act on: an abandoned branch is indistinguishable from a
|
|
/// live one.
|
|
///
|
|
/// An overlay rather than an exported copy: there is no second tree to keep in
|
|
/// sync, so nothing can go stale, and no code path has to remember to refresh
|
|
/// it.
|
|
///
|
|
/// Applied only when the bind's host source has a `.git` entry. nspawn creates
|
|
/// the mount point inside the read-only bind, and a read-only mount refuses
|
|
/// that (`EROFS`) where nothing exists to mount over, which fails the whole
|
|
/// container start while the first clone is missing. A container started
|
|
/// before that clone has no overlay, so it sees the `.git` that arrives until
|
|
/// its next start rewrites these flags. The entry is looked up without
|
|
/// following a symlink, and a symlink named `.git` counts as present.
|
|
///
|
|
/// ⚠️ Ordering matters: these must be appended **after** the `--bind` flags so
|
|
/// nspawn mounts them on top of the already-mounted trees.
|
|
fn git_overlay_flags(binds: &[BindMount]) -> Vec<String> {
|
|
binds
|
|
.iter()
|
|
.filter(|b| b.container_path == "/knowledge")
|
|
.filter(|b| std::fs::symlink_metadata(Path::new(&b.host_path).join(".git")).is_ok())
|
|
.map(|b| format!("--tmpfs={}/.git", b.container_path))
|
|
.collect()
|
|
}
|
|
|
|
/// What a start of the container asks of nspawn besides its binds, as
|
|
/// [`write_nspawn_flags`] writes it.
|
|
#[derive(Clone, Copy)]
|
|
struct StartMode {
|
|
/// `--ephemeral`, once [`crate::rootfs`] has made the root fit for it.
|
|
ephemeral: bool,
|
|
/// The container runs in its own user namespace.
|
|
userns: bool,
|
|
/// `None` keeps the choice the conf's last write made.
|
|
nix_builds: Option<NixBuilds>,
|
|
}
|
|
|
|
/// Update `/etc/nixos-containers/<container>.conf`: strip old network vars
|
|
/// (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`),
|
|
/// write the current network-isolation settings, then append
|
|
/// `EXTRA_NSPAWN_FLAGS`. Always writes `PRIVATE_NETWORK=1` + veth
|
|
/// wiring — isolation is the only mode, so there is no branch that
|
|
/// leaves a container on the host's network namespace.
|
|
fn write_nspawn_flags(
|
|
container: &str,
|
|
binds: &[BindMount],
|
|
isolation: &NetworkIsolation,
|
|
load_credentials: &[CredentialMount],
|
|
hive_facts: &str,
|
|
mode: StartMode,
|
|
) -> Result<()> {
|
|
use std::fmt::Write as _;
|
|
use std::os::unix::fs::MetadataExt as _;
|
|
let StartMode {
|
|
ephemeral,
|
|
userns,
|
|
nix_builds,
|
|
} = mode;
|
|
let path = format!("/etc/nixos-containers/{container}.conf");
|
|
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
|
|
let lines: Vec<&str> = original
|
|
.lines()
|
|
.filter(|line| !is_owned_var(line))
|
|
.collect();
|
|
let mut out = lines.join("\n");
|
|
if !out.is_empty() {
|
|
out.push('\n');
|
|
}
|
|
out.push_str(private_users_line(userns));
|
|
{
|
|
let iso = isolation;
|
|
out.push_str("PRIVATE_NETWORK=1\n");
|
|
// HOST_ADDRESS = the bridge gateway IP. nixos-container's
|
|
// container-side setup only installs a default route
|
|
// (`ip route add default via $HOST_ADDRESS`) when HOST_ADDRESS is
|
|
// non-empty; leaving it blank gave the container an address but no
|
|
// route off the bridge subnet (no internet, no api.anthropic.com).
|
|
// In bridge mode (HOST_BRIDGE set) the host-side address/route
|
|
// setup is skipped, so this only affects the container's route —
|
|
// exactly what we want.
|
|
let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip);
|
|
// LOCAL_ADDRESS is intentionally empty: containers take their IP by
|
|
// DHCP from the bridge dnsmasq pool. (Why HOST_ADDRESS is still
|
|
// written: the comment directly above.)
|
|
out.push_str("LOCAL_ADDRESS=\n");
|
|
out.push_str("HOST_ADDRESS6=\n");
|
|
out.push_str("LOCAL_ADDRESS6=\n");
|
|
let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge);
|
|
}
|
|
let nix_builds = chosen_nix_builds(nix_builds, &original);
|
|
let flags_joined =
|
|
nspawn_flags(binds, load_credentials, ephemeral, userns, nix_builds).join(" ");
|
|
let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
|
|
// Keeps the owner and mode `nixos-container` created the file with; this
|
|
// helper only rewrites its lines.
|
|
let meta = std::fs::metadata(&path).with_context(|| format!("stat {path}"))?;
|
|
publish_file(
|
|
Path::new(&path),
|
|
out.as_bytes(),
|
|
meta.mode() & 0o7777,
|
|
Some((meta.uid(), meta.gid())),
|
|
)?;
|
|
|
|
// DNS marker for the in-container resolver oneshot: the oneshot only
|
|
// rewrites the container's resolv.conf when this marker exists, and the
|
|
// marker carries the gateway IP so the container need not re-derive it.
|
|
// Always written — every container is isolated, so there is no mode in
|
|
// which the marker should be absent. Why the rewrite is needed at all,
|
|
// and which unit does it: `docs/4-implementation/networking/network.md` § *How the isolated
|
|
// container gets its resolver*.
|
|
write_bridge_dns_marker(container, isolation)?;
|
|
// Read by the agent's environment generator at boot
|
|
// (`nix/agent-modules/hive-facts.nix`).
|
|
write_container_etc_file_in(&container_rootfs(container), HIVE_FACTS_FILE, hive_facts)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Whether a line of the container conf is one `write_nspawn_flags` writes
|
|
/// itself, and so drops before writing it again. Dropped for every container,
|
|
/// not only those it writes it for: a `PRIVATE_USERS=pick` left behind would
|
|
/// keep a container namespaced after it left the list.
|
|
fn is_owned_var(line: &str) -> bool {
|
|
let t = line.trim_start();
|
|
[
|
|
"EXTRA_NSPAWN_FLAGS=",
|
|
"PRIVATE_NETWORK=",
|
|
"PRIVATE_USERS=",
|
|
"HOST_ADDRESS=",
|
|
"LOCAL_ADDRESS=",
|
|
"HOST_ADDRESS6=",
|
|
"LOCAL_ADDRESS6=",
|
|
"HOST_BRIDGE=",
|
|
]
|
|
.iter()
|
|
.any(|var| t.starts_with(var))
|
|
}
|
|
|
|
/// The conf line that makes nixos-container pass `--private-users=pick`.
|
|
fn private_users_line(userns: bool) -> &'static str {
|
|
if userns { "PRIVATE_USERS=pick\n" } else { "" }
|
|
}
|
|
|
|
/// An empty tmpfs over `/nix/var/nix/daemon-socket`, the directory nixos-containers
|
|
/// always binds the host's nix-daemon socket through, so the container finds no
|
|
/// socket to connect to. nspawn mounts its custom mounts sorted by destination
|
|
/// and then by kind, and a tmpfs sorts after a bind, so this lands on top of the
|
|
/// start script's bind whatever order the flags are in.
|
|
const NIX_DAEMON_SHADOW_FLAG: &str = "--tmpfs=/nix/var/nix/daemon-socket";
|
|
|
|
/// What the previous write of the conf chose: [`NixBuilds::Disabled`] when its
|
|
/// `EXTRA_NSPAWN_FLAGS` line carries [`NIX_DAEMON_SHADOW_FLAG`], else
|
|
/// [`NixBuilds::Host`], which is also what a conf never written gets.
|
|
fn previous_nix_builds(conf: &str) -> NixBuilds {
|
|
let shadowed = conf
|
|
.lines()
|
|
.filter_map(|line| line.trim_start().strip_prefix("EXTRA_NSPAWN_FLAGS="))
|
|
.any(|flags| {
|
|
flags
|
|
.split(|c: char| c.is_whitespace() || c == '"')
|
|
.any(|flag| flag == NIX_DAEMON_SHADOW_FLAG)
|
|
});
|
|
if shadowed {
|
|
NixBuilds::Disabled
|
|
} else {
|
|
NixBuilds::Host
|
|
}
|
|
}
|
|
|
|
/// The mode a write uses: the one asked for, else what the conf's last write
|
|
/// chose.
|
|
fn chosen_nix_builds(requested: Option<NixBuilds>, conf: &str) -> NixBuilds {
|
|
requested.unwrap_or_else(|| previous_nix_builds(conf))
|
|
}
|
|
|
|
/// The flags `EXTRA_NSPAWN_FLAGS` carries, in the order nspawn needs them.
|
|
///
|
|
/// `--ephemeral` runs the container on a snapshot of its root and drops the
|
|
/// snapshot at stop; `crate::rootfs` has made the root fit for that before
|
|
/// the flag is asked for. The git overlays and credentials follow the binds
|
|
/// they refer to.
|
|
///
|
|
/// `nix_builds` set to [`NixBuilds::Disabled`] adds [`NIX_DAEMON_SHADOW_FLAG`].
|
|
///
|
|
/// `userns` mounts every bind `owneridmap` and idmaps the root instead of
|
|
/// chowning it, so a different `pick` range never walks the tree.
|
|
fn nspawn_flags(
|
|
binds: &[BindMount],
|
|
load_credentials: &[CredentialMount],
|
|
ephemeral: bool,
|
|
userns: bool,
|
|
nix_builds: NixBuilds,
|
|
) -> Vec<String> {
|
|
let mut flags: Vec<String> = ephemeral
|
|
.then(|| "--ephemeral".to_owned())
|
|
.into_iter()
|
|
.collect();
|
|
if userns {
|
|
flags.push("--private-users-ownership=map".to_owned());
|
|
}
|
|
let idmap = if userns { ":owneridmap" } else { "" };
|
|
flags.extend(binds.iter().map(|b| {
|
|
let flag = if b.read_only { "--bind-ro" } else { "--bind" };
|
|
format!("{flag}={}:{}{idmap}", b.host_path, b.container_path)
|
|
}));
|
|
flags.extend(git_overlay_flags(binds));
|
|
if nix_builds == NixBuilds::Disabled {
|
|
flags.push(NIX_DAEMON_SHADOW_FLAG.to_owned());
|
|
}
|
|
// Credential forwarding: nspawn loads each host secret into the
|
|
// container's credential store under `<name>`; inner units inherit it
|
|
// via `LoadCredential=<name>`. Validated (name charset + bind-path
|
|
// rules) in handle_write_nspawn_flags.
|
|
flags.extend(
|
|
load_credentials
|
|
.iter()
|
|
.map(|cred| format!("--load-credential={}:{}", cred.name, cred.host_path)),
|
|
);
|
|
flags
|
|
}
|
|
|
|
/// Filename of the in-container DNS marker, in the container's own `/etc`.
|
|
const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns";
|
|
|
|
/// Filename of the hive-facts environment file, in the container's own `/etc`.
|
|
const HIVE_FACTS_FILE: &str = "hyperhive-hive-facts";
|
|
|
|
/// The container's `/` on the host.
|
|
pub(crate) fn container_rootfs(container: &str) -> PathBuf {
|
|
PathBuf::from(format!("/var/lib/nixos-containers/{container}"))
|
|
}
|
|
|
|
/// Render [`HiveFacts`] as the systemd environment file the agent loads,
|
|
/// one `KEY=value` line per present fact.
|
|
///
|
|
/// Each value is refused, never escaped, unless it is made only of the
|
|
/// characters its fact can hold. None of those sets admits a newline (a
|
|
/// second assignment), whitespace, a quote, `\` or `$`: the agent hands each
|
|
/// line to systemd and its shells, which would read those as syntax.
|
|
fn render_hive_facts(facts: &HiveFacts) -> Result<String> {
|
|
use std::fmt::Write as _;
|
|
let dns = |b: u8| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_');
|
|
let port = facts.dashboard_port.map(|p| p.to_string());
|
|
let mut out = String::new();
|
|
for (key, value, max_len, allowed) in [
|
|
(
|
|
"HYPERHIVE_HIVE_NAME",
|
|
facts.name.as_ref(),
|
|
253,
|
|
&dns as &dyn Fn(u8) -> bool,
|
|
),
|
|
("HYPERHIVE_HIVE_DOMAIN", facts.domain.as_ref(), 253, &dns),
|
|
("HIVE_DASHBOARD_PORT", port.as_ref(), 5, &|b: u8| {
|
|
b.is_ascii_digit()
|
|
}),
|
|
// A store path's name characters, plus the `/` of its prefix; the
|
|
// prefix itself is checked below.
|
|
(
|
|
"HYPERHIVE_CLAUDE_CODE_PATH",
|
|
facts.claude_code_path.as_ref(),
|
|
256,
|
|
&|b: u8| dns(b) || matches!(b, b'+' | b'?' | b'=' | b'/'),
|
|
),
|
|
// `k=v,k=v`, values percent-encoded as the OTEL spec has them.
|
|
(
|
|
"HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES",
|
|
facts.otel_extra_resource_attributes.as_ref(),
|
|
1024,
|
|
&|b: u8| dns(b) || matches!(b, b'=' | b',' | b'/' | b':' | b'%' | b'+' | b'@'),
|
|
),
|
|
// `<scheme>://<host>:<port>`.
|
|
(
|
|
"HIVE_AGENT_NATS_URL",
|
|
facts.agent_nats_url.as_ref(),
|
|
253,
|
|
&|b: u8| dns(b) || matches!(b, b':' | b'/'),
|
|
),
|
|
// `<ipv4>:<port>`.
|
|
(
|
|
"HIVE_SQUID_PROXY",
|
|
facts.squid_proxy.as_ref(),
|
|
21,
|
|
&|b: u8| b.is_ascii_digit() || matches!(b, b'.' | b':'),
|
|
),
|
|
// `host,host`: bare names and addresses.
|
|
(
|
|
"HIVE_EGRESS_NO_PROXY",
|
|
facts.egress_no_proxy.as_ref(),
|
|
4096,
|
|
&|b: u8| dns(b) || b == b',',
|
|
),
|
|
] {
|
|
let Some(value) = value else { continue };
|
|
if value.is_empty() || value.len() > max_len || !value.bytes().all(allowed) {
|
|
bail!(
|
|
"invalid {key} value {value:?}: must be 1-{max_len} chars of its fact's character set"
|
|
);
|
|
}
|
|
if key == "HYPERHIVE_CLAUDE_CODE_PATH"
|
|
&& !value
|
|
.strip_prefix("/nix/store/")
|
|
.is_some_and(|name| !name.is_empty() && !name.contains('/'))
|
|
{
|
|
bail!("invalid {key} value {value:?}: must be a top-level /nix/store path");
|
|
}
|
|
let _ = writeln!(out, "{key}={value}");
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
/// Write the bridge-DNS marker the `hyperhive-isolated-dns` oneshot keys
|
|
/// off. The marker file contains just the gateway IP. Always written:
|
|
/// every container is isolated, so there is no host-netns case that
|
|
/// wants the marker absent.
|
|
fn write_bridge_dns_marker(container: &str, isolation: &NetworkIsolation) -> Result<()> {
|
|
write_bridge_dns_marker_in(&container_rootfs(container), &isolation.gateway_ip)
|
|
}
|
|
|
|
/// Write `<rootfs>/etc/hyperhive-bridge-dns`; see [`write_container_etc_file_in`].
|
|
fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> {
|
|
write_container_etc_file_in(rootfs, BRIDGE_DNS_MARKER, &format!("{gateway_ip}\n"))
|
|
}
|
|
|
|
/// Write `<rootfs>/etc/<name>` without following a symlink.
|
|
///
|
|
/// Root inside the container owns its `/etc`, and this write runs as host
|
|
/// root, where an absolute symlink planted in the container resolves against
|
|
/// the host's `/`. So this opens `etc` with `O_DIRECTORY|O_NOFOLLOW` (a
|
|
/// symlinked `etc` fails with `ELOOP`) and publishes the file through a
|
|
/// [`StagedFile`] relative to that fd: whatever the container planted at the
|
|
/// leaf, symlink or FIFO or device node, is replaced by the rename, never
|
|
/// written through or opened.
|
|
///
|
|
/// On a fresh install the container's `/etc` may not exist yet (nixos-container
|
|
/// materialises the rootfs on the first start), so this creates it first; the
|
|
/// dir and the file persist through that start. Mode 0644: nothing written
|
|
/// here is a secret.
|
|
pub(crate) fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> {
|
|
use std::os::unix::fs::OpenOptionsExt as _;
|
|
|
|
let etc_path = rootfs.join("etc");
|
|
// The rootfs dir is the container's `/`, which the container cannot
|
|
// replace, so creating it by path is safe.
|
|
std::fs::create_dir_all(rootfs)
|
|
.with_context(|| format!("create container rootfs {}", rootfs.display()))?;
|
|
let root = std::fs::OpenOptions::new()
|
|
.read(true)
|
|
.custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW)
|
|
.open(rootfs)
|
|
.with_context(|| format!("open container rootfs {}", rootfs.display()))?;
|
|
// SAFETY: `root` is an open directory fd and the name a NUL-terminated
|
|
// literal.
|
|
if unsafe { libc::mkdirat(root.as_raw_fd(), c"etc".as_ptr(), 0o755) } != 0 {
|
|
let e = std::io::Error::last_os_error();
|
|
if e.kind() != std::io::ErrorKind::AlreadyExists {
|
|
return Err(e).with_context(|| format!("create {}", etc_path.display()));
|
|
}
|
|
}
|
|
// SAFETY: as above; this checks the result before wrapping it.
|
|
let etc_fd = unsafe {
|
|
libc::openat(
|
|
root.as_raw_fd(),
|
|
c"etc".as_ptr(),
|
|
libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC,
|
|
)
|
|
};
|
|
if etc_fd < 0 {
|
|
return Err(std::io::Error::last_os_error())
|
|
.with_context(|| format!("open (no-follow) {}", etc_path.display()));
|
|
}
|
|
// SAFETY: `etc_fd` is a fresh, valid fd that nothing else owns.
|
|
let etc = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(etc_fd) });
|
|
StagedFile::write(&etc, &etc_path, name, contents.as_bytes(), 0o644)?.publish()
|
|
}
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{
|
|
BindMount, CredentialMount, HiveFacts, NIX_DAEMON_SHADOW_FLAG, NetworkIsolation, NixBuilds,
|
|
Path, chosen_nix_builds, configured_exe, git_overlay_flags, handle_write_nspawn_flags,
|
|
hive_priv_exe, is_owned_var, limits_dropin_body, nspawn_flags, pick_exe,
|
|
previous_nix_builds, private_users_line, publish_file, remove_service_dropin,
|
|
render_hive_facts, rootfs_check_command, running_exe, validate_bind_path, validate_bridge,
|
|
validate_container_system_name, validate_cpu_quota, validate_gateway_ip,
|
|
validate_memory_max, write_bridge_dns_marker_in, write_resource_limits,
|
|
};
|
|
use crate::shared::test_support::{entries, scratch};
|
|
|
|
/// A stand-in for the pre-start check's command line.
|
|
const CHECK: &str = "/store/hive-priv/bin/hive-priv check-rootfs h-iris";
|
|
|
|
fn bind(container_path: &str) -> BindMount {
|
|
BindMount {
|
|
host_path: "/var/lib/hyperhive/whatever".to_owned(),
|
|
container_path: container_path.to_owned(),
|
|
read_only: true,
|
|
}
|
|
}
|
|
|
|
/// A source dir holding a `.git` directory, as a repo checkout does.
|
|
fn repo_source() -> String {
|
|
let dir = scratch();
|
|
std::fs::create_dir(dir.join(".git")).unwrap();
|
|
dir.to_string_lossy().into_owned()
|
|
}
|
|
|
|
fn bind_from(host_path: &str, container_path: &str) -> BindMount {
|
|
BindMount {
|
|
host_path: host_path.to_owned(),
|
|
..bind(container_path)
|
|
}
|
|
}
|
|
|
|
/// The knowledge tree gets its `.git` overlaid when its source holds one.
|
|
/// Without one the overlay is left out, since nspawn cannot create the
|
|
/// mount point inside the read-only bind and the container would not start.
|
|
#[test]
|
|
fn the_knowledge_dot_git_is_hidden_only_where_there_is_one() {
|
|
let flags = git_overlay_flags(&[bind_from(&repo_source(), "/knowledge")]);
|
|
assert_eq!(flags, ["--tmpfs=/knowledge/.git"]);
|
|
let empty = scratch().to_string_lossy().into_owned();
|
|
let flags = git_overlay_flags(&[bind_from(&empty, "/knowledge")]);
|
|
assert!(
|
|
flags.is_empty(),
|
|
"overlaid a source with no .git: {flags:?}"
|
|
);
|
|
}
|
|
|
|
/// A symlink named `.git` is not followed: it counts as present, even
|
|
/// dangling, so the path it leads to is hidden rather than skipped.
|
|
#[test]
|
|
fn a_dangling_dot_git_symlink_still_gets_the_overlay() {
|
|
let dir = scratch();
|
|
std::os::unix::fs::symlink("/nonexistent", dir.join(".git")).unwrap();
|
|
let src = dir.to_string_lossy().into_owned();
|
|
let flags = git_overlay_flags(&[bind_from(&src, "/knowledge")]);
|
|
assert_eq!(flags, ["--tmpfs=/knowledge/.git"]);
|
|
}
|
|
|
|
/// No other mount is overlaid. A blanket "overlay .git on every bind"
|
|
/// would mask a real `.git` under `state/`, where an agent legitimately
|
|
/// keeps working clones of its own.
|
|
#[test]
|
|
fn other_mounts_are_left_alone() {
|
|
let src = repo_source();
|
|
let flags = git_overlay_flags(&[
|
|
bind_from(&src, "/agents/atlas/state"),
|
|
bind_from(&src, "/work"),
|
|
bind_from(&src, "/agents/atlas/config"),
|
|
]);
|
|
assert!(
|
|
flags.is_empty(),
|
|
"overlaid a non-knowledge mount: {flags:?}"
|
|
);
|
|
}
|
|
|
|
/// The whole rendered body, pinned literally — including the values,
|
|
/// so changing the restart policy is a visible test edit rather than a
|
|
/// silent one.
|
|
///
|
|
/// Placement is the part most worth pinning: `StartLimit*` are `[Unit]`
|
|
/// settings and systemd **silently ignores** them under `[Service]`, so
|
|
/// a bound that moved sections would look configured and do nothing.
|
|
/// An unset weight still emits no line at all, which is what keeps a
|
|
/// hive-c0re older than that field from changing what lands on disk.
|
|
#[test]
|
|
fn dropin_body_is_pinned_exactly() {
|
|
assert_eq!(
|
|
limits_dropin_body(
|
|
"/run/hyperhive/agents/iris",
|
|
CHECK,
|
|
"4G",
|
|
"200%",
|
|
None,
|
|
None,
|
|
None
|
|
),
|
|
"[Unit]\n\
|
|
ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\
|
|
StartLimitIntervalSec=600\n\
|
|
StartLimitBurst=3\n\
|
|
\n\
|
|
[Service]\n\
|
|
ExecStartPre=/store/hive-priv/bin/hive-priv check-rootfs h-iris\n\
|
|
RestartSec=5\n\
|
|
MemoryMax=4G\n\
|
|
MemorySwapMax=0\n\
|
|
CPUQuota=200%\n"
|
|
);
|
|
}
|
|
|
|
/// The bound is hive-wide **policy**, not a per-agent parameter: it is
|
|
/// rendered from constants and no caller-supplied value can omit or
|
|
/// alter it. This is the property that justifies keeping it out of the
|
|
/// wire protocol — if it ever varies by request, that argument is gone.
|
|
#[test]
|
|
fn the_start_limit_is_present_whatever_the_caller_passes() {
|
|
for (mem, cpu, cw, iw) in [
|
|
("4G", "200%", None, None),
|
|
("512M", "50%", Some(10), Some(10)),
|
|
("infinity", "infinity", Some(10_000), None),
|
|
] {
|
|
let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw, None);
|
|
let unit = body
|
|
.split("[Service]")
|
|
.next()
|
|
.expect("the body always has a [Unit] section before [Service]");
|
|
assert!(
|
|
unit.contains("StartLimitIntervalSec=600") && unit.contains("StartLimitBurst=3"),
|
|
"start limit missing from [Unit] for ({mem}, {cpu}): {body}"
|
|
);
|
|
assert!(
|
|
body.contains("\nRestartSec=5\n"),
|
|
"restart backoff missing for ({mem}, {cpu}): {body}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A budgeted container gets no swap; an unbudgeted one keeps it. The
|
|
/// finite arm is the presence control for the infinity arm's absence.
|
|
#[test]
|
|
fn swap_is_off_exactly_when_the_memory_is_budgeted() {
|
|
for mem in ["4G", "512M", "50%", "8589934592"] {
|
|
let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None, None);
|
|
assert!(
|
|
body.contains(&format!("\nMemoryMax={mem}\nMemorySwapMax=0\n")),
|
|
"no swap line under a budget of {mem}: {body}"
|
|
);
|
|
assert_eq!(body.matches("MemorySwapMax=").count(), 1, "{body}");
|
|
}
|
|
let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None, None);
|
|
assert!(
|
|
unbudgeted.contains("\nMemoryMax=infinity\n"),
|
|
"{unbudgeted}"
|
|
);
|
|
assert!(!unbudgeted.contains("MemorySwapMax"), "{unbudgeted}");
|
|
}
|
|
|
|
/// `MemoryHigh=` is the byte count the caller sends, between `MemoryMax=`
|
|
/// and the swap line, and absent when none is sent.
|
|
#[test]
|
|
fn memory_high_is_rendered_only_when_sent() {
|
|
let with = limits_dropin_body(
|
|
"/rt/x",
|
|
CHECK,
|
|
"4G",
|
|
"200%",
|
|
None,
|
|
None,
|
|
Some(3_865_470_566),
|
|
);
|
|
assert!(
|
|
with.contains("\nMemoryMax=4G\nMemoryHigh=3865470566\nMemorySwapMax=0\n"),
|
|
"{with}"
|
|
);
|
|
let without = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, None, None);
|
|
assert!(!without.contains("MemoryHigh"), "{without}");
|
|
}
|
|
/// Weights are appended to the `[Service]` section, each omitted
|
|
/// independently when `None`.
|
|
#[test]
|
|
fn weights_are_emitted_only_when_set() {
|
|
let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80), None);
|
|
assert!(
|
|
both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"),
|
|
"{both}"
|
|
);
|
|
|
|
let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None, None);
|
|
assert!(
|
|
cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"),
|
|
"{cpu_only}"
|
|
);
|
|
assert!(!cpu_only.contains("IOWeight"), "{cpu_only}");
|
|
|
|
let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80), None);
|
|
assert!(
|
|
io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"),
|
|
"{io_only}"
|
|
);
|
|
assert!(!io_only.contains("CPUWeight"), "{io_only}");
|
|
}
|
|
|
|
/// The pre-start check is the first `[Service]` line, whatever else the
|
|
/// caller sets, and a drop-in without it is what an old hive-priv wrote.
|
|
#[test]
|
|
fn the_rootfs_check_leads_the_service_section_whatever_the_caller_passes() {
|
|
for (cw, iw) in [(None, None), (Some(80), Some(80))] {
|
|
let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw, None);
|
|
let service = body
|
|
.split("[Service]\n")
|
|
.nth(1)
|
|
.expect("the body always has a [Service] section");
|
|
assert_eq!(
|
|
service.lines().next(),
|
|
Some("ExecStartPre=/store/hive-priv/bin/hive-priv check-rootfs h-iris")
|
|
);
|
|
assert_eq!(body.matches("ExecStartPre=").count(), 1, "{body}");
|
|
}
|
|
}
|
|
|
|
/// The command names an absolute path to this very binary, the
|
|
/// sub-command that binary implements and the container.
|
|
#[test]
|
|
fn the_check_command_names_this_binary_the_subcommand_and_the_container() {
|
|
let exe = std::env::current_exe().unwrap();
|
|
let cmd = rootfs_check_command(&hive_priv_exe().unwrap(), "h-iris");
|
|
assert_eq!(cmd, format!("{} check-rootfs h-iris", exe.display()));
|
|
assert!(cmd.starts_with('/'), "{cmd}");
|
|
}
|
|
|
|
/// A configured link is used as written, even though it is not this
|
|
/// process's own binary, and the process's own path is never consulted.
|
|
#[test]
|
|
fn a_configured_exe_is_used_as_written() {
|
|
let dir = scratch();
|
|
let link = dir.join("hive-rootfs-check");
|
|
std::os::unix::fs::symlink(std::env::current_exe().unwrap(), &link).unwrap();
|
|
let got = configured_exe(link.as_os_str()).unwrap();
|
|
assert_eq!(got, link.to_str().unwrap());
|
|
|
|
let broken = Err(std::io::Error::other("no /proc"));
|
|
assert_eq!(pick_exe(Some(&got), broken).unwrap(), got);
|
|
let own = std::env::current_exe().unwrap();
|
|
assert_eq!(
|
|
pick_exe(None, Ok(own.clone())).unwrap(),
|
|
own.to_str().unwrap()
|
|
);
|
|
}
|
|
|
|
/// A configured value that is unsafe in a unit file, relative, empty or
|
|
/// missing stops the start, each for its own reason; the existing link is
|
|
/// the control that a good value passes.
|
|
#[test]
|
|
fn a_bad_configured_exe_is_refused_at_start() {
|
|
use std::ffi::OsStr;
|
|
let dir = scratch();
|
|
let good = dir.join("good");
|
|
std::fs::write(&good, b"").unwrap();
|
|
assert!(configured_exe(good.as_os_str()).is_ok());
|
|
|
|
for bad in [
|
|
"/store/hive priv/bin/hive-priv",
|
|
"/s/hive-priv%h",
|
|
"/s/$x",
|
|
"relative/hive-priv",
|
|
"",
|
|
] {
|
|
let err = configured_exe(OsStr::new(bad)).unwrap_err().to_string();
|
|
assert!(err.contains("is not safe in a unit file"), "{bad:?}: {err}");
|
|
}
|
|
let missing = dir.join("absent");
|
|
let err = configured_exe(missing.as_os_str()).unwrap_err().to_string();
|
|
assert!(err.contains("does not exist"), "{err}");
|
|
}
|
|
|
|
/// Without a configured value the running binary's path is used; once the
|
|
/// kernel marks it unlinked the error names the cause and the fix, and a
|
|
/// path unsafe for another reason keeps the generic message.
|
|
#[test]
|
|
fn the_running_exe_is_the_fallback_and_an_unlinked_one_is_named() {
|
|
use std::path::PathBuf;
|
|
let ok = PathBuf::from("/nix/store/aaa-hive-priv/bin/hive-priv");
|
|
assert_eq!(
|
|
running_exe(Ok(ok)).unwrap(),
|
|
"/nix/store/aaa-hive-priv/bin/hive-priv"
|
|
);
|
|
|
|
let gone = PathBuf::from("/nix/store/aaa-hive-priv/bin/hive-priv (deleted)");
|
|
let err = running_exe(Ok(gone)).unwrap_err().to_string();
|
|
assert!(err.contains("was unlinked"), "{err}");
|
|
assert!(err.contains("restart hive-priv.service"), "{err}");
|
|
|
|
let spaced = PathBuf::from("/nix/store/a b/hive-priv");
|
|
let err = running_exe(Ok(spaced)).unwrap_err().to_string();
|
|
assert!(err.contains("is not safe in a unit file"), "{err}");
|
|
}
|
|
|
|
/// The mode lands exactly as asked, whatever the umask would have made
|
|
/// of it (0666 under the usual 022 would otherwise come out 0644), and
|
|
/// the requested owner is on the file.
|
|
#[test]
|
|
fn publish_file_applies_mode_and_owner() {
|
|
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _};
|
|
let dir = scratch();
|
|
let path = dir.join("c.conf");
|
|
let owner = std::fs::metadata(&dir).unwrap();
|
|
|
|
publish_file(&path, b"x\n", 0o666, Some((owner.uid(), owner.gid()))).unwrap();
|
|
let meta = std::fs::metadata(&path).unwrap();
|
|
assert_eq!(meta.permissions().mode() & 0o7777, 0o666);
|
|
assert_eq!((meta.uid(), meta.gid()), (owner.uid(), owner.gid()));
|
|
|
|
publish_file(&path, b"y\n", 0o600, None).unwrap();
|
|
let meta = std::fs::metadata(&path).unwrap();
|
|
assert_eq!(meta.permissions().mode() & 0o7777, 0o600);
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), "y\n");
|
|
assert_eq!(entries(&dir), ["c.conf"], "temp left behind");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The container plants a symlink where the bridge-DNS marker goes; the
|
|
/// rename replaces it without writing through it.
|
|
#[test]
|
|
fn bridge_dns_marker_replaces_symlink_leaf() {
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(rootfs.join("etc")).unwrap();
|
|
let target = dir.join("target");
|
|
std::fs::write(&target, "original").unwrap();
|
|
let leaf = rootfs.join("etc/hyperhive-bridge-dns");
|
|
std::os::unix::fs::symlink(&target, &leaf).unwrap();
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap();
|
|
assert_eq!(
|
|
std::fs::read_to_string(&target).unwrap(),
|
|
"original",
|
|
"symlink target must be untouched"
|
|
);
|
|
assert!(
|
|
std::fs::symlink_metadata(&leaf)
|
|
.unwrap()
|
|
.file_type()
|
|
.is_file()
|
|
);
|
|
assert_eq!(std::fs::read_to_string(&leaf).unwrap(), "10.0.0.1\n");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The container replaces its whole `/etc` with a symlink, so the leaf
|
|
/// name resolves inside a directory of its choosing.
|
|
#[test]
|
|
fn bridge_dns_marker_refuses_symlink_etc() {
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(&rootfs).unwrap();
|
|
let elsewhere = dir.join("elsewhere");
|
|
std::fs::create_dir_all(&elsewhere).unwrap();
|
|
let target = elsewhere.join("hyperhive-bridge-dns");
|
|
std::fs::write(&target, "original").unwrap();
|
|
std::os::unix::fs::symlink(&elsewhere, rootfs.join("etc")).unwrap();
|
|
|
|
let res = write_bridge_dns_marker_in(&rootfs, "10.0.0.1");
|
|
assert!(res.is_err(), "O_NOFOLLOW must refuse a symlinked etc");
|
|
assert_eq!(
|
|
std::fs::read_to_string(&target).unwrap(),
|
|
"original",
|
|
"file behind the symlinked etc must be untouched"
|
|
);
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A FIFO at the leaf is replaced, never opened, so it cannot block the
|
|
/// helper.
|
|
#[test]
|
|
fn bridge_dns_marker_replaces_fifo_leaf() {
|
|
use std::os::unix::ffi::OsStrExt as _;
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(rootfs.join("etc")).unwrap();
|
|
let fifo = rootfs.join("etc/hyperhive-bridge-dns");
|
|
let c_fifo = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap();
|
|
// SAFETY: `c_fifo` is a valid NUL-terminated path.
|
|
assert_eq!(unsafe { libc::mkfifo(c_fifo.as_ptr(), 0o644) }, 0);
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap();
|
|
assert!(
|
|
std::fs::symlink_metadata(&fifo)
|
|
.unwrap()
|
|
.file_type()
|
|
.is_file()
|
|
);
|
|
assert_eq!(std::fs::read_to_string(&fifo).unwrap(), "10.0.0.1\n");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The control: a plain rootfs with no `etc` yet (a fresh install) gets
|
|
/// the dir and a 0644 marker, and a second write replaces it.
|
|
#[test]
|
|
fn bridge_dns_marker_written_into_plain_etc() {
|
|
use std::os::unix::fs::PermissionsExt as _;
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(&rootfs).unwrap();
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap();
|
|
let path = rootfs.join("etc/hyperhive-bridge-dns");
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.0.0.1\n");
|
|
let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
|
|
assert_eq!(mode, 0o644, "marker must stay readable in the container");
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.9.8.7").unwrap();
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.9.8.7\n");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// Each present fact is one environment-file line; an absent one writes
|
|
/// nothing, so the agent sees the variable unset rather than empty.
|
|
#[test]
|
|
fn hive_facts_render_one_line_per_present_fact() {
|
|
let both = HiveFacts {
|
|
name: Some("pr1ma".to_owned()),
|
|
domain: Some("pr1ma.darkest.space".to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert_eq!(
|
|
render_hive_facts(&both).unwrap(),
|
|
"HYPERHIVE_HIVE_NAME=pr1ma\nHYPERHIVE_HIVE_DOMAIN=pr1ma.darkest.space\n"
|
|
);
|
|
let domain_only = HiveFacts {
|
|
domain: Some("h1.example.test".to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert_eq!(
|
|
render_hive_facts(&domain_only).unwrap(),
|
|
"HYPERHIVE_HIVE_DOMAIN=h1.example.test\n"
|
|
);
|
|
assert_eq!(render_hive_facts(&HiveFacts::default()).unwrap(), "");
|
|
}
|
|
|
|
/// A value that could end its line or be read as syntax is refused, not
|
|
/// escaped: a newline would add an assignment of the caller's choosing.
|
|
#[test]
|
|
fn hive_facts_refuse_values_outside_dns_characters() {
|
|
for bad in ["", "pr1ma\nLD_PRELOAD=/x", "a b", "a\"b", "$(x)"] {
|
|
let facts = HiveFacts {
|
|
name: Some(bad.to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert!(
|
|
render_hive_facts(&facts).is_err(),
|
|
"{bad:?} must be refused"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The per-host values agents used to get at build time arrive here, one
|
|
/// line each.
|
|
#[test]
|
|
fn hive_facts_render_the_per_host_values() {
|
|
let facts = HiveFacts {
|
|
dashboard_port: Some(7123),
|
|
claude_code_path: Some("/nix/store/0abc-claude-code-2.1.220".to_owned()),
|
|
otel_extra_resource_attributes: Some(
|
|
"deployment.environment=prod,team=a%2Fb".to_owned(),
|
|
),
|
|
agent_nats_url: Some("tls://nats.example.test:4222".to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert_eq!(
|
|
render_hive_facts(&facts).unwrap(),
|
|
"HIVE_DASHBOARD_PORT=7123\n\
|
|
HYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code-2.1.220\n\
|
|
HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=deployment.environment=prod,team=a%2Fb\n\
|
|
HIVE_AGENT_NATS_URL=tls://nats.example.test:4222\n"
|
|
);
|
|
}
|
|
|
|
/// The egress facts: the proxy as `<ip>:<port>`, and the hosts that skip it.
|
|
#[test]
|
|
fn hive_facts_render_the_egress_proxy() {
|
|
let facts = HiveFacts {
|
|
squid_proxy: Some("10.42.0.1:3128".to_owned()),
|
|
egress_no_proxy: Some("10.42.0.1,pr1ma.example.test,forge.example.test".to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert_eq!(
|
|
render_hive_facts(&facts).unwrap(),
|
|
"HIVE_SQUID_PROXY=10.42.0.1:3128\n\
|
|
HIVE_EGRESS_NO_PROXY=10.42.0.1,pr1ma.example.test,forge.example.test\n"
|
|
);
|
|
}
|
|
|
|
/// Neither egress fact may carry a second assignment, a space, a quote or a
|
|
/// URL: the agent builds its proxy URL and `NO_PROXY` from them.
|
|
#[test]
|
|
fn hive_facts_refuse_bad_egress_values() {
|
|
for bad in [
|
|
"",
|
|
"10.42.0.1:3128\nHIVE_AGENT_NATS_URL=x",
|
|
"http://10.42.0.1:3128",
|
|
"10.42.0.1:3128 ",
|
|
"u:p@10.42.0.1:3128",
|
|
] {
|
|
let facts = HiveFacts {
|
|
squid_proxy: Some(bad.to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert!(
|
|
render_hive_facts(&facts).is_err(),
|
|
"{bad:?} must be refused"
|
|
);
|
|
}
|
|
for bad in [
|
|
"",
|
|
"a.test,b.test\nPATH=/x",
|
|
"a.test b.test",
|
|
"\"a.test\"",
|
|
"$(x)",
|
|
] {
|
|
let facts = HiveFacts {
|
|
egress_no_proxy: Some(bad.to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert!(
|
|
render_hive_facts(&facts).is_err(),
|
|
"{bad:?} must be refused"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The claude path is a top-level store path or nothing, and neither
|
|
/// value may carry generator syntax.
|
|
#[test]
|
|
fn hive_facts_refuse_bad_claude_paths_and_resource_attributes() {
|
|
for bad in [
|
|
"/nix/store/",
|
|
"/nix/store/abc-claude/bin/claude",
|
|
"/tmp/claude",
|
|
"nix/store/abc-claude",
|
|
"/nix/store/abc-claude\nLD_PRELOAD=/x",
|
|
] {
|
|
let facts = HiveFacts {
|
|
claude_code_path: Some(bad.to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert!(
|
|
render_hive_facts(&facts).is_err(),
|
|
"{bad:?} must be refused"
|
|
);
|
|
}
|
|
for bad in [
|
|
"a=b\nPATH=/x",
|
|
"a=${HOME}",
|
|
"a=b c",
|
|
"a=\"b\"",
|
|
"a=b\\c",
|
|
"",
|
|
] {
|
|
let facts = HiveFacts {
|
|
otel_extra_resource_attributes: Some(bad.to_owned()),
|
|
..HiveFacts::default()
|
|
};
|
|
assert!(
|
|
render_hive_facts(&facts).is_err(),
|
|
"{bad:?} must be refused"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// `--ephemeral` is asked for, not assumed: present exactly when the root has
|
|
/// been dealt with, ahead of the binds it applies to. The binds are the
|
|
/// control that the lookup finds what is there.
|
|
#[test]
|
|
fn ephemeral_is_passed_only_when_asked() {
|
|
let binds = [bind("/agents/atlas/work")];
|
|
let with = nspawn_flags(&binds, &[], true, false, NixBuilds::Host);
|
|
assert_eq!(with[0], "--ephemeral", "{with:?}");
|
|
assert!(with.iter().any(|f| f.starts_with("--bind-ro=")), "{with:?}");
|
|
let without = nspawn_flags(&binds, &[], false, false, NixBuilds::Host);
|
|
assert!(!without.iter().any(|f| f == "--ephemeral"), "{without:?}");
|
|
assert_eq!(without.len() + 1, with.len());
|
|
}
|
|
|
|
/// Every bind, read-only ones included, is mounted `owneridmap` for a
|
|
/// namespaced agent and none is for any other; the root is idmapped, not
|
|
/// chowned. The `false` arm is the control that the lookup finds the binds.
|
|
#[test]
|
|
fn binds_are_owneridmap_only_for_a_namespaced_agent() {
|
|
let binds = [bind("/knowledge"), bind("/agents/a/config")];
|
|
let plain = nspawn_flags(&binds, &[], false, false, NixBuilds::Host);
|
|
assert!(
|
|
plain.iter().any(|f| f.starts_with("--bind-ro=")),
|
|
"{plain:?}"
|
|
);
|
|
assert!(
|
|
plain
|
|
.iter()
|
|
.all(|f| !f.contains("idmap") && !f.contains("ownership")),
|
|
"{plain:?}"
|
|
);
|
|
let userns = nspawn_flags(&binds, &[], false, true, NixBuilds::Host);
|
|
assert!(
|
|
userns.contains(&"--private-users-ownership=map".to_owned()),
|
|
"{userns:?}"
|
|
);
|
|
let mounts: Vec<&String> = userns.iter().filter(|f| f.starts_with("--bind")).collect();
|
|
assert_eq!(mounts.len(), 2, "{userns:?}");
|
|
assert!(
|
|
mounts.iter().all(|f| f.ends_with(":owneridmap")),
|
|
"{userns:?}"
|
|
);
|
|
}
|
|
|
|
/// `Disabled` puts the tmpfs over the daemon socket directory after the
|
|
/// binds; `Host` leaves it out. The binds are the control that the lookup
|
|
/// finds what is there, and the flag is the whole of the difference.
|
|
#[test]
|
|
fn nix_builds_none_shadows_the_daemon_socket_dir() {
|
|
let binds = [bind("/agents/atlas/work")];
|
|
let host = nspawn_flags(&binds, &[], false, false, NixBuilds::Host);
|
|
let none = nspawn_flags(&binds, &[], false, false, NixBuilds::Disabled);
|
|
assert!(host.iter().any(|f| f.starts_with("--bind-ro=")), "{host:?}");
|
|
assert!(
|
|
!host.iter().any(|f| f == NIX_DAEMON_SHADOW_FLAG),
|
|
"{host:?}"
|
|
);
|
|
assert_eq!(
|
|
none.last().map(String::as_str),
|
|
Some(NIX_DAEMON_SHADOW_FLAG)
|
|
);
|
|
assert_eq!(none.len(), host.len() + 1);
|
|
assert_eq!(NIX_DAEMON_SHADOW_FLAG, "--tmpfs=/nix/var/nix/daemon-socket");
|
|
}
|
|
|
|
/// An unspecified mode keeps what the conf's last write chose, whatever
|
|
/// else the line carries; a conf with no such line is a host-builds agent.
|
|
#[test]
|
|
fn an_unspecified_nix_builds_mode_keeps_the_last_writes_choice() {
|
|
let binds = [bind("/knowledge")];
|
|
for want in [NixBuilds::Host, NixBuilds::Disabled] {
|
|
let flags = nspawn_flags(&binds, &[], true, true, want).join(" ");
|
|
let conf = format!("AUTO_START=1\nEXTRA_NSPAWN_FLAGS=\"{flags}\"\n");
|
|
assert_eq!(previous_nix_builds(&conf), want, "{conf}");
|
|
}
|
|
assert_eq!(previous_nix_builds(""), NixBuilds::Host);
|
|
// A mode asked for wins over the conf; none asked for keeps it.
|
|
let shadowed = format!("EXTRA_NSPAWN_FLAGS=\"{NIX_DAEMON_SHADOW_FLAG}\"\n");
|
|
assert_eq!(chosen_nix_builds(None, &shadowed), NixBuilds::Disabled);
|
|
assert_eq!(chosen_nix_builds(None, ""), NixBuilds::Host);
|
|
assert_eq!(
|
|
chosen_nix_builds(Some(NixBuilds::Host), &shadowed),
|
|
NixBuilds::Host
|
|
);
|
|
assert_eq!(
|
|
chosen_nix_builds(Some(NixBuilds::Disabled), ""),
|
|
NixBuilds::Disabled
|
|
);
|
|
// The flag has to be a flag of its own, on the flags line.
|
|
assert_eq!(
|
|
previous_nix_builds(&format!(
|
|
"EXTRA_NSPAWN_FLAGS=\"{NIX_DAEMON_SHADOW_FLAG}/x\"\n"
|
|
)),
|
|
NixBuilds::Host
|
|
);
|
|
assert_eq!(
|
|
previous_nix_builds(&format!("OTHER=\"{NIX_DAEMON_SHADOW_FLAG}\"\n")),
|
|
NixBuilds::Host
|
|
);
|
|
}
|
|
|
|
/// `PRIVATE_USERS=` is dropped from the conf whatever the agent now is, so
|
|
/// leaving the list cannot leave the line behind. `AUTO_START=` is the
|
|
/// control: a line that is not ours stays.
|
|
#[test]
|
|
fn the_private_users_line_is_ours_to_drop_and_to_write() {
|
|
assert!(is_owned_var("PRIVATE_USERS=pick"));
|
|
assert!(!is_owned_var("AUTO_START=1"));
|
|
assert_eq!(private_users_line(true), "PRIVATE_USERS=pick\n");
|
|
assert_eq!(private_users_line(false), "");
|
|
}
|
|
|
|
/// The swarm's forge and matrix containers are not the hive's: every
|
|
/// request that names a container as a string refuses them before
|
|
/// touching the filesystem. Control: an agent container name passes.
|
|
#[test]
|
|
fn container_name_requests_refuse_swarm_service_containers() {
|
|
let isolation = NetworkIsolation {
|
|
bridge: "hive0".to_owned(),
|
|
gateway_ip: "10.42.0.1".to_owned(),
|
|
};
|
|
let facts = HiveFacts::default();
|
|
for name in ["hive-forge", "hive-matrix"] {
|
|
let errs = [
|
|
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None, None)
|
|
.unwrap_err(),
|
|
write_resource_limits(name, "4G", "200%", None, None, None).unwrap_err(),
|
|
remove_service_dropin(name).unwrap_err(),
|
|
];
|
|
for e in errs {
|
|
assert!(
|
|
e.to_string().contains("is not managed by hive"),
|
|
"{name}: {e:#}"
|
|
);
|
|
}
|
|
}
|
|
assert_eq!(validate_container_system_name("h-atlas").unwrap(), "atlas");
|
|
}
|
|
|
|
/// A container name that exists nowhere on the host, so a refusal that
|
|
/// comes before every write leaves these paths absent.
|
|
const PROBE: &str = "h-privvalidate";
|
|
|
|
fn probe_paths() -> [String; 2] {
|
|
[
|
|
format!("/etc/nixos-containers/{PROBE}.conf"),
|
|
format!("/run/systemd/system/container@{PROBE}.service.d"),
|
|
]
|
|
}
|
|
|
|
fn assert_nothing_written() {
|
|
for path in probe_paths() {
|
|
assert!(!Path::new(&path).exists(), "{path} was created");
|
|
}
|
|
}
|
|
|
|
fn isolation(bridge: &str, gateway_ip: &str) -> NetworkIsolation {
|
|
NetworkIsolation {
|
|
bridge: bridge.to_owned(),
|
|
gateway_ip: gateway_ip.to_owned(),
|
|
}
|
|
}
|
|
|
|
/// Limits that break out of their drop-in line, or that systemd would
|
|
/// ignore, are refused by name before any file is made. Control: every
|
|
/// shape the c0re validators and the nix defaults produce passes.
|
|
#[test]
|
|
fn resource_limits_refuse_what_is_not_a_size_a_share_or_infinity() {
|
|
for bad in [
|
|
"1G\n[Service]\nExecStartPre=/bin/true",
|
|
"1G\r\nExecStartPre=/bin/true",
|
|
"1G ExecStartPre=/bin/true",
|
|
"1G\0",
|
|
"",
|
|
"G",
|
|
"%",
|
|
".5G",
|
|
"1.G",
|
|
"1.5.5G",
|
|
"-1G",
|
|
"+1G",
|
|
"1e3",
|
|
"8GB",
|
|
"8g",
|
|
"0x10",
|
|
"Infinity",
|
|
"50 %",
|
|
"5%%",
|
|
"5G%",
|
|
"$(id)",
|
|
"%t",
|
|
] {
|
|
let err = write_resource_limits(PROBE, bad, "200%", None, None, None).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("invalid memory_max"),
|
|
"{bad:?}: {err:#}"
|
|
);
|
|
}
|
|
for bad in [
|
|
"400%\nExecStartPre=/bin/true",
|
|
"400% ",
|
|
"400",
|
|
"",
|
|
"%",
|
|
"1e3%",
|
|
"+5%",
|
|
".5%",
|
|
"1.%",
|
|
"infinity",
|
|
"4G",
|
|
"%t",
|
|
] {
|
|
let err = write_resource_limits(PROBE, "4G", bad, None, None, None).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("invalid cpu_quota"),
|
|
"{bad:?}: {err:#}"
|
|
);
|
|
}
|
|
assert_nothing_written();
|
|
|
|
for ok in [
|
|
"8G", "4G", "512M", "4096K", "1T", "1048576", "1048576B", "1.5G", "0.5K", "3P", "2E",
|
|
"50%", "12.34%", "100%", "infinity",
|
|
] {
|
|
validate_memory_max(ok).unwrap_or_else(|e| panic!("{ok:?}: {e:#}"));
|
|
}
|
|
for ok in ["200%", "400%", "100%", "0.5%", "21474836.47%", "1500.25%"] {
|
|
validate_cpu_quota(ok).unwrap_or_else(|e| panic!("{ok:?}: {e:#}"));
|
|
}
|
|
}
|
|
|
|
/// Paths that would add an nspawn argument, or be read by the shell that
|
|
/// expands `EXTRA_NSPAWN_FLAGS`, are refused whichever side or list
|
|
/// carries them, before the conf is touched. Control: plain paths and
|
|
/// the store-path alphabet pass.
|
|
#[test]
|
|
fn bind_paths_refuse_whitespace_control_bytes_and_shell_syntax() {
|
|
let iso = isolation("hive-br0", "10.42.0.1");
|
|
let facts = HiveFacts::default();
|
|
for bad in [
|
|
"/srv/x --bind=/etc:/e",
|
|
"/srv/x\t--bind=/etc",
|
|
"/srv/x\n--bind=/etc",
|
|
"/srv/x\r",
|
|
"/srv/x\x0b",
|
|
"/srv/x\x7f",
|
|
"/srv/x\0",
|
|
"/srv/x\\ y",
|
|
"/srv/$HOME",
|
|
"/srv/${HOME}",
|
|
"/srv/`id`",
|
|
"/srv/\"x",
|
|
"/srv/a:b",
|
|
"srv/x",
|
|
"",
|
|
" /srv/x",
|
|
] {
|
|
let on_host = [bind_from(bad, "/data")];
|
|
let on_container = [bind_from("/srv/x", bad)];
|
|
let credential = [CredentialMount {
|
|
name: "otel-headers".to_owned(),
|
|
host_path: bad.to_owned(),
|
|
}];
|
|
for (binds, credentials) in [
|
|
(&on_host[..], &[][..]),
|
|
(&on_container[..], &[][..]),
|
|
(&[][..], &credential[..]),
|
|
] {
|
|
for nix_builds in [None, Some(NixBuilds::Host), Some(NixBuilds::Disabled)] {
|
|
let err = handle_write_nspawn_flags(
|
|
PROBE,
|
|
binds,
|
|
&iso,
|
|
credentials,
|
|
&facts,
|
|
None,
|
|
nix_builds,
|
|
)
|
|
.unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("invalid bind path"),
|
|
"{bad:?}: {err:#}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
assert_nothing_written();
|
|
|
|
for ok in [
|
|
"/srv/x",
|
|
"/var/lib/hyperhive/agents/atlas/state",
|
|
"/nix/store/abc123-name_1.2+x=y?",
|
|
"/run/hive/secrets/otel-headers",
|
|
"/run/hive-agent/iris",
|
|
"/agents/iris/sockets",
|
|
] {
|
|
validate_bind_path(ok).unwrap_or_else(|e| panic!("{ok:?}: {e:#}"));
|
|
}
|
|
}
|
|
|
|
/// A bridge or gateway that adds a conf assignment, or is not an
|
|
/// interface name or an IPv4 literal, is refused before the conf is
|
|
/// touched. Control: the default bridge, the example, and the gateways
|
|
/// the c0re tests use pass.
|
|
#[test]
|
|
fn network_isolation_refuses_what_is_not_an_interface_name_and_an_ipv4_literal() {
|
|
let facts = HiveFacts::default();
|
|
for (bridge, gateway_ip, want) in [
|
|
("br0\nSYSTEM_PATH=/x", "10.42.0.1", "invalid bridge"),
|
|
("br0 x", "10.42.0.1", "invalid bridge"),
|
|
("br0\r", "10.42.0.1", "invalid bridge"),
|
|
("br$0", "10.42.0.1", "invalid bridge"),
|
|
("br\"0", "10.42.0.1", "invalid bridge"),
|
|
("br0/../x", "10.42.0.1", "invalid bridge"),
|
|
("", "10.42.0.1", "invalid bridge"),
|
|
("sixteen-chars-xx", "10.42.0.1", "invalid bridge"),
|
|
(
|
|
"hive-br0",
|
|
"10.42.0.1\nSYSTEM_PATH=/x",
|
|
"invalid gateway_ip",
|
|
),
|
|
("hive-br0", "10.42.0.1 x", "invalid gateway_ip"),
|
|
("hive-br0", "10.42.0.1/24", "invalid gateway_ip"),
|
|
("hive-br0", "10.42.0", "invalid gateway_ip"),
|
|
("hive-br0", "10.42.0.999", "invalid gateway_ip"),
|
|
("hive-br0", "$(id)", "invalid gateway_ip"),
|
|
("hive-br0", "fd00::1", "invalid gateway_ip"),
|
|
("hive-br0", "", "invalid gateway_ip"),
|
|
] {
|
|
let iso = isolation(bridge, gateway_ip);
|
|
let err =
|
|
handle_write_nspawn_flags(PROBE, &[], &iso, &[], &facts, None, None).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains(want),
|
|
"{bridge:?} {gateway_ip:?}: {err:#}"
|
|
);
|
|
}
|
|
assert_nothing_written();
|
|
|
|
for bridge in ["hive-br0", "h0", "hive0", "br_0.1", "fifteen-chars-x"] {
|
|
validate_bridge(bridge).unwrap_or_else(|e| panic!("{bridge:?}: {e:#}"));
|
|
}
|
|
for gateway_ip in ["10.42.0.1", "10.42.0.254", "172.30.0.1", "192.168.1.1"] {
|
|
validate_gateway_ip(gateway_ip).unwrap_or_else(|e| panic!("{gateway_ip:?}: {e:#}"));
|
|
}
|
|
}
|
|
}
|