vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(#702): drop hive-c0re from root to hive-core user (privsep phase 2)

Browse source
This commit is contained in:
damocles 2026-06-02 22:14:07 +02:00 • committed by mara
commit 0bec3af933
2 changed files with 128 additions and 261 deletions
Download patch file Download diff file Expand all files Collapse all files

360
hive-c0re/src/lifecycle.rs
View file

@ -3,6 +3,7 @@
use std::path::Path; use std::path::Path;
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use hive_sh4re::priv_proto::BindMount;
use tokio::process::Command; use tokio::process::Command;
/// Sub-agent container prefix. `nixos-container` caps the total container name /// Sub-agent container prefix. `nixos-container` caps the total container name
@ -206,12 +207,11 @@ pub async fn spawn(
) )
.await?; .await?;
let container = container_name(name); let container = container_name(name);
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display()); priv_run("create", name).await?;
run(&["create", &container, "--flake", &flake_ref]).await?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_resource_limits(&container).await?;
set_resource_limits(&container)?;
systemd_daemon_reload().await?; systemd_daemon_reload().await?;
run(&["start", &container]).await priv_run("start", name).await
} }
/// Build the `AgentSpec` list for the meta flake from `nixos-container /// Build the `AgentSpec` list for the meta flake from `nixos-container
@ -280,14 +280,12 @@ pub async fn container_exists(name: &str) -> bool {
pub async fn kill(name: &str) -> Result<()> { pub async fn kill(name: &str) -> Result<()> {
validate(name)?; validate(name)?;
let container = container_name(name); priv_run("stop", name).await
run(&["stop", &container]).await
} }
pub async fn start(name: &str) -> Result<()> { pub async fn start(name: &str) -> Result<()> {
validate(name)?; validate(name)?;
let container = container_name(name); priv_run("start", name).await
run(&["start", &container]).await
} }
/// Stop + start without regenerating any config. For "kick the container" /// Stop + start without regenerating any config. For "kick the container"
@ -317,12 +315,12 @@ pub async fn destroy(name: &str) -> Result<()> {
let container = container_name(name); let container = container_name(name);
// nixos-container destroy handles stop + removal of /var/lib/nixos-containers/<C> // nixos-container destroy handles stop + removal of /var/lib/nixos-containers/<C>
// and /etc/nixos-containers/<C>.conf. Tolerate "no such container". // and /etc/nixos-containers/<C>.conf. Tolerate "no such container".
if let Err(e) = run(&["destroy", &container]).await { if let Err(e) = priv_run("destroy", name).await {
tracing::warn!(error = ?e, "nixos-container destroy returned an error; continuing cleanup"); tracing::warn!(error = ?e, "nixos-container destroy returned an error; continuing cleanup");
} }
let dropin_dir = format!("/run/systemd/system/container@{container}.service.d"); // Remove the systemd resource-limits drop-in via hive-priv.
if std::path::Path::new(&dropin_dir).exists() { if let Err(e) = crate::priv_client::remove_service_dropin(&container).await {
std::fs::remove_dir_all(&dropin_dir).with_context(|| format!("remove {dropin_dir}"))?; tracing::warn!(error = ?e, "remove service drop-in failed (non-fatal)");
} }
Ok(()) Ok(())
} }
@ -399,43 +397,43 @@ pub async fn rebuild_no_meta(
// Rebuild strategy: stop-before-update + pre-build. // Rebuild strategy: stop-before-update + pre-build.
// See `docs/coordinator.md::Container lifecycle`. // See `docs/coordinator.md::Container lifecycle`.
let was_running = is_running(name).await; let was_running = is_running(name).await;
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
set_resource_limits(&container)?; set_resource_limits(&container).await?;
systemd_daemon_reload().await?; systemd_daemon_reload().await?;
if was_running { if was_running {
on_step("nix build"); on_step("nix build");
prebuild_toplevel(name, &flake_ref).await?; prebuild_toplevel(name, &flake_ref).await?;
on_step("nixos-container stop"); on_step("nixos-container stop");
run(&["stop", &container]).await?; priv_run("stop", name).await?;
} }
on_step("nixos-container update"); on_step("nixos-container update");
run(&["update", &container, "--flake", &flake_ref]).await?; priv_run("update", name).await?;
if was_running { if was_running {
// Cold-start fallback on activation errors. // Cold-start fallback on activation errors.
// See `docs/coordinator.md::Cold-start fallback`. // See `docs/coordinator.md::Cold-start fallback`.
on_step("nixos-container start"); on_step("nixos-container start");
if let Err(start_err) = run(&["start", &container]).await { if let Err(start_err) = priv_run("start", name).await {
tracing::warn!( tracing::warn!(
container = %container, container = %container,
error = %start_err, error = %start_err,
"start after rebuild failed (possible activation error); \ "start after rebuild failed (possible activation error); \
retrying via stop + kill + start" retrying via stop + kill + start"
); );
run(&["stop", &container]).await.unwrap_or_else(|e| { priv_run("stop", name).await.unwrap_or_else(|e| {
tracing::warn!( tracing::warn!(
container = %container, container = %container,
error = %e, error = %e,
"stop before cold-start retry failed (ignored)" "stop before cold-start retry failed (ignored)"
); );
}); });
run(&["kill", &container]).await.unwrap_or_else(|e| { priv_run("kill", name).await.unwrap_or_else(|e| {
tracing::warn!( tracing::warn!(
container = %container, container = %container,
error = %e, error = %e,
"kill before cold-start retry failed (ignored)" "kill before cold-start retry failed (ignored)"
); );
}); });
run(&["start", &container]).await.map_err(|e| { priv_run("start", name).await.map_err(|e| {
anyhow::anyhow!( anyhow::anyhow!(
"cold-start fallback also failed: {e:#} \ "cold-start fallback also failed: {e:#} \
(original start error: {start_err:#})" (original start error: {start_err:#})"
@ -451,12 +449,12 @@ pub async fn rebuild_no_meta(
// Spawn path: create is atomic, no prebuild needed. // Spawn path: create is atomic, no prebuild needed.
// See `docs/coordinator.md::Spawn path`. // See `docs/coordinator.md::Spawn path`.
on_step("nixos-container create"); on_step("nixos-container create");
run(&["create", &container, "--flake", &flake_ref]).await?; priv_run("create", name).await?;
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
set_resource_limits(&container)?; set_resource_limits(&container).await?;
systemd_daemon_reload().await?; systemd_daemon_reload().await?;
on_step("nixos-container start"); on_step("nixos-container start");
run(&["start", &container]).await priv_run("start", name).await
} }
} }
@ -570,21 +568,8 @@ async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
} }
pub async fn list() -> Result<Vec<String>> { pub async fn list() -> Result<Vec<String>> {
let out = Command::new("nixos-container") let stdout = crate::priv_client::list_containers().await?;
.arg("list") Ok(stdout.lines().map(str::trim)
.output()
.await
.context("invoke nixos-container list")?;
if !out.status.success() {
bail!(
"nixos-container list exited with status {}: {}",
out.status,
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(String::from_utf8_lossy(&out.stdout)
.lines()
.map(str::trim)
.filter(|line| line.starts_with(AGENT_PREFIX)) .filter(|line| line.starts_with(AGENT_PREFIX))
.map(str::to_owned) .map(str::to_owned)
.collect()) .collect())
@ -983,36 +968,12 @@ pub async fn git_update_ref(dir: &Path, refname: &str, target: &str) -> Result<(
/// Write a systemd drop-in for `container@<container>.service` that applies /// Write a systemd drop-in for `container@<container>.service` that applies
/// our default resource caps. Goes under `/run/systemd/system/...` so it's /// our default resource caps. Goes under `/run/systemd/system/...` so it's
/// ephemeral (regenerated on every spawn / rebuild). /// ephemeral (regenerated on every spawn / rebuild).
fn set_resource_limits(container: &str) -> Result<()> { async fn set_resource_limits(container: &str) -> Result<()> {
let dir = format!("/run/systemd/system/container@{container}.service.d"); crate::priv_client::write_resource_limits(container, DEFAULT_MEMORY_MAX, DEFAULT_CPU_QUOTA).await
std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?;
let path = format!("{dir}/hyperhive-limits.conf");
let content =
format!("[Service]\nMemoryMax={DEFAULT_MEMORY_MAX}\nCPUQuota={DEFAULT_CPU_QUOTA}\n");
std::fs::write(&path, content).with_context(|| format!("write {path}"))?;
tracing::info!(
%path,
memory_max = DEFAULT_MEMORY_MAX,
cpu_quota = DEFAULT_CPU_QUOTA,
"wrote resource limits drop-in"
);
Ok(())
} }
async fn systemd_daemon_reload() -> Result<()> { async fn systemd_daemon_reload() -> Result<()> {
let out = Command::new("systemctl") crate::priv_client::daemon_reload().await
.arg("daemon-reload")
.output()
.await
.context("invoke systemctl daemon-reload")?;
if !out.status.success() {
bail!(
"systemctl daemon-reload failed ({}): {}",
out.status,
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(())
} }
/// Idempotently rewrite the lines in `/etc/nixos-containers/<container>.conf` /// Idempotently rewrite the lines in `/etc/nixos-containers/<container>.conf`
@ -1052,35 +1013,28 @@ const HOST_SHARED_ROOT: &str = "/var/lib/hyperhive/shared";
/// `binds`. All three are RW so the parent can read/write state and /// `binds`. All three are RW so the parent can read/write state and
/// submit config-change requests. Creates missing host-side directories /// submit config-change requests. Creates missing host-side directories
/// so nspawn doesn't refuse to start; missing dirs are non-fatal. /// so nspawn doesn't refuse to start; missing dirs are non-fatal.
fn bind_child_agent_dirs(child: &str, binds: &mut String) { fn bind_child_agent_dirs(child: &str, binds: &mut Vec<BindMount>) {
use std::fmt::Write as _;
let state_dir = format!("{HOST_AGENTS_ROOT}/{child}/state"); let state_dir = format!("{HOST_AGENTS_ROOT}/{child}/state");
let harness_dir = format!("{HOST_AGENTS_ROOT}/{child}/harness"); let harness_dir = format!("{HOST_AGENTS_ROOT}/{child}/harness");
let config_dir = format!("{HOST_AGENTS_ROOT}/{child}/config"); let config_dir = format!("{HOST_AGENTS_ROOT}/{child}/config");
for dir in [&state_dir, &harness_dir, &config_dir] { for dir in [&state_dir, &harness_dir, &config_dir] {
let _ = std::fs::create_dir_all(dir); let _ = std::fs::create_dir_all(dir);
} }
let _ = write!(binds, " --bind={state_dir}:/agents/{child}/state"); binds.push(BindMount { host_path: state_dir, container_path: format!("/agents/{child}/state"), read_only: false });
let _ = write!(binds, " --bind={harness_dir}:/agents/{child}/harness"); binds.push(BindMount { host_path: harness_dir, container_path: format!("/agents/{child}/harness"), read_only: false });
let _ = write!(binds, " --bind={config_dir}:/agents/{child}/config"); binds.push(BindMount { host_path: config_dir, container_path: format!("/agents/{child}/config"), read_only: false });
} }
#[allow(clippy::too_many_lines)] async fn set_nspawn_flags(
fn set_nspawn_flags(
container: &str, container: &str,
runtime_dir: &Path, runtime_dir: &Path,
claude_dir: &Path, claude_dir: &Path,
notes_dir: &Path, notes_dir: &Path,
) -> Result<()> { ) -> Result<()> {
use std::fmt::Write as _;
// Ensure /shared directory exists before binding. systemd-nspawn requires the bind source to exist. // Ensure /shared directory exists before binding. systemd-nspawn requires the bind source to exist.
std::fs::create_dir_all(HOST_SHARED_ROOT) std::fs::create_dir_all(HOST_SHARED_ROOT)
.with_context(|| format!("create {HOST_SHARED_ROOT}"))?; .with_context(|| format!("create {HOST_SHARED_ROOT}"))?;
let path = format!("/etc/nixos-containers/{container}.conf");
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
// Logical agent name — strip the `h-` prefix. // Logical agent name — strip the `h-` prefix.
// For the manager: `h-ruth` → `ruth`. For sub-agents: `h-iris` → `iris`. // For the manager: `h-ruth` → `ruth`. For sub-agents: `h-iris` → `iris`.
let agent_name = container.strip_prefix(AGENT_PREFIX).unwrap_or(container); let agent_name = container.strip_prefix(AGENT_PREFIX).unwrap_or(container);
@ -1092,37 +1046,26 @@ fn set_nspawn_flags(
// is needed here — the bind alone is enough. // is needed here — the bind alone is enough.
let claude_mount = container_claude_mount(agent_name); let claude_mount = container_claude_mount(agent_name);
let mut binds = format!( let mut binds: Vec<BindMount> = vec![
"--bind={runtime}:{CONTAINER_RUNTIME_MOUNT} --bind={claude}:{claude_mount} --bind={shared}:{CONTAINER_SHARED_MOUNT}", BindMount { host_path: runtime_dir.to_string_lossy().into_owned(), container_path: CONTAINER_RUNTIME_MOUNT.to_owned(), read_only: false },
runtime = runtime_dir.display(), BindMount { host_path: claude_dir.to_string_lossy().into_owned(), container_path: claude_mount, read_only: false },
claude = claude_dir.display(), BindMount { host_path: HOST_SHARED_ROOT.to_owned(), container_path: CONTAINER_SHARED_MOUNT.to_owned(), read_only: false },
shared = HOST_SHARED_ROOT, ];
);
// Own state, harness, and config dirs — same for every agent including // Own state, harness, and config dirs — same for every agent including
// root. Config is RO: an agent must not edit its own config; changes // the manager. Config is RO: an agent must not edit its own config; changes
// only ever flow through the approval queue. // only ever flow through the approval queue.
{ binds.push(BindMount { host_path: notes_dir.to_string_lossy().into_owned(), container_path: format!("/agents/{agent_name}/state"), read_only: false });
let _ = write!( if let Some(state_parent) = notes_dir.parent() {
binds, let harness_dir = state_parent.join("harness");
" --bind={notes}:/agents/{agent_name}/state", if !harness_dir.exists() {
notes = notes_dir.display(), let _ = std::fs::create_dir_all(&harness_dir);
);
if let Some(state_parent) = notes_dir.parent() {
let harness_dir = state_parent.join("harness");
if !harness_dir.exists() {
let _ = std::fs::create_dir_all(&harness_dir);
}
let _ = write!(
binds,
" --bind={harness}:/agents/{agent_name}/harness",
harness = harness_dir.display(),
);
} }
let own_config = format!("{HOST_AGENTS_ROOT}/{agent_name}/config"); binds.push(BindMount { host_path: harness_dir.to_string_lossy().into_owned(), container_path: format!("/agents/{agent_name}/harness"), read_only: false });
std::fs::create_dir_all(&own_config).with_context(|| format!("create {own_config}"))?;
let _ = write!(binds, " --bind-ro={own_config}:/agents/{agent_name}/config");
} }
let own_config = format!("{HOST_AGENTS_ROOT}/{agent_name}/config");
std::fs::create_dir_all(&own_config).with_context(|| format!("create {own_config}"))?;
binds.push(BindMount { host_path: own_config, container_path: format!("/agents/{agent_name}/config"), read_only: true });
// Topology-driven child mounts: every direct child of this agent gets // Topology-driven child mounts: every direct child of this agent gets
// its state, harness, and config dirs bind-mounted RW so the parent // its state, harness, and config dirs bind-mounted RW so the parent
@ -1153,16 +1096,10 @@ fn set_nspawn_flags(
// fires first (e.g. cold start with no agents). // fires first (e.g. cold start with no agents).
std::fs::create_dir_all(HOST_META_ROOT) std::fs::create_dir_all(HOST_META_ROOT)
.with_context(|| format!("create {HOST_META_ROOT}"))?; .with_context(|| format!("create {HOST_META_ROOT}"))?;
let _ = write!( binds.push(BindMount { host_path: HOST_APPLIED_ROOT.to_owned(), container_path: CONTAINER_MANAGER_APPLIED_MOUNT.to_owned(), read_only: true });
binds, binds.push(BindMount { host_path: HOST_META_ROOT.to_owned(), container_path: crate::meta::CONTAINER_MANAGER_META_MOUNT.to_owned(), read_only: true });
" --bind-ro={HOST_APPLIED_ROOT}:{CONTAINER_MANAGER_APPLIED_MOUNT}",
);
let _ = write!(
binds,
" --bind-ro={HOST_META_ROOT}:{mount}",
mount = crate::meta::CONTAINER_MANAGER_META_MOUNT,
);
} }
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the // Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
// container so the harness can bind `web.sock` there and the host-side // container so the harness can bind `web.sock` there and the host-side
// gateway sees it. Subdir bind (not socket file) keeps the inode // gateway sees it. Subdir bind (not socket file) keeps the inode
@ -1175,159 +1112,74 @@ fn set_nspawn_flags(
// Falls back to 0777 on first spawn when uid lookup returns None // Falls back to 0777 on first spawn when uid lookup returns None
// (container /etc/passwd not yet rendered). // (container /etc/passwd not yet rendered).
if let Some((uid, gid)) = agent_uid_gid(agent_name) { if let Some((uid, gid)) = agent_uid_gid(agent_name) {
std::os::unix::fs::chown(&socket_dir, Some(uid), Some(gid)) if let Err(e) = crate::priv_client::chown_socket_dir(agent_name, uid, gid).await {
.with_context(|| format!("chown {} to {uid}:{gid}", socket_dir.display()))?; tracing::warn!(%agent_name, error = ?e, "chown socket dir failed");
} else { }
use std::os::unix::fs::PermissionsExt; } else if let Err(e) = crate::priv_client::chmod_socket_dir(agent_name, 0o777).await {
std::fs::set_permissions(&socket_dir, std::fs::Permissions::from_mode(0o777)) tracing::warn!(%agent_name, error = ?e, "chmod socket dir failed");
.with_context(|| format!("chmod 0777 {}", socket_dir.display()))?;
} }
let _ = write!( binds.push(BindMount { host_path: socket_dir.to_string_lossy().into_owned(), container_path: socket_dir.to_string_lossy().into_owned(), read_only: false });
binds,
" --bind={socket_dir}:{socket_dir}", // Delegate the actual conf-file rewrite to hive-priv (runs as root).
socket_dir = socket_dir.display(), crate::priv_client::write_nspawn_flags(container, &binds).await
);
let bind_flag = format!("EXTRA_NSPAWN_FLAGS=\"{binds}\"");
let mut lines: Vec<String> = original
.lines()
.filter(|line| {
let trimmed = line.trim_start();
// Strip any network-namespace knobs nixos-container's create
// might have populated. The start script adds `--network-veth`
// whenever HOST_ADDRESS / LOCAL_ADDRESS (or their IPv6 cousins)
// are non-empty — and veth implies a private netns, hiding our
// web-UI port from the host. Force host netns.
!trimmed.starts_with("EXTRA_NSPAWN_FLAGS=")
&& !trimmed.starts_with("PRIVATE_NETWORK=")
&& !trimmed.starts_with("HOST_ADDRESS=")
&& !trimmed.starts_with("LOCAL_ADDRESS=")
&& !trimmed.starts_with("HOST_ADDRESS6=")
&& !trimmed.starts_with("LOCAL_ADDRESS6=")
&& !trimmed.starts_with("HOST_BRIDGE=")
})
.map(str::to_owned)
.collect();
lines.push("PRIVATE_NETWORK=0".to_owned());
lines.push("HOST_ADDRESS=".to_owned());
lines.push("LOCAL_ADDRESS=".to_owned());
lines.push("HOST_ADDRESS6=".to_owned());
lines.push("LOCAL_ADDRESS6=".to_owned());
lines.push("HOST_BRIDGE=".to_owned());
lines.push(bind_flag);
let mut content = lines.join("\n");
content.push('\n');
std::fs::write(&path, content).with_context(|| format!("write {path}"))?;
tracing::info!(%path, "set PRIVATE_NETWORK=0 + EXTRA_NSPAWN_FLAGS");
Ok(())
} }
/// Spawn `nixos-container <args>` and pipe its stdout + stderr into /// Execute a container operation via hive-priv and integrate with
/// `tracing` one line at a time so a long-running command (most /// build_logs.sqlite. hive-priv runs as root and logs output to
/// notably `update`, which kicks off a full nix build that can run /// journald as it arrives; this function captures the final stdout +
/// for minutes on a stale flake) shows progress in journald as it /// stderr into build_logs for the dashboard after the operation
/// happens. The buffered `.output()` we used before only flushed the /// completes. For `create` and `update` (the long-running ops)
/// summary at exit, which made "slow" and "stuck" look identical to /// hive-priv already logs each line to its own journald stream —
/// the operator watching `journalctl -u hive-c0re -f`. /// streaming into build_logs is deferred to a follow-up that adds a
/// /// streaming mode to the priv protocol.
/// stdout lines log at INFO, stderr at WARN. The same lines are async fn priv_run(kind: &str, name: &str) -> Result<()> {
/// captured per-attempt into `build_logs.sqlite` so the dashboard let container = container_name(name);
/// can surface the full stream to the operator; on failure we bail let cmdline = format!("nixos-container {kind} {container}");
/// with a `see build log #<id>` pointer instead of the legacy
/// 32-line ring-buffer tail that routinely truncated eval errors.
async fn run(args: &[&str]) -> Result<()> {
use tokio::io::{AsyncBufReadExt, BufReader};
let cmdline = args.join(" ");
// Convention: `nixos-container <verb> <container> ...` — the
// verb is `args[0]` (kind) and the container is `args[1]`
// (h-<name> | root | hive-matrix | ...) for every long-running
// case we care about. Strip the `h-` prefix for sub-agents so the
// build_logs row's `agent` column matches the agent's bare name
// (`alice` rather than `h-alice`) — that's what the dashboard
// groups by. Manager + sibling containers pass through as-is.
let kind = args.first().copied().unwrap_or("nixos-container");
let agent = args
.get(1)
.copied()
.map_or_else(|| "<unknown>".to_string(), |c| c.strip_prefix(AGENT_PREFIX).unwrap_or(c).to_string());
let logs = crate::build_logs::global(); let logs = crate::build_logs::global();
let log_id = logs.as_ref().and_then(|h| { let log_id = logs.as_ref().and_then(|h| {
h.start(&agent, kind, &cmdline) h.start(name, kind, &cmdline)
.map_err(|e| { .map_err(|e| {
tracing::warn!(error = ?e, "build_logs: start failed (nixos-container log dropped)"); tracing::warn!(error = ?e, "build_logs: start failed (priv_run log dropped)");
}) })
.ok() .ok()
}); });
let mut child = Command::new("nixos-container") let result: Result<(String, String)> = match kind {
.args(args) "create" => crate::priv_client::create_container(name).await,
.stdout(std::process::Stdio::piped()) "update" => crate::priv_client::update_container(name).await,
.stderr(std::process::Stdio::piped()) "start" => crate::priv_client::start_container(name).await.map(|()| (String::new(), String::new())),
.spawn() "stop" => crate::priv_client::stop_container(name).await.map(|()| (String::new(), String::new())),
.with_context(|| format!("invoke nixos-container {cmdline}"))?; "kill" => crate::priv_client::kill_container(name).await.map(|()| (String::new(), String::new())),
"destroy" => crate::priv_client::destroy_container(name).await.map(|()| (String::new(), String::new())),
other => Err(anyhow::anyhow!("unknown container op: {other}")),
};
let stdout = child.stdout.take().expect("piped stdout"); let ok = result.is_ok();
let stderr = child.stderr.take().expect("piped stderr");
let stdout_cmdline = cmdline.clone();
let stdout_logs = logs.clone();
let pump_stdout = tokio::spawn(async move {
let mut lines = BufReader::new(stdout).lines();
while let Ok(Some(line)) = lines.next_line().await {
tracing::info!(target: "nixos-container", cmdline = %stdout_cmdline, "{line}");
if let (Some(h), Some(id)) = (&stdout_logs, log_id) {
h.append_stdout(id, &line);
}
}
});
let stderr_cmdline = cmdline.clone();
let stderr_logs = logs.clone();
let pump_stderr = tokio::spawn(async move {
let mut lines = BufReader::new(stderr).lines();
while let Ok(Some(line)) = lines.next_line().await {
tracing::warn!(target: "nixos-container", cmdline = %stderr_cmdline, "{line}");
if let (Some(h), Some(id)) = (&stderr_logs, log_id) {
h.append_stderr(id, &line);
}
}
});
let status = child
.wait()
.await
.with_context(|| format!("wait nixos-container {cmdline}"))?;
let _ = pump_stdout.await;
let _ = pump_stderr.await;
let ok = status.success();
if let (Some(h), Some(id)) = (&logs, log_id) { if let (Some(h), Some(id)) = (&logs, log_id) {
h.finish( if let Ok((ref stdout, ref stderr)) = result {
id, for line in stdout.lines() {
if ok { tracing::info!(target: "nixos-container", cmdline = %cmdline, "{line}");
crate::build_logs::BuildStatus::Ok h.append_stdout(id, line);
} else { }
crate::build_logs::BuildStatus::Fail for line in stderr.lines() {
}, tracing::warn!(target: "nixos-container", cmdline = %cmdline, "{line}");
); h.append_stderr(id, line);
}
}
h.finish(id, if ok { crate::build_logs::BuildStatus::Ok } else { crate::build_logs::BuildStatus::Fail });
} }
if !ok {
// `container_journal_tail` is best-effort + only fires on match result {
// `update`; the captured build log holds the full host-side Ok(_) => Ok(()),
// stderr regardless, so the bail message can stay terse: a Err(e) => {
// pointer to the log id + the journal tail (when available) let journal = if kind == "update" { container_journal_tail(&container).await } else { String::new() };
// is enough for the operator to drill in without flooding match log_id {
// every notification with the eval-error verbatim. Some(id) => bail!("{e:#}; see build log #{id}{journal}"),
let journal = container_journal_tail(args).await; None => bail!("{e:#}{journal}"),
match log_id {
Some(id) => {
bail!("nixos-container {cmdline} failed ({status}); see build log #{id}{journal}")
} }
None => bail!("nixos-container {cmdline} failed ({status}){journal}"),
} }
} }
Ok(())
} }
/// On a failed `nixos-container update`, the stderr nixos-container /// On a failed `nixos-container update`, the stderr nixos-container
@ -1342,13 +1194,7 @@ async fn run(args: &[&str]) -> Result<()> {
/// `journalctl -M` works. Best-effort — returns "" for other verbs /// `journalctl -M` works. Best-effort — returns "" for other verbs
/// or when the journal can't be read (machine gone, journalctl /// or when the journal can't be read (machine gone, journalctl
/// missing); it never produces an error of its own. /// missing); it never produces an error of its own.
async fn container_journal_tail(args: &[&str]) -> String { async fn container_journal_tail(container: &str) -> String {
if args.first().copied() != Some("update") {
return String::new();
}
let Some(container) = args.get(1) else {
return String::new();
};
let out = Command::new("journalctl") let out = Command::new("journalctl")
.args(["-M", container, "-n", "40", "--no-pager", "--output=short"]) .args(["-M", container, "-n", "40", "--no-pager", "--output=short"])
.output() .output()

33
nix/modules/hive-c0re.nix
View file

@ -315,6 +315,17 @@ in
managerToplevel managerToplevel
]; ];
# Unprivileged coordinator user. hive-c0re runs as this user
# (privsep phase 2); privileged operations are delegated to
# hive-priv which runs as root, socket-activated at
# /run/hive/priv.sock.
users.users.hive-core = {
isSystemUser = true;
group = "hive-core";
description = "hive-c0re coordinator daemon user";
};
users.groups.hive-core = { };
# Open the per-agent web-port range when the gateway is *off* — # Open the per-agent web-port range when the gateway is *off* —
# otherwise the gateway nginx is the sole external entry point. # otherwise the gateway nginx is the sole external entry point.
# See `docs/gateway.md::Firewall posture (host-level)`. # See `docs/gateway.md::Firewall posture (host-level)`.
@ -412,12 +423,22 @@ in
}; };
serviceConfig = { serviceConfig = {
ExecStart = "${cfg.package}/bin/hive-c0re --socket /run/hyperhive/host.sock serve --hyperhive-flake ${cfg.hyperhiveFlake} --nixpkgs-flake ${cfg.nixpkgsFlake} --nixpkgs-unstable-flake ${cfg.nixpkgsUnstableFlake} --dashboard-port ${toString cfg.dashboardPort} --operator-pronouns ${lib.escapeShellArg cfg.operatorPronouns} --context-window-tokens ${lib.escapeShellArg (builtins.toJSON cfg.contextWindowTokens)}"; ExecStart = "${cfg.package}/bin/hive-c0re --socket /run/hyperhive/host.sock serve --hyperhive-flake ${cfg.hyperhiveFlake} --nixpkgs-flake ${cfg.nixpkgsFlake} --nixpkgs-unstable-flake ${cfg.nixpkgsUnstableFlake} --dashboard-port ${toString cfg.dashboardPort} --operator-pronouns ${lib.escapeShellArg cfg.operatorPronouns} --context-window-tokens ${lib.escapeShellArg (builtins.toJSON cfg.contextWindowTokens)}";
# One-time migration: chown existing state tree to the service
# user after upgrading from a root-run install. The `+` prefix
# runs this step as root even though User = hive-core. systemd's
# StateDirectory chowns the top-level dir at every start, but
# pre-existing files inside may still be root-owned.
ExecStartPre = "+${pkgs.coreutils}/bin/sh -c 'chown -R hive-core:hive-core /var/lib/hyperhive || true'";
Restart = "on-failure"; Restart = "on-failure";
RestartSec = 2; RestartSec = 2;
User = "hive-core";
Group = "hive-core";
SupplementaryGroups = [ "systemd-journal" ];
RuntimeDirectory = "hyperhive"; RuntimeDirectory = "hyperhive";
RuntimeDirectoryMode = "0750"; RuntimeDirectoryMode = "0750";
RuntimeDirectoryPreserve = "yes"; RuntimeDirectoryPreserve = "yes";
StateDirectory = "hyperhive"; StateDirectory = "hyperhive";
StateDirectoryMode = "0750";
}; };
}; };
@ -451,18 +472,18 @@ in
# privileged operations on behalf of hive-c0re. Systemd creates and # privileged operations on behalf of hive-c0re. Systemd creates and
# holds `/run/hive/priv.sock` before the first connection arrives. # holds `/run/hive/priv.sock` before the first connection arrives.
# #
# Mode 0660 root:root is correct for phase 1 (hive-c0re still runs as # Mode 0660 hive-core:hive-core: only the hive-c0re service user can
# root and is the only caller). Phase 2 (privsep: hive-c0re drops to a # connect. hive-priv (server) runs as root and validates every request
# non-root user) will add `SocketGroup = hive-core` so the unprivileged # against a strict allowlist before executing any privileged op.
# hive-c0re process can still connect.
systemd.sockets.hive-priv = { systemd.sockets.hive-priv = {
description = "hive-priv privileged helper socket"; description = "hive-priv privileged helper socket";
wantedBy = [ "sockets.target" ]; wantedBy = [ "sockets.target" ];
socketConfig = { socketConfig = {
ListenStream = "/run/hive/priv.sock"; ListenStream = "/run/hive/priv.sock";
SocketMode = "0660"; SocketMode = "0660";
# Create /run/hive/ if absent; 0755 so future unprivileged callers SocketGroup = "hive-core";
# can traverse into it to reach the socket. # Create /run/hive/ if absent; 0755 so the hive-core user can
# traverse into it to reach the socket.
DirectoryMode = "0755"; DirectoryMode = "0755";
}; };
}; };