mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

remove the hive's local build of agent closures

Browse source 
A rebuild or create downloads the closure the swarm built and applies it; with none to deploy it fails and changes nothing. hive-priv takes a store path instead of an internal/meta commit and builds nothing.
This commit is contained in:
atlas 2026-10-29 22:17:07 +01:00
commit 4590b04cac
2 changed files with 632 additions and 1351 deletions
Download patch file Download diff file Expand all files Collapse all files

233
hive-c0re/src/lifecycle/mod.rs
View file

@ -234,9 +234,9 @@ pub async fn provision_container(name: &str, paths: &AgentPaths) -> Result<()> {
/// The `nixos-container create` proper — the job queue's `Create` node.
/// Assumes `provision_container` already registered the agent.
/// `meta_rev` is the `internal/meta` commit to build from (`crate::meta_rev`).
pub async fn create_only(name: &str, meta_rev: &str) -> Result<()> {
priv_run_inner("create", name, Some(meta_rev), None).await
/// `toplevel` is the store path of the system closure to create it from.
pub async fn create_only(name: &str, toplevel: &str) -> Result<()> {
priv_run_inner("create", name, Some(toplevel), None).await
}
/// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on
@ -285,20 +285,20 @@ pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()>
}
/// Profile-swap for an existing, stopped container: re-apply the
/// drop-ins, then `nixos-container update`. The job queue's `Swap`
/// node. Requires the container stopped (the queue's `StopForUpdate`
/// upstream); does NOT start it — the DAG's tail `Reconcile` owns
/// bringing the agent back to its wanted power state. `meta_rev` is as for
/// drop-ins, then point the container at the closure `toplevel`. The job
/// queue's `Swap` node. Requires the container stopped (the queue's
/// `StopForUpdate` upstream); does NOT start it — the DAG's tail `Reconcile` owns
/// bringing the agent back to its wanted power state. `toplevel` is as for
/// [`create_only`].
pub async fn swap_update(
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
meta_rev: &str,
toplevel: &str,
node_id: Option<u64>,
) -> Result<()> {
write_dropins(name, hive, paths).await?;
priv_run_inner("update", name, Some(meta_rev), node_id).await
priv_run_inner("update", name, Some(toplevel), node_id).await
}
/// Build the `AgentSpec` list for [`crate::agent_set::refresh`] from
@ -720,195 +720,6 @@ fn confirm_gone_after_failed_destroy(
}
}
/// Wall-clock bound on [`prebuild_toplevel`]'s `nix build`. Four times CI's
/// observed cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`),
/// which compiles this workspace's crates, the part of an agent toplevel no
/// binary cache holds. A stalled download is bounded by nix's own
/// `stalled-download-timeout`; this covers the rest, such as a wedged
/// nix-daemon holding the rebuild's build slot forever.
const PREBUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1);
/// Wait for `child` until `limit`. `None` means it was still running, and its
/// whole process group was `SIGKILL`ed and the child reaped, so a grandchild
/// holding the daemon connection dies with it. The child must have been
/// spawned with `process_group(0)`.
async fn wait_bounded(
child: &mut tokio::process::Child,
limit: std::time::Duration,
) -> std::io::Result<Option<std::process::ExitStatus>> {
// Group leader, so its pid is the pgid. Read before waiting: a reaped
// child has no id.
let pgid = child.id().and_then(|pid| i32::try_from(pid).ok());
if let Ok(status) = tokio::time::timeout(limit, child.wait()).await {
return status.map(Some);
}
if let Some(pgid) = pgid {
// SAFETY: plain `kill(2)`; `-pgid` targets the child's own process
// group, which `process_group(0)` made it lead.
unsafe {
libc::kill(-pgid, libc::SIGKILL);
}
}
child.wait().await?;
Ok(None)
}
/// Log and record a prebuild's stdout; the last non-empty line is the
/// toplevel `nix build --print-out-paths` printed.
async fn pump_prebuild_stdout(
stdout: tokio::process::ChildStdout,
cmdline: String,
logs: Option<std::sync::Arc<crate::build_logs::BuildLogs>>,
log_id: Option<i64>,
) -> Option<String> {
use tokio::io::{AsyncBufReadExt, BufReader};
let mut lines = BufReader::new(stdout).lines();
let mut out_path = None;
while let Ok(Some(line)) = lines.next_line().await {
tracing::info!(target: "nix-prebuild", cmdline = %cmdline, "{line}");
if !line.trim().is_empty() {
out_path = Some(line.trim().to_owned());
}
if let (Some(h), Some(id)) = (&logs, log_id) {
h.append_stdout(id, &line);
}
}
out_path
}
/// Pre-build `system.build.toplevel` against `meta#<name>` so the
/// subsequent `nixos-container update` finds the result cached and
/// skips straight to the profile-swap. Store-warming only — container
/// is untouched. See `docs/scheduler/coordinator.md::Rebuild path` for why
/// the prebuild happens before stop, and `docs/scheduler/coordinator.md::Prebuild
/// attr path` for why the explicit nixosConfigurations attr is required.
///
/// `node_id` is the queue node this build belongs to, when there is one —
/// it is stored on the `build_logs` row so the dashboard can find the log
/// from the node. Pass `None` for builds that run outside the queue.
///
/// Returns the toplevel `nix build` printed, `None` if it printed nothing.
pub async fn prebuild_toplevel(
name: &str,
flake_ref: &str,
node_id: Option<u64>,
) -> Result<Option<String>> {
use tokio::io::{AsyncBufReadExt, BufReader};
// Split `<root>#<name>` so we can re-emit with the explicit
// `nixosConfigurations.<name>` segment. The flake_ref shape is
// constructed by the caller and always contains exactly one
// `#`; `split_once` returning None here would be a programmer
// error we'd want to surface loudly rather than paper over.
let (flake_root, fragment) = flake_ref
.split_once('#')
.with_context(|| format!("flake_ref {flake_ref:?} missing '#<name>' fragment"))?;
// Sanity-check the fragment matches the agent name we were
// passed — guards against future calls that pass a divergent
// pair (no current callsite does, but the pair is redundant
// and worth checking once).
if fragment != name {
anyhow::bail!("prebuild_toplevel: flake_ref fragment '{fragment}' ≠ agent name '{name}'");
}
let attr = format!("{flake_root}#nixosConfigurations.{name}.config.system.build.toplevel");
let args = vec![
"--extra-experimental-features",
"nix-command flakes",
"build",
"--no-link",
"--print-out-paths",
&attr,
];
let cmdline = format!("nix {}", args.join(" "));
tracing::info!(%name, %cmdline, "prebuild: warming system toplevel");
// Open a build_logs row for this attempt (best-effort — None when
// the global handle hasn't been installed, e.g. early startup
// or standalone tests). Lines pumped from stdout/stderr append
// into the row; `finish` lands the terminal status before we bail.
let logs = crate::build_logs::global();
let log_id = logs.as_ref().and_then(|h| {
h.start(name, "prebuild", &cmdline, node_id)
.map_err(|e| {
tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)");
})
.ok()
});
let mut child = Command::new("nix")
.args(&args)
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.process_group(0)
.spawn()
.with_context(|| format!("spawn {cmdline}"))?;
let stdout = child.stdout.take().expect("piped stdout");
let stderr = child.stderr.take().expect("piped stderr");
let pump_stdout = tokio::spawn(pump_prebuild_stdout(
stdout,
cmdline.clone(),
logs.clone(),
log_id,
));
let stderr_cmdline = cmdline.clone();
let stderr_logs = logs.clone();
let pump_stderr = tokio::spawn(async move {
let mut lines = BufReader::new(stderr).lines();
while let Ok(Some(line)) = lines.next_line().await {
if is_actionable_nix_line(&line) {
tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}");
} else {
tracing::info!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}");
}
if let (Some(h), Some(id)) = (&stderr_logs, log_id) {
h.append_stderr(id, &line);
}
}
});
let Some(status) = wait_bounded(&mut child, PREBUILD_TIMEOUT)
.await
.with_context(|| format!("wait {cmdline}"))?
else {
// A process that left the group can still hold the pipes open.
pump_stdout.abort();
pump_stderr.abort();
if let (Some(h), Some(id)) = (&logs, log_id) {
h.finish(id, crate::build_logs::BuildStatus::Fail);
}
let secs = PREBUILD_TIMEOUT.as_secs();
match log_id {
Some(id) => {
bail!("prebuild {cmdline} timed out after {secs}s; killed it; see build log #{id}")
}
None => bail!("prebuild {cmdline} timed out after {secs}s; killed it"),
}
};
let out_path = pump_stdout.await.ok().flatten();
let _ = pump_stderr.await;
let ok = status.success();
if let (Some(h), Some(id)) = (&logs, log_id) {
h.finish(
id,
if ok {
crate::build_logs::BuildStatus::Ok
} else {
crate::build_logs::BuildStatus::Fail
},
);
}
if !ok {
match log_id {
Some(id) => bail!("prebuild {cmdline} failed ({status}); see build log #{id}"),
None => bail!("prebuild {cmdline} failed ({status})"),
}
}
Ok(out_path)
}
pub async fn list() -> Result<Vec<String>> {
let stdout = crate::priv_client::list_containers().await?;
Ok(stdout
@ -990,11 +801,11 @@ fn make_log_callback(
}
}
/// A `create` or `update` asked for without an `internal/meta` commit. The
/// public entry points take the commit, so this guards the string dispatch
/// A `create` or `update` asked for without a closure. The
/// public entry points take the closure, so this guards the string dispatch
/// below only.
fn missing_meta_rev(kind: &str, name: &str) -> anyhow::Error {
anyhow::anyhow!("{name}: {kind} needs an internal/meta commit to build from")
fn missing_toplevel(kind: &str, name: &str) -> anyhow::Error {
anyhow::anyhow!("{name}: {kind} needs a closure to apply")
}
/// Execute a container operation via hive-priv and integrate with
@ -1016,7 +827,7 @@ async fn priv_run(kind: &str, name: &str) -> Result<()> {
async fn priv_run_inner(
kind: &str,
name: &str,
meta_rev: Option<&str>,
toplevel: Option<&str>,
node_id: Option<u64>,
) -> Result<()> {
let container = container_name(name);
@ -1034,27 +845,27 @@ async fn priv_run_inner(
// For long-running ops use the streaming protocol so build_logs
// receives lines in real time rather than as a batch at completion.
let result: Result<()> = match kind {
"create" => match meta_rev {
Some(rev) => {
"create" => match toplevel {
Some(toplevel) => {
crate::priv_client::create_container_streaming(
name,
rev,
toplevel,
make_log_callback(logs.clone(), log_id, cmdline.clone()),
)
.await
}
None => Err(missing_meta_rev(kind, name)),
None => Err(missing_toplevel(kind, name)),
},
"update" => match meta_rev {
Some(rev) => {
"update" => match toplevel {
Some(toplevel) => {
crate::priv_client::update_container_streaming(
name,
rev,
toplevel,
make_log_callback(logs.clone(), log_id, cmdline.clone()),
)
.await
}
None => Err(missing_meta_rev(kind, name)),
None => Err(missing_toplevel(kind, name)),
},
"start" => crate::priv_client::start_container(name).await,
"stop" => crate::priv_client::stop_container(name).await,

62
hive-c0re/src/lifecycle/tests.rs
View file

@ -349,68 +349,6 @@ fn agent_names_propagates_an_unreadable_list() {
assert!(format!("{err:#}").contains("hive-priv"), "{err:#}");
}
fn is_dead(pid: &str) -> bool {
// A killed process that is not our child stays a zombie until its own
// parent reaps it, so a zombie counts as dead.
match std::fs::read_to_string(format!("/proc/{pid}/stat")) {
Err(_) => true,
Ok(stat) => stat
.rsplit_once(") ")
.is_some_and(|(_, rest)| rest.starts_with('Z')),
}
}
/// A child still running at the limit comes back `None`, and the kill reaches
/// its whole process group: the backgrounded grandchild dies too, not just the
/// shell that started it.
#[tokio::test]
async fn a_prebuild_past_its_limit_is_killed_with_its_group() {
use std::time::{Duration, Instant};
let dir = tempfile::tempdir().expect("tempdir");
let pidfile = dir.path().join("grandchild");
let mut child = Command::new("sh")
.arg("-c")
.arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display()))
.process_group(0)
.spawn()
.expect("spawn sh");
let started = Instant::now();
let status = wait_bounded(&mut child, Duration::from_secs(2))
.await
.expect("wait_bounded");
assert!(status.is_none(), "must time out, got {status:?}");
assert!(
started.elapsed() < Duration::from_mins(1),
"the limit did not bound the wait"
);
let pid = std::fs::read_to_string(&pidfile).expect("grandchild pid");
let pid = pid.trim();
let deadline = Instant::now() + Duration::from_secs(5);
while !is_dead(pid) && Instant::now() < deadline {
tokio::time::sleep(Duration::from_millis(50)).await;
}
assert!(is_dead(pid), "grandchild {pid} outlived the timeout");
}
/// The control: a child that exits inside its limit comes back with its
/// status.
#[tokio::test]
async fn a_prebuild_inside_its_limit_returns_its_status() {
let mut child = Command::new("sh")
.arg("-c")
.arg("exit 3")
.process_group(0)
.spawn()
.expect("spawn sh");
let status = wait_bounded(&mut child, std::time::Duration::from_mins(1))
.await
.expect("wait_bounded")
.expect("a fast child must not time out");
assert_eq!(status.code(), Some(3));
}
/// The setup path creates the agent's `work/` dir beside `state/`, so the
/// bind source exists before nspawn starts the container.
#[test]