feat(#2290): maintain /etc/tmpfiles.d/hyperhive-agents.conf for boot safety

Root cause of the boot outage: container@h-* units try to start before
hive-c0re reaches ensure_runtime, so bind-mount source dirs are missing.

Fix: hive-c0re (via hive-priv, which runs as root) writes
/etc/tmpfiles.d/hyperhive-agents.conf whenever the agent set changes.
systemd-tmpfiles-setup.service (sysinit.target) reads it at every boot
BEFORE any container units start, pre-creating:

  /run/hyperhive/agents/<name>  — MCP socket dir (bind -> /run/hive)
  /run/hive-agent/<name>        — web socket dir (bind -> /run/hive-agent)

This alone removes the outage class: even if hive-c0re is slow to start,
the bind-mount sources exist and container units can activate.

Added:
- PrivRequest::SyncAgentTmpfiles { agents } in hive-sh4re
- sync_agent_tmpfiles() in hive-priv: generates content, writes atomically,
  calls systemd-tmpfiles --create to apply immediately
- priv_client::sync_agent_tmpfiles() wrapper
- lifecycle::sync_tmpfiles() best-effort helper (list + priv call)
- Call sites: hive-c0re startup, handle_spawn success, destroy success
This commit is contained in:
atlas 2026-07-08 22:59:04 +02:00 committed by mara
commit 9d1f5ebe76
7 changed files with 128 additions and 0 deletions

View file

@ -967,6 +967,9 @@ pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Resul
// roster, so any schedule that still targets the just-destroyed agent
// now drops that ghost column live (no page reload needed).
coord.emit_schedules_snapshot();
// Update tmpfiles.d to remove the destroyed agent's dirs from the
// boot-time pre-creation list. Best-effort: failure is logged only.
tokio::spawn(async { lifecycle::sync_tmpfiles().await });
Ok(())
}

View file

@ -722,6 +722,33 @@ pub async fn list() -> Result<Vec<String>> {
.collect())
}
/// Sync `/etc/tmpfiles.d/hyperhive-agents.conf` with the currently-known
/// agent set (from `nixos-container list`). Strips the `h-` prefix to get
/// logical names. Best-effort: errors are logged but never propagated — a
/// failed tmpfiles write shouldn't block a spawn or destroy.
///
/// Called at hive-c0re startup and after each spawn / destroy so the file
/// always reflects the live agent set. `systemd-tmpfiles-setup.service`
/// reads the file at boot (before any container units start), pre-creating
/// bind-mount source dirs so container@h-* units don't race hive-c0re.
pub async fn sync_tmpfiles() {
let agents = match list().await {
Ok(containers) => containers
.into_iter()
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
.collect::<Vec<_>>(),
Err(e) => {
tracing::warn!(error = ?e, "sync_tmpfiles: list failed; skipping");
return;
}
};
if let Err(e) = crate::priv_client::sync_agent_tmpfiles(&agents).await {
tracing::warn!(error = ?e, "sync_tmpfiles: priv call failed");
} else {
tracing::debug!(count = agents.len(), "sync_tmpfiles: ok");
}
}
/// Build the per-line callback for `create_container_streaming` /
/// `update_container_streaming`. Both ops share identical dispatch logic
/// (stdout → info + `append_stdout`, stderr → warn + `append_stderr`); this

View file

@ -283,6 +283,12 @@ async fn cmd_serve(
if let Err(e) = auto_update::ensure_root_agent(&coord).await {
tracing::warn!(error = ?e, "auto-spawn root agent failed");
}
// Sync /etc/tmpfiles.d/hyperhive-agents.conf so agent runtime dirs are
// pre-declared for the next boot. Best-effort background task — a failure
// here must not block hive-c0re startup. See lifecycle::sync_tmpfiles.
tokio::spawn(async {
hive_c0re::lifecycle::sync_tmpfiles().await;
});
// Auto-update in the background — don't block service start.
// Sub-agent rebuilds can take tens of seconds; we want the admin
// socket up immediately.

View file

@ -389,6 +389,16 @@ pub async fn upgrade_agent_subvolume(agent_name: &str) -> Result<()> {
.await?)
}
/// Write `/etc/tmpfiles.d/hyperhive-agents.conf` for `agents` (logical names,
/// e.g. `"atlas"`) and immediately apply it with `systemd-tmpfiles --create`.
/// See [`PrivRequest::SyncAgentTmpfiles`] for the full semantics.
pub async fn sync_agent_tmpfiles(agents: &[String]) -> Result<()> {
ok(call(&PrivRequest::SyncAgentTmpfiles {
agents: agents.to_vec(),
})
.await?)
}
/// Parse `(referenced, exclusive)` bytes from `btrfs qgroup show -f --raw`
/// output (a qgroup row is `<id-with-slash> <rfer> <excl> …`).
///

View file

@ -216,6 +216,8 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
note: None,
sha: None,
});
// Update tmpfiles.d so the new agent's dirs survive a reboot.
tokio::spawn(async { lifecycle::sync_tmpfiles().await });
}
Err(e) => {
// Roll back socket registration if container creation failed.

View file

@ -326,6 +326,10 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
validate_agent_name(agent_name)?;
upgrade_agent_subvolume(agent_name).await
}
PrivRequest::SyncAgentTmpfiles { ref agents } => {
sync_agent_tmpfiles(agents).await
}
}
}
@ -1493,3 +1497,64 @@ fn write_bridge_dns_marker(container: &str, isolation: Option<&NetworkIsolation>
}
Ok(())
}
/// `SyncAgentTmpfiles` — write `/etc/tmpfiles.d/hyperhive-agents.conf` for
/// the given agent set and immediately apply it with `systemd-tmpfiles --create`.
///
/// Each call atomically replaces the file with entries for all current agents,
/// then creates any missing dirs on the running host. The file survives reboots
/// and is read by `systemd-tmpfiles-setup.service` (runs in `sysinit.target`,
/// before any container units can start), so bind-mount source dirs are always
/// pre-created regardless of whether hive-c0re has reached `ensure_runtime`.
///
/// Directories written per agent:
/// - `/run/hyperhive/agents/<name>` (MCP socket dir, bind-mounted into container
/// as `/run/hive`)
/// - `/run/hive-agent/<name>` (web socket dir, bind-mounted into container)
const TMPFILES_PATH: &str = "/etc/tmpfiles.d/hyperhive-agents.conf";
const AGENT_RUNTIME_ROOT: &str = "/run/hyperhive/agents";
async fn sync_agent_tmpfiles(agents: &[String]) -> Result<(String, String)> {
for name in agents {
validate_agent_name(name)?;
}
// Build tmpfiles.d content. Root dirs first, then per-agent.
let mut content = String::from(
"# managed by hive-c0re — do not edit (regenerated on spawn/destroy)\n",
);
// Parent dirs — created with permissive mode so hive-c0re can make subdirs.
// /run/hyperhive itself is also a RuntimeDirectory of hive-c0re.service; the
// tmpfiles.d entry here ensures it exists before hive-c0re starts (boot race).
content.push_str("d /run/hyperhive 0750 hive-core hive-core -\n");
content.push_str(&format!("d {AGENT_RUNTIME_ROOT} 0755 hive-core hive-core -\n"));
content.push_str(&format!("d {SOCKET_DIR_ROOT} 0755 root root -\n"));
// Per-agent dirs.
for name in agents {
content.push_str(&format!("d {AGENT_RUNTIME_ROOT}/{name} 0755 hive-core hive-core -\n"));
content.push_str(&format!("d {SOCKET_DIR_ROOT}/{name} 0755 root root -\n"));
}
// Atomic write: write to a tmp file then rename so a concurrent reader
// always sees a complete file.
let tmp = format!("{TMPFILES_PATH}.tmp");
std::fs::write(&tmp, &content).with_context(|| format!("write {tmp}"))?;
std::fs::rename(&tmp, TMPFILES_PATH)
.with_context(|| format!("rename {TMPFILES_PATH}.tmp -> {TMPFILES_PATH}"))?;
tracing::info!(agents = agents.len(), "tmpfiles.d: wrote {TMPFILES_PATH}");
// Apply immediately so dirs exist on the running host, not just after next boot.
let out = Command::new("systemd-tmpfiles")
.args(["--create", TMPFILES_PATH])
.output()
.await
.context("systemd-tmpfiles --create")?;
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr).trim().to_owned();
anyhow::bail!(
"systemd-tmpfiles --create failed ({}): {stderr}",
out.status
);
}
Ok((String::new(), String::new()))
}

View file

@ -512,6 +512,21 @@ pub enum PrivRequest {
/// Logical agent name (validated by `validate_agent_name`).
agent_name: String,
},
/// Write `/etc/tmpfiles.d/hyperhive-agents.conf` for the given agent set
/// and immediately apply it with `systemd-tmpfiles --create`. Each entry
/// declares the per-agent runtime dirs (`/run/hyperhive/agents/<name>` and
/// `/run/hive-agent/<name>`) so systemd recreates them at every boot before
/// any container units start — preventing bind-mount source missing errors
/// when container@h-* units race hive-c0re after a reboot.
///
/// Called at hive-c0re startup and after every agent spawn / destroy.
/// Agents are logical names (validated by `validate_agent_name`).
SyncAgentTmpfiles {
/// Logical agent names (e.g. `"atlas"`, `"ruth"`). hive-priv validates
/// each name before writing any path component derived from it.
agents: Vec<String>,
},
}
/// Response from the privileged helper.