feat(#2290): maintain /etc/tmpfiles.d/hyperhive-agents.conf for boot safety
Root cause of the boot outage: container@h-* units try to start before
hive-c0re reaches ensure_runtime, so bind-mount source dirs are missing.
Fix: hive-c0re (via hive-priv, which runs as root) writes
/etc/tmpfiles.d/hyperhive-agents.conf whenever the agent set changes.
systemd-tmpfiles-setup.service (sysinit.target) reads it at every boot
BEFORE any container units start, pre-creating:
/run/hyperhive/agents/<name> — MCP socket dir (bind -> /run/hive)
/run/hive-agent/<name> — web socket dir (bind -> /run/hive-agent)
This alone removes the outage class: even if hive-c0re is slow to start,
the bind-mount sources exist and container units can activate.
Added:
- PrivRequest::SyncAgentTmpfiles { agents } in hive-sh4re
- sync_agent_tmpfiles() in hive-priv: generates content, writes atomically,
calls systemd-tmpfiles --create to apply immediately
- priv_client::sync_agent_tmpfiles() wrapper
- lifecycle::sync_tmpfiles() best-effort helper (list + priv call)
- Call sites: hive-c0re startup, handle_spawn success, destroy success
This commit is contained in:
parent
9cd408de8b
commit
9d1f5ebe76
7 changed files with 128 additions and 0 deletions
|
|
@ -967,6 +967,9 @@ pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Resul
|
|||
// roster, so any schedule that still targets the just-destroyed agent
|
||||
// now drops that ghost column live (no page reload needed).
|
||||
coord.emit_schedules_snapshot();
|
||||
// Update tmpfiles.d to remove the destroyed agent's dirs from the
|
||||
// boot-time pre-creation list. Best-effort: failure is logged only.
|
||||
tokio::spawn(async { lifecycle::sync_tmpfiles().await });
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -722,6 +722,33 @@ pub async fn list() -> Result<Vec<String>> {
|
|||
.collect())
|
||||
}
|
||||
|
||||
/// Sync `/etc/tmpfiles.d/hyperhive-agents.conf` with the currently-known
|
||||
/// agent set (from `nixos-container list`). Strips the `h-` prefix to get
|
||||
/// logical names. Best-effort: errors are logged but never propagated — a
|
||||
/// failed tmpfiles write shouldn't block a spawn or destroy.
|
||||
///
|
||||
/// Called at hive-c0re startup and after each spawn / destroy so the file
|
||||
/// always reflects the live agent set. `systemd-tmpfiles-setup.service`
|
||||
/// reads the file at boot (before any container units start), pre-creating
|
||||
/// bind-mount source dirs so container@h-* units don't race hive-c0re.
|
||||
pub async fn sync_tmpfiles() {
|
||||
let agents = match list().await {
|
||||
Ok(containers) => containers
|
||||
.into_iter()
|
||||
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
|
||||
.collect::<Vec<_>>(),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "sync_tmpfiles: list failed; skipping");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Err(e) = crate::priv_client::sync_agent_tmpfiles(&agents).await {
|
||||
tracing::warn!(error = ?e, "sync_tmpfiles: priv call failed");
|
||||
} else {
|
||||
tracing::debug!(count = agents.len(), "sync_tmpfiles: ok");
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the per-line callback for `create_container_streaming` /
|
||||
/// `update_container_streaming`. Both ops share identical dispatch logic
|
||||
/// (stdout → info + `append_stdout`, stderr → warn + `append_stderr`); this
|
||||
|
|
|
|||
|
|
@ -283,6 +283,12 @@ async fn cmd_serve(
|
|||
if let Err(e) = auto_update::ensure_root_agent(&coord).await {
|
||||
tracing::warn!(error = ?e, "auto-spawn root agent failed");
|
||||
}
|
||||
// Sync /etc/tmpfiles.d/hyperhive-agents.conf so agent runtime dirs are
|
||||
// pre-declared for the next boot. Best-effort background task — a failure
|
||||
// here must not block hive-c0re startup. See lifecycle::sync_tmpfiles.
|
||||
tokio::spawn(async {
|
||||
hive_c0re::lifecycle::sync_tmpfiles().await;
|
||||
});
|
||||
// Auto-update in the background — don't block service start.
|
||||
// Sub-agent rebuilds can take tens of seconds; we want the admin
|
||||
// socket up immediately.
|
||||
|
|
|
|||
|
|
@ -389,6 +389,16 @@ pub async fn upgrade_agent_subvolume(agent_name: &str) -> Result<()> {
|
|||
.await?)
|
||||
}
|
||||
|
||||
/// Write `/etc/tmpfiles.d/hyperhive-agents.conf` for `agents` (logical names,
|
||||
/// e.g. `"atlas"`) and immediately apply it with `systemd-tmpfiles --create`.
|
||||
/// See [`PrivRequest::SyncAgentTmpfiles`] for the full semantics.
|
||||
pub async fn sync_agent_tmpfiles(agents: &[String]) -> Result<()> {
|
||||
ok(call(&PrivRequest::SyncAgentTmpfiles {
|
||||
agents: agents.to_vec(),
|
||||
})
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Parse `(referenced, exclusive)` bytes from `btrfs qgroup show -f --raw`
|
||||
/// output (a qgroup row is `<id-with-slash> <rfer> <excl> …`).
|
||||
///
|
||||
|
|
|
|||
|
|
@ -216,6 +216,8 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
|
|||
note: None,
|
||||
sha: None,
|
||||
});
|
||||
// Update tmpfiles.d so the new agent's dirs survive a reboot.
|
||||
tokio::spawn(async { lifecycle::sync_tmpfiles().await });
|
||||
}
|
||||
Err(e) => {
|
||||
// Roll back socket registration if container creation failed.
|
||||
|
|
|
|||
Loading…
Reference in a new issue