refactor(#2290): replace mcp_sockets poll with event-driven register_agent
mara: the background worker is redundant if c0re knows when its own sockets go missing. damocles: 10s poll latency and redundancy are two faces of the same issue — poll adds a reconnect window and does redundant work when c0re could react directly. design: c0re owns the MCP listener lifecycle, so the only time a listener disappears without c0re knowing is when c0re itself restarts. - replace spawn_poll (recurring 10s loop) with sync_on_start (one-shot sweep at daemon boot): re-registers all running agents on startup after /run/hyperhive/agents/ is cleared by the tmpfs reset. - run_reconcile (reconcile-start path): add coord.register_agent(name) immediately after start_with_fallback — event-driven, no poll delay. - run_create already calls register_agent eagerly; kill/destroy paths already call unregister_agent — no changes needed there. tracker: #2290
This commit is contained in:
parent
44dd9d45f0
commit
73f1020a7e
3 changed files with 38 additions and 43 deletions
|
|
@ -99,7 +99,7 @@ async fn run_prebuild(
|
||||||
let name = &claim.agent;
|
let name = &claim.agent;
|
||||||
// Prebuild runs while the agent is still up — the runtime dir and
|
// Prebuild runs while the agent is still up — the runtime dir and
|
||||||
// MCP listener already exist. Use the pure path accessor; no need
|
// MCP listener already exist. Use the pure path accessor; no need
|
||||||
// to re-register the listener (the mcp_sockets supervisor owns that).
|
// to re-register the listener (event-driven: registered at start/create).
|
||||||
let agent_dir = Coordinator::agent_dir(name);
|
let agent_dir = Coordinator::agent_dir(name);
|
||||||
let hive = coord.hive_env();
|
let hive = coord.hive_env();
|
||||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||||
|
|
@ -190,8 +190,8 @@ async fn run_create(coord: &Arc<Coordinator>, claim: &Claim, ctx: &Ctx<'_>) -> R
|
||||||
// deploy-window gate so that commit can't land inside another
|
// deploy-window gate so that commit can't land inside another
|
||||||
// node's staged deploy window.
|
// node's staged deploy window.
|
||||||
// Runtime dir creation and MCP listener registration are deferred to
|
// Runtime dir creation and MCP listener registration are deferred to
|
||||||
// the tail Reconcile's converge_start_preamble / mcp_sockets supervisor
|
// the tail Reconcile (converge_start_preamble + register_agent) so this
|
||||||
// so this node stays purely "provision + create", not "create + start".
|
// node stays purely "provision + create", not "create + start".
|
||||||
let _window = crate::meta::exclusive().await;
|
let _window = crate::meta::exclusive().await;
|
||||||
crate::lifecycle::create_container(name, &hive, &paths).await?;
|
crate::lifecycle::create_container(name, &hive, &paths).await?;
|
||||||
Ok(NodeOutput::default())
|
Ok(NodeOutput::default())
|
||||||
|
|
@ -255,15 +255,18 @@ async fn run_reconcile(
|
||||||
// exists and writes the nspawn/resource-limits drop-ins.
|
// exists and writes the nspawn/resource-limits drop-ins.
|
||||||
// The returned StartableAgent token is the only way to call
|
// The returned StartableAgent token is the only way to call
|
||||||
// start_with_fallback — omitting this becomes a compile error.
|
// start_with_fallback — omitting this becomes a compile error.
|
||||||
// MCP listener registration is handled by mcp_sockets::spawn_poll
|
|
||||||
// (first tick immediate); the container boot takes longer than
|
|
||||||
// the 10 s interval so the listener is ready in time.
|
|
||||||
let agent_dir = Coordinator::agent_dir(name);
|
let agent_dir = Coordinator::agent_dir(name);
|
||||||
let hive = coord.hive_env();
|
let hive = coord.hive_env();
|
||||||
let paths = Coordinator::agent_paths(name, agent_dir);
|
let paths = Coordinator::agent_paths(name, agent_dir);
|
||||||
let token = crate::lifecycle::converge_start_preamble(name, &hive, &paths).await?;
|
let token = crate::lifecycle::converge_start_preamble(name, &hive, &paths).await?;
|
||||||
ctx.step("nixos-container start");
|
ctx.step("nixos-container start");
|
||||||
crate::lifecycle::start_with_fallback(token).await?;
|
crate::lifecycle::start_with_fallback(token).await?;
|
||||||
|
// Bind the MCP listener immediately after starting the container.
|
||||||
|
// The preamble created the runtime dir; the container is now
|
||||||
|
// coming up and will connect to this socket on its first turn.
|
||||||
|
// Event-driven (no background poll) — c0re owns the listener
|
||||||
|
// lifecycle, so register here rather than waiting for a sweep.
|
||||||
|
coord.register_agent(name)?;
|
||||||
coord.kick_agent(name, "container started");
|
coord.kick_agent(name, "container started");
|
||||||
coord.rescan_containers_and_emit().await;
|
coord.rescan_containers_and_emit().await;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -420,13 +420,13 @@ async fn cmd_serve(
|
||||||
// is one stat per agent per tick.
|
// is one stat per agent per tick.
|
||||||
// See `docs/gateway.md::Per-agent unix-socket upstream`.
|
// See `docs/gateway.md::Per-agent unix-socket upstream`.
|
||||||
agent_sockets::spawn_poll();
|
agent_sockets::spawn_poll();
|
||||||
// MCP socket listener reconcile loop: every 10s re-registers any
|
// MCP socket listener startup sync: one-shot sweep that re-registers any
|
||||||
// running agent that lost its host-side MCP listener (e.g. after a
|
// running agent container whose MCP listener was lost when hive-c0re
|
||||||
// hive-c0re restart cleared /run/hyperhive/agents/). First tick fires
|
// restarted (Coordinator starts empty; /run/hyperhive/agents/ is tmpfs).
|
||||||
// immediately so restarts re-register all running agents without delay.
|
// After this, listener registration is event-driven: run_create /
|
||||||
// Decouples listener registration from the start path — start only needs
|
// run_reconcile call register_agent on start; kill/destroy call
|
||||||
// lifecycle::ensure_agent_runtime_dir; the supervisor converges the rest.
|
// unregister_agent. No recurring poll needed — c0re owns the listeners.
|
||||||
mcp_sockets::spawn_poll(coord.clone());
|
mcp_sockets::sync_on_start(coord.clone()).await;
|
||||||
// Reminder scheduler: drains due reminders + handles
|
// Reminder scheduler: drains due reminders + handles
|
||||||
// file_path payload persistence. See reminder_scheduler.rs.
|
// file_path payload persistence. See reminder_scheduler.rs.
|
||||||
reminder_scheduler::spawn(coord.clone());
|
reminder_scheduler::spawn(coord.clone());
|
||||||
|
|
|
||||||
|
|
@ -1,42 +1,34 @@
|
||||||
//! MCP socket listener reconcile loop.
|
//! MCP socket listener boot sync.
|
||||||
//!
|
//!
|
||||||
//! Periodically checks that every running agent container has a bound
|
//! On hive-c0re startup, any agent containers that survived the daemon restart
|
||||||
//! MCP listener registered in the `Coordinator`. Any agent that is running
|
//! still have their bind-mount source dirs but no live MCP listener (the
|
||||||
//! but whose listener has gone (e.g. after a hive-c0re restart that cleared
|
//! `Coordinator` is freshly empty). `sync_on_start` does a one-shot sweep to
|
||||||
//! `/run/hyperhive/agents/`) gets re-registered automatically.
|
//! re-register all running agents.
|
||||||
//!
|
//!
|
||||||
//! Same shape as `agent_sockets::spawn_poll` — a simple 10 s tick loop that
|
//! After startup, listeners are managed event-driven:
|
||||||
//! converges "agent container running ⇒ MCP listener bound". The self-healing
|
//! - `run_create` calls `register_agent` eagerly on first-spawn.
|
||||||
//! guarantee means no callsite needs to call `register_agent` directly;
|
//! - `run_reconcile` calls `register_agent` immediately after `start_with_fallback`.
|
||||||
//! `lifecycle::ensure_agent_runtime_dir` (called inside `lifecycle::spawn`
|
//! - `kill`/`destroy` paths call `unregister_agent`.
|
||||||
//! and `converge_start_preamble`) creates the bind-mount source, and the
|
//!
|
||||||
//! reconcile loop picks up the listener binding on the next tick.
|
//! No recurring poll is needed because c0re owns the listener lifecycle —
|
||||||
|
//! a listener can only disappear when c0re itself restarts, which is exactly
|
||||||
|
//! the case `sync_on_start` covers.
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::coordinator::Coordinator;
|
use crate::coordinator::Coordinator;
|
||||||
|
|
||||||
/// Spawn the MCP socket listener reconcile loop.
|
/// One-shot MCP listener sync run at daemon startup.
|
||||||
///
|
///
|
||||||
/// Every 10 s the loop lists running agent containers and calls
|
/// Iterates all currently-running agent containers and calls `register_agent`
|
||||||
/// `register_agent` for any that lack a bound listener. The first tick fires
|
/// for any that have no live listener in the `Coordinator`. Safe to call
|
||||||
/// immediately so hive-c0re restarts re-register all running agents without
|
/// concurrently with the rest of startup — `register_agent` is idempotent
|
||||||
/// waiting a full interval.
|
/// (drops and rebinds) and the coordinator lock serialises concurrent calls.
|
||||||
pub fn spawn_poll(coord: Arc<Coordinator>) {
|
pub async fn sync_on_start(coord: Arc<Coordinator>) {
|
||||||
tokio::spawn(async move {
|
|
||||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(10));
|
|
||||||
loop {
|
|
||||||
interval.tick().await;
|
|
||||||
reconcile_once(&coord).await;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reconcile_once(coord: &Arc<Coordinator>) {
|
|
||||||
let running = match crate::lifecycle::list().await {
|
let running = match crate::lifecycle::list().await {
|
||||||
Ok(names) => names,
|
Ok(names) => names,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::debug!(error = ?e, "mcp_sockets poll: failed to list agents");
|
tracing::warn!(error = ?e, "mcp_sockets: startup sync failed to list agents; MCP listeners may be missing until next start");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -50,10 +42,10 @@ async fn reconcile_once(coord: &Arc<Coordinator>) {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
agent = %name,
|
agent = %name,
|
||||||
error = ?e,
|
error = ?e,
|
||||||
"mcp_sockets poll: register_agent failed"
|
"mcp_sockets: startup register_agent failed"
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
tracing::debug!(agent = %name, "mcp_sockets poll: registered missing listener");
|
tracing::debug!(agent = %name, "mcp_sockets: registered listener on startup");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue