vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor(#2439): build hive-wide stop/start/restart DAGs dynamically

Browse source 
Hive-wide `stop` / `start` / `restart` emit ONE DAG with a per-agent
subgraph each (concurrent on their own leases) instead of N DAGs — and each
subgraph is now built dynamically from the agent's live running state rather
than a fixed template shape:

- online agent: the full stop→reconcile (restart: stop-for-update→reconcile)
  chain; `graceful` prepends signal→drain.
- offline agent: just `SetWanted → Reconcile` (nothing to quiesce/stop; a
  restart of a down agent is really a start).

The head `SetWanted` (intent) and tail `Reconcile` (convergence guarantee)
are always present; only the mechanical `Signal`/`Drain`/`StopForUpdate`
nodes are state-conditional. Keeping `Reconcile` in every shape closes the
TOCTOU window — a race-up between the `is_running` read and node exec is
still converged in-DAG (with `StopForUpdate`-noop as the backstop) — with no
reliance on an external reconcile sweep.

The state-aware assembly needs an async `is_running` read, so it moves out
of the pure/sync `templates.rs` into `submit.rs`, layered as pure
`*_chain(running)` → pure `*_spec(targets)` (the unit-test seam) → async
`*_many` (reads live state + submits). `templates.rs` keeps only the shared
pure primitives (`node`/`after_ok`/`rebuild_nodes`).

Callers await the now-async submit fns (server, dashboard, socket_server).
Tests exercise both the online and offline shapes via the pure `*_spec`
seam. docs/coordinator.md shapes updated.
This commit is contained in:
atlas 2026-07-14 23:34:56 +02:00
commit 860484a193
8 changed files with 574 additions and 335 deletions
Download patch file Download diff file Expand all files Collapse all files

107
hive-c0re/src/server.rs
View file

@ -97,8 +97,8 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
tracing::info!(%id, %name, "spawn approval queued");
HostResponse::success()
}
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill),
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart),
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill).await,
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart).await,
HostRequest::RestartAll => handle_restart_all(&coord).await?,
HostRequest::RestartScoped { scope, graceful } => {
handle_restart_scoped(&coord, scope, *graceful).await?
@ -143,7 +143,7 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
actions::destroy(&coord, name, *purge).await?;
HostResponse::success()
}
HostRequest::Rebuild { name } => submit_single(&coord, name, Verb::Rebuild),
HostRequest::Rebuild { name } => submit_single(&coord, name, Verb::Rebuild).await,
HostRequest::QueueDag { id } => {
// The polled DAG first, then its live fan-out children.
let dags = coord
@ -558,7 +558,7 @@ enum Verb {
Rebuild,
}
fn submit_single(coord: &Arc<Coordinator>, name: &str, verb: Verb) -> HostResponse {
async fn submit_single(coord: &Arc<Coordinator>, name: &str, verb: Verb) -> HostResponse {
use crate::job_queue::{Source, submit};
let id = match verb {
Verb::Kill => {
@ -569,6 +569,7 @@ fn submit_single(coord: &Arc<Coordinator>, name: &str, verb: Verb) -> HostRespon
Source::Manual,
"manual kill via hivectl".to_owned(),
)
.await
}
Verb::Restart => {
tracing::info!(%name, "restart");
@ -578,6 +579,7 @@ fn submit_single(coord: &Arc<Coordinator>, name: &str, verb: Verb) -> HostRespon
Source::Manual,
"manual restart via hivectl".to_owned(),
)
.await
}
Verb::Rebuild => {
tracing::info!(%name, "rebuild");
@ -606,13 +608,16 @@ async fn handle_restart_all(coord: &Arc<Coordinator>) -> Result<HostResponse> {
let queued = if agents.is_empty() {
Vec::new()
} else {
vec![crate::job_queue::submit::restart_many(
coord,
&agents,
false,
crate::job_queue::Source::Manual,
"manual restart via hivectl restart-all".to_owned(),
)]
vec![
crate::job_queue::submit::restart_many(
coord,
&agents,
false,
crate::job_queue::Source::Manual,
"manual restart via hivectl restart-all".to_owned(),
)
.await,
]
};
let mut resp = HostResponse::list(agents);
resp.queued_dags = Some(queued);
@ -644,29 +649,27 @@ async fn handle_stop(
let mut errors: Vec<String> = Vec::new();
let mut queued: Vec<u64> = Vec::new();
for agent in agents {
// One DAG for all targeted agents — a per-agent stop subgraph each
// (`SetWanted(Offline) → [Signal → Drain →] Reconcile`), independent
// roots that run concurrently on their own leases. A hive-wide
// `hivectl stop` is now a single DAG, not N.
if !agents.is_empty() {
let reason = if graceful {
"manual via hivectl graceful stop"
} else {
"manual via hivectl stop"
};
let id = if graceful {
crate::job_queue::submit::graceful_stop(
queued.push(
crate::job_queue::submit::stop_many(
coord,
agent,
agents,
graceful,
crate::job_queue::Source::Manual,
reason.to_owned(),
)
} else {
crate::job_queue::submit::stop(
coord,
agent,
crate::job_queue::Source::Manual,
reason.to_owned(),
)
};
queued.push(id);
ok_items.push(agent.clone());
.await,
);
ok_items.extend(agents.iter().cloned());
}
// Agents go down before infra so they're not mid-request against a
@ -739,18 +742,25 @@ async fn handle_start(
}
}
// One DAG for all targeted agents — a per-agent start subgraph each
// (`SetWanted(Up) → Reconcile`, or a rebuild-then-start for a stale
// rev), independent roots that run concurrently on their own leases. A
// hive-wide `hivectl start` is now a single DAG, not N. Through the
// queue: persists `wanted = Up`, per-agent stale-rev upgrade to a full
// rebuild, serializes on each agent's lease. The id rides back for
// hivectl's wait loop.
let mut queued: Vec<u64> = Vec::new();
for agent in agents {
// Through the queue: persists `wanted = Up`, upgrades a
// stale-rev start to a full rebuild, and serializes on the
// agent's lease. Ids ride back for hivectl's wait loop.
queued.push(crate::job_queue::submit::start(
coord,
agent,
crate::job_queue::Source::Manual,
"manual via hivectl start".to_owned(),
));
ok_items.push(agent.clone());
if !agents.is_empty() {
queued.push(
crate::job_queue::submit::start_many(
coord,
agents,
crate::job_queue::Source::Manual,
"manual via hivectl start".to_owned(),
)
.await,
);
ok_items.extend(agents.iter().cloned());
}
let mut resp = finish_lifecycle(ok_items, &errors);
@ -786,17 +796,20 @@ async fn handle_restart_scoped(
// client-side stop-then-start composition — the whole restart survives
// a dropped connection because the DAG owns it.
if !agents.is_empty() {
queued.push(crate::job_queue::submit::restart_many(
coord,
&agents,
graceful,
crate::job_queue::Source::Manual,
if graceful {
"manual via hivectl restart --graceful".to_owned()
} else {
"manual restart via hivectl restart".to_owned()
},
));
queued.push(
crate::job_queue::submit::restart_many(
coord,
&agents,
graceful,
crate::job_queue::Source::Manual,
if graceful {
"manual via hivectl restart --graceful".to_owned()
} else {
"manual restart via hivectl restart".to_owned()
},
)
.await,
);
ok_items.extend(agents.iter().cloned());
}