Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
407965b6e1 | ||
|
|
901ab6a779 | ||
|
|
4cfa040154 |
8 changed files with 182 additions and 11 deletions
|
|
@ -78,6 +78,7 @@ container build:
|
||||||
rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a)
|
rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(after-any) Reconcile(a)
|
||||||
graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
||||||
restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
||||||
|
graceful-restart(a): [wanted=Up] Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
|
||||||
start(a): [wanted=Up] Reconcile(a) (stale rev ⇒ upgraded to rebuild)
|
start(a): [wanted=Up] Reconcile(a) (stale rev ⇒ upgraded to rebuild)
|
||||||
stop(a): [wanted=Offline] Reconcile(a)
|
stop(a): [wanted=Offline] Reconcile(a)
|
||||||
spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
|
spawn(a): [wanted=Up] Create(a) → WriteDropin(a) → Reconcile(a)
|
||||||
|
|
|
||||||
|
|
@ -1642,22 +1642,33 @@ async fn start(socket: &Path, scope: hive_host_sock::LifecycleScope, no_wait: bo
|
||||||
rendered
|
rendered
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restart = `stop` then `start` over the same scope, composed client-side
|
/// Restart — one `RestartScoped` daemon call, server-side DAG-based (see
|
||||||
/// from the two daemon ops (no dedicated wire op). The stop phase honours
|
/// issue tracker "dagify hivectl commands"). Each targeted agent rides
|
||||||
/// `--graceful`; if it reports a failure (`stop` returns `Err`) the `?`
|
/// exactly one atomic DAG queued up front — `Restart` (mechanical stop +
|
||||||
/// short-circuits before the start phase, so a half-stopped hive isn't
|
/// reconcile), or `GracefulRestart` with `--graceful` (signal → drain →
|
||||||
/// blindly started over — the operator sees the stop errors and can recover.
|
/// mechanical stop → reconcile); infra containers restart synchronously.
|
||||||
|
/// No "submit one DAG, wait for it, submit another" composition on either
|
||||||
|
/// side of the wire: unlike the old client-side stop-then-start compose, a
|
||||||
|
/// dropped `hivectl` connection mid-restart no longer leaves an agent
|
||||||
|
/// stopped with no automatic follow-up — the daemon owns the whole
|
||||||
|
/// sequence once this call is made.
|
||||||
///
|
///
|
||||||
/// No `--no-wait` here on purpose: the stop DAGs must complete before
|
/// No `--no-wait` here on purpose, same as before: the operator wants to
|
||||||
/// the start submits, otherwise the start's `wanted = Up` write would
|
/// see the restart actually land, not just get queued.
|
||||||
/// land before the queued stops execute and turn them into noops.
|
|
||||||
async fn restart(
|
async fn restart(
|
||||||
socket: &Path,
|
socket: &Path,
|
||||||
scope: hive_host_sock::LifecycleScope,
|
scope: hive_host_sock::LifecycleScope,
|
||||||
graceful: bool,
|
graceful: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
stop(socket, scope.clone(), graceful, false).await?;
|
let resp = hive_c0re::client::request(
|
||||||
start(socket, scope, false).await
|
socket,
|
||||||
|
hive_host_sock::HostRequest::RestartScoped { scope, graceful },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||||
|
let rendered = render_lifecycle(&resp, "restart queued");
|
||||||
|
wait_for_dags(socket, resp.queued_dags.unwrap_or_default(), false).await?;
|
||||||
|
rendered
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A [`LifecycleScope`](hive_host_sock::LifecycleScope) targeting exactly one
|
/// A [`LifecycleScope`](hive_host_sock::LifecycleScope) targeting exactly one
|
||||||
|
|
|
||||||
|
|
@ -507,7 +507,11 @@ pub(super) async fn on_dag_terminal(coord: &Arc<Coordinator>, terminal: &Termina
|
||||||
if terminal.state == State::Cancelled
|
if terminal.state == State::Cancelled
|
||||||
&& matches!(
|
&& matches!(
|
||||||
terminal.template,
|
terminal.template,
|
||||||
Template::Start | Template::Stop | Template::GracefulStop | Template::Restart
|
Template::Start
|
||||||
|
| Template::Stop
|
||||||
|
| Template::GracefulStop
|
||||||
|
| Template::Restart
|
||||||
|
| Template::GracefulRestart
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
let running = crate::lifecycle::is_running(&terminal.agent).await;
|
let running = crate::lifecycle::is_running(&terminal.agent).await;
|
||||||
|
|
|
||||||
|
|
@ -100,6 +100,19 @@ pub fn graceful_stop(coord: &Arc<Coordinator>, agent: &str, source: Source, reas
|
||||||
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
|
submit_and_emit(coord, templates::graceful_stop(agent, source, reason))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Graceful restart: persist `wanted = Up`, then signal → drain →
|
||||||
|
/// mechanical stop → reconcile (starts it back up) — one atomic DAG,
|
||||||
|
/// no client-side "await the stop DAG then submit a start DAG" split.
|
||||||
|
pub fn graceful_restart(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
agent: &str,
|
||||||
|
source: Source,
|
||||||
|
reason: String,
|
||||||
|
) -> u64 {
|
||||||
|
set_wanted(coord, agent, Wanted::Up);
|
||||||
|
submit_and_emit(coord, templates::graceful_restart(agent, source, reason))
|
||||||
|
}
|
||||||
|
|
||||||
/// Perm change: commit the JSON file(s) then rebuild.
|
/// Perm change: commit the JSON file(s) then rebuild.
|
||||||
pub fn perm_change(
|
pub fn perm_change(
|
||||||
coord: &Arc<Coordinator>,
|
coord: &Arc<Coordinator>,
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@
|
||||||
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
|
//! rebuild(a): Prebuild(a) → StopForUpdate(a) → Swap(a) →(any) Reconcile(a)
|
||||||
//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
//! graceful-stop(a): [wanted=Offline] Signal(a) → Drain(a) → Reconcile(a)
|
||||||
//! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
//! restart(a): [wanted=Up] StopForUpdate(a) → Reconcile(a)
|
||||||
|
//! graceful-restart(a): [wanted=Up] Signal(a) → Drain(a) → StopForUpdate(a) → Reconcile(a)
|
||||||
//! start(a): [wanted=Up] Reconcile(a)
|
//! start(a): [wanted=Up] Reconcile(a)
|
||||||
//! stop(a): [wanted=Offline] Reconcile(a)
|
//! stop(a): [wanted=Offline] Reconcile(a)
|
||||||
//! spawn(a): [wanted=Up] Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)
|
//! spawn(a): [wanted=Up] Provision(a) → Create(a) → WriteDropin(a) → Reconcile(a)
|
||||||
|
|
@ -168,6 +169,46 @@ pub fn restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Graceful restart: signal → drain → mechanical stop → converge to
|
||||||
|
/// `wanted` — the caller writes `wanted = Up` first, same as `restart`.
|
||||||
|
/// One atomic DAG start to finish (no client- or server-side "submit
|
||||||
|
/// one DAG, await it, submit the next" composition): the `Drain` node
|
||||||
|
/// is the same bounded harness-checkpoint wait `graceful_stop` uses,
|
||||||
|
/// then `StopForUpdate` (mechanical, ignores `wanted`) and the tail
|
||||||
|
/// `Reconcile` (converges to `wanted = Up`, i.e. starts it back up)
|
||||||
|
/// chain exactly like `restart`'s tail.
|
||||||
|
pub fn graceful_restart(agent: &str, source: Source, reason: String) -> DagSpec {
|
||||||
|
DagSpec {
|
||||||
|
template: Template::GracefulRestart,
|
||||||
|
agent: agent.to_owned(),
|
||||||
|
source,
|
||||||
|
reason,
|
||||||
|
parent_id: None,
|
||||||
|
approval_id: None,
|
||||||
|
inputs: Vec::new(),
|
||||||
|
perm_payload: None,
|
||||||
|
transient: Some(TransientKind::Restarting),
|
||||||
|
nodes: vec![
|
||||||
|
NodeSpec {
|
||||||
|
kind: NodeKind::Signal,
|
||||||
|
deps: Vec::new(),
|
||||||
|
},
|
||||||
|
NodeSpec {
|
||||||
|
kind: NodeKind::Drain,
|
||||||
|
deps: after_ok(0),
|
||||||
|
},
|
||||||
|
NodeSpec {
|
||||||
|
kind: NodeKind::StopForUpdate,
|
||||||
|
deps: after_ok(1),
|
||||||
|
},
|
||||||
|
NodeSpec {
|
||||||
|
kind: NodeKind::Reconcile,
|
||||||
|
deps: after_ok(2),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
|
/// Single-`Reconcile` DAG: `Start` / `Stop` (caller writes `wanted`
|
||||||
/// first) and the boot-time `Reconcile` converge (wanted untouched).
|
/// first) and the boot-time `Reconcile` converge (wanted untouched).
|
||||||
pub fn reconcile_only(
|
pub fn reconcile_only(
|
||||||
|
|
|
||||||
|
|
@ -94,6 +94,9 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
||||||
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill),
|
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill),
|
||||||
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart),
|
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart),
|
||||||
HostRequest::RestartAll => handle_restart_all(&coord).await?,
|
HostRequest::RestartAll => handle_restart_all(&coord).await?,
|
||||||
|
HostRequest::RestartScoped { scope, graceful } => {
|
||||||
|
handle_restart_scoped(&coord, scope, *graceful).await?
|
||||||
|
}
|
||||||
HostRequest::Stop { scope, graceful } => {
|
HostRequest::Stop { scope, graceful } => {
|
||||||
// Resolve the scope to explicit container names at the entry
|
// Resolve the scope to explicit container names at the entry
|
||||||
// point, then operate on names — never pass the bare "all
|
// point, then operate on names — never pass the bare "all
|
||||||
|
|
@ -631,6 +634,76 @@ async fn handle_start(
|
||||||
Ok(resp)
|
Ok(resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Restart containers hive-wide (`hivectl restart`) — the DAG-based
|
||||||
|
/// sibling of [`handle_stop`]/[`handle_start`], replacing the old
|
||||||
|
/// client-side stop-then-start composition (issue tracker "dagify hivectl
|
||||||
|
/// commands"). Each targeted agent gets exactly one atomic DAG submitted
|
||||||
|
/// up front: the `Restart` template (mechanical stop + reconcile) in the
|
||||||
|
/// common case, or `GracefulRestart` (signal → drain → mechanical stop →
|
||||||
|
/// reconcile) with `graceful` set — no "submit a DAG, wait for it, submit
|
||||||
|
/// another" composition on either path, so a dropped `hivectl` connection
|
||||||
|
/// never strands an agent, the same way `handle_restart_all` already
|
||||||
|
/// avoids it. Infra containers have no lease/DAG and restart
|
||||||
|
/// synchronously (stop then start).
|
||||||
|
async fn handle_restart_scoped(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
scope: &LifecycleScope,
|
||||||
|
graceful: bool,
|
||||||
|
) -> Result<HostResponse> {
|
||||||
|
tracing::info!(?scope, graceful, "restart");
|
||||||
|
let agents = scoped_agents(scope).await?;
|
||||||
|
let infra = scoped_infra(scope);
|
||||||
|
let mut ok_items: Vec<String> = Vec::new();
|
||||||
|
let mut errors: Vec<String> = Vec::new();
|
||||||
|
let mut queued: Vec<u64> = Vec::new();
|
||||||
|
|
||||||
|
// One atomic DAG per agent, submitted up front — `Restart`
|
||||||
|
// (mechanical stop + reconcile) or, with `--graceful`,
|
||||||
|
// `GracefulRestart` (signal → drain → mechanical stop → reconcile).
|
||||||
|
// No client- or server-side "submit a stop DAG, await it, then
|
||||||
|
// submit a start DAG" composition: that window is exactly the
|
||||||
|
// dropped-connection gap this DAG-based path exists to close.
|
||||||
|
for agent in &agents {
|
||||||
|
let id = if graceful {
|
||||||
|
crate::job_queue::submit::graceful_restart(
|
||||||
|
coord,
|
||||||
|
agent,
|
||||||
|
crate::job_queue::Source::Manual,
|
||||||
|
"manual via hivectl restart --graceful".to_owned(),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
crate::job_queue::submit::restart(
|
||||||
|
coord,
|
||||||
|
agent,
|
||||||
|
crate::job_queue::Source::Manual,
|
||||||
|
"manual restart via hivectl restart".to_owned(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
queued.push(id);
|
||||||
|
ok_items.push(agent.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
for &container in &infra {
|
||||||
|
let name = container.unit_name();
|
||||||
|
let res = async {
|
||||||
|
crate::priv_client::control_infra_container(container, InfraAction::Stop).await?;
|
||||||
|
crate::priv_client::control_infra_container(container, InfraAction::Start).await
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
match res {
|
||||||
|
Ok(()) => ok_items.push(name.to_owned()),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(%name, error = ?e, "restart: infra restart failed");
|
||||||
|
errors.push(format!("{name}: {e:#}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut resp = finish_lifecycle(ok_items, &errors);
|
||||||
|
resp.queued_dags = Some(queued);
|
||||||
|
Ok(resp)
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolve which sub-agent logical names a scope targets: every live
|
/// Resolve which sub-agent logical names a scope targets: every live
|
||||||
/// container (from `lifecycle::list`) when `agents` is set or the scope is
|
/// container (from `lifecycle::list`) when `agents` is set or the scope is
|
||||||
/// "everything", plus any explicit `agent_names`. Returns de-duplicated
|
/// "everything", plus any explicit `agent_names`. Returns de-duplicated
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,28 @@ pub enum HostRequest {
|
||||||
/// wrapper for `hivectl agents restart-all`; iterates the live
|
/// wrapper for `hivectl agents restart-all`; iterates the live
|
||||||
/// container list and restarts each one.
|
/// container list and restarts each one.
|
||||||
RestartAll,
|
RestartAll,
|
||||||
|
/// Restart containers hive-wide (`hivectl restart`), scoped like
|
||||||
|
/// `Stop`/`Start`. Each targeted agent rides exactly one DAG
|
||||||
|
/// server-side — the `Restart` template (mechanical stop + reconcile),
|
||||||
|
/// or, when `graceful` is set, the `GracefulRestart` template (signal →
|
||||||
|
/// drain → mechanical stop → reconcile) — rather than the old
|
||||||
|
/// client-side stop-then-start composition (and, briefly, a server-side
|
||||||
|
/// "submit stop DAG, await it, submit start DAG" composition): a
|
||||||
|
/// dropped `hivectl` connection mid-way, or a crash between the two
|
||||||
|
/// submits, used to leave the agent stopped with no automatic
|
||||||
|
/// follow-up, since nothing durable remembered "finish the restart"
|
||||||
|
/// once the calling process/turn was gone. `GracefulRestart` closes
|
||||||
|
/// that gap the same way `Restart` already does — one DAG, queued up
|
||||||
|
/// front, that owns the whole sequence. Infra containers have no
|
||||||
|
/// lease/DAG and restart synchronously (stop then start), same as
|
||||||
|
/// before. Scope semantics match `Stop`/`Start` (all-false =
|
||||||
|
/// everything).
|
||||||
|
RestartScoped {
|
||||||
|
#[serde(default)]
|
||||||
|
scope: LifecycleScope,
|
||||||
|
#[serde(default)]
|
||||||
|
graceful: bool,
|
||||||
|
},
|
||||||
/// Apply pending config to a managed container.
|
/// Apply pending config to a managed container.
|
||||||
Rebuild { name: String },
|
Rebuild { name: String },
|
||||||
/// List managed containers.
|
/// List managed containers.
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,11 @@ pub enum Template {
|
||||||
StartupSweep,
|
StartupSweep,
|
||||||
/// Mechanical stop + converge to `wanted = Up` (a restart).
|
/// Mechanical stop + converge to `wanted = Up` (a restart).
|
||||||
Restart,
|
Restart,
|
||||||
|
/// Signal → drain → mechanical stop → converge to `wanted = Up` — a
|
||||||
|
/// graceful restart as one atomic DAG (drains the harness before the
|
||||||
|
/// stop, same as `GracefulStop`, but then reconciles back up instead
|
||||||
|
/// of staying down).
|
||||||
|
GracefulRestart,
|
||||||
/// Perm-file commit followed by the rebuild subgraph.
|
/// Perm-file commit followed by the rebuild subgraph.
|
||||||
PermChange,
|
PermChange,
|
||||||
/// Quiesce the harness, drain, then stop (`wanted = Offline`).
|
/// Quiesce the harness, drain, then stop (`wanted = Offline`).
|
||||||
|
|
@ -56,6 +61,7 @@ impl Template {
|
||||||
Template::Destroy => "destroy",
|
Template::Destroy => "destroy",
|
||||||
Template::StartupSweep => "startup_sweep",
|
Template::StartupSweep => "startup_sweep",
|
||||||
Template::Restart => "restart",
|
Template::Restart => "restart",
|
||||||
|
Template::GracefulRestart => "graceful_restart",
|
||||||
Template::PermChange => "perm_change",
|
Template::PermChange => "perm_change",
|
||||||
Template::GracefulStop => "graceful_stop",
|
Template::GracefulStop => "graceful_stop",
|
||||||
Template::Start => "start",
|
Template::Start => "start",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue