fix(#1957): persist hivectl-start running-agents snapshot in the broker kv table
This commit is contained in:
parent
e5da5b900e
commit
f7d84f4847
2 changed files with 110 additions and 5 deletions
|
|
@ -22,6 +22,12 @@ use crate::socket_server::{self, AgentSocket};
|
|||
/// fine, the seq dedupe makes a reconnect resync safe.
|
||||
const DASHBOARD_CHANNEL: usize = 256;
|
||||
|
||||
/// Broker `kv` key under which the broad-stop running-agents snapshot is
|
||||
/// persisted, so a `hivectl start` after a hive-c0re restart can still
|
||||
/// restore only the previously-running agents. See
|
||||
/// `Coordinator::set_last_stopped_running`.
|
||||
const LAST_STOPPED_RUNNING_KEY: &str = "last_stopped_running";
|
||||
|
||||
const AGENT_RUNTIME_ROOT: &str = "/run/hyperhive/agents";
|
||||
/// Manager-editable per-agent config repos. Bind-mounted RW into the manager
|
||||
/// container as `/agents/<name>/`. Hive-c0re only writes to these on first
|
||||
|
|
@ -137,8 +143,10 @@ pub struct Coordinator {
|
|||
/// every configured container, so agents the operator intentionally
|
||||
/// left stopped stay stopped. `None` when no broad stop has happened
|
||||
/// since the last start (or since daemon boot) — start then falls
|
||||
/// back to "start all". In-daemon memory only (hive-c0re survives
|
||||
/// `hivectl stop`); host-reboot persistence is a separate follow-up.
|
||||
/// back to "start all". Mirrored to the broker `kv` table
|
||||
/// (`last_stopped_running` key) so the snapshot also survives a
|
||||
/// hive-c0re restart between the stop-all and the start: this
|
||||
/// in-memory copy is the fast path, the persisted copy the backstop.
|
||||
last_stopped_running: Mutex<Option<Vec<String>>>,
|
||||
/// Unified wire-facing event channel feeding the dashboard SSE
|
||||
/// stream. Carries broker messages (mirrored from `broker.subscribe`
|
||||
|
|
@ -1185,16 +1193,44 @@ impl Coordinator {
|
|||
|
||||
/// Record the set of agents that were running at a broad-scope
|
||||
/// `hivectl stop`, so the next broad-scope `start` restores exactly
|
||||
/// this set. See the `last_stopped_running` field doc.
|
||||
/// this set. See the `last_stopped_running` field doc. Persists a
|
||||
/// copy to the broker `kv` table (best-effort) so the snapshot
|
||||
/// survives a hive-c0re restart; the in-memory copy is the fast path.
|
||||
pub fn set_last_stopped_running(&self, agents: Vec<String>) {
|
||||
match serde_json::to_string(&agents) {
|
||||
Ok(json) => {
|
||||
if let Err(e) = self.broker.kv_set(LAST_STOPPED_RUNNING_KEY, &json) {
|
||||
tracing::warn!(error = ?e, "persist last_stopped_running failed (in-memory copy still set)");
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = ?e, "serialise last_stopped_running failed"),
|
||||
}
|
||||
*self.last_stopped_running.lock().unwrap() = Some(agents);
|
||||
}
|
||||
|
||||
/// Take (and clear) the recorded broad-stop running set, if any. A
|
||||
/// broad-scope `start` uses this to restore only the previously
|
||||
/// running agents; `None` means "no record — start all".
|
||||
/// running agents; `None` means "no record — start all". Falls back
|
||||
/// to the persisted `kv` copy when the in-memory snapshot is empty
|
||||
/// (hive-c0re restarted between the stop-all and the start). Clears
|
||||
/// the persisted copy either way — the snapshot is one-shot.
|
||||
pub fn take_last_stopped_running(&self) -> Option<Vec<String>> {
|
||||
self.last_stopped_running.lock().unwrap().take()
|
||||
let result = self
|
||||
.last_stopped_running
|
||||
.lock()
|
||||
.unwrap()
|
||||
.take()
|
||||
.or_else(|| {
|
||||
self.broker
|
||||
.kv_get(LAST_STOPPED_RUNNING_KEY)
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|json| serde_json::from_str::<Vec<String>>(&json).ok())
|
||||
});
|
||||
if let Err(e) = self.broker.kv_delete(LAST_STOPPED_RUNNING_KEY) {
|
||||
tracing::warn!(error = ?e, "clear persisted last_stopped_running failed");
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Set of agents whose transient was cleared within the last
|
||||
|
|
|
|||
Loading…
Reference in a new issue