//! Tombstone rows for the dashboard: state dirs surviving a //! destroy-without-purge. Builds `TombstoneView`s for the snapshot, //! emits the `TombstonesChanged` event after mutations, and handles //! `POST /api/purge-tombstone/{name}`. use std::path::Path; use std::sync::Arc; use axum::{ extract::{Path as AxumPath, State}, http::StatusCode, response::{IntoResponse, Response}, }; use serde::Serialize; use crate::container_view::{ContainerView, claude_has_session}; use crate::coordinator::Coordinator; use crate::lifecycle; use super::{AppState, Ident, error_response}; #[derive(Serialize, Clone, Debug)] pub struct TombstoneView { pub name: String, /// Bytes used by the state dir tree. Cheap-ish to compute; let the /// operator know how much they're holding onto. pub state_bytes: u64, /// Mtime (unix seconds) of the state dir; rough "last seen". pub last_seen: i64, pub has_creds: bool, } /// State-dir names that don't appear in the live container list. Each /// one surfaces in the dashboard as a row with R3V1V3 + PURG3 actions. /// /// ⚠️ **This lists every agent whose container is absent, not only destroyed /// ones** — a mid-spawn agent (state dir seeded by `Provision`, container not /// yet made by `Create`) is indistinguishable from a tombstone here, because /// *nothing records a destroy*. Every definition-side artifact — state /// subvolume, proposed + applied repos, `deployed/0`, meta registration, /// topology entry — is written before the container exists and survives /// `lifecycle::destroy`, which removes the container and leaves the rest. /// /// This used to be papered over by treating agents with in-flight transient /// work as live. That made the page's contents a function of the **job /// graph** (`transient_snapshot` is derived from running nodes), which is the /// wrong dependency for "what state is on disk" — so the filter is gone and /// the page says what it actually shows. The real fix is to record the /// destroy rather than infer it from an absence, deferred to the /// swarm-controller / snapshot-storage rework where this changes shape anyway. pub(super) fn build_tombstone_views( coord: &Coordinator, containers: &[ContainerView], ) -> Vec { let _ = coord; // kept_state_names is a free fn but takes &self by future plan let live: std::collections::HashSet<&str> = containers.iter().map(|c| c.name.as_str()).collect(); Coordinator::kept_state_names() .into_iter() .filter(|name| !live.contains(name.as_str())) .map(|name| { let root = crate::paths::agent_state_dir(&name); let state_bytes = dir_size_bytes(&root); let last_seen = std::fs::metadata(&root) .and_then(|m| m.modified()) .ok() .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) .and_then(|d| i64::try_from(d.as_secs()).ok()) .unwrap_or(0); let has_creds = claude_has_session(&Coordinator::agent_claude_dir(&name)); TombstoneView { name: name.into_string(), state_bytes, last_seen, has_creds, } }) .collect() } /// Sum the byte size of every regular file under `root`. Cheap to compute /// for typical agent state (config repo + claude creds + notes file — /// usually a few MB); fine to do inline on each /api/state. Returns 0 on /// any error. fn dir_size_bytes(root: &Path) -> u64 { fn walk(p: &Path, acc: &mut u64) { let Ok(rd) = std::fs::read_dir(p) else { return }; for entry in rd.flatten() { let Ok(ft) = entry.file_type() else { continue }; if ft.is_dir() { walk(&entry.path(), acc); } else if ft.is_file() && let Ok(meta) = entry.metadata() { *acc += meta.len(); } } } let mut total = 0u64; walk(root, &mut total); total } /// Snapshot the current tombstone list and emit a /// `TombstonesChanged` event. Call after any mutation that could /// add or remove a tombstone (`actions::destroy`, /// `post_purge_tombstone`, spawn finalisation). Cheap — the list /// is tiny. pub(crate) async fn emit_tombstones_snapshot(coord: &Arc) { let containers = coord.containers_snapshot().await; let tombstones = build_tombstone_views(coord, &containers); coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::TombstonesChanged { seq: coord.next_seq(), tombstones, }); } /// Wipe a tombstoned agent's /// retained state dir + applied config dir entirely. #[utoipa::path( post, path = "/api/purge-tombstone/{name}", params(("name" = String, Path, description = "agent name")), responses( (status = 200, description = "purged", body = String), (status = 400, description = "bad agent name"), (status = 500, description = "a live container still exists, or a dir removal failed"), ), tag = "tombstones" )] pub(super) async fn post_purge_tombstone( State(state): State, AxumPath(name): AxumPath, ) -> Response { // Format guard FIRST so a name like `..` can't traverse into the // parent of `/var/lib/hyperhive/agents/{name}` and have // `remove_dir_all` wipe `/var/lib/hyperhive/` itself. Existing // manager + live-container checks below don't catch `..` — only // the whitelist does. Existence check via // `containers_snapshot()` is deliberately NOT used here: // tombstoned agents are gone from the snapshot by design; that's // the whole point of this endpoint. let name = match Ident::parse(&name) { Ok(n) => n, Err(reason) => { return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(); } }; // Sanity: refuse to purge if a live container still exists with this // name. The dashboard already filters tombstones to non-live names, // but the operator could send a stale POST. let live = lifecycle::list().await.unwrap_or_default(); if live .iter() .any(|c| c == &format!("{}{name}", lifecycle::AGENT_PREFIX) || c.as_str() == name.as_str()) { return error_response(&format!( "refusing to purge {name}: container still exists — use DESTR0Y first" )); } let mut errors = Vec::new(); for dir in [ crate::paths::agent_state_dir(&name), crate::paths::applied_dir(name.as_str()), ] { if dir.exists() && let Err(e) = std::fs::remove_dir_all(&dir) { errors.push(format!("{}: {e}", dir.display())); } } let _ = state .coord .approvals .fail_pending_for_agent(name.as_str(), "agent state purged"); if errors.is_empty() { tracing::info!(%name, "tombstone purged"); // Fire the post-purge tombstones snapshot so dashboards // drop the row live; matching form carries // `data-no-refresh`. emit_tombstones_snapshot(&state.coord).await; (StatusCode::OK, "ok").into_response() } else { error_response(&format!("purge {name} partial: {}", errors.join(", "))) } }