hyperhive/hive-c0re/src/dashboard/tombstones.rs

//! Tombstone rows for the dashboard: state dirs surviving a
//! destroy-without-purge. Builds `TombstoneView`s for the snapshot,
//! emits the `TombstonesChanged` event after mutations, and handles
//! `POST /api/purge-tombstone/{name}`.

use std::path::Path;
use std::sync::Arc;

use axum::{
    extract::{Path as AxumPath, State},
    http::StatusCode,
    response::{IntoResponse, Response},
};
use serde::Serialize;

use crate::container_view::{ContainerView, claude_has_session};
use crate::coordinator::Coordinator;
use crate::lifecycle;

use super::{AppState, Ident, error_response};

#[derive(Serialize, Clone, Debug)]
pub struct TombstoneView {
    pub name: String,
    /// Bytes used by the state dir tree. Cheap-ish to compute; let the
    /// operator know how much they're holding onto.
    pub state_bytes: u64,
    /// Mtime (unix seconds) of the state dir; rough "last seen".
    pub last_seen: i64,
    pub has_creds: bool,
}

/// State-dir names that don't appear in the live container list. Each
/// one surfaces in the dashboard as a row with R3V1V3 + PURG3 actions.
///
/// ⚠️ **This lists every agent whose container is absent, not only destroyed
/// ones** — a mid-spawn agent (state dir seeded by `Provision`, container not
/// yet made by `Create`) is indistinguishable from a tombstone here, because
/// *nothing records a destroy*. Every definition-side artifact — state
/// subvolume, proposed + applied repos, `deployed/0`, meta registration,
/// topology entry — is written before the container exists and survives
/// `lifecycle::destroy`, which removes the container and leaves the rest.
///
/// This used to be papered over by treating agents with in-flight transient
/// work as live. That made the page's contents a function of the **job
/// graph** (`transient_snapshot` is derived from running nodes), which is the
/// wrong dependency for "what state is on disk" — so the filter is gone and
/// the page says what it actually shows. The real fix is to record the
/// destroy rather than infer it from an absence, deferred to the
/// swarm-controller / snapshot-storage rework where this changes shape anyway.
pub(super) fn build_tombstone_views(
    coord: &Coordinator,
    containers: &[ContainerView],
) -> Vec<TombstoneView> {
    let _ = coord; // kept_state_names is a free fn but takes &self by future plan
    let live: std::collections::HashSet<&str> =
        containers.iter().map(|c| c.name.as_str()).collect();
    Coordinator::kept_state_names()
        .into_iter()
        .filter(|name| !live.contains(name.as_str()))
        .map(|name| {
            let root = crate::paths::agent_state_dir(&name);
            let state_bytes = dir_size_bytes(&root);
            let last_seen = std::fs::metadata(&root)
                .and_then(|m| m.modified())
                .ok()
                .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
                .and_then(|d| i64::try_from(d.as_secs()).ok())
                .unwrap_or(0);
            let has_creds = claude_has_session(&Coordinator::agent_claude_dir(&name));
            TombstoneView {
                name: name.into_string(),
                state_bytes,
                last_seen,
                has_creds,
            }
        })
        .collect()
}

/// Sum the byte size of every regular file under `root`. Cheap to compute
/// for typical agent state (config repo + claude creds + notes file —
/// usually a few MB); fine to do inline on each /api/state. Returns 0 on
/// any error.
fn dir_size_bytes(root: &Path) -> u64 {
    fn walk(p: &Path, acc: &mut u64) {
        let Ok(rd) = std::fs::read_dir(p) else { return };
        for entry in rd.flatten() {
            let Ok(ft) = entry.file_type() else { continue };
            if ft.is_dir() {
                walk(&entry.path(), acc);
            } else if ft.is_file()
                && let Ok(meta) = entry.metadata()
            {
                *acc += meta.len();
            }
        }
    }
    let mut total = 0u64;
    walk(root, &mut total);
    total
}

/// Snapshot the current tombstone list and emit a
/// `TombstonesChanged` event. Call after any mutation that could
/// add or remove a tombstone (`actions::destroy`,
/// `post_purge_tombstone`, spawn finalisation). Cheap — the list
/// is tiny.
pub(crate) async fn emit_tombstones_snapshot(coord: &Arc<Coordinator>) {
    let containers = coord.containers_snapshot().await;
    let tombstones = build_tombstone_views(coord, &containers);
    coord.emit_dashboard_event(crate::dashboard_events::DashboardEvent::TombstonesChanged {
        seq: coord.next_seq(),
        tombstones,
    });
}

/// Decide whether `post_purge_tombstone` may proceed, given the outcome of
/// `lifecycle::list()`. `Ok` only when the list was read and names no live
/// container for `name`; an unreadable list proves nothing about whether a
/// container is live, so it refuses too rather than reading absence into it.
fn check_not_live(
    name: &hive_types::Ident,
    listed: anyhow::Result<Vec<String>>,
) -> Result<(), String> {
    match listed {
        Ok(live)
            if live.iter().any(|c| {
                c == &format!("{}{name}", lifecycle::AGENT_PREFIX) || c.as_str() == name.as_str()
            }) =>
        {
            Err(format!(
                "refusing to purge {name}: container still exists — use DESTR0Y first"
            ))
        }
        Ok(_) => Ok(()),
        Err(e) => Err(format!(
            "refusing to purge {name}: cannot confirm no live container exists: {e:#}"
        )),
    }
}

/// Wipe a tombstoned agent's
/// retained state dir + applied config dir entirely.
#[utoipa::path(
    post,
    path = "/api/purge-tombstone/{name}",
    params(("name" = String, Path, description = "agent name")),
    responses(
        (status = 200, description = "purged", body = String),
        (status = 400, description = "bad agent name"),
        (status = 500, description = "a live container still exists, or a dir removal failed"),
    ),
    tag = "tombstones"
)]
pub(super) async fn post_purge_tombstone(
    State(state): State<AppState>,
    AxumPath(name): AxumPath<String>,
) -> Response {
    // Format guard FIRST so a name like `..` can't traverse into the
    // parent of `/var/lib/hyperhive/agents/{name}` and have
    // `remove_dir_all` wipe `/var/lib/hyperhive/` itself. Existing
    // manager + live-container checks below don't catch `..` — only
    // the whitelist does. Existence check via
    // `containers_snapshot()` is deliberately NOT used here:
    // tombstoned agents are gone from the snapshot by design; that's
    // the whole point of this endpoint.
    let name = match Ident::parse(&name) {
        Ok(n) => n,
        Err(reason) => {
            return (StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response();
        }
    };
    // Sanity: refuse to purge if a live container still exists with this
    // name. The dashboard already filters tombstones to non-live names,
    // but the operator could send a stale POST. An unreadable list is not
    // proof of absence — see `check_not_live`.
    if let Err(reason) = check_not_live(&name, lifecycle::list().await) {
        return error_response(&reason);
    }
    let mut errors = Vec::new();
    for dir in [
        crate::paths::agent_state_dir(&name),
        crate::paths::applied_dir(name.as_str()),
    ] {
        if dir.exists()
            && let Err(e) = std::fs::remove_dir_all(&dir)
        {
            errors.push(format!("{}: {e}", dir.display()));
        }
    }
    let _ = state
        .coord
        .approvals
        .fail_pending_for_agent(name.as_str(), "agent state purged");
    if errors.is_empty() {
        tracing::info!(%name, "tombstone purged");
        // Fire the post-purge tombstones snapshot so dashboards
        // drop the row live; matching form carries
        // `data-no-refresh`.
        emit_tombstones_snapshot(&state.coord).await;
        (StatusCode::OK, "ok").into_response()
    } else {
        error_response(&format!("purge {name} partial: {}", errors.join(", ")))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn ident(s: &str) -> hive_types::Ident {
        hive_types::Ident::parse(s).unwrap()
    }

    /// A live container by that name in the list refuses the purge.
    #[test]
    fn live_container_in_list_refuses() {
        let name = ident("doomed");
        let err = check_not_live(&name, Ok(vec!["h-doomed".to_owned()]))
            .expect_err("a listed container must refuse the purge");
        assert!(err.contains("still exists"), "{err}");
    }

    /// No matching name in a readable list allows the purge.
    #[test]
    fn absent_from_list_allows() {
        let name = ident("doomed");
        check_not_live(&name, Ok(vec!["h-other".to_owned()]))
            .expect("an absent container must allow the purge");
        // Control: the same name present in the list still refuses, so the
        // pass above comes from the absence, not from the helper accepting
        // anything.
        assert!(check_not_live(&name, Ok(vec!["h-doomed".to_owned()])).is_err());
    }

    /// An unreadable list proves nothing, so it must refuse — this is the
    /// case that regresses to a silent delete under `unwrap_or_default()`.
    #[test]
    fn unreadable_list_refuses() {
        let name = ident("doomed");
        let err = check_not_live(&name, Err(anyhow::anyhow!("connect to hive-priv socket")))
            .expect_err("an unreadable list must not read as an absent container");
        assert!(err.contains("cannot confirm"), "{err}");
    }
}