hyperhive/hive-c0re/src/actions.rs

//! Operations that are exposed through more than one surface (the host admin
//! socket *and* the dashboard's POST endpoints). Each function takes a
//! `&Coordinator` and the request parameters; callers stitch the response
//! shape they want (HTTP redirect vs JSON).

use std::sync::Arc;

use anyhow::{Context as _, Result, bail};
use hive_sh4re::{ApprovalKind, ApprovalStatus, HelperEvent};

use crate::coordinator::{Coordinator, TransientKind};
use crate::lifecycle;

/// Approve a pending request. Marks the approval row durably, then
/// either runs the work inline (`InitConfig`, sub-second git ops) or
/// submits it to the job queue so the dashboard POST returns
/// immediately while the long-running pipeline runs off-thread
/// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`).
///
/// Dispatch:
/// - `ApplyCommit` / `MergeConfigPr` → a single-node `ApprovalDeploy`
///   DAG (the two-phase meta deploy stays opaque in v1; ~30-90s)
/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
/// - `Spawn`       → a `Spawn` DAG (`Create → WriteDropin → Reconcile`)
/// - `InitConfig`  → inline (<1s; queue card would be noise)
///
/// `ApprovalDeploy` resolves the approval inside its pipeline; the
/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`]
/// when their DAG settles terminal.
pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
    let approval = coord.approvals.mark_approved(id)?;
    tracing::info!(
        %approval.id,
        %approval.agent,
        kind = ?approval.kind,
        %approval.commit_ref,
        "approval: dispatching",
    );
    match approval.kind {
        ApprovalKind::InitConfig => {
            // Sub-second git seed + forge-remote wire. Routing through
            // the queue would surface a queue card that's gone before
            // the operator's eyes refocus. Run inline.
            let proposed_dir = Coordinator::agent_proposed_dir(&approval.agent);
            let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
            let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
            run_approval_init_config(&coord, approval, proposed_dir, claude_dir, notes_dir).await
        }
        ApprovalKind::ApplyCommit => {
            enqueue_approval_rebuild(
                &coord,
                &approval.agent,
                id,
                format!("approval #{id} apply commit"),
            );
            Ok(())
        }
        ApprovalKind::UpdateMetaInputs => {
            // Inputs JSON-encoded into commit_ref by the manager's
            // submit path — surface them on the DAG so the dashboard
            // can show *which* inputs are about to bump. The cascade
            // rebuilds fan out when the lock bump lands (so they build
            // against the post-bump lock, and a failed bump fans out
            // nothing).
            let inputs: Vec<String> =
                serde_json::from_str(&approval.commit_ref).unwrap_or_default();
            let submitted = coord
                .job_queue
                .submit(crate::job_queue::templates::meta_update(
                    inputs,
                    crate::job_queue::Source::Approval,
                    format!("approval #{id} meta input update"),
                    Some(id),
                ));
            if let Err(e) = submitted {
                return Err(e.context("submit meta-update dag"));
            }
            coord.emit_rebuild_queue_snapshot();
            Ok(())
        }
        ApprovalKind::Spawn => {
            // The spawn's tail `Reconcile` starts the container, so the
            // new agent's power intent is `Up` from the outset.
            if let Err(e) = coord.power.set(&approval.agent, crate::power::Wanted::Up) {
                tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
            }
            let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn(
                &approval.agent,
                id,
                format!("approval #{id} spawn"),
            ));
            if let Err(e) = submitted {
                return Err(e.context("submit spawn dag"));
            }
            coord.emit_rebuild_queue_snapshot();
            Ok(())
        }
        ApprovalKind::SchedulePrompt => {
            // No queue card for SchedulePrompt — the work is a single
            // sqlite insert, the actual "running" lifetime lives on
            // the scheduled-prompts surface itself (worker fires it
            // at the scheduled time). Run inline + fire
            // `ApprovalResolved` so the approval row leaves Pending
            // immediately.
            let result = run_approval_schedule_prompt(&coord, approval).await;
            if result.is_ok() {
                coord.emit_schedules_snapshot();
            }
            result
        }
        ApprovalKind::MergeConfigPr => {
            // Like ApplyCommit, the work ends in a container rebuild, so
            // route it through the rebuild queue. The queue worker
            // dispatches MergeConfigPr approvals to `run_merge_config_pr`
            // (verify the reviewed PR head, ff the forge config repo's
            // main to it, mark merged, then the shared deploy tail).
            enqueue_approval_rebuild(
                &coord,
                &approval.agent,
                id,
                format!("approval #{id} merge config pr"),
            );
            Ok(())
        }
    }
}

/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id.
/// Shared by the `ApplyCommit` and `MergeConfigPr` dispatch arms — both
/// end in a container rebuild routed through the queue, differing only
/// in the `reason`. The node executor branches on the approval's kind
/// to pick the right handler.
fn enqueue_approval_rebuild(
    coord: &Arc<Coordinator>,
    agent: &str,
    approval_id: i64,
    reason: String,
) {
    if let Err(e) = coord
        .job_queue
        .submit(crate::job_queue::templates::approval_deploy(
            agent,
            approval_id,
            reason,
        ))
    {
        tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed");
    }
    coord.emit_rebuild_queue_snapshot();
}

/// Worker entry point for `ApprovalKind::ApplyCommit` queue entries.
/// Re-fetches the approval row, runs the commit pipeline, and fires
/// `ApprovalResolved` + the lifecycle event (`Rebuilt` / `Spawned`
/// for first-spawn).
pub async fn run_approval_apply_commit(
    coord: &Arc<Coordinator>,
    queue_entry_id: Option<u64>,
    approval_id: i64,
) -> Result<()> {
    let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::ApplyCommit)?;
    // Runtime dir creation is handled inside lifecycle::rebuild_no_meta's
    // spawn path (first-spawn) or is already present for rebuilds.
    let agent_dir = crate::paths::agent_runtime_dir(&approval.agent);
    let applied_dir = crate::paths::applied_dir(&approval.agent);
    coord.set_queue_step(queue_entry_id, "apply commit");
    let (result, terminal_tag, is_first_spawn) =
        run_apply_commit(coord, &approval, &agent_dir, &applied_dir, queue_entry_id).await;
    coord.set_queue_step(queue_entry_id, "forge push");
    if let Err(e) = crate::forge::push_config(&approval.agent).await {
        tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after apply failed");
    }
    if is_first_spawn && result.is_ok() {
        coord.set_queue_step(queue_entry_id, "first-spawn forge bootstrap");
        forge_after_first_spawn(coord, &approval.agent).await;
    }
    // `finish_approval` returns the original `result` so the queue
    // worker sees Ok/Err and marks the queue entry accordingly. The
    // approval row + helper events have already been fanned out.
    finish_approval(coord, &approval, result, terminal_tag, is_first_spawn)
}

/// Worker entry point for `ApprovalKind::MergeConfigPr` queue entries —
/// the PR-based config flow's counterpart to `run_approval_apply_commit`.
/// Re-fetches the approval row, runs the merge pipeline, and fires
/// `ApprovalResolved` + the `Rebuilt` lifecycle event via `finish_approval`.
/// `run_merge_config_pr` already fast-forwarded the forge repo's `main` to the
/// reviewed head (that IS the merge), so `push_config`'s `main` refspec is a
/// no-op — but it still mirrors the `deployed/<id>` / `failed/<id>` tag the
/// deploy tail plants onto the merged sha, giving the merged commit a
/// forge-visible deploy marker. A `MergeConfigPr` is never a first spawn (the
/// agent already exists).
pub async fn run_approval_merge_config_pr(
    coord: &Arc<Coordinator>,
    queue_entry_id: Option<u64>,
    approval_id: i64,
) -> Result<()> {
    let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::MergeConfigPr)?;
    let agent_dir = crate::paths::agent_runtime_dir(&approval.agent);
    let applied_dir = crate::paths::applied_dir(&approval.agent);
    // Captured up front to scope the failure-comment's build-log lookup to
    // rows this deploy produced (see `post_merge_failure_to_pr`).
    let since_ts = hive_sh4re::wire_time::now_unix();
    coord.set_queue_step(queue_entry_id, "merge config pr");
    let (result, terminal_tag) =
        run_merge_config_pr(coord, &approval, &agent_dir, &applied_dir, queue_entry_id).await;
    // Mirror the deploy bookkeeping tag (`deployed/<id>` or `failed/<id>`) the
    // deploy tail planted onto the merged sha to the forge config repo, so the
    // merged commit carries a forge-visible deploy marker. `main` is already
    // ff'd by the merge, so only the tag refspec actually lands; best-effort,
    // never fails the approval.
    coord.set_queue_step(queue_entry_id, "forge push");
    if let Err(e) = crate::forge::push_config(&approval.agent).await {
        tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after merge failed");
    }
    // On a failed deploy, surface the failing build log back onto the PR so
    // the manager sees why it was rejected without leaving the forge.
    if let Err(e) = &result {
        post_merge_failure_to_pr(coord, &approval, since_ts, e).await;
    }
    finish_approval(coord, &approval, result, terminal_tag, false)
}

/// Max stderr bytes to inline in a PR failure comment. Keeps the comment
/// readable and under forge's size limits while still carrying the tail
/// where the nix/build error actually surfaces.
const PR_FAIL_LOG_TAIL_BYTES: usize = 4000;

/// On a failed `MergeConfigPr` deploy, post the failing build log back to the
/// config PR as a comment so the manager sees the rejection reason on the PR
/// itself. Best-effort: any error here is logged, never allowed to disturb the
/// approval-resolution path.
///
/// The failing `build_log` row is located heuristically: the most recent `fail`
/// row for this agent that started at/after `since_ts` (the caller's function
/// entry). Because deploys are serialised per agent through the queue, that is
/// the step which just failed — `verify`, `prepare-deploy`, `prebuild`, or the
/// container rebuild. Pre-build failures (drift gate, fetch) create no `build_log`
/// row, so the comment then carries only the error text.
async fn post_merge_failure_to_pr(
    coord: &Arc<Coordinator>,
    approval: &hive_sh4re::Approval,
    since_ts: i64,
    err: &anyhow::Error,
) {
    let Ok(pr) = approval.commit_ref.parse::<u64>() else {
        return;
    };
    let repo = crate::forge::config_repo(&approval.agent);

    let log_section = coord
        .build_logs
        .list_recent_for_agent(&approval.agent, 10)
        .ok()
        .and_then(|rows| {
            rows.into_iter()
                .find(|r| r.status.as_deref() == Some("fail") && r.started_at >= since_ts)
        })
        .and_then(|row| coord.build_logs.get_full(row.id).ok().flatten())
        .map(|full| {
            let tail = tail_bytes(full.stderr.trim_end(), PR_FAIL_LOG_TAIL_BYTES);
            format!(
                "\n\n**Failing step:** `{}` (build log #{})\n\n```\n{tail}\n```",
                full.header.kind, full.header.id
            )
        })
        .unwrap_or_default();

    let body = format!(
        "## ⚠️ config deploy failed\n\n\
         Approval #{} to merge this PR could not be deployed:\n\n\
         ```\n{err:#}\n```{log_section}",
        approval.id
    );

    if let Err(e) = crate::forge::post_pr_comment(&repo, pr, &body).await {
        tracing::warn!(agent = %approval.agent, %pr, error = ?e, "post merge-failure comment to PR failed");
    }
}

/// Return the last `max_bytes` of `s`, snapped to a char boundary, prefixed
/// with an elision marker when truncated.
fn tail_bytes(s: &str, max_bytes: usize) -> String {
    if s.len() <= max_bytes {
        return s.to_owned();
    }
    let mut start = s.len() - max_bytes;
    while start < s.len() && !s.is_char_boundary(start) {
        start += 1;
    }
    format!("[… truncated …]\n{}", &s[start..])
}

/// PR-merge config pipeline. `approval.commit_ref` is the PR number;
/// `approval.fetched_sha` is the PR head sha the operator reviewed. Steps:
/// 1. drift gate — re-read the live PR head; if it moved since review, abort
///    WITHOUT mutating anything (the operator must re-review the new head);
/// 2. fetch the reviewed head into the applied repo so later git ops resolve
///    it locally;
/// 3. eval-verify the reviewed commit against the meta flake BEFORE the
///    irreversible push (same gate `run_apply_commit` uses);
/// 4. fast-forward the forge repo's `main` to the reviewed head — THE merge;
/// 5. mark the PR merged (best-effort: `main` is already at the head, so a
///    failure here is logged, not fatal);
/// 6. run the shared deploy tail (`deploy_applied_target`): ff applied/main,
///    meta deploy, container rebuild, finalize/rollback.
///
/// Returns `(build result, terminal tag)` like `deploy_applied_target`. Any
/// pre-merge abort returns `Err` with no mutation; the operator re-reviews.
async fn run_merge_config_pr(
    coord: &Arc<Coordinator>,
    approval: &hive_sh4re::Approval,
    agent_dir: &std::path::Path,
    applied_dir: &std::path::Path,
    queue_entry_id: Option<u64>,
) -> (Result<()>, Option<String>) {
    let id = approval.id;
    let pr: u64 = match approval.commit_ref.parse() {
        Ok(n) => n,
        Err(e) => {
            return (
                Err(anyhow::anyhow!(
                    "parse PR number from commit_ref {:?}: {e}",
                    approval.commit_ref
                )),
                None,
            );
        }
    };
    let reviewed = match approval.fetched_sha.as_deref() {
        Some(s) => s.to_owned(),
        None => {
            return (
                Err(anyhow::anyhow!(
                    "merge config pr approval {id} has no reviewed head sha"
                )),
                None,
            );
        }
    };
    let repo = crate::forge::config_repo(&approval.agent);

    // 1. Drift gate: the live PR head must still equal what was reviewed.
    coord.set_queue_step(queue_entry_id, "verify PR head");
    let head = match crate::forge::pr_head_sha(&repo, pr).await {
        Ok(h) => h,
        Err(e) => return (Err(anyhow::anyhow!("read PR #{pr} head: {e}")), None),
    };
    if head != reviewed {
        return (
            Err(anyhow::anyhow!(
                "PR #{pr} head drifted since review (reviewed {reviewed}, now {head}); re-review before merging"
            )),
            None,
        );
    }

    // 2. Fetch the reviewed head into applied so ff/verify/deploy resolve it.
    coord.set_queue_step(queue_entry_id, "fetch PR head");
    if let Err(e) = crate::forge::fetch_pr_head_into_applied(&repo, pr).await {
        return (
            Err(anyhow::anyhow!("fetch PR #{pr} head into applied: {e}")),
            None,
        );
    }

    // 3. Eval-verify BEFORE the irreversible push (bad nix fails fast here).
    coord.set_queue_step(queue_entry_id, "verify proposal (eval)");
    if let Err(e) = crate::meta::verify_commit(&approval.agent, applied_dir, &reviewed).await {
        return (
            Err(anyhow::anyhow!("verify merge head {reviewed}: {e:#}")),
            None,
        );
    }

    // Capture the currently-deployed sha for the deploy tail's rollback.
    let prev_main_sha = match lifecycle::git_rev_parse(applied_dir, "refs/heads/main").await {
        Ok(s) => s,
        Err(e) => return (Err(anyhow::anyhow!("read applied/main: {e:#}")), None),
    };

    // 4. THE merge: fast-forward-only merge the reviewed head to `main` via the
    // forge API, pinned to the reviewed sha (`head_commit_id`). This one call
    // both advances `main` to the reviewed head and marks the PR merged — no
    // direct push to the protected branch. Unlike the old push-then-mark split,
    // a failure here means `main` was NOT advanced, so it's fatal: we must not
    // deploy a head the forge didn't merge.
    coord.set_queue_step(queue_entry_id, "fast-forward-merge PR");
    match crate::forge::merge_config_pr_ff(&repo, pr, &reviewed).await {
        Ok(()) => {}
        Err(crate::forge::ForgeMergeError::HeadDrift { expected, actual }) => {
            return (
                Err(anyhow::anyhow!(
                    "PR #{pr} head drifted before merge (reviewed {expected}, now {actual}); re-review before merging"
                )),
                None,
            );
        }
        Err(e) => return (Err(anyhow::anyhow!("ff-merge PR #{pr}: {e}")), None),
    }

    // 5. Shared deploy tail. target == finalize == the reviewed head;
    // never a first spawn (the agent already exists).
    deploy_applied_target(
        coord,
        &approval.agent,
        agent_dir,
        applied_dir,
        &reviewed,
        &reviewed,
        id,
        &prev_main_sha,
        false,
        queue_entry_id,
    )
    .await
}

/// Inline (non-queued) handler for `ApprovalKind::SchedulePrompt`.
/// On approve, decode the `SchedulePromptPayload` JSON from the
/// approval's `commit_ref`, insert a row into `scheduled_prompts`
/// (with `source = Approval { id }`), and fire `ApprovalResolved`.
/// The worker takes over from here — fan-out at fire time.
async fn run_approval_schedule_prompt(
    coord: &Coordinator,
    approval: hive_sh4re::Approval,
) -> Result<()> {
    let result: Result<()> = async {
        let payload: hive_sh4re::SchedulePromptPayload = serde_json::from_str(&approval.commit_ref)
            .context("decode SchedulePromptPayload from approval.commit_ref")?;
        coord
            .scheduled_prompts
            .submit(&crate::scheduled_prompts::NewSchedule {
                owner: approval.agent.clone(),
                targets: payload.targets,
                body: payload.body,
                first_fire_at_unix: payload.first_fire_at_unix,
                interval_seconds: payload.interval_seconds,
                description: payload.description,
                source: crate::scheduled_prompts::ScheduleSource::Approval { id: approval.id },
            })
            .map(|_| ())
            .context("insert scheduled prompt")
    }
    .await;
    finish_approval(coord, &approval, result, None, false)
}

/// Terminal hook for approval-carrying DAGs — the job queue's
/// scheduler calls this exactly once when such a DAG settles terminal.
/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is
/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves
/// *inside* its node, so its DAG is skipped — unless it was cancelled
/// while still queued, in which case the node never ran and the row
/// would otherwise dangle forever.
pub(crate) async fn resolve_approval_dag(
    coord: &Arc<Coordinator>,
    terminal: &crate::job_queue::TerminalDag,
) {
    use crate::job_queue::{State, Template};
    let Some(approval_id) = terminal.approval_id else {
        return;
    };
    if terminal.template == Template::Rebuild && terminal.state != State::Cancelled {
        return; // ApprovalDeploy resolved inside the node.
    }
    let approval = match coord.approvals.get(approval_id) {
        Ok(Some(a)) => a,
        Ok(None) => {
            tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
            return;
        }
        Err(e) => {
            tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
            return;
        }
    };
    let result: Result<()> = match terminal.state {
        State::Done => Ok(()),
        State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")),
        _ => Err(anyhow::anyhow!(
            "{}",
            terminal
                .error
                .clone()
                .unwrap_or_else(|| "job dag failed".to_owned())
        )),
    };
    if approval.kind == ApprovalKind::Spawn {
        // Post-spawn forge bookkeeping (user, config repo mirror, meta
        // access) — warn-only, then the resolution events + a rescan so
        // the dashboard reflects the post-spawn state either way.
        if result.is_ok() {
            forge_after_first_spawn(coord, &approval.agent).await;
        } else {
            coord.rescan_containers_and_emit().await;
            crate::dashboard::emit_tombstones_snapshot(coord).await;
        }
    }
    if let Err(e) = finish_approval(coord, &approval, result, None, false) {
        tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
    }
}

/// Re-fetch an approval row from sqlite for a queue-worker dispatch.
/// Bails if the row is gone (deny race), if its kind doesn't match,
/// or if the lookup itself fails. The kind check is defensive — the
/// queue's `dispatch` already routes by `QueueKind`, but the approval
/// kind is the authoritative source of truth and a mismatch points
/// at a deeper bug we'd want to surface.
fn fetch_approval_for_worker(
    coord: &Coordinator,
    approval_id: i64,
    expected_kind: ApprovalKind,
) -> Result<hive_sh4re::Approval> {
    let approval = coord
        .approvals
        .get(approval_id)
        .map_err(|e| anyhow::anyhow!("read approval {approval_id}: {e:#}"))?
        .ok_or_else(|| anyhow::anyhow!("approval {approval_id} no longer exists"))?;
    if approval.kind != expected_kind {
        bail!(
            "approval {approval_id} kind mismatch: queue expected {expected_kind:?}, row is {actual:?}",
            actual = approval.kind
        );
    }
    Ok(approval)
}

/// Forge bookkeeping run once after the very first container spawn:
/// create the per-agent forge user, mirror the applied repo, and grant
/// read access to core/meta. Also rescans containers so the dashboard
/// reflects the post-spawn state.
async fn forge_after_first_spawn(coord: &Arc<Coordinator>, agent: &str) {
    if let Err(e) = crate::forge::ensure_user_for(agent).await {
        tracing::warn!(%agent, error = ?e, "forge: ensure_user after first spawn failed");
    }
    if let Err(e) = crate::forge::ensure_config_repo(agent).await {
        tracing::warn!(%agent, error = ?e, "forge: ensure_config_repo after first spawn failed");
    }
    if let Some(core_token) = crate::forge::core_token()
        && let Err(e) = crate::forge::meta_read_access(agent, &core_token).await
    {
        tracing::warn!(%agent, error = ?e, "forge: meta_read_access after first spawn failed");
    }
    if let Err(e) = crate::forge::ensure_meta_remote(agent).await {
        tracing::warn!(%agent, error = ?e, "forge: ensure_meta_remote after first spawn failed");
    }
    coord.rescan_containers_and_emit().await;
    crate::dashboard::emit_tombstones_snapshot(coord).await;
}

/// Inline (non-queued) handler for `ApprovalKind::InitConfig`. Just
/// seeds the proposed git repo + the per-agent dirs — sub-second
/// work that doesn't justify a queue card.
async fn run_approval_init_config(
    coord: &Coordinator,
    approval: hive_sh4re::Approval,
    proposed_dir: std::path::PathBuf,
    claude_dir: std::path::PathBuf,
    notes_dir: std::path::PathBuf,
) -> Result<()> {
    let result: Result<()> = async {
        // Place the new child under its requesting parent (carried in
        // commit_ref by submit_init_config). An empty commit_ref means
        // no explicit parent was named (privileged manager socket, or an
        // approval queued before the new-child feature) — write no edge
        // and let `topology::reconcile` assign the default position on
        // first spawn, so this path never names a specific root agent.
        if !approval.commit_ref.is_empty() {
            crate::topology::add_child(&approval.agent, &approval.commit_ref)
                .map_err(|e| anyhow::anyhow!("topology add_child: {e}"))?;
        }
        // Create the agent's state root as a btrfs subvolume FIRST, before
        // any dir-seed touches it. `ensure_agent_state_subvolume` is
        // progressive ("root exists → skip"), and `setup_proposed` does a
        // `create_dir_all` on the proposed-config path which would
        // materialise the state root as a plain directory — after which
        // the subvolume create is silently skipped and the agent never
        // lands on a subvolume (no quota, no snapshot). Order matters.
        lifecycle::ensure_agent_state_subvolume(&approval.agent).await?;
        lifecycle::setup_proposed(&proposed_dir, &approval.agent).await?;
        lifecycle::ensure_claude_dir(&claude_dir)?;
        lifecycle::ensure_state_dir(&notes_dir)?;
        Ok(())
    }
    .await;
    if result.is_ok()
        && let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await
    {
        tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after init_config failed");
    }
    finish_approval(coord, &approval, result, None, false)
}

fn finish_approval(
    coord: &Coordinator,
    approval: &hive_sh4re::Approval,
    result: Result<()>,
    terminal_tag: Option<String>,
    is_first_spawn: bool,
) -> Result<()> {
    let (status, note, ok) = match &result {
        Ok(()) => (ApprovalStatus::Approved, None, true),
        Err(e) => {
            let note = format!("{e:#}");
            let _ = coord.approvals.mark_failed(approval.id, &note);
            (ApprovalStatus::Failed, Some(note), false)
        }
    };
    coord.notify_submitter(
        approval.id,
        &HelperEvent::ApprovalResolved {
            id: approval.id,
            agent: approval.agent.clone(),
            commit_ref: approval.commit_ref.clone(),
            status,
            note: note.clone(),
            sha: approval.fetched_sha.clone(),
            tag: terminal_tag.clone(),
        },
    );
    // Phase 5b: also fire on the dashboard event channel so the
    // browser moves the row out of pending into history without a
    // snapshot refetch. `approved` rows that succeed get the
    // approval's logged resolved_at indirectly via `now_unix()`;
    // failures already wrote it via mark_failed above.
    let approval_kind = approval.kind.as_str();
    let sha_short = approval
        .fetched_sha
        .as_deref()
        .map(|s| s[..s.len().min(12)].to_owned());
    let status_str = if ok { "approved" } else { "failed" };
    coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
        id: approval.id,
        agent: &approval.agent,
        approval_kind,
        sha_short,
        status: status_str,
        note: note.clone(),
        description: approval.description.clone(),
    });
    // For spawn/rebuild/init_config approvals, also surface the underlying
    // action so the manager knows whether the lifecycle step succeeded.
    // The ApprovalResolved event already carries the same `ok` signal but
    // separating it lets the manager react to the lifecycle change
    // without having to special-case approvals.
    match approval.kind {
        ApprovalKind::InitConfig => {
            if ok {
                coord.notify_submitter(
                    approval.id,
                    &HelperEvent::ConfigReady {
                        agent: approval.agent.clone(),
                    },
                );
            }
        }
        ApprovalKind::Spawn => coord.notify_submitter(
            approval.id,
            &HelperEvent::Spawned {
                agent: approval.agent.clone(),
                ok,
                note,
                sha: approval.fetched_sha.clone(),
            },
        ),
        ApprovalKind::ApplyCommit if is_first_spawn => {
            coord.notify_submitter(
                approval.id,
                &HelperEvent::Spawned {
                    agent: approval.agent.clone(),
                    ok,
                    note,
                    sha: approval.fetched_sha.clone(),
                },
            );
        }
        // MergeConfigPr ends in a container rebuild just like a
        // non-first-spawn ApplyCommit, so both surface the same Rebuilt
        // lifecycle event. (MergeConfigPr is never a first spawn — the
        // agent already exists — so it never hits the Spawned arm above.)
        ApprovalKind::ApplyCommit | ApprovalKind::MergeConfigPr => {
            coord.notify_submitter(
                approval.id,
                &HelperEvent::Rebuilt {
                    agent: approval.agent.clone(),
                    ok,
                    note,
                    sha: approval.fetched_sha.clone(),
                    tag: terminal_tag,
                },
            );
        }
        // UpdateMetaInputs / SchedulePrompt: ApprovalResolved already
        // carries the result. No separate lifecycle event needed.
        ApprovalKind::UpdateMetaInputs | ApprovalKind::SchedulePrompt => {}
    }
    result
}

/// Tag-driven `ApplyCommit` handler. Walks the approval through the tag
/// state machine documented in `docs/approvals.md`: stamp
/// `approved/<id>` and `building/<id>` first so the audit trail
/// captures intent, then drop the candidate tree into the working dir
/// without moving HEAD, run the rebuild, and either fast-forward
/// `applied/main` to the proposal commit on success
/// (`deployed/<id>`) or annotate `failed/<id>` with the build error
/// and reset the working tree back to the last known-good main. main
/// never advances on a failed build, so a crash-and-recover doesn't
/// leave the agent pointing at a tree it can't evaluate. The shared
/// ff/deploy/rebuild/finalize tail lives in `deploy_applied_target`.
async fn run_apply_commit(
    coord: &Arc<Coordinator>,
    approval: &hive_sh4re::Approval,
    agent_dir: &std::path::Path,
    applied_dir: &std::path::Path,
    queue_entry_id: Option<u64>,
) -> (Result<()>, Option<String>, bool) {
    let id = approval.id;
    let proposal_ref = format!("refs/tags/proposal/{id}");

    // Detect first spawn before we touch anything so we can branch on it
    // throughout this function.
    let is_first_spawn = !lifecycle::container_exists(&approval.agent).await;

    // Defensive: submit-time should have planted proposal/<id>, but if
    // the row was migrated from an older schema or the tag got pruned
    // we fail early with a clear note rather than building a stale
    // tree.
    if let Err(e) = lifecycle::git_rev_parse(applied_dir, &proposal_ref).await {
        return (
            Err(anyhow::anyhow!(
                "missing proposal tag {proposal_ref}: {e:#}"
            )),
            None,
            is_first_spawn,
        );
    }

    // Capture the currently-deployed sha so we can roll applied/main
    // (and the meta lock indirectly) back if the build fails.
    let prev_main_sha = match lifecycle::git_rev_parse(applied_dir, "refs/heads/main").await {
        Ok(s) => s,
        Err(e) => {
            return (
                Err(anyhow::anyhow!("read applied/main: {e:#}")),
                None,
                is_first_spawn,
            );
        }
    };

    // Pre-flight eval-verify the proposal commit against the meta flake
    // WITHOUT mutating applied/main or the meta lock, so an evaluation
    // error (bad nix, missing module option, unresolvable lock) fails
    // fast here instead of after we've fast-forwarded main and have to
    // roll it back. Skipped on first spawn: the agent has no
    // `agent-<name>` meta input to override yet (sync_agents adds it
    // below). This is the reusable verify primitive the PR-based config
    // flow gates its irreversible ff-push on.
    if !is_first_spawn {
        let proposal_sha = match lifecycle::git_rev_parse(applied_dir, &proposal_ref).await {
            Ok(s) => s,
            Err(e) => {
                return (
                    Err(anyhow::anyhow!("rev-parse {proposal_ref}: {e:#}")),
                    None,
                    is_first_spawn,
                );
            }
        };
        coord.set_queue_step(queue_entry_id, "verify proposal (eval)");
        if let Err(e) =
            crate::meta::verify_commit(&approval.agent, applied_dir, &proposal_sha).await
        {
            return (
                Err(anyhow::anyhow!("verify proposal {proposal_ref}: {e:#}")),
                None,
                is_first_spawn,
            );
        }
    }

    coord.set_queue_step(queue_entry_id, "plant tags");
    if let Err(e) = lifecycle::git_tag(applied_dir, &format!("approved/{id}"), &proposal_ref).await
    {
        return (
            Err(anyhow::anyhow!("plant approved/{id}: {e:#}")),
            None,
            is_first_spawn,
        );
    }
    if let Err(e) = lifecycle::git_tag(applied_dir, &format!("building/{id}"), &proposal_ref).await
    {
        return (
            Err(anyhow::anyhow!("plant building/{id}: {e:#}")),
            None,
            is_first_spawn,
        );
    }

    // Fast-forward applied/main to the proposal, run the meta deploy +
    // container rebuild, and finalize/roll-back — the tail shared with the
    // PR-merge flow. ApplyCommit's target == finalize sha source is
    // `fetched_sha` (or the proposal ref when unset), matching the prior
    // inline behavior exactly.
    let (result, tag) = deploy_applied_target(
        coord,
        &approval.agent,
        agent_dir,
        applied_dir,
        &proposal_ref,
        approval.fetched_sha.as_deref().unwrap_or(&proposal_ref),
        id,
        &prev_main_sha,
        is_first_spawn,
        queue_entry_id,
    )
    .await;
    (result, tag, is_first_spawn)
}

/// Shared deploy tail for config-applying approvals (`ApplyCommit` + the
/// PR-merge flow). Fast-forwards `applied/main` to `target_ref`, syncs the
/// working tree, runs the meta two-phase deploy + container rebuild, and
/// plants the `deployed/<tag_base>` / `failed/<tag_base>` bookkeeping tags.
/// On build failure it rolls `applied/main` back to `prev_main_sha` and aborts
/// the staged meta lock so the agent stays on its last-good tree. Returns the
/// build result + the terminal tag name.
///
/// Caller-specific bits stay OUT of here: the source fetch (proposal tag vs
/// forge fetch), the `approved/building` tags, `verify_commit`, and any forge
/// ff-push / mark-merged. `is_first_spawn` gates the one-time meta
/// `sync_agents` step (only `ApplyCommit`'s first spawn passes `true`;
/// the PR-merge flow always passes `false` — the agent already exists).
/// `finalize_sha` is the sha recorded by `meta::finalize_deploy`; `target_ref`
/// is what `applied/main` fast-forwards to (a proposal ref or a commit sha).
#[allow(
    clippy::too_many_arguments,
    clippy::too_many_lines,
    reason = "one sequential ff/deploy/rebuild/finalize pipeline shared by both \
              config-apply callers; splitting it would obscure the linear flow"
)]
async fn deploy_applied_target(
    coord: &Arc<Coordinator>,
    agent: &str,
    agent_dir: &std::path::Path,
    applied_dir: &std::path::Path,
    target_ref: &str,
    finalize_sha: &str,
    tag_base: i64,
    prev_main_sha: &str,
    is_first_spawn: bool,
    queue_entry_id: Option<u64>,
) -> (Result<()>, Option<String>) {
    let id = tag_base;

    coord.set_queue_step(queue_entry_id, "fast-forward applied/main");
    // Fast-forward applied/main to target_ref + sync the working tree.
    // Meta input pins `?ref=main`, so this is what makes nix re-lock to
    // the target commit on the prepare_deploy step below. On build
    // failure we roll main back to prev_main_sha so a crash leaves the
    // agent on its last-good tree.
    if let Err(e) = lifecycle::git_update_ref(applied_dir, "refs/heads/main", target_ref).await {
        return (Err(anyhow::anyhow!("ff main to {target_ref}: {e:#}")), None);
    }
    if let Err(e) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
        // main is ahead; working tree didn't sync. Roll main back to
        // keep the two consistent before bailing.
        let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await;
        return (Err(anyhow::anyhow!("read-tree to main: {e:#}")), None);
    }

    // First spawn: sync_agents must add this agent to the meta flake
    // before prepare_deploy can update its input lock (which won't
    // exist yet if this is the agent's first deploy).
    if is_first_spawn {
        coord.set_queue_step(queue_entry_id, "meta sync_agents (first spawn)");
        let agents = match lifecycle::agents_for_meta_listing_with(agent).await {
            Ok(a) => a,
            Err(e) => {
                let _ =
                    lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await;
                let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
                return (
                    Err(anyhow::anyhow!("agents_for_meta_listing_with: {e:#}")),
                    None,
                );
            }
        };
        if let Err(e) = crate::meta::sync_agents(&coord.hive_env(), &agents).await {
            let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await;
            let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
            return (
                Err(anyhow::anyhow!("meta sync_agents for first spawn: {e:#}")),
                None,
            );
        }
    }

    coord.set_queue_step(queue_entry_id, "meta prepare_deploy");
    // Phase 1 of the meta two-phase deploy: relock without committing.
    if let Err(e) = crate::meta::prepare_deploy(agent).await {
        let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await;
        let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
        return (Err(anyhow::anyhow!("meta prepare_deploy: {e:#}")), None);
    }

    // Container-level rebuild (or first-time create) against meta#<name>.
    // Step labels are emitted inside rebuild_no_meta via the callback so
    // the dashboard reflects actual phase progress rather than a static
    // "nixos-container update" label for the whole multi-minute window.
    let hive = coord.hive_env();
    let paths = Coordinator::agent_paths(agent, agent_dir.to_path_buf());
    let build_result = lifecycle::rebuild_no_meta(
        agent,
        &hive,
        &paths,
        // Inline start: the apply-commit flow verifies the agent comes
        // back up before finalizing the deploy tag, so the start stays
        // part of this entry rather than a deferred fast-lane follow-up.
        false,
        &|step| coord.set_queue_step(queue_entry_id, step),
        &|log_id| coord.set_queue_build_log(queue_entry_id, log_id),
    )
    .await;

    match build_result {
        Ok(_) => {
            coord.set_queue_step(queue_entry_id, "finalize deploy");
            let tag = format!("deployed/{id}");
            if let Err(e) = lifecycle::git_tag(applied_dir, &tag, target_ref).await {
                tracing::warn!(%agent, %id, error = ?e, "plant deployed tag failed");
            }
            if let Err(e) = crate::meta::finalize_deploy(agent, finalize_sha, &tag).await {
                // The build itself succeeded — meta lock landed but
                // couldn't be committed. Surface as a soft warn so the
                // operator can git-commit by hand if they care.
                tracing::warn!(%agent, %id, error = ?e, "meta finalize_deploy failed");
            }
            // Wake the agent on its next turn so claude sees the
            // config change took effect. Same hint pattern as
            // auto_update::rebuild_agent — manager approved a
            // proposal, agent picks up where it left off with the
            // new env / packages.
            coord.kick_agent(agent, "config update applied");
            (Ok(()), Some(tag))
        }
        Err(e) => {
            let tag = format!("failed/{id}");
            let body = format!("{e:#}");
            if let Err(te) =
                lifecycle::git_tag_annotated(applied_dir, &tag, target_ref, &body).await
            {
                tracing::warn!(%agent, %id, error = ?te, "annotate failed tag failed");
            }
            // Roll main back to last known-good so the on-disk state
            // matches what nixos-container last successfully built.
            if let Err(re) =
                lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await
            {
                tracing::warn!(%agent, %id, error = ?re, "main rollback failed");
            }
            if let Err(re) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
                tracing::warn!(%agent, %id, error = ?re, "rollback read-tree failed");
            }
            // Drop the staged meta lock change so the deploy log
            // only ever shows successes.
            if let Err(ae) = crate::meta::abort_deploy().await {
                tracing::warn!(%agent, %id, error = ?ae, "meta abort_deploy failed");
            }
            (Err(e), Some(tag))
        }
    }
}

/// Tear down a sub-agent container. By default this is non-destructive to
/// persistent state: the proposed/applied config repos and the Claude
/// credentials dir under `/var/lib/hyperhive/{agents,applied}/<name>/` are
/// kept, so recreating an agent of the same name reuses prior config + creds
/// (no re-login). The ephemeral runtime dir under `/run/hyperhive/agents/`
/// is cleared because its contents (the mcp socket) don't survive restarts
/// anyway. With `purge=true` the persistent trees are also wiped — config
/// history, claude creds, notes — there is no undo.
/// The bootstrap/root container is destroyable like any other: it's
/// imperative infra that `auto_update::ensure_root_agent` recreates on the
/// next hive-c0re startup if absent, so destroying it is transient rather
/// than something to refuse at the API.
pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Result<()> {
    tracing::info!(%name, purge, "destroy");
    // Guard auto-clears on the success path's final scope exit and on
    // every early-return / cancellation along the way.
    let guard = coord.transient_guard(name, TransientKind::Destroying);
    lifecycle::destroy(name).await?;
    coord.unregister_agent(name);
    let runtime = crate::paths::agent_runtime_dir(name);
    if runtime.exists() {
        let _ = std::fs::remove_dir_all(&runtime);
    }
    if purge {
        // The state root may be a btrfs subvolume: a subvolume root
        // can't be removed with rmdir/`remove_dir_all`, so delete it via
        // hive-priv (root) first. No-op for plain-dir agents — the loop below
        // then handles the plain-dir state root plus the applied dir.
        if let Err(e) = crate::priv_client::delete_agent_subvolume(name).await {
            tracing::warn!(error = ?e, %name, "purge: delete state subvolume failed");
        }
        for dir in [
            crate::paths::agent_state_dir(name),
            crate::paths::applied_dir(name),
        ] {
            if dir.exists()
                && let Err(e) = std::fs::remove_dir_all(&dir)
            {
                tracing::warn!(error = ?e, dir = %dir.display(), "purge: remove failed");
            }
        }
    }
    // Meta flake: drop the agent's input + nixosConfiguration so a
    // future spawn under the same name re-seeds cleanly, and so the
    // meta lock doesn't reference a vanished applied repo. Log + keep
    // going on failure — destroy already succeeded at the
    // nixos-container level, the meta repo is just bookkeeping.
    if let Err(e) = sync_meta_after_lifecycle(coord).await {
        tracing::warn!(error = ?e, %name, "meta sync after destroy failed");
    }
    let _ = coord.approvals.fail_pending_for_agent(
        name,
        if purge {
            "agent purged"
        } else {
            "agent destroyed"
        },
    );
    // Drop the durable power intent — a future agent of the same name
    // seeds fresh from its observed state.
    if let Err(e) = coord.power.remove(name) {
        tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed");
    }
    drop(guard);
    coord.notify_manager(&HelperEvent::Destroyed {
        agent: name.to_owned(),
    });
    // Container row disappeared — rescan so the dashboard fires
    // `ContainerRemoved` for the gone row, then emit the
    // tombstones snapshot (gained one on destroy, lost one on
    // purge — recompute either way).
    coord.rescan_containers_and_emit().await;
    crate::dashboard::emit_tombstones_snapshot(coord).await;
    // Re-emit the schedules snapshot: the rescan above refreshed the live
    // roster, so any schedule that still targets the just-destroyed agent
    // now drops that ghost column live (no page reload needed).
    coord.emit_schedules_snapshot();
    // Update tmpfiles.d to remove the destroyed agent's dirs from the
    // boot-time pre-creation list. Best-effort: failure is logged only.
    tokio::spawn(lifecycle::sync_tmpfiles());
    Ok(())
}

/// Rerender the meta flake from whatever containers still exist on
/// disk. Called after lifecycle ops that change the agent set (today:
/// destroy). Idempotent — a no-op when nothing changed.
async fn sync_meta_after_lifecycle(coord: &Coordinator) -> Result<()> {
    let agents = lifecycle::agents_for_meta_listing().await?;
    crate::meta::sync_agents(&coord.hive_env(), &agents).await
}

pub async fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
    let approval = coord.approvals.get(id)?;
    coord.approvals.mark_denied(id, note)?;
    tracing::info!(%id, note, "approval denied");
    let mut tag = None;
    if let Some(a) = approval {
        let sha = a.fetched_sha.clone();
        // ApplyCommit approvals leave a `denied/<id>` tag on the
        // proposal commit so rejected configs are first-class git
        // objects — `git show denied/<id>` in the manager's applied
        // mount yields both the tree the operator rejected and (in
        // the annotated body) the reason. Spawn approvals have no
        // commit to tag, so they fall through unannotated.
        if matches!(a.kind, ApprovalKind::ApplyCommit) {
            let applied_dir = crate::paths::applied_dir(&a.agent);
            let proposal_ref = format!("refs/tags/proposal/{id}");
            if lifecycle::git_rev_parse(&applied_dir, &proposal_ref)
                .await
                .is_ok()
            {
                let tag_name = format!("denied/{id}");
                let body = note.unwrap_or("").to_owned();
                if let Err(e) =
                    lifecycle::git_tag_annotated(&applied_dir, &tag_name, &proposal_ref, &body)
                        .await
                {
                    tracing::warn!(%id, error = ?e, "plant denied tag failed");
                } else {
                    tag = Some(tag_name);
                }
            }
            // Mirror the denied/<id> tag to the forge.
            if let Err(e) = crate::forge::push_config(&a.agent).await {
                tracing::warn!(%id, agent = %a.agent, error = ?e, "forge: push_config after deny failed");
            }
        }
        let approval_kind = a.kind.as_str();
        let sha_short = sha.as_deref().map(|s| s[..s.len().min(12)].to_owned());
        let description = a.description.clone();
        let agent_owned = a.agent.clone();
        coord.notify_submitter(
            a.id,
            &HelperEvent::ApprovalResolved {
                id: a.id,
                agent: a.agent,
                commit_ref: a.commit_ref,
                status: ApprovalStatus::Denied,
                note: note.map(String::from),
                sha,
                tag,
            },
        );
        coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
            id,
            agent: &agent_owned,
            approval_kind,
            sha_short,
            status: "denied",
            note: note.map(String::from),
            description,
        });
    }
    Ok(())
}