//! Operations that are exposed through more than one surface (the host admin //! socket *and* the dashboard's POST endpoints). Each function takes a //! `&Coordinator` and the request parameters; callers stitch the response //! shape they want (HTTP redirect vs JSON). use std::sync::Arc; use anyhow::{Context as _, Result, bail}; use hive_sh4re::{ApprovalKind, ApprovalStatus, HelperEvent}; use crate::coordinator::{Coordinator, TransientKind}; use crate::lifecycle; /// Approve a pending request. Marks the approval row durably, then /// either runs the work inline (`InitConfig`, sub-second git ops) or /// submits it to the job queue so the dashboard POST returns /// immediately while the long-running pipeline runs off-thread /// (operator no longer blocks on a 30-90s spinner for `MergeConfigPr`). /// /// Dispatch: /// - `MergeConfigPr` → a single-node `ApprovalDeploy` /// DAG (the two-phase meta deploy stays opaque in v1; ~30-90s) /// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion) /// - `Spawn` → a `Spawn` DAG (`Create → WriteDropin → Reconcile`) /// - `InitConfig` → inline (<1s; queue card would be noise) /// /// `ApprovalDeploy` resolves the approval inside its pipeline; the /// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`] /// when their DAG settles terminal. pub async fn approve(coord: Arc, id: i64) -> Result<()> { let approval = coord.approvals.mark_approved(id)?; tracing::info!( %approval.id, %approval.agent, kind = ?approval.kind, %approval.commit_ref, "approval: dispatching", ); match approval.kind { ApprovalKind::InitConfig => { // Sub-second git seed + forge-remote wire. Routing through // the queue would surface a queue card that's gone before // the operator's eyes refocus. Run inline. let proposed_dir = Coordinator::agent_proposed_dir(&approval.agent); let claude_dir = Coordinator::agent_claude_dir(&approval.agent); let notes_dir = Coordinator::agent_notes_dir(&approval.agent); run_approval_init_config(&coord, approval, proposed_dir, claude_dir, notes_dir).await } ApprovalKind::UpdateMetaInputs => { // Inputs JSON-encoded into commit_ref by the manager's // submit path — surface them on the DAG so the dashboard // can show *which* inputs are about to bump. The cascade // rebuilds fan out when the lock bump lands (so they build // against the post-bump lock, and a failed bump fans out // nothing). let inputs: Vec = serde_json::from_str(&approval.commit_ref).unwrap_or_default(); let submitted = coord .job_queue .submit(crate::job_queue::templates::meta_update( inputs, crate::job_queue::Source::Approval, format!("approval #{id} meta input update"), Some(id), )); if let Err(e) = submitted { return Err(e.context("submit meta-update dag")); } coord.emit_rebuild_queue_snapshot(); Ok(()) } ApprovalKind::Spawn => { // The spawn's tail `Reconcile` starts the container, so the // new agent's power intent is `Up` from the outset. if let Err(e) = coord .power .set(approval.agent.as_str(), crate::power::Wanted::Up) { tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed"); } let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn( approval.agent.as_str(), id, format!("approval #{id} spawn"), )); if let Err(e) = submitted { return Err(e.context("submit spawn dag")); } coord.emit_rebuild_queue_snapshot(); Ok(()) } ApprovalKind::SchedulePrompt => { // No queue card for SchedulePrompt — the work is a single // sqlite insert, the actual "running" lifetime lives on // the scheduled-prompts surface itself (worker fires it // at the scheduled time). Run inline + fire // `ApprovalResolved` so the approval row leaves Pending // immediately. let result = run_approval_schedule_prompt(&coord, approval).await; if result.is_ok() { coord.emit_schedules_snapshot(); } result } ApprovalKind::MergeConfigPr => { // The work ends in a container rebuild, so route it through the // rebuild queue. The queue worker dispatches MergeConfigPr // approvals to `run_merge_config_pr` (verify the reviewed PR head, // ff the forge config repo's main to it, mark merged, then the // deploy tail). enqueue_approval_rebuild( &coord, approval.agent.as_str(), id, format!("approval #{id} merge config pr"), ); Ok(()) } } } /// Submit the single-node `ApprovalDeploy` DAG tied to an approval id. /// Used by the `MergeConfigPr` dispatch arm — the work ends in a container /// rebuild routed through the queue; the node executor runs /// `run_merge_config_pr`. fn enqueue_approval_rebuild( coord: &Arc, agent: &str, approval_id: i64, reason: String, ) { if let Err(e) = coord .job_queue .submit(crate::job_queue::templates::approval_deploy( agent, approval_id, reason, )) { tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed"); } coord.emit_rebuild_queue_snapshot(); } /// Worker entry point for `ApprovalKind::MergeConfigPr` queue entries — the /// config-change flow's deploy worker. Re-fetches the approval row, runs the /// merge pipeline, and fires /// `ApprovalResolved` + the `Rebuilt` lifecycle event via `finish_approval`. /// `run_merge_config_pr` already fast-forwarded the forge repo's `main` to the /// reviewed head (that IS the merge), so `push_config`'s `main` refspec is a /// no-op — but it still mirrors the `deployed/` / `failed/` tag the /// deploy tail plants onto the merged sha, giving the merged commit a /// forge-visible deploy marker. A `MergeConfigPr` is never a first spawn (the /// agent already exists). pub async fn run_approval_merge_config_pr( coord: &Arc, queue_entry_id: Option, approval_id: i64, ) -> Result<()> { let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::MergeConfigPr)?; let agent_dir = crate::paths::agent_runtime_dir(approval.agent.as_str()); let applied_dir = crate::paths::applied_dir(approval.agent.as_str()); // Captured up front to scope the failure-comment's build-log lookup to // rows this deploy produced (see `post_merge_failure_to_pr`). let since_ts = hive_sh4re::wire_time::now_unix(); coord.set_queue_step(queue_entry_id, "merge config pr"); let (result, terminal_tag) = run_merge_config_pr(coord, &approval, &agent_dir, &applied_dir, queue_entry_id).await; // Mirror the deploy bookkeeping tag (`deployed/` or `failed/`) the // deploy tail planted onto the merged sha to the forge config repo, so the // merged commit carries a forge-visible deploy marker. `main` is already // ff'd by the merge, so only the tag refspec actually lands; best-effort, // never fails the approval. coord.set_queue_step(queue_entry_id, "forge push"); if let Err(e) = crate::forge::push_config(approval.agent.as_str()).await { tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after merge failed"); } // On a failed deploy, surface the failing build log back onto the PR so // the manager sees why it was rejected without leaving the forge. if let Err(e) = &result { post_merge_failure_to_pr(coord, &approval, since_ts, e).await; } finish_approval(coord, &approval, result, terminal_tag) } /// Max stderr bytes to inline in a PR failure comment. Keeps the comment /// readable and under forge's size limits while still carrying the tail /// where the nix/build error actually surfaces. const PR_FAIL_LOG_TAIL_BYTES: usize = 4000; /// On a failed `MergeConfigPr` deploy, post the failing build log back to the /// config PR as a comment so the manager sees the rejection reason on the PR /// itself. Best-effort: any error here is logged, never allowed to disturb the /// approval-resolution path. /// /// The failing `build_log` row is located heuristically: the most recent `fail` /// row for this agent that started at/after `since_ts` (the caller's function /// entry). Because deploys are serialised per agent through the queue, that is /// the step which just failed — `verify`, `prepare-deploy`, `prebuild`, or the /// container rebuild. Pre-build failures (drift gate, fetch) create no `build_log` /// row, so the comment then carries only the error text. async fn post_merge_failure_to_pr( coord: &Arc, approval: &hive_sh4re::Approval, since_ts: i64, err: &anyhow::Error, ) { let Ok(pr) = approval.commit_ref.parse::() else { return; }; let repo = crate::forge::config_repo(approval.agent.as_str()); let log_section = coord .build_logs .list_recent_for_agent(approval.agent.as_str(), 10) .ok() .and_then(|rows| { rows.into_iter() .find(|r| r.status.as_deref() == Some("fail") && r.started_at >= since_ts) }) .and_then(|row| coord.build_logs.get_full(row.id).ok().flatten()) .map(|full| { let tail = tail_bytes(full.stderr.trim_end(), PR_FAIL_LOG_TAIL_BYTES); format!( "\n\n**Failing step:** `{}` (build log #{})\n\n```\n{tail}\n```", full.header.kind, full.header.id ) }) .unwrap_or_default(); let body = format!( "## ⚠️ config deploy failed\n\n\ Approval #{} to merge this PR could not be deployed:\n\n\ ```\n{err:#}\n```{log_section}", approval.id ); if let Err(e) = crate::forge::post_pr_comment(&repo, pr, &body).await { tracing::warn!(agent = %approval.agent, %pr, error = ?e, "post merge-failure comment to PR failed"); } } /// Return the last `max_bytes` of `s`, snapped to a char boundary, prefixed /// with an elision marker when truncated. fn tail_bytes(s: &str, max_bytes: usize) -> String { if s.len() <= max_bytes { return s.to_owned(); } let mut start = s.len() - max_bytes; while start < s.len() && !s.is_char_boundary(start) { start += 1; } format!("[… truncated …]\n{}", &s[start..]) } /// PR-merge config pipeline. `approval.commit_ref` is the PR number; /// `approval.fetched_sha` is the PR head sha the operator reviewed. Steps: /// 1. drift gate — re-read the live PR head; if it moved since review, abort /// WITHOUT mutating anything (the operator must re-review the new head); /// 2. fetch the reviewed head into the applied repo so later git ops resolve /// it locally; /// 3. eval-verify the reviewed commit against the meta flake BEFORE the /// irreversible push; /// 4. fast-forward the forge repo's `main` to the reviewed head — THE merge; /// 5. mark the PR merged (best-effort: `main` is already at the head, so a /// failure here is logged, not fatal); /// 6. run the shared deploy tail (`deploy_applied_target`): ff applied/main, /// meta deploy, container rebuild, finalize/rollback. /// /// Returns `(build result, terminal tag)` like `deploy_applied_target`. Any /// pre-merge abort returns `Err` with no mutation; the operator re-reviews. async fn run_merge_config_pr( coord: &Arc, approval: &hive_sh4re::Approval, agent_dir: &std::path::Path, applied_dir: &std::path::Path, queue_entry_id: Option, ) -> (Result<()>, Option) { let id = approval.id; let pr: u64 = match approval.commit_ref.parse() { Ok(n) => n, Err(e) => { return ( Err(anyhow::anyhow!( "parse PR number from commit_ref {:?}: {e}", approval.commit_ref )), None, ); } }; let reviewed = match approval.fetched_sha.as_deref() { Some(s) => s.to_owned(), None => { return ( Err(anyhow::anyhow!( "merge config pr approval {id} has no reviewed head sha" )), None, ); } }; let repo = crate::forge::config_repo(approval.agent.as_str()); // 1. Drift gate: the live PR head must still equal what was reviewed. coord.set_queue_step(queue_entry_id, "verify PR head"); let head = match crate::forge::pr_head_sha(&repo, pr).await { Ok(h) => h, Err(e) => return (Err(anyhow::anyhow!("read PR #{pr} head: {e}")), None), }; if head != reviewed { return ( Err(anyhow::anyhow!( "PR #{pr} head drifted since review (reviewed {reviewed}, now {head}); re-review before merging" )), None, ); } // 2. Fetch the reviewed head into applied so ff/verify/deploy resolve it. coord.set_queue_step(queue_entry_id, "fetch PR head"); if let Err(e) = crate::forge::fetch_pr_head_into_applied(&repo, pr).await { return ( Err(anyhow::anyhow!("fetch PR #{pr} head into applied: {e}")), None, ); } // 3. Eval-verify BEFORE the irreversible push (bad nix fails fast here). coord.set_queue_step(queue_entry_id, "verify proposal (eval)"); if let Err(e) = crate::meta::verify_commit(approval.agent.as_str(), applied_dir, &reviewed).await { return ( Err(anyhow::anyhow!("verify merge head {reviewed}: {e:#}")), None, ); } // Capture the currently-deployed sha for the deploy tail's rollback. let prev_main_sha = match lifecycle::git_rev_parse(applied_dir, "refs/heads/main").await { Ok(s) => s, Err(e) => return (Err(anyhow::anyhow!("read applied/main: {e:#}")), None), }; // 4. THE merge: fast-forward-only merge the reviewed head to `main` via the // forge API, pinned to the reviewed sha (`head_commit_id`). This one call // both advances `main` to the reviewed head and marks the PR merged — no // direct push to the protected branch. Unlike the old push-then-mark split, // a failure here means `main` was NOT advanced, so it's fatal: we must not // deploy a head the forge didn't merge. coord.set_queue_step(queue_entry_id, "fast-forward-merge PR"); match crate::forge::merge_config_pr_ff(&repo, pr, &reviewed).await { Ok(()) => {} Err(crate::forge::ForgeMergeError::HeadDrift { expected, actual }) => { return ( Err(anyhow::anyhow!( "PR #{pr} head drifted before merge (reviewed {expected}, now {actual}); re-review before merging" )), None, ); } Err(e) => return (Err(anyhow::anyhow!("ff-merge PR #{pr}: {e}")), None), } // 5. Deploy tail. target == finalize == the reviewed head. deploy_applied_target( coord, approval.agent.as_str(), agent_dir, applied_dir, &reviewed, &reviewed, id, &prev_main_sha, queue_entry_id, ) .await } /// Inline (non-queued) handler for `ApprovalKind::SchedulePrompt`. /// On approve, decode the `SchedulePromptPayload` JSON from the /// approval's `commit_ref`, insert a row into `scheduled_prompts` /// (with `source = Approval { id }`), and fire `ApprovalResolved`. /// The worker takes over from here — fan-out at fire time. async fn run_approval_schedule_prompt( coord: &Coordinator, approval: hive_sh4re::Approval, ) -> Result<()> { let result: Result<()> = async { let payload: hive_sh4re::SchedulePromptPayload = serde_json::from_str(&approval.commit_ref) .context("decode SchedulePromptPayload from approval.commit_ref")?; coord .scheduled_prompts .submit(&crate::scheduled_prompts::NewSchedule { owner: approval.agent.to_string(), targets: payload.targets, body: payload.body, first_fire_at_unix: payload.first_fire_at_unix, interval_seconds: payload.interval_seconds, description: payload.description, source: crate::scheduled_prompts::ScheduleSource::Approval { id: approval.id }, }) .map(|_| ()) .context("insert scheduled prompt") } .await; finish_approval(coord, &approval, result, None) } /// Terminal hook for approval-carrying DAGs — the job queue's /// scheduler calls this exactly once when such a DAG settles terminal. /// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is /// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves /// *inside* its node, so its DAG is skipped — unless it was cancelled /// while still queued, in which case the node never ran and the row /// would otherwise dangle forever. pub(crate) async fn resolve_approval_dag( coord: &Arc, terminal: &crate::job_queue::TerminalDag, ) { use crate::job_queue::{State, Template}; let Some(approval_id) = terminal.approval_id else { return; }; if terminal.template == Template::Rebuild && terminal.state != State::Cancelled { return; // ApprovalDeploy resolved inside the node. } let approval = match coord.approvals.get(approval_id) { Ok(Some(a)) => a, Ok(None) => { tracing::warn!(approval_id, "approval dag terminal: row no longer exists"); return; } Err(e) => { tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed"); return; } }; let result: Result<()> = match terminal.state { State::Done => Ok(()), State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")), _ => Err(anyhow::anyhow!( "{}", terminal .error .clone() .unwrap_or_else(|| "job dag failed".to_owned()) )), }; if approval.kind == ApprovalKind::Spawn { // Post-spawn forge bookkeeping (user, config repo mirror, meta // access) — warn-only, then the resolution events + a rescan so // the dashboard reflects the post-spawn state either way. if result.is_ok() { forge_after_first_spawn(coord, approval.agent.as_str()).await; } else { coord.rescan_containers_and_emit().await; crate::dashboard::emit_tombstones_snapshot(coord).await; } } if let Err(e) = finish_approval(coord, &approval, result, None) { tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure"); } } /// Re-fetch an approval row from sqlite for a queue-worker dispatch. /// Bails if the row is gone (deny race), if its kind doesn't match, /// or if the lookup itself fails. The kind check is defensive — the /// queue's `dispatch` already routes by `QueueKind`, but the approval /// kind is the authoritative source of truth and a mismatch points /// at a deeper bug we'd want to surface. fn fetch_approval_for_worker( coord: &Coordinator, approval_id: i64, expected_kind: ApprovalKind, ) -> Result { let approval = coord .approvals .get(approval_id) .map_err(|e| anyhow::anyhow!("read approval {approval_id}: {e:#}"))? .ok_or_else(|| anyhow::anyhow!("approval {approval_id} no longer exists"))?; if approval.kind != expected_kind { bail!( "approval {approval_id} kind mismatch: queue expected {expected_kind:?}, row is {actual:?}", actual = approval.kind ); } Ok(approval) } /// Forge bookkeeping run once after the very first container spawn: /// create the per-agent forge user, mirror the applied repo, and grant /// read access to core/meta. Also rescans containers so the dashboard /// reflects the post-spawn state. async fn forge_after_first_spawn(coord: &Arc, agent: &str) { if let Err(e) = crate::forge::ensure_user_for(agent).await { tracing::warn!(%agent, error = ?e, "forge: ensure_user after first spawn failed"); } if let Err(e) = crate::forge::ensure_config_repo(agent).await { tracing::warn!(%agent, error = ?e, "forge: ensure_config_repo after first spawn failed"); } if let Some(core_token) = crate::forge::core_token() && let Err(e) = crate::forge::meta_read_access(agent, &core_token).await { tracing::warn!(%agent, error = ?e, "forge: meta_read_access after first spawn failed"); } if let Err(e) = crate::forge::ensure_meta_remote(agent).await { tracing::warn!(%agent, error = ?e, "forge: ensure_meta_remote after first spawn failed"); } coord.rescan_containers_and_emit().await; crate::dashboard::emit_tombstones_snapshot(coord).await; } /// Inline (non-queued) handler for `ApprovalKind::InitConfig`. Just /// seeds the proposed git repo + the per-agent dirs — sub-second /// work that doesn't justify a queue card. async fn run_approval_init_config( coord: &Coordinator, approval: hive_sh4re::Approval, proposed_dir: std::path::PathBuf, claude_dir: std::path::PathBuf, notes_dir: std::path::PathBuf, ) -> Result<()> { let result: Result<()> = async { // Place the new child under its requesting parent (carried in // commit_ref by submit_init_config). An empty commit_ref means // no explicit parent was named (privileged manager socket, or an // approval queued before the new-child feature) — write no edge // and let `topology::reconcile` assign the default position on // first spawn, so this path never names a specific root agent. if !approval.commit_ref.is_empty() { crate::topology::add_child(approval.agent.as_str(), &approval.commit_ref) .map_err(|e| anyhow::anyhow!("topology add_child: {e}"))?; } // Create the agent's state root as a btrfs subvolume FIRST, before // any dir-seed touches it. `ensure_agent_state_subvolume` is // progressive ("root exists → skip"), and `setup_proposed` does a // `create_dir_all` on the proposed-config path which would // materialise the state root as a plain directory — after which // the subvolume create is silently skipped and the agent never // lands on a subvolume (no quota, no snapshot). Order matters. lifecycle::ensure_agent_state_subvolume(approval.agent.as_str()).await?; lifecycle::setup_proposed(&proposed_dir, approval.agent.as_str()).await?; lifecycle::ensure_claude_dir(&claude_dir)?; lifecycle::ensure_state_dir(¬es_dir)?; Ok(()) } .await; if result.is_ok() && let Err(e) = crate::forge::ensure_meta_remote(approval.agent.as_str()).await { tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after init_config failed"); } finish_approval(coord, &approval, result, None) } fn finish_approval( coord: &Coordinator, approval: &hive_sh4re::Approval, result: Result<()>, terminal_tag: Option, ) -> Result<()> { let (status, note, ok) = match &result { Ok(()) => (ApprovalStatus::Approved, None, true), Err(e) => { let note = format!("{e:#}"); let _ = coord.approvals.mark_failed(approval.id, ¬e); (ApprovalStatus::Failed, Some(note), false) } }; coord.notify_submitter( approval.id, &HelperEvent::ApprovalResolved { id: approval.id, agent: approval.agent.to_string(), commit_ref: approval.commit_ref.clone(), status, note: note.clone(), sha: approval.fetched_sha.clone(), tag: terminal_tag.clone(), }, ); // Phase 5b: also fire on the dashboard event channel so the // browser moves the row out of pending into history without a // snapshot refetch. `approved` rows that succeed get the // approval's logged resolved_at indirectly via `now_unix()`; // failures already wrote it via mark_failed above. let approval_kind = approval.kind.as_str(); let sha_short = approval .fetched_sha .as_deref() .map(|s| s[..s.len().min(12)].to_owned()); let status_str = if ok { "approved" } else { "failed" }; coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { id: approval.id, agent: approval.agent.as_str(), approval_kind, sha_short, status: status_str, note: note.clone(), description: approval.description.clone(), }); // For spawn/rebuild/init_config approvals, also surface the underlying // action so the manager knows whether the lifecycle step succeeded. // The ApprovalResolved event already carries the same `ok` signal but // separating it lets the manager react to the lifecycle change // without having to special-case approvals. match approval.kind { ApprovalKind::InitConfig => { if ok { coord.notify_submitter( approval.id, &HelperEvent::ConfigReady { agent: approval.agent.to_string(), }, ); } } ApprovalKind::Spawn => coord.notify_submitter( approval.id, &HelperEvent::Spawned { agent: approval.agent.to_string(), ok, note, }, ), // MergeConfigPr ends in a container rebuild — surface a Rebuilt // lifecycle event. (It is never a first spawn — the agent already // exists — so it never needs the Spawned arm above.) ApprovalKind::MergeConfigPr => { coord.notify_submitter( approval.id, &HelperEvent::Rebuilt { agent: approval.agent.to_string(), ok, note, sha: approval.fetched_sha.clone(), tag: terminal_tag, }, ); } // UpdateMetaInputs / SchedulePrompt: ApprovalResolved already // carries the result. No separate lifecycle event needed. ApprovalKind::UpdateMetaInputs | ApprovalKind::SchedulePrompt => {} } result } /// Deploy tail for the config-PR merge flow. Fast-forwards `applied/main` to /// `target_ref`, syncs the working tree, runs the meta two-phase deploy + /// container rebuild, and plants the `deployed/` / /// `failed/` bookkeeping tags. On build failure it rolls /// `applied/main` back to `prev_main_sha` and aborts the staged meta lock so /// the agent stays on its last-good tree. Returns the build result + the /// terminal tag name. /// /// Caller-specific bits stay OUT of here: fetching the PR head, the /// `verify_commit` gate, the ff-merge, and forge mark-merged. `finalize_sha` /// is the sha recorded by `meta::finalize_deploy`; `target_ref` is what /// `applied/main` fast-forwards to. The agent always already exists here (a /// merge is never a first spawn), so there's no `sync_agents` step — the /// operator `Spawn` flow owns first-time meta registration. #[allow( clippy::too_many_arguments, clippy::too_many_lines, reason = "one sequential ff/deploy/rebuild/finalize pipeline; splitting it \ would obscure the linear flow" )] async fn deploy_applied_target( coord: &Arc, agent: &str, agent_dir: &std::path::Path, applied_dir: &std::path::Path, target_ref: &str, finalize_sha: &str, tag_base: i64, prev_main_sha: &str, queue_entry_id: Option, ) -> (Result<()>, Option) { let id = tag_base; coord.set_queue_step(queue_entry_id, "fast-forward applied/main"); // Fast-forward applied/main to target_ref + sync the working tree. // Meta input pins `?ref=main`, so this is what makes nix re-lock to // the target commit on the prepare_deploy step below. On build // failure we roll main back to prev_main_sha so a crash leaves the // agent on its last-good tree. if let Err(e) = lifecycle::git_update_ref(applied_dir, "refs/heads/main", target_ref).await { return (Err(anyhow::anyhow!("ff main to {target_ref}: {e:#}")), None); } if let Err(e) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await { // main is ahead; working tree didn't sync. Roll main back to // keep the two consistent before bailing. let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await; return (Err(anyhow::anyhow!("read-tree to main: {e:#}")), None); } coord.set_queue_step(queue_entry_id, "meta prepare_deploy"); // Phase 1 of the meta two-phase deploy: relock without committing. if let Err(e) = crate::meta::prepare_deploy(agent).await { let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await; let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await; return (Err(anyhow::anyhow!("meta prepare_deploy: {e:#}")), None); } // Container-level rebuild (or first-time create) against meta#. // Step labels are emitted inside rebuild_no_meta via the callback so // the dashboard reflects actual phase progress rather than a static // "nixos-container update" label for the whole multi-minute window. let hive = coord.hive_env(); let paths = Coordinator::agent_paths(agent, agent_dir.to_path_buf()); let build_result = lifecycle::rebuild_no_meta( agent, &hive, &paths, // Inline start: the apply-commit flow verifies the agent comes // back up before finalizing the deploy tag, so the start stays // part of this entry rather than a deferred fast-lane follow-up. false, &|step| coord.set_queue_step(queue_entry_id, step), &|log_id| coord.set_queue_build_log(queue_entry_id, log_id), ) .await; match build_result { Ok(_) => { coord.set_queue_step(queue_entry_id, "finalize deploy"); let tag = format!("deployed/{id}"); if let Err(e) = lifecycle::git_tag(applied_dir, &tag, target_ref).await { tracing::warn!(%agent, %id, error = ?e, "plant deployed tag failed"); } if let Err(e) = crate::meta::finalize_deploy(agent, finalize_sha, &tag).await { // The build itself succeeded — meta lock landed but // couldn't be committed. Surface as a soft warn so the // operator can git-commit by hand if they care. tracing::warn!(%agent, %id, error = ?e, "meta finalize_deploy failed"); } // Wake the agent on its next turn so claude sees the // config change took effect. Same hint pattern as // auto_update::rebuild_agent — manager approved a // proposal, agent picks up where it left off with the // new env / packages. coord.kick_agent(agent, "config update applied"); (Ok(()), Some(tag)) } Err(e) => { let tag = format!("failed/{id}"); let body = format!("{e:#}"); if let Err(te) = lifecycle::git_tag_annotated(applied_dir, &tag, target_ref, &body).await { tracing::warn!(%agent, %id, error = ?te, "annotate failed tag failed"); } // Roll main back to last known-good so the on-disk state // matches what nixos-container last successfully built. if let Err(re) = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await { tracing::warn!(%agent, %id, error = ?re, "main rollback failed"); } if let Err(re) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await { tracing::warn!(%agent, %id, error = ?re, "rollback read-tree failed"); } // Drop the staged meta lock change so the deploy log // only ever shows successes. if let Err(ae) = crate::meta::abort_deploy().await { tracing::warn!(%agent, %id, error = ?ae, "meta abort_deploy failed"); } (Err(e), Some(tag)) } } } /// Tear down a sub-agent container. By default this is non-destructive to /// persistent state: the proposed/applied config repos and the Claude /// credentials dir under `/var/lib/hyperhive/{agents,applied}//` are /// kept, so recreating an agent of the same name reuses prior config + creds /// (no re-login). The ephemeral runtime dir under `/run/hyperhive/agents/` /// is cleared because its contents (the mcp socket) don't survive restarts /// anyway. With `purge=true` the persistent trees are also wiped — config /// history, claude creds, notes — there is no undo. /// The bootstrap/root container is destroyable like any other: it's /// imperative infra that `auto_update::ensure_root_agent` recreates on the /// next hive-c0re startup if absent, so destroying it is transient rather /// than something to refuse at the API. pub async fn destroy(coord: &Arc, name: &str, purge: bool) -> Result<()> { tracing::info!(%name, purge, "destroy"); // Guard auto-clears on the success path's final scope exit and on // every early-return / cancellation along the way. let guard = coord.transient_guard(name, TransientKind::Destroying); lifecycle::destroy(name).await?; coord.unregister_agent(name); let runtime = crate::paths::agent_runtime_dir(name); if runtime.exists() { let _ = std::fs::remove_dir_all(&runtime); } if purge { // The state root may be a btrfs subvolume: a subvolume root // can't be removed with rmdir/`remove_dir_all`, so delete it via // hive-priv (root) first. No-op for plain-dir agents — the loop below // then handles the plain-dir state root plus the applied dir. if let Err(e) = crate::priv_client::delete_agent_subvolume(name).await { tracing::warn!(error = ?e, %name, "purge: delete state subvolume failed"); } // A malformed name can't have a persistent state tree (the state dir // is only ever created under a validated Ident), so its removal is a // no-op — skip the state-dir sweep and just clear the applied dir. let state_dir = hive_types::Ident::parse(name) .ok() .map(|id| crate::paths::agent_state_dir(&id)); for dir in state_dir .into_iter() .chain([crate::paths::applied_dir(name)]) { if dir.exists() && let Err(e) = std::fs::remove_dir_all(&dir) { tracing::warn!(error = ?e, dir = %dir.display(), "purge: remove failed"); } } } // Meta flake: drop the agent's input + nixosConfiguration so a // future spawn under the same name re-seeds cleanly, and so the // meta lock doesn't reference a vanished applied repo. Log + keep // going on failure — destroy already succeeded at the // nixos-container level, the meta repo is just bookkeeping. if let Err(e) = sync_meta_after_lifecycle(coord).await { tracing::warn!(error = ?e, %name, "meta sync after destroy failed"); } let _ = coord.approvals.fail_pending_for_agent( name, if purge { "agent purged" } else { "agent destroyed" }, ); // Drop the durable power intent — a future agent of the same name // seeds fresh from its observed state. if let Err(e) = coord.power.remove(name) { tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed"); } drop(guard); coord.notify_manager(&HelperEvent::Destroyed { agent: name.to_owned(), }); // Container row disappeared — rescan so the dashboard fires // `ContainerRemoved` for the gone row, then emit the // tombstones snapshot (gained one on destroy, lost one on // purge — recompute either way). coord.rescan_containers_and_emit().await; crate::dashboard::emit_tombstones_snapshot(coord).await; // Re-emit the schedules snapshot: the rescan above refreshed the live // roster, so any schedule that still targets the just-destroyed agent // now drops that ghost column live (no page reload needed). coord.emit_schedules_snapshot(); // Update tmpfiles.d to remove the destroyed agent's dirs from the // boot-time pre-creation list. Best-effort: failure is logged only. tokio::spawn(lifecycle::sync_tmpfiles()); Ok(()) } /// Rerender the meta flake from whatever containers still exist on /// disk. Called after lifecycle ops that change the agent set (today: /// destroy). Idempotent — a no-op when nothing changed. async fn sync_meta_after_lifecycle(coord: &Coordinator) -> Result<()> { let agents = lifecycle::agents_for_meta_listing().await?; crate::meta::sync_agents(&coord.hive_env(), &agents).await } pub fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> { let approval = coord.approvals.get(id)?; coord.approvals.mark_denied(id, note)?; tracing::info!(%id, note, "approval denied"); if let Some(a) = approval { let sha = a.fetched_sha.clone(); let approval_kind = a.kind.as_str(); let sha_short = sha.as_deref().map(|s| s[..s.len().min(12)].to_owned()); let description = a.description.clone(); let agent_owned = a.agent.clone(); coord.notify_submitter( a.id, &HelperEvent::ApprovalResolved { id: a.id, agent: a.agent.to_string(), commit_ref: a.commit_ref, status: ApprovalStatus::Denied, note: note.map(String::from), sha, // A denied config PR carries no git tag — it stays open on the forge. tag: None, }, ); coord.emit_approval_resolved(crate::coordinator::ApprovalResolved { id, agent: agent_owned.as_str(), approval_kind, sha_short, status: "denied", note: note.map(String::from), description, }); } Ok(()) }