188f3ea5ec
Part of the docs-migration chore (issue #708). Remove GitHub issue numbers from inline comments, option descriptions, and rustdoc — these are contextless noise for anyone reading the code without access to the original discussions. Replace with prose that captures the same rationale directly. No functional change. Build still clean (cargo check passes).
776 lines
32 KiB
Rust
776 lines
32 KiB
Rust
//! Operations that are exposed through more than one surface (the host admin
|
|
//! socket *and* the dashboard's POST endpoints). Each function takes a
|
|
//! `&Coordinator` and the request parameters; callers stitch the response
|
|
//! shape they want (HTTP redirect vs JSON).
|
|
|
|
use std::sync::Arc;
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_sh4re::{ApprovalKind, ApprovalStatus, HelperEvent, MANAGER_AGENT};
|
|
|
|
use crate::coordinator::{Coordinator, TransientKind};
|
|
use crate::lifecycle::{self, MANAGER_NAME};
|
|
|
|
/// Approve a pending request. Marks the approval row durably, then
|
|
/// either runs the work inline (`InitConfig`, sub-second git ops) or
|
|
/// enqueues it into `rebuild_queue` so the dashboard POST returns
|
|
/// immediately while the long-running pipeline runs off-thread
|
|
/// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`).
|
|
///
|
|
/// Dispatch:
|
|
/// - `ApplyCommit` → `QueueKind::Rebuild` (~30-90s wall time)
|
|
/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s)
|
|
/// - `Spawn` → `QueueKind::Spawn` (~30-90s)
|
|
/// - `InitConfig` → inline (<1s; queue card would be noise)
|
|
///
|
|
/// The queue worker re-fetches the approval row on dispatch, runs
|
|
/// the kind-specific pipeline, and fires `ApprovalResolved` /
|
|
/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`.
|
|
pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
|
let approval = coord.approvals.mark_approved(id)?;
|
|
tracing::info!(
|
|
%approval.id,
|
|
%approval.agent,
|
|
kind = ?approval.kind,
|
|
%approval.commit_ref,
|
|
"approval: dispatching",
|
|
);
|
|
match approval.kind {
|
|
ApprovalKind::InitConfig => {
|
|
// Sub-second git seed + forge-remote wire. Routing through
|
|
// the queue would surface a queue card that's gone before
|
|
// the operator's eyes refocus. Run inline.
|
|
let proposed_dir = Coordinator::agent_proposed_dir(&approval.agent);
|
|
let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
|
|
let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
|
|
run_approval_init_config(&coord, approval, proposed_dir, claude_dir, notes_dir).await
|
|
}
|
|
ApprovalKind::ApplyCommit => {
|
|
coord.rebuild_queue.enqueue_full(
|
|
crate::rebuild_queue::QueueKind::Rebuild,
|
|
approval.agent.clone(),
|
|
crate::rebuild_queue::QueueSource::Approval,
|
|
format!("approval #{id} apply commit"),
|
|
None,
|
|
Vec::new(),
|
|
Some(id),
|
|
);
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::UpdateMetaInputs => {
|
|
// Inputs JSON-encoded into commit_ref by the manager's
|
|
// submit path — surface them on the queue entry so the
|
|
// dashboard can show *which* inputs are about to bump.
|
|
let inputs: Vec<String> =
|
|
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
|
coord.rebuild_queue.enqueue_full(
|
|
crate::rebuild_queue::QueueKind::MetaUpdate,
|
|
approval.agent.clone(),
|
|
crate::rebuild_queue::QueueSource::Approval,
|
|
format!("approval #{id} meta input update"),
|
|
None,
|
|
inputs,
|
|
Some(id),
|
|
);
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::Spawn => {
|
|
coord.rebuild_queue.enqueue_full(
|
|
crate::rebuild_queue::QueueKind::Spawn,
|
|
approval.agent.clone(),
|
|
crate::rebuild_queue::QueueSource::Approval,
|
|
format!("approval #{id} spawn"),
|
|
None,
|
|
Vec::new(),
|
|
Some(id),
|
|
);
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::SchedulePrompt => {
|
|
// No queue card for SchedulePrompt — the work is a single
|
|
// sqlite insert, the actual "running" lifetime lives on
|
|
// the scheduled-prompts surface itself (worker fires it
|
|
// at the scheduled time). Run inline + fire
|
|
// `ApprovalResolved` so the approval row leaves Pending
|
|
// immediately.
|
|
run_approval_schedule_prompt(&coord, approval).await
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Worker entry point for `ApprovalKind::ApplyCommit` queue entries.
|
|
/// Re-fetches the approval row, runs the commit pipeline, and fires
|
|
/// `ApprovalResolved` + the lifecycle event (`Rebuilt` / `Spawned`
|
|
/// for first-spawn).
|
|
pub async fn run_approval_apply_commit(
|
|
coord: &Arc<Coordinator>,
|
|
queue_entry_id: Option<u64>,
|
|
approval_id: i64,
|
|
) -> Result<()> {
|
|
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::ApplyCommit)?;
|
|
let agent_dir = coord.ensure_runtime(&approval.agent)?;
|
|
let applied_dir = Coordinator::agent_applied_dir(&approval.agent);
|
|
let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
|
|
let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
|
|
coord.set_queue_step(queue_entry_id, "apply commit");
|
|
let (result, terminal_tag, is_first_spawn) = run_apply_commit(
|
|
coord,
|
|
&approval,
|
|
&agent_dir,
|
|
&applied_dir,
|
|
&claude_dir,
|
|
¬es_dir,
|
|
queue_entry_id,
|
|
)
|
|
.await;
|
|
coord.set_queue_step(queue_entry_id, "forge push");
|
|
if let Err(e) = crate::forge::push_config(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after apply failed");
|
|
}
|
|
if is_first_spawn && result.is_ok() {
|
|
coord.set_queue_step(queue_entry_id, "first-spawn forge bootstrap");
|
|
forge_after_first_spawn(coord, &approval.agent).await;
|
|
}
|
|
// `finish_approval` returns the original `result` so the queue
|
|
// worker sees Ok/Err and marks the queue entry accordingly. The
|
|
// approval row + helper events have already been fanned out.
|
|
finish_approval(coord, &approval, result, terminal_tag, is_first_spawn)
|
|
}
|
|
|
|
/// Inline (non-queued) handler for `ApprovalKind::SchedulePrompt`.
|
|
/// On approve, decode the `SchedulePromptPayload` JSON from the
|
|
/// approval's `commit_ref`, insert a row into `scheduled_prompts`
|
|
/// (with `source = Approval { id }`), and fire `ApprovalResolved`.
|
|
/// The worker takes over from here — fan-out at fire time.
|
|
async fn run_approval_schedule_prompt(
|
|
coord: &Coordinator,
|
|
approval: hive_sh4re::Approval,
|
|
) -> Result<()> {
|
|
let result: Result<()> = async {
|
|
let payload: hive_sh4re::SchedulePromptPayload = serde_json::from_str(&approval.commit_ref)
|
|
.context("decode SchedulePromptPayload from approval.commit_ref")?;
|
|
coord
|
|
.scheduled_prompts
|
|
.submit(&crate::scheduled_prompts::NewSchedule {
|
|
owner: approval.agent.clone(),
|
|
targets: payload.targets,
|
|
body: payload.body,
|
|
first_fire_at_unix: payload.first_fire_at_unix,
|
|
interval_seconds: payload.interval_seconds,
|
|
description: payload.description,
|
|
source: crate::scheduled_prompts::ScheduleSource::Approval { id: approval.id },
|
|
})
|
|
.map(|_| ())
|
|
.context("insert scheduled prompt")
|
|
}
|
|
.await;
|
|
finish_approval(coord, &approval, result, None, false)
|
|
}
|
|
|
|
/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue
|
|
/// entries. Inputs come from the approval row's `commit_ref` field
|
|
/// (JSON-encoded by the manager submit path), not the queue entry's
|
|
/// `inputs` — the queue copy is for dashboard display only.
|
|
pub async fn run_approval_update_meta_inputs(
|
|
coord: &Arc<Coordinator>,
|
|
queue_entry_id: Option<u64>,
|
|
approval_id: i64,
|
|
) -> Result<()> {
|
|
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?;
|
|
let inputs: Vec<String> = serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
|
coord.set_queue_step(queue_entry_id, "nix flake update");
|
|
let result = crate::meta::lock_update(&inputs).await;
|
|
finish_approval(coord, &approval, result, None, false)
|
|
}
|
|
|
|
/// Worker entry point for `ApprovalKind::Spawn` queue entries.
|
|
/// Differs from `run_approval_apply_commit` only in routing through
|
|
/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous
|
|
/// in the queue worker — the previous `tokio::spawn` wrapper is gone
|
|
/// (the queue worker itself is the async task).
|
|
pub async fn run_approval_spawn(
|
|
coord: &Arc<Coordinator>,
|
|
queue_entry_id: Option<u64>,
|
|
approval_id: i64,
|
|
) -> Result<()> {
|
|
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?;
|
|
let agent_dir = coord.ensure_runtime(&approval.agent)?;
|
|
let proposed_dir = Coordinator::agent_proposed_dir(&approval.agent);
|
|
let applied_dir = Coordinator::agent_applied_dir(&approval.agent);
|
|
let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
|
|
let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
|
|
// Transient guard keeps the per-container "Spawning" pill lit while
|
|
// the worker is doing the actual nixos-container create. Auto-clears
|
|
// on the function's scope exit (success or panic).
|
|
let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning);
|
|
coord.set_queue_step(queue_entry_id, "lifecycle::spawn");
|
|
let result = lifecycle::spawn(
|
|
&approval.agent,
|
|
&coord.hyperhive_flake,
|
|
&agent_dir,
|
|
&proposed_dir,
|
|
&applied_dir,
|
|
&claude_dir,
|
|
¬es_dir,
|
|
coord.dashboard_port,
|
|
&coord.operator_pronouns,
|
|
&coord.context_window_tokens,
|
|
)
|
|
.await;
|
|
if result.is_ok() {
|
|
coord.set_queue_step(queue_entry_id, "forge user");
|
|
if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed");
|
|
}
|
|
coord.set_queue_step(queue_entry_id, "forge config repo");
|
|
if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed");
|
|
}
|
|
coord.set_queue_step(queue_entry_id, "forge push");
|
|
if let Err(e) = crate::forge::push_config(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed");
|
|
}
|
|
coord.set_queue_step(queue_entry_id, "forge meta access");
|
|
if let Some(core_token) = crate::forge::core_token()
|
|
&& let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed");
|
|
}
|
|
}
|
|
let final_result = finish_approval(coord, &approval, result, None, false);
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
final_result
|
|
}
|
|
|
|
/// Re-fetch an approval row from sqlite for a queue-worker dispatch.
|
|
/// Bails if the row is gone (deny race), if its kind doesn't match,
|
|
/// or if the lookup itself fails. The kind check is defensive — the
|
|
/// queue's `dispatch` already routes by `QueueKind`, but the approval
|
|
/// kind is the authoritative source of truth and a mismatch points
|
|
/// at a deeper bug we'd want to surface.
|
|
fn fetch_approval_for_worker(
|
|
coord: &Coordinator,
|
|
approval_id: i64,
|
|
expected_kind: ApprovalKind,
|
|
) -> Result<hive_sh4re::Approval> {
|
|
let approval = coord
|
|
.approvals
|
|
.get(approval_id)
|
|
.map_err(|e| anyhow::anyhow!("read approval {approval_id}: {e:#}"))?
|
|
.ok_or_else(|| anyhow::anyhow!("approval {approval_id} no longer exists"))?;
|
|
if approval.kind != expected_kind {
|
|
bail!(
|
|
"approval {approval_id} kind mismatch: queue expected {expected_kind:?}, row is {actual:?}",
|
|
actual = approval.kind
|
|
);
|
|
}
|
|
Ok(approval)
|
|
}
|
|
|
|
/// Forge bookkeeping run once after the very first container spawn:
|
|
/// create the per-agent forge user, mirror the applied repo, and grant
|
|
/// read access to core/meta. Also rescans containers so the dashboard
|
|
/// reflects the post-spawn state.
|
|
async fn forge_after_first_spawn(coord: &Arc<Coordinator>, agent: &str) {
|
|
if let Err(e) = crate::forge::ensure_user_for(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_user after first spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_config_repo(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_config_repo after first spawn failed");
|
|
}
|
|
if let Some(core_token) = crate::forge::core_token()
|
|
&& let Err(e) = crate::forge::meta_read_access(agent, &core_token).await
|
|
{
|
|
tracing::warn!(%agent, error = ?e, "forge: meta_read_access after first spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_meta_remote(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_meta_remote after first spawn failed");
|
|
}
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
}
|
|
|
|
/// Inline (non-queued) handler for `ApprovalKind::InitConfig`. Just
|
|
/// seeds the proposed git repo + the per-agent dirs — sub-second
|
|
/// work that doesn't justify a queue card.
|
|
async fn run_approval_init_config(
|
|
coord: &Coordinator,
|
|
approval: hive_sh4re::Approval,
|
|
proposed_dir: std::path::PathBuf,
|
|
claude_dir: std::path::PathBuf,
|
|
notes_dir: std::path::PathBuf,
|
|
) -> Result<()> {
|
|
let result: Result<()> = async {
|
|
lifecycle::setup_proposed(&proposed_dir, &approval.agent).await?;
|
|
lifecycle::ensure_claude_dir(&claude_dir)?;
|
|
lifecycle::ensure_state_dir(¬es_dir)?;
|
|
Ok(())
|
|
}
|
|
.await;
|
|
if result.is_ok()
|
|
&& let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after init_config failed");
|
|
}
|
|
finish_approval(coord, &approval, result, None, false)
|
|
}
|
|
|
|
fn finish_approval(
|
|
coord: &Coordinator,
|
|
approval: &hive_sh4re::Approval,
|
|
result: Result<()>,
|
|
terminal_tag: Option<String>,
|
|
is_first_spawn: bool,
|
|
) -> Result<()> {
|
|
let (status, note, ok) = match &result {
|
|
Ok(()) => (ApprovalStatus::Approved, None, true),
|
|
Err(e) => {
|
|
let note = format!("{e:#}");
|
|
let _ = coord.approvals.mark_failed(approval.id, ¬e);
|
|
(ApprovalStatus::Failed, Some(note), false)
|
|
}
|
|
};
|
|
coord.notify_manager(&HelperEvent::ApprovalResolved {
|
|
id: approval.id,
|
|
agent: approval.agent.clone(),
|
|
commit_ref: approval.commit_ref.clone(),
|
|
status,
|
|
note: note.clone(),
|
|
sha: approval.fetched_sha.clone(),
|
|
tag: terminal_tag.clone(),
|
|
});
|
|
// Phase 5b: also fire on the dashboard event channel so the
|
|
// browser moves the row out of pending into history without a
|
|
// snapshot refetch. `approved` rows that succeed get the
|
|
// approval's logged resolved_at indirectly via `now_unix()`;
|
|
// failures already wrote it via mark_failed above.
|
|
let approval_kind = match approval.kind {
|
|
ApprovalKind::Spawn => "spawn",
|
|
ApprovalKind::ApplyCommit => "apply_commit",
|
|
ApprovalKind::InitConfig => "init_config",
|
|
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
|
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
|
};
|
|
let sha_short = approval
|
|
.fetched_sha
|
|
.as_deref()
|
|
.map(|s| s[..s.len().min(12)].to_owned());
|
|
let status_str = if ok { "approved" } else { "failed" };
|
|
coord.emit_approval_resolved(
|
|
approval.id,
|
|
&approval.agent,
|
|
approval_kind,
|
|
sha_short,
|
|
status_str,
|
|
note.clone(),
|
|
approval.description.clone(),
|
|
);
|
|
// For spawn/rebuild/init_config approvals, also surface the underlying
|
|
// action so the manager knows whether the lifecycle step succeeded.
|
|
// The ApprovalResolved event already carries the same `ok` signal but
|
|
// separating it lets the manager react to the lifecycle change
|
|
// without having to special-case approvals.
|
|
match approval.kind {
|
|
ApprovalKind::InitConfig => {
|
|
if ok {
|
|
coord.notify_manager(&HelperEvent::ConfigReady {
|
|
agent: approval.agent.clone(),
|
|
});
|
|
}
|
|
}
|
|
ApprovalKind::Spawn => coord.notify_manager(&HelperEvent::Spawned {
|
|
agent: approval.agent.clone(),
|
|
ok,
|
|
note,
|
|
sha: approval.fetched_sha.clone(),
|
|
}),
|
|
ApprovalKind::ApplyCommit if is_first_spawn => {
|
|
coord.notify_manager(&HelperEvent::Spawned {
|
|
agent: approval.agent.clone(),
|
|
ok,
|
|
note,
|
|
sha: approval.fetched_sha.clone(),
|
|
});
|
|
}
|
|
ApprovalKind::ApplyCommit => coord.notify_manager(&HelperEvent::Rebuilt {
|
|
agent: approval.agent.clone(),
|
|
ok,
|
|
note,
|
|
sha: approval.fetched_sha.clone(),
|
|
tag: terminal_tag,
|
|
}),
|
|
// UpdateMetaInputs / SchedulePrompt: ApprovalResolved already
|
|
// carries the result. No separate lifecycle event needed.
|
|
ApprovalKind::UpdateMetaInputs | ApprovalKind::SchedulePrompt => {}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// Tag-driven `ApplyCommit` handler. Walks the approval through the tag
|
|
/// state machine documented in `docs/approvals.md`: stamp
|
|
/// `approved/<id>` and `building/<id>` first so the audit trail
|
|
/// captures intent, then drop the candidate tree into the working dir
|
|
/// without moving HEAD, run the rebuild, and either fast-forward
|
|
/// `applied/main` to the proposal commit on success
|
|
/// (`deployed/<id>`) or annotate `failed/<id>` with the build error
|
|
/// and reset the working tree back to the last known-good main. main
|
|
/// never advances on a failed build, so a crash-and-recover doesn't
|
|
/// leave the agent pointing at a tree it can't evaluate.
|
|
#[allow(clippy::too_many_lines)] // sequential build/tag/notify pipeline; splitting would obscure the flow
|
|
async fn run_apply_commit(
|
|
coord: &Arc<Coordinator>,
|
|
approval: &hive_sh4re::Approval,
|
|
agent_dir: &std::path::Path,
|
|
applied_dir: &std::path::Path,
|
|
claude_dir: &std::path::Path,
|
|
notes_dir: &std::path::Path,
|
|
queue_entry_id: Option<u64>,
|
|
) -> (Result<()>, Option<String>, bool) {
|
|
let id = approval.id;
|
|
let proposal_ref = format!("refs/tags/proposal/{id}");
|
|
|
|
// Detect first spawn before we touch anything so we can branch on it
|
|
// throughout this function.
|
|
let is_first_spawn = !lifecycle::container_exists(&approval.agent).await;
|
|
|
|
// Defensive: submit-time should have planted proposal/<id>, but if
|
|
// the row was migrated from an older schema or the tag got pruned
|
|
// we fail early with a clear note rather than building a stale
|
|
// tree.
|
|
if let Err(e) = lifecycle::git_rev_parse(applied_dir, &proposal_ref).await {
|
|
return (
|
|
Err(anyhow::anyhow!(
|
|
"missing proposal tag {proposal_ref}: {e:#}"
|
|
)),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
// Capture the currently-deployed sha so we can roll applied/main
|
|
// (and the meta lock indirectly) back if the build fails.
|
|
let prev_main_sha = match lifecycle::git_rev_parse(applied_dir, "refs/heads/main").await {
|
|
Ok(s) => s,
|
|
Err(e) => {
|
|
return (
|
|
Err(anyhow::anyhow!("read applied/main: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
};
|
|
|
|
coord.set_queue_step(queue_entry_id, "plant tags");
|
|
if let Err(e) = lifecycle::git_tag(applied_dir, &format!("approved/{id}"), &proposal_ref).await
|
|
{
|
|
return (
|
|
Err(anyhow::anyhow!("plant approved/{id}: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
if let Err(e) = lifecycle::git_tag(applied_dir, &format!("building/{id}"), &proposal_ref).await
|
|
{
|
|
return (
|
|
Err(anyhow::anyhow!("plant building/{id}: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
coord.set_queue_step(queue_entry_id, "fast-forward applied/main");
|
|
// Fast-forward applied/main to proposal/<id> + sync the working
|
|
// tree. Meta input pins `?ref=main`, so this is what makes nix
|
|
// re-lock to the proposal commit on the prepare_deploy step
|
|
// below. On build failure we roll main back to prev_main_sha so
|
|
// a crash leaves the agent on its last-good tree.
|
|
if let Err(e) = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &proposal_ref).await {
|
|
return (
|
|
Err(anyhow::anyhow!("ff main to {proposal_ref}: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
if let Err(e) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
|
|
// main is ahead; working tree didn't sync. Roll main back to
|
|
// keep the two consistent before bailing.
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
return (
|
|
Err(anyhow::anyhow!("read-tree to main: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
// First spawn: sync_agents must add this agent to the meta flake
|
|
// before prepare_deploy can update its input lock (which won't
|
|
// exist yet if this is the agent's first deploy).
|
|
if is_first_spawn {
|
|
coord.set_queue_step(queue_entry_id, "meta sync_agents (first spawn)");
|
|
let agents = match lifecycle::agents_for_meta_listing_with(&approval.agent).await {
|
|
Ok(a) => a,
|
|
Err(e) => {
|
|
let _ =
|
|
lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
|
|
return (
|
|
Err(anyhow::anyhow!("agents_for_meta_listing_with: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
};
|
|
if let Err(e) = crate::meta::sync_agents(
|
|
&coord.hyperhive_flake,
|
|
coord.dashboard_port,
|
|
&coord.operator_pronouns,
|
|
&coord.context_window_tokens,
|
|
&agents,
|
|
)
|
|
.await
|
|
{
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
|
|
return (
|
|
Err(anyhow::anyhow!("meta sync_agents for first spawn: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
}
|
|
|
|
coord.set_queue_step(queue_entry_id, "meta prepare_deploy");
|
|
// Phase 1 of the meta two-phase deploy: relock without committing.
|
|
if let Err(e) = crate::meta::prepare_deploy(&approval.agent).await {
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
|
|
return (
|
|
Err(anyhow::anyhow!("meta prepare_deploy: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
// Container-level rebuild (or first-time create) against meta#<name>.
|
|
// Step labels are emitted inside rebuild_no_meta via the callback so
|
|
// the dashboard reflects actual phase progress rather than a static
|
|
// "nixos-container update" label for the whole multi-minute window.
|
|
let build_result = lifecycle::rebuild_no_meta(
|
|
&approval.agent,
|
|
agent_dir,
|
|
applied_dir,
|
|
claude_dir,
|
|
notes_dir,
|
|
&|step| coord.set_queue_step(queue_entry_id, step),
|
|
)
|
|
.await;
|
|
|
|
match build_result {
|
|
Ok(()) => {
|
|
coord.set_queue_step(queue_entry_id, "finalize deploy");
|
|
let tag = format!("deployed/{id}");
|
|
if let Err(e) = lifecycle::git_tag(applied_dir, &tag, &proposal_ref).await {
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?e, "plant deployed tag failed");
|
|
}
|
|
if let Err(e) = crate::meta::finalize_deploy(
|
|
&approval.agent,
|
|
approval.fetched_sha.as_deref().unwrap_or(&proposal_ref),
|
|
&tag,
|
|
)
|
|
.await
|
|
{
|
|
// The build itself succeeded — meta lock landed but
|
|
// couldn't be committed. Surface as a soft warn so the
|
|
// operator can git-commit by hand if they care.
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?e, "meta finalize_deploy failed");
|
|
}
|
|
// Wake the agent on its next turn so claude sees the
|
|
// config change took effect. Same hint pattern as
|
|
// auto_update::rebuild_agent — manager approved a
|
|
// proposal, agent picks up where it left off with the
|
|
// new env / packages.
|
|
coord.kick_agent(&approval.agent, "config update applied");
|
|
(Ok(()), Some(tag), is_first_spawn)
|
|
}
|
|
Err(e) => {
|
|
let tag = format!("failed/{id}");
|
|
let body = format!("{e:#}");
|
|
if let Err(te) =
|
|
lifecycle::git_tag_annotated(applied_dir, &tag, &proposal_ref, &body).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?te, "annotate failed tag failed");
|
|
}
|
|
// Roll main back to last known-good so the on-disk state
|
|
// matches what nixos-container last successfully built.
|
|
if let Err(re) =
|
|
lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?re, "main rollback failed");
|
|
}
|
|
if let Err(re) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?re, "rollback read-tree failed");
|
|
}
|
|
// Drop the staged meta lock change so the deploy log
|
|
// only ever shows successes.
|
|
if let Err(ae) = crate::meta::abort_deploy().await {
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?ae, "meta abort_deploy failed");
|
|
}
|
|
let _ = coord;
|
|
(Err(e), Some(tag), is_first_spawn)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Tear down a sub-agent container. By default this is non-destructive to
|
|
/// persistent state: the proposed/applied config repos and the Claude
|
|
/// credentials dir under `/var/lib/hyperhive/{agents,applied}/<name>/` are
|
|
/// kept, so recreating an agent of the same name reuses prior config + creds
|
|
/// (no re-login). The ephemeral runtime dir under `/run/hyperhive/agents/`
|
|
/// is cleared because its contents (the mcp socket) don't survive restarts
|
|
/// anyway. With `purge=true` the persistent trees are also wiped — config
|
|
/// history, claude creds, notes — there is no undo.
|
|
/// Refuses the manager (declarative; would fight with the host's nixos config).
|
|
pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Result<()> {
|
|
if name == MANAGER_NAME || name == MANAGER_AGENT {
|
|
bail!("refusing to destroy the manager ({name})");
|
|
}
|
|
tracing::info!(%name, purge, "destroy");
|
|
// Guard auto-clears on the success path's final scope exit and on
|
|
// every early-return / cancellation along the way.
|
|
let guard = coord.transient_guard(name, TransientKind::Destroying);
|
|
lifecycle::destroy(name).await?;
|
|
coord.unregister_agent(name);
|
|
let runtime = Coordinator::agent_dir(name);
|
|
if runtime.exists() {
|
|
let _ = std::fs::remove_dir_all(&runtime);
|
|
}
|
|
if purge {
|
|
for dir in [
|
|
Coordinator::agent_state_root(name),
|
|
Coordinator::agent_applied_dir(name),
|
|
] {
|
|
if dir.exists()
|
|
&& let Err(e) = std::fs::remove_dir_all(&dir)
|
|
{
|
|
tracing::warn!(error = ?e, dir = %dir.display(), "purge: remove failed");
|
|
}
|
|
}
|
|
}
|
|
// Meta flake: drop the agent's input + nixosConfiguration so a
|
|
// future spawn under the same name re-seeds cleanly, and so the
|
|
// meta lock doesn't reference a vanished applied repo. Log + keep
|
|
// going on failure — destroy already succeeded at the
|
|
// nixos-container level, the meta repo is just bookkeeping.
|
|
if let Err(e) = sync_meta_after_lifecycle(coord).await {
|
|
tracing::warn!(error = ?e, %name, "meta sync after destroy failed");
|
|
}
|
|
let _ = coord.approvals.fail_pending_for_agent(
|
|
name,
|
|
if purge {
|
|
"agent purged"
|
|
} else {
|
|
"agent destroyed"
|
|
},
|
|
);
|
|
drop(guard);
|
|
coord.notify_manager(&HelperEvent::Destroyed {
|
|
agent: name.to_owned(),
|
|
});
|
|
// Container row disappeared — rescan so the dashboard fires
|
|
// `ContainerRemoved` for the gone row, then emit the
|
|
// tombstones snapshot (gained one on destroy, lost one on
|
|
// purge — recompute either way).
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
Ok(())
|
|
}
|
|
|
|
/// Rerender the meta flake from whatever containers still exist on
|
|
/// disk. Called after lifecycle ops that change the agent set (today:
|
|
/// destroy). Idempotent — a no-op when nothing changed.
|
|
async fn sync_meta_after_lifecycle(coord: &Coordinator) -> Result<()> {
|
|
let agents = lifecycle::agents_for_meta_listing().await?;
|
|
crate::meta::sync_agents(
|
|
&coord.hyperhive_flake,
|
|
coord.dashboard_port,
|
|
&coord.operator_pronouns,
|
|
&coord.context_window_tokens,
|
|
&agents,
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
|
|
let approval = coord.approvals.get(id)?;
|
|
coord.approvals.mark_denied(id, note)?;
|
|
tracing::info!(%id, note, "approval denied");
|
|
let mut tag = None;
|
|
if let Some(a) = approval {
|
|
let sha = a.fetched_sha.clone();
|
|
// ApplyCommit approvals leave a `denied/<id>` tag on the
|
|
// proposal commit so rejected configs are first-class git
|
|
// objects — `git show denied/<id>` in the manager's applied
|
|
// mount yields both the tree the operator rejected and (in
|
|
// the annotated body) the reason. Spawn approvals have no
|
|
// commit to tag, so they fall through unannotated.
|
|
if matches!(a.kind, ApprovalKind::ApplyCommit) {
|
|
let applied_dir = Coordinator::agent_applied_dir(&a.agent);
|
|
let proposal_ref = format!("refs/tags/proposal/{id}");
|
|
if lifecycle::git_rev_parse(&applied_dir, &proposal_ref)
|
|
.await
|
|
.is_ok()
|
|
{
|
|
let tag_name = format!("denied/{id}");
|
|
let body = note.unwrap_or("").to_owned();
|
|
if let Err(e) =
|
|
lifecycle::git_tag_annotated(&applied_dir, &tag_name, &proposal_ref, &body)
|
|
.await
|
|
{
|
|
tracing::warn!(%id, error = ?e, "plant denied tag failed");
|
|
} else {
|
|
tag = Some(tag_name);
|
|
}
|
|
}
|
|
// Mirror the denied/<id> tag to the forge.
|
|
if let Err(e) = crate::forge::push_config(&a.agent).await {
|
|
tracing::warn!(%id, agent = %a.agent, error = ?e, "forge: push_config after deny failed");
|
|
}
|
|
}
|
|
let approval_kind = match a.kind {
|
|
ApprovalKind::Spawn => "spawn",
|
|
ApprovalKind::ApplyCommit => "apply_commit",
|
|
ApprovalKind::InitConfig => "init_config",
|
|
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
|
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
|
};
|
|
let sha_short = sha.as_deref().map(|s| s[..s.len().min(12)].to_owned());
|
|
let description = a.description.clone();
|
|
let agent_owned = a.agent.clone();
|
|
coord.notify_manager(&HelperEvent::ApprovalResolved {
|
|
id: a.id,
|
|
agent: a.agent,
|
|
commit_ref: a.commit_ref,
|
|
status: ApprovalStatus::Denied,
|
|
note: note.map(String::from),
|
|
sha,
|
|
tag,
|
|
});
|
|
coord.emit_approval_resolved(
|
|
id,
|
|
&agent_owned,
|
|
approval_kind,
|
|
sha_short,
|
|
"denied",
|
|
note.map(String::from),
|
|
description,
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|