eae0e875cf
add QueueKind::PermChange — dashboard tool-group and capability handlers no longer write the shared JSON files inline. instead they enqueue a PermChange entry; the FIFO worker applies the file write then calls rebuild_agent so the updated env var takes effect. concurrent batch-apply actions for different agents previously raced on tool-groups.json / capabilities.json (last write wins, earlier change silently dropped). serialising through the queue prevents this. dedup check extended with perm-type discriminant so tool-groups and capabilities changes for the same agent are kept as distinct entries and never collapse into one slot.
799 lines
34 KiB
Rust
799 lines
34 KiB
Rust
//! Operations that are exposed through more than one surface (the host admin
|
|
//! socket *and* the dashboard's POST endpoints). Each function takes a
|
|
//! `&Coordinator` and the request parameters; callers stitch the response
|
|
//! shape they want (HTTP redirect vs JSON).
|
|
|
|
use std::sync::Arc;
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_sh4re::{ApprovalKind, ApprovalStatus, HelperEvent, MANAGER_AGENT};
|
|
|
|
use crate::coordinator::{Coordinator, TransientKind};
|
|
use crate::lifecycle::{self, MANAGER_NAME};
|
|
|
|
/// Approve a pending request. Marks the approval row durably, then
|
|
/// either runs the work inline (`InitConfig`, sub-second git ops) or
|
|
/// enqueues it into `rebuild_queue` so the dashboard POST returns
|
|
/// immediately while the long-running pipeline runs off-thread
|
|
/// (operator no longer blocks on a 30-90s spinner for `ApplyCommit`).
|
|
///
|
|
/// Dispatch:
|
|
/// - `ApplyCommit` → `QueueKind::Rebuild` (~30-90s wall time)
|
|
/// - `UpdateMetaInputs` → `QueueKind::MetaUpdate` (~3-15s)
|
|
/// - `Spawn` → `QueueKind::Spawn` (~30-90s)
|
|
/// - `InitConfig` → inline (<1s; queue card would be noise)
|
|
///
|
|
/// The queue worker re-fetches the approval row on dispatch, runs
|
|
/// the kind-specific pipeline, and fires `ApprovalResolved` /
|
|
/// `Spawned` / `Rebuilt` / `ConfigReady` via `finish_approval`.
|
|
pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
|
let approval = coord.approvals.mark_approved(id)?;
|
|
tracing::info!(
|
|
%approval.id,
|
|
%approval.agent,
|
|
kind = ?approval.kind,
|
|
%approval.commit_ref,
|
|
"approval: dispatching",
|
|
);
|
|
match approval.kind {
|
|
ApprovalKind::InitConfig => {
|
|
// Sub-second git seed + forge-remote wire. Routing through
|
|
// the queue would surface a queue card that's gone before
|
|
// the operator's eyes refocus. Run inline.
|
|
let proposed_dir = Coordinator::agent_proposed_dir(&approval.agent);
|
|
let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
|
|
let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
|
|
run_approval_init_config(&coord, approval, proposed_dir, claude_dir, notes_dir).await
|
|
}
|
|
ApprovalKind::ApplyCommit => {
|
|
coord.rebuild_queue.enqueue_full(
|
|
crate::rebuild_queue::QueueKind::Rebuild,
|
|
approval.agent.clone(),
|
|
crate::rebuild_queue::QueueSource::Approval,
|
|
format!("approval #{id} apply commit"),
|
|
None,
|
|
Vec::new(),
|
|
Some(id),
|
|
None,
|
|
);
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::UpdateMetaInputs => {
|
|
// Inputs JSON-encoded into commit_ref by the manager's
|
|
// submit path — surface them on the queue entry so the
|
|
// dashboard can show *which* inputs are about to bump.
|
|
let inputs: Vec<String> =
|
|
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
|
let parent_id = coord.rebuild_queue.enqueue_full(
|
|
crate::rebuild_queue::QueueKind::MetaUpdate,
|
|
approval.agent.clone(),
|
|
crate::rebuild_queue::QueueSource::Approval,
|
|
format!("approval #{id} meta input update"),
|
|
None,
|
|
inputs.clone(),
|
|
Some(id),
|
|
None,
|
|
);
|
|
// Pre-enqueue cascade rebuilds in topological order so
|
|
// agents depending on updated inputs are rebuilt after the
|
|
// lock bump, matching the dashboard post_meta_update path.
|
|
let cascade_agents = crate::rebuild_queue::meta_update_cascade_agents(&inputs).await;
|
|
let cascade_reason = format!("approval #{id} meta input cascade");
|
|
for name in cascade_agents {
|
|
coord.rebuild_queue.enqueue(
|
|
crate::rebuild_queue::QueueKind::Rebuild,
|
|
name,
|
|
crate::rebuild_queue::QueueSource::MetaUpdate,
|
|
cascade_reason.clone(),
|
|
Some(parent_id),
|
|
);
|
|
}
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::Spawn => {
|
|
coord.rebuild_queue.enqueue_full(
|
|
crate::rebuild_queue::QueueKind::Spawn,
|
|
approval.agent.clone(),
|
|
crate::rebuild_queue::QueueSource::Approval,
|
|
format!("approval #{id} spawn"),
|
|
None,
|
|
Vec::new(),
|
|
Some(id),
|
|
None,
|
|
);
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::SchedulePrompt => {
|
|
// No queue card for SchedulePrompt — the work is a single
|
|
// sqlite insert, the actual "running" lifetime lives on
|
|
// the scheduled-prompts surface itself (worker fires it
|
|
// at the scheduled time). Run inline + fire
|
|
// `ApprovalResolved` so the approval row leaves Pending
|
|
// immediately.
|
|
run_approval_schedule_prompt(&coord, approval).await
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Worker entry point for `ApprovalKind::ApplyCommit` queue entries.
|
|
/// Re-fetches the approval row, runs the commit pipeline, and fires
|
|
/// `ApprovalResolved` + the lifecycle event (`Rebuilt` / `Spawned`
|
|
/// for first-spawn).
|
|
pub async fn run_approval_apply_commit(
|
|
coord: &Arc<Coordinator>,
|
|
queue_entry_id: Option<u64>,
|
|
approval_id: i64,
|
|
) -> Result<()> {
|
|
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::ApplyCommit)?;
|
|
let agent_dir = coord.ensure_runtime(&approval.agent)?;
|
|
let applied_dir = Coordinator::agent_applied_dir(&approval.agent);
|
|
let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
|
|
let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
|
|
coord.set_queue_step(queue_entry_id, "apply commit");
|
|
let (result, terminal_tag, is_first_spawn) = run_apply_commit(
|
|
coord,
|
|
&approval,
|
|
&agent_dir,
|
|
&applied_dir,
|
|
&claude_dir,
|
|
¬es_dir,
|
|
queue_entry_id,
|
|
)
|
|
.await;
|
|
coord.set_queue_step(queue_entry_id, "forge push");
|
|
if let Err(e) = crate::forge::push_config(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after apply failed");
|
|
}
|
|
if is_first_spawn && result.is_ok() {
|
|
coord.set_queue_step(queue_entry_id, "first-spawn forge bootstrap");
|
|
forge_after_first_spawn(coord, &approval.agent).await;
|
|
}
|
|
// `finish_approval` returns the original `result` so the queue
|
|
// worker sees Ok/Err and marks the queue entry accordingly. The
|
|
// approval row + helper events have already been fanned out.
|
|
finish_approval(coord, &approval, result, terminal_tag, is_first_spawn)
|
|
}
|
|
|
|
/// Inline (non-queued) handler for `ApprovalKind::SchedulePrompt`.
|
|
/// On approve, decode the `SchedulePromptPayload` JSON from the
|
|
/// approval's `commit_ref`, insert a row into `scheduled_prompts`
|
|
/// (with `source = Approval { id }`), and fire `ApprovalResolved`.
|
|
/// The worker takes over from here — fan-out at fire time.
|
|
async fn run_approval_schedule_prompt(
|
|
coord: &Coordinator,
|
|
approval: hive_sh4re::Approval,
|
|
) -> Result<()> {
|
|
let result: Result<()> = async {
|
|
let payload: hive_sh4re::SchedulePromptPayload = serde_json::from_str(&approval.commit_ref)
|
|
.context("decode SchedulePromptPayload from approval.commit_ref")?;
|
|
coord
|
|
.scheduled_prompts
|
|
.submit(&crate::scheduled_prompts::NewSchedule {
|
|
owner: approval.agent.clone(),
|
|
targets: payload.targets,
|
|
body: payload.body,
|
|
first_fire_at_unix: payload.first_fire_at_unix,
|
|
interval_seconds: payload.interval_seconds,
|
|
description: payload.description,
|
|
source: crate::scheduled_prompts::ScheduleSource::Approval { id: approval.id },
|
|
})
|
|
.map(|_| ())
|
|
.context("insert scheduled prompt")
|
|
}
|
|
.await;
|
|
finish_approval(coord, &approval, result, None, false)
|
|
}
|
|
|
|
/// Worker entry point for `ApprovalKind::UpdateMetaInputs` queue
|
|
/// entries. Inputs come from the approval row's `commit_ref` field
|
|
/// (JSON-encoded by the manager submit path), not the queue entry's
|
|
/// `inputs` — the queue copy is for dashboard display only.
|
|
pub async fn run_approval_update_meta_inputs(
|
|
coord: &Arc<Coordinator>,
|
|
queue_entry_id: Option<u64>,
|
|
approval_id: i64,
|
|
) -> Result<()> {
|
|
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::UpdateMetaInputs)?;
|
|
let inputs: Vec<String> = serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
|
coord.set_queue_step(queue_entry_id, "nix flake update");
|
|
let result = crate::meta::lock_update(&inputs).await;
|
|
finish_approval(coord, &approval, result, None, false)
|
|
}
|
|
|
|
/// Worker entry point for `ApprovalKind::Spawn` queue entries.
|
|
/// Differs from `run_approval_apply_commit` only in routing through
|
|
/// `lifecycle::spawn` (the deprecated direct-spawn path). Synchronous
|
|
/// in the queue worker — the previous `tokio::spawn` wrapper is gone
|
|
/// (the queue worker itself is the async task).
|
|
pub async fn run_approval_spawn(
|
|
coord: &Arc<Coordinator>,
|
|
queue_entry_id: Option<u64>,
|
|
approval_id: i64,
|
|
) -> Result<()> {
|
|
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::Spawn)?;
|
|
let agent_dir = coord.ensure_runtime(&approval.agent)?;
|
|
let proposed_dir = Coordinator::agent_proposed_dir(&approval.agent);
|
|
let applied_dir = Coordinator::agent_applied_dir(&approval.agent);
|
|
let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
|
|
let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
|
|
// Transient guard keeps the per-container "Spawning" pill lit while
|
|
// the worker is doing the actual nixos-container create. Auto-clears
|
|
// on the function's scope exit (success or panic).
|
|
let _guard = coord.transient_guard(&approval.agent, TransientKind::Spawning);
|
|
coord.set_queue_step(queue_entry_id, "lifecycle::spawn");
|
|
let result = lifecycle::spawn(
|
|
&approval.agent,
|
|
&coord.hyperhive_flake,
|
|
&coord.nixpkgs_flake,
|
|
&coord.nixpkgs_unstable_flake,
|
|
&agent_dir,
|
|
&proposed_dir,
|
|
&applied_dir,
|
|
&claude_dir,
|
|
¬es_dir,
|
|
coord.dashboard_port,
|
|
&coord.operator_pronouns,
|
|
&coord.context_window_tokens,
|
|
)
|
|
.await;
|
|
if result.is_ok() {
|
|
coord.set_queue_step(queue_entry_id, "forge user");
|
|
if let Err(e) = crate::forge::ensure_user_for(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_user after spawn failed");
|
|
}
|
|
coord.set_queue_step(queue_entry_id, "forge config repo");
|
|
if let Err(e) = crate::forge::ensure_config_repo(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_config_repo after spawn failed");
|
|
}
|
|
coord.set_queue_step(queue_entry_id, "forge push");
|
|
if let Err(e) = crate::forge::push_config(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after spawn failed");
|
|
}
|
|
coord.set_queue_step(queue_entry_id, "forge meta access");
|
|
if let Some(core_token) = crate::forge::core_token()
|
|
&& let Err(e) = crate::forge::meta_read_access(&approval.agent, &core_token).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: meta_read_access after spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after spawn failed");
|
|
}
|
|
}
|
|
let final_result = finish_approval(coord, &approval, result, None, false);
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
final_result
|
|
}
|
|
|
|
/// Re-fetch an approval row from sqlite for a queue-worker dispatch.
|
|
/// Bails if the row is gone (deny race), if its kind doesn't match,
|
|
/// or if the lookup itself fails. The kind check is defensive — the
|
|
/// queue's `dispatch` already routes by `QueueKind`, but the approval
|
|
/// kind is the authoritative source of truth and a mismatch points
|
|
/// at a deeper bug we'd want to surface.
|
|
fn fetch_approval_for_worker(
|
|
coord: &Coordinator,
|
|
approval_id: i64,
|
|
expected_kind: ApprovalKind,
|
|
) -> Result<hive_sh4re::Approval> {
|
|
let approval = coord
|
|
.approvals
|
|
.get(approval_id)
|
|
.map_err(|e| anyhow::anyhow!("read approval {approval_id}: {e:#}"))?
|
|
.ok_or_else(|| anyhow::anyhow!("approval {approval_id} no longer exists"))?;
|
|
if approval.kind != expected_kind {
|
|
bail!(
|
|
"approval {approval_id} kind mismatch: queue expected {expected_kind:?}, row is {actual:?}",
|
|
actual = approval.kind
|
|
);
|
|
}
|
|
Ok(approval)
|
|
}
|
|
|
|
/// Forge bookkeeping run once after the very first container spawn:
|
|
/// create the per-agent forge user, mirror the applied repo, and grant
|
|
/// read access to core/meta. Also rescans containers so the dashboard
|
|
/// reflects the post-spawn state.
|
|
async fn forge_after_first_spawn(coord: &Arc<Coordinator>, agent: &str) {
|
|
if let Err(e) = crate::forge::ensure_user_for(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_user after first spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_config_repo(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_config_repo after first spawn failed");
|
|
}
|
|
if let Some(core_token) = crate::forge::core_token()
|
|
&& let Err(e) = crate::forge::meta_read_access(agent, &core_token).await
|
|
{
|
|
tracing::warn!(%agent, error = ?e, "forge: meta_read_access after first spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_meta_remote(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_meta_remote after first spawn failed");
|
|
}
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
}
|
|
|
|
/// Inline (non-queued) handler for `ApprovalKind::InitConfig`. Just
|
|
/// seeds the proposed git repo + the per-agent dirs — sub-second
|
|
/// work that doesn't justify a queue card.
|
|
async fn run_approval_init_config(
|
|
coord: &Coordinator,
|
|
approval: hive_sh4re::Approval,
|
|
proposed_dir: std::path::PathBuf,
|
|
claude_dir: std::path::PathBuf,
|
|
notes_dir: std::path::PathBuf,
|
|
) -> Result<()> {
|
|
let result: Result<()> = async {
|
|
lifecycle::setup_proposed(&proposed_dir, &approval.agent).await?;
|
|
lifecycle::ensure_claude_dir(&claude_dir)?;
|
|
lifecycle::ensure_state_dir(¬es_dir)?;
|
|
Ok(())
|
|
}
|
|
.await;
|
|
if result.is_ok()
|
|
&& let Err(e) = crate::forge::ensure_meta_remote(&approval.agent).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after init_config failed");
|
|
}
|
|
finish_approval(coord, &approval, result, None, false)
|
|
}
|
|
|
|
fn finish_approval(
|
|
coord: &Coordinator,
|
|
approval: &hive_sh4re::Approval,
|
|
result: Result<()>,
|
|
terminal_tag: Option<String>,
|
|
is_first_spawn: bool,
|
|
) -> Result<()> {
|
|
let (status, note, ok) = match &result {
|
|
Ok(()) => (ApprovalStatus::Approved, None, true),
|
|
Err(e) => {
|
|
let note = format!("{e:#}");
|
|
let _ = coord.approvals.mark_failed(approval.id, ¬e);
|
|
(ApprovalStatus::Failed, Some(note), false)
|
|
}
|
|
};
|
|
coord.notify_manager(&HelperEvent::ApprovalResolved {
|
|
id: approval.id,
|
|
agent: approval.agent.clone(),
|
|
commit_ref: approval.commit_ref.clone(),
|
|
status,
|
|
note: note.clone(),
|
|
sha: approval.fetched_sha.clone(),
|
|
tag: terminal_tag.clone(),
|
|
});
|
|
// Phase 5b: also fire on the dashboard event channel so the
|
|
// browser moves the row out of pending into history without a
|
|
// snapshot refetch. `approved` rows that succeed get the
|
|
// approval's logged resolved_at indirectly via `now_unix()`;
|
|
// failures already wrote it via mark_failed above.
|
|
let approval_kind = match approval.kind {
|
|
ApprovalKind::Spawn => "spawn",
|
|
ApprovalKind::ApplyCommit => "apply_commit",
|
|
ApprovalKind::InitConfig => "init_config",
|
|
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
|
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
|
};
|
|
let sha_short = approval
|
|
.fetched_sha
|
|
.as_deref()
|
|
.map(|s| s[..s.len().min(12)].to_owned());
|
|
let status_str = if ok { "approved" } else { "failed" };
|
|
coord.emit_approval_resolved(
|
|
approval.id,
|
|
&approval.agent,
|
|
approval_kind,
|
|
sha_short,
|
|
status_str,
|
|
note.clone(),
|
|
approval.description.clone(),
|
|
);
|
|
// For spawn/rebuild/init_config approvals, also surface the underlying
|
|
// action so the manager knows whether the lifecycle step succeeded.
|
|
// The ApprovalResolved event already carries the same `ok` signal but
|
|
// separating it lets the manager react to the lifecycle change
|
|
// without having to special-case approvals.
|
|
match approval.kind {
|
|
ApprovalKind::InitConfig => {
|
|
if ok {
|
|
coord.notify_manager(&HelperEvent::ConfigReady {
|
|
agent: approval.agent.clone(),
|
|
});
|
|
}
|
|
}
|
|
ApprovalKind::Spawn => coord.notify_manager(&HelperEvent::Spawned {
|
|
agent: approval.agent.clone(),
|
|
ok,
|
|
note,
|
|
sha: approval.fetched_sha.clone(),
|
|
}),
|
|
ApprovalKind::ApplyCommit if is_first_spawn => {
|
|
coord.notify_manager(&HelperEvent::Spawned {
|
|
agent: approval.agent.clone(),
|
|
ok,
|
|
note,
|
|
sha: approval.fetched_sha.clone(),
|
|
});
|
|
}
|
|
ApprovalKind::ApplyCommit => coord.notify_manager(&HelperEvent::Rebuilt {
|
|
agent: approval.agent.clone(),
|
|
ok,
|
|
note,
|
|
sha: approval.fetched_sha.clone(),
|
|
tag: terminal_tag,
|
|
}),
|
|
// UpdateMetaInputs / SchedulePrompt: ApprovalResolved already
|
|
// carries the result. No separate lifecycle event needed.
|
|
ApprovalKind::UpdateMetaInputs | ApprovalKind::SchedulePrompt => {}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// Tag-driven `ApplyCommit` handler. Walks the approval through the tag
|
|
/// state machine documented in `docs/approvals.md`: stamp
|
|
/// `approved/<id>` and `building/<id>` first so the audit trail
|
|
/// captures intent, then drop the candidate tree into the working dir
|
|
/// without moving HEAD, run the rebuild, and either fast-forward
|
|
/// `applied/main` to the proposal commit on success
|
|
/// (`deployed/<id>`) or annotate `failed/<id>` with the build error
|
|
/// and reset the working tree back to the last known-good main. main
|
|
/// never advances on a failed build, so a crash-and-recover doesn't
|
|
/// leave the agent pointing at a tree it can't evaluate.
|
|
#[allow(clippy::too_many_lines)] // sequential build/tag/notify pipeline; splitting would obscure the flow
|
|
async fn run_apply_commit(
|
|
coord: &Arc<Coordinator>,
|
|
approval: &hive_sh4re::Approval,
|
|
agent_dir: &std::path::Path,
|
|
applied_dir: &std::path::Path,
|
|
claude_dir: &std::path::Path,
|
|
notes_dir: &std::path::Path,
|
|
queue_entry_id: Option<u64>,
|
|
) -> (Result<()>, Option<String>, bool) {
|
|
let id = approval.id;
|
|
let proposal_ref = format!("refs/tags/proposal/{id}");
|
|
|
|
// Detect first spawn before we touch anything so we can branch on it
|
|
// throughout this function.
|
|
let is_first_spawn = !lifecycle::container_exists(&approval.agent).await;
|
|
|
|
// Defensive: submit-time should have planted proposal/<id>, but if
|
|
// the row was migrated from an older schema or the tag got pruned
|
|
// we fail early with a clear note rather than building a stale
|
|
// tree.
|
|
if let Err(e) = lifecycle::git_rev_parse(applied_dir, &proposal_ref).await {
|
|
return (
|
|
Err(anyhow::anyhow!(
|
|
"missing proposal tag {proposal_ref}: {e:#}"
|
|
)),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
// Capture the currently-deployed sha so we can roll applied/main
|
|
// (and the meta lock indirectly) back if the build fails.
|
|
let prev_main_sha = match lifecycle::git_rev_parse(applied_dir, "refs/heads/main").await {
|
|
Ok(s) => s,
|
|
Err(e) => {
|
|
return (
|
|
Err(anyhow::anyhow!("read applied/main: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
};
|
|
|
|
coord.set_queue_step(queue_entry_id, "plant tags");
|
|
if let Err(e) = lifecycle::git_tag(applied_dir, &format!("approved/{id}"), &proposal_ref).await
|
|
{
|
|
return (
|
|
Err(anyhow::anyhow!("plant approved/{id}: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
if let Err(e) = lifecycle::git_tag(applied_dir, &format!("building/{id}"), &proposal_ref).await
|
|
{
|
|
return (
|
|
Err(anyhow::anyhow!("plant building/{id}: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
coord.set_queue_step(queue_entry_id, "fast-forward applied/main");
|
|
// Fast-forward applied/main to proposal/<id> + sync the working
|
|
// tree. Meta input pins `?ref=main`, so this is what makes nix
|
|
// re-lock to the proposal commit on the prepare_deploy step
|
|
// below. On build failure we roll main back to prev_main_sha so
|
|
// a crash leaves the agent on its last-good tree.
|
|
if let Err(e) = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &proposal_ref).await {
|
|
return (
|
|
Err(anyhow::anyhow!("ff main to {proposal_ref}: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
if let Err(e) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
|
|
// main is ahead; working tree didn't sync. Roll main back to
|
|
// keep the two consistent before bailing.
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
return (
|
|
Err(anyhow::anyhow!("read-tree to main: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
// First spawn: sync_agents must add this agent to the meta flake
|
|
// before prepare_deploy can update its input lock (which won't
|
|
// exist yet if this is the agent's first deploy).
|
|
if is_first_spawn {
|
|
coord.set_queue_step(queue_entry_id, "meta sync_agents (first spawn)");
|
|
let agents = match lifecycle::agents_for_meta_listing_with(&approval.agent).await {
|
|
Ok(a) => a,
|
|
Err(e) => {
|
|
let _ =
|
|
lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
|
|
return (
|
|
Err(anyhow::anyhow!("agents_for_meta_listing_with: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
};
|
|
if let Err(e) = crate::meta::sync_agents(
|
|
&coord.hyperhive_flake,
|
|
&coord.nixpkgs_flake,
|
|
&coord.nixpkgs_unstable_flake,
|
|
coord.dashboard_port,
|
|
&coord.operator_pronouns,
|
|
&coord.context_window_tokens,
|
|
&agents,
|
|
)
|
|
.await
|
|
{
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
|
|
return (
|
|
Err(anyhow::anyhow!("meta sync_agents for first spawn: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
}
|
|
|
|
coord.set_queue_step(queue_entry_id, "meta prepare_deploy");
|
|
// Phase 1 of the meta two-phase deploy: relock without committing.
|
|
if let Err(e) = crate::meta::prepare_deploy(&approval.agent).await {
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await;
|
|
let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
|
|
return (
|
|
Err(anyhow::anyhow!("meta prepare_deploy: {e:#}")),
|
|
None,
|
|
is_first_spawn,
|
|
);
|
|
}
|
|
|
|
// Container-level rebuild (or first-time create) against meta#<name>.
|
|
// Step labels are emitted inside rebuild_no_meta via the callback so
|
|
// the dashboard reflects actual phase progress rather than a static
|
|
// "nixos-container update" label for the whole multi-minute window.
|
|
let build_result = lifecycle::rebuild_no_meta(
|
|
&approval.agent,
|
|
agent_dir,
|
|
applied_dir,
|
|
claude_dir,
|
|
notes_dir,
|
|
&|step| coord.set_queue_step(queue_entry_id, step),
|
|
)
|
|
.await;
|
|
|
|
match build_result {
|
|
Ok(()) => {
|
|
coord.set_queue_step(queue_entry_id, "finalize deploy");
|
|
let tag = format!("deployed/{id}");
|
|
if let Err(e) = lifecycle::git_tag(applied_dir, &tag, &proposal_ref).await {
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?e, "plant deployed tag failed");
|
|
}
|
|
if let Err(e) = crate::meta::finalize_deploy(
|
|
&approval.agent,
|
|
approval.fetched_sha.as_deref().unwrap_or(&proposal_ref),
|
|
&tag,
|
|
)
|
|
.await
|
|
{
|
|
// The build itself succeeded — meta lock landed but
|
|
// couldn't be committed. Surface as a soft warn so the
|
|
// operator can git-commit by hand if they care.
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?e, "meta finalize_deploy failed");
|
|
}
|
|
// Wake the agent on its next turn so claude sees the
|
|
// config change took effect. Same hint pattern as
|
|
// auto_update::rebuild_agent — manager approved a
|
|
// proposal, agent picks up where it left off with the
|
|
// new env / packages.
|
|
coord.kick_agent(&approval.agent, "config update applied");
|
|
(Ok(()), Some(tag), is_first_spawn)
|
|
}
|
|
Err(e) => {
|
|
let tag = format!("failed/{id}");
|
|
let body = format!("{e:#}");
|
|
if let Err(te) =
|
|
lifecycle::git_tag_annotated(applied_dir, &tag, &proposal_ref, &body).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?te, "annotate failed tag failed");
|
|
}
|
|
// Roll main back to last known-good so the on-disk state
|
|
// matches what nixos-container last successfully built.
|
|
if let Err(re) =
|
|
lifecycle::git_update_ref(applied_dir, "refs/heads/main", &prev_main_sha).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?re, "main rollback failed");
|
|
}
|
|
if let Err(re) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?re, "rollback read-tree failed");
|
|
}
|
|
// Drop the staged meta lock change so the deploy log
|
|
// only ever shows successes.
|
|
if let Err(ae) = crate::meta::abort_deploy().await {
|
|
tracing::warn!(agent = %approval.agent, %id, error = ?ae, "meta abort_deploy failed");
|
|
}
|
|
let _ = coord;
|
|
(Err(e), Some(tag), is_first_spawn)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Tear down a sub-agent container. By default this is non-destructive to
|
|
/// persistent state: the proposed/applied config repos and the Claude
|
|
/// credentials dir under `/var/lib/hyperhive/{agents,applied}/<name>/` are
|
|
/// kept, so recreating an agent of the same name reuses prior config + creds
|
|
/// (no re-login). The ephemeral runtime dir under `/run/hyperhive/agents/`
|
|
/// is cleared because its contents (the mcp socket) don't survive restarts
|
|
/// anyway. With `purge=true` the persistent trees are also wiped — config
|
|
/// history, claude creds, notes — there is no undo.
|
|
/// Refuses the manager (declarative; would fight with the host's nixos config).
|
|
pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Result<()> {
|
|
if name == MANAGER_NAME || name == MANAGER_AGENT {
|
|
bail!("refusing to destroy the manager ({name})");
|
|
}
|
|
tracing::info!(%name, purge, "destroy");
|
|
// Guard auto-clears on the success path's final scope exit and on
|
|
// every early-return / cancellation along the way.
|
|
let guard = coord.transient_guard(name, TransientKind::Destroying);
|
|
lifecycle::destroy(name).await?;
|
|
coord.unregister_agent(name);
|
|
let runtime = Coordinator::agent_dir(name);
|
|
if runtime.exists() {
|
|
let _ = std::fs::remove_dir_all(&runtime);
|
|
}
|
|
if purge {
|
|
for dir in [
|
|
Coordinator::agent_state_root(name),
|
|
Coordinator::agent_applied_dir(name),
|
|
] {
|
|
if dir.exists()
|
|
&& let Err(e) = std::fs::remove_dir_all(&dir)
|
|
{
|
|
tracing::warn!(error = ?e, dir = %dir.display(), "purge: remove failed");
|
|
}
|
|
}
|
|
}
|
|
// Meta flake: drop the agent's input + nixosConfiguration so a
|
|
// future spawn under the same name re-seeds cleanly, and so the
|
|
// meta lock doesn't reference a vanished applied repo. Log + keep
|
|
// going on failure — destroy already succeeded at the
|
|
// nixos-container level, the meta repo is just bookkeeping.
|
|
if let Err(e) = sync_meta_after_lifecycle(coord).await {
|
|
tracing::warn!(error = ?e, %name, "meta sync after destroy failed");
|
|
}
|
|
let _ = coord.approvals.fail_pending_for_agent(
|
|
name,
|
|
if purge {
|
|
"agent purged"
|
|
} else {
|
|
"agent destroyed"
|
|
},
|
|
);
|
|
drop(guard);
|
|
coord.notify_manager(&HelperEvent::Destroyed {
|
|
agent: name.to_owned(),
|
|
});
|
|
// Container row disappeared — rescan so the dashboard fires
|
|
// `ContainerRemoved` for the gone row, then emit the
|
|
// tombstones snapshot (gained one on destroy, lost one on
|
|
// purge — recompute either way).
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
Ok(())
|
|
}
|
|
|
|
/// Rerender the meta flake from whatever containers still exist on
|
|
/// disk. Called after lifecycle ops that change the agent set (today:
|
|
/// destroy). Idempotent — a no-op when nothing changed.
|
|
async fn sync_meta_after_lifecycle(coord: &Coordinator) -> Result<()> {
|
|
let agents = lifecycle::agents_for_meta_listing().await?;
|
|
crate::meta::sync_agents(
|
|
&coord.hyperhive_flake,
|
|
&coord.nixpkgs_flake,
|
|
&coord.nixpkgs_unstable_flake,
|
|
coord.dashboard_port,
|
|
&coord.operator_pronouns,
|
|
&coord.context_window_tokens,
|
|
&agents,
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
|
|
let approval = coord.approvals.get(id)?;
|
|
coord.approvals.mark_denied(id, note)?;
|
|
tracing::info!(%id, note, "approval denied");
|
|
let mut tag = None;
|
|
if let Some(a) = approval {
|
|
let sha = a.fetched_sha.clone();
|
|
// ApplyCommit approvals leave a `denied/<id>` tag on the
|
|
// proposal commit so rejected configs are first-class git
|
|
// objects — `git show denied/<id>` in the manager's applied
|
|
// mount yields both the tree the operator rejected and (in
|
|
// the annotated body) the reason. Spawn approvals have no
|
|
// commit to tag, so they fall through unannotated.
|
|
if matches!(a.kind, ApprovalKind::ApplyCommit) {
|
|
let applied_dir = Coordinator::agent_applied_dir(&a.agent);
|
|
let proposal_ref = format!("refs/tags/proposal/{id}");
|
|
if lifecycle::git_rev_parse(&applied_dir, &proposal_ref)
|
|
.await
|
|
.is_ok()
|
|
{
|
|
let tag_name = format!("denied/{id}");
|
|
let body = note.unwrap_or("").to_owned();
|
|
if let Err(e) =
|
|
lifecycle::git_tag_annotated(&applied_dir, &tag_name, &proposal_ref, &body)
|
|
.await
|
|
{
|
|
tracing::warn!(%id, error = ?e, "plant denied tag failed");
|
|
} else {
|
|
tag = Some(tag_name);
|
|
}
|
|
}
|
|
// Mirror the denied/<id> tag to the forge.
|
|
if let Err(e) = crate::forge::push_config(&a.agent).await {
|
|
tracing::warn!(%id, agent = %a.agent, error = ?e, "forge: push_config after deny failed");
|
|
}
|
|
}
|
|
let approval_kind = match a.kind {
|
|
ApprovalKind::Spawn => "spawn",
|
|
ApprovalKind::ApplyCommit => "apply_commit",
|
|
ApprovalKind::InitConfig => "init_config",
|
|
ApprovalKind::UpdateMetaInputs => "update_meta_inputs",
|
|
ApprovalKind::SchedulePrompt => "schedule_prompt",
|
|
};
|
|
let sha_short = sha.as_deref().map(|s| s[..s.len().min(12)].to_owned());
|
|
let description = a.description.clone();
|
|
let agent_owned = a.agent.clone();
|
|
coord.notify_manager(&HelperEvent::ApprovalResolved {
|
|
id: a.id,
|
|
agent: a.agent,
|
|
commit_ref: a.commit_ref,
|
|
status: ApprovalStatus::Denied,
|
|
note: note.map(String::from),
|
|
sha,
|
|
tag,
|
|
});
|
|
coord.emit_approval_resolved(
|
|
id,
|
|
&agent_owned,
|
|
approval_kind,
|
|
sha_short,
|
|
"denied",
|
|
note.map(String::from),
|
|
description,
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|