906 lines
39 KiB
Rust
906 lines
39 KiB
Rust
//! Operations that are exposed through more than one surface (the host admin
|
|
//! socket *and* the dashboard's POST endpoints). Each function takes a
|
|
//! `&Coordinator` and the request parameters; callers stitch the response
|
|
//! shape they want (HTTP redirect vs JSON).
|
|
|
|
use std::sync::Arc;
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_sh4re::{ApprovalKind, ApprovalStatus, HelperEvent};
|
|
|
|
use crate::coordinator::{Coordinator, TransientKind};
|
|
use crate::lifecycle;
|
|
|
|
/// Approve a pending request. Marks the approval row durably, then
|
|
/// either runs the work inline (`InitConfig`, sub-second git ops) or
|
|
/// submits it to the job queue so the dashboard POST returns
|
|
/// immediately while the long-running pipeline runs off-thread
|
|
/// (operator no longer blocks on a 30-90s spinner for `MergeConfigPr`).
|
|
///
|
|
/// Dispatch:
|
|
/// - `MergeConfigPr` → a single-node `ApprovalDeploy`
|
|
/// DAG (the two-phase meta deploy stays opaque in v1; ~30-90s)
|
|
/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
|
|
/// - `Spawn` → a `Spawn` DAG (`Create → WriteDropin → Reconcile`)
|
|
/// - `InitConfig` → inline (<1s; queue card would be noise)
|
|
///
|
|
/// `ApprovalDeploy` resolves the approval inside its pipeline; the
|
|
/// `MetaUpdate` / `Spawn` DAGs resolve via [`resolve_approval_dag`]
|
|
/// when their DAG settles terminal.
|
|
pub async fn approve(coord: Arc<Coordinator>, id: i64) -> Result<()> {
|
|
let approval = coord.approvals.mark_approved(id)?;
|
|
tracing::info!(
|
|
%approval.id,
|
|
%approval.agent,
|
|
kind = ?approval.kind,
|
|
%approval.commit_ref,
|
|
"approval: dispatching",
|
|
);
|
|
match approval.kind {
|
|
ApprovalKind::InitConfig => {
|
|
// Sub-second git seed + forge-remote wire. Routing through
|
|
// the queue would surface a queue card that's gone before
|
|
// the operator's eyes refocus. Run inline.
|
|
let proposed_dir = Coordinator::agent_proposed_dir(&approval.agent);
|
|
let claude_dir = Coordinator::agent_claude_dir(&approval.agent);
|
|
let notes_dir = Coordinator::agent_notes_dir(&approval.agent);
|
|
run_approval_init_config(&coord, approval, proposed_dir, claude_dir, notes_dir).await
|
|
}
|
|
ApprovalKind::UpdateMetaInputs => {
|
|
// Inputs JSON-encoded into commit_ref by the manager's
|
|
// submit path — surface them on the DAG so the dashboard
|
|
// can show *which* inputs are about to bump. The cascade
|
|
// rebuilds fan out when the lock bump lands (so they build
|
|
// against the post-bump lock, and a failed bump fans out
|
|
// nothing).
|
|
let inputs: Vec<String> =
|
|
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
|
let submitted = coord
|
|
.job_queue
|
|
.submit(crate::job_queue::templates::meta_update(
|
|
inputs,
|
|
crate::job_queue::Source::Approval,
|
|
format!("approval #{id} meta input update"),
|
|
Some(id),
|
|
));
|
|
if let Err(e) = submitted {
|
|
return Err(e.context("submit meta-update dag"));
|
|
}
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::Spawn => {
|
|
// The spawn's tail `Reconcile` starts the container, so the
|
|
// new agent's power intent is `Up` from the outset.
|
|
if let Err(e) = coord
|
|
.power
|
|
.set(approval.agent.as_str(), crate::power::Wanted::Up)
|
|
{
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "agent_power: seed on spawn failed");
|
|
}
|
|
let submitted = coord.job_queue.submit(crate::job_queue::templates::spawn(
|
|
approval.agent.as_str(),
|
|
id,
|
|
format!("approval #{id} spawn"),
|
|
));
|
|
if let Err(e) = submitted {
|
|
return Err(e.context("submit spawn dag"));
|
|
}
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
ApprovalKind::SchedulePrompt => {
|
|
// No queue card for SchedulePrompt — the work is a single
|
|
// sqlite insert, the actual "running" lifetime lives on
|
|
// the scheduled-prompts surface itself (worker fires it
|
|
// at the scheduled time). Run inline + fire
|
|
// `ApprovalResolved` so the approval row leaves Pending
|
|
// immediately.
|
|
let result = run_approval_schedule_prompt(&coord, approval).await;
|
|
if result.is_ok() {
|
|
coord.emit_schedules_snapshot();
|
|
}
|
|
result
|
|
}
|
|
ApprovalKind::MergeConfigPr => {
|
|
// The work ends in a container rebuild, so route it through the
|
|
// rebuild queue. The queue worker dispatches MergeConfigPr
|
|
// approvals to `run_merge_config_pr` (verify the reviewed PR head,
|
|
// ff the forge config repo's main to it, mark merged, then the
|
|
// deploy tail).
|
|
enqueue_approval_rebuild(
|
|
&coord,
|
|
approval.agent.as_str(),
|
|
id,
|
|
format!("approval #{id} merge config pr"),
|
|
);
|
|
Ok(())
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Submit the single-node `ApprovalDeploy` DAG tied to an approval id.
|
|
/// Used by the `MergeConfigPr` dispatch arm — the work ends in a container
|
|
/// rebuild routed through the queue; the node executor runs
|
|
/// `run_merge_config_pr`.
|
|
fn enqueue_approval_rebuild(
|
|
coord: &Arc<Coordinator>,
|
|
agent: &str,
|
|
approval_id: i64,
|
|
reason: String,
|
|
) {
|
|
if let Err(e) = coord
|
|
.job_queue
|
|
.submit(crate::job_queue::templates::approval_deploy(
|
|
agent,
|
|
approval_id,
|
|
reason,
|
|
))
|
|
{
|
|
tracing::error!(%agent, approval_id, error = ?e, "submit approval deploy dag failed");
|
|
}
|
|
coord.emit_rebuild_queue_snapshot();
|
|
}
|
|
|
|
/// Worker entry point for `ApprovalKind::MergeConfigPr` queue entries — the
|
|
/// config-change flow's deploy worker. Re-fetches the approval row, runs the
|
|
/// merge pipeline, and fires
|
|
/// `ApprovalResolved` + the `Rebuilt` lifecycle event via `finish_approval`.
|
|
/// `run_merge_config_pr` already fast-forwarded the forge repo's `main` to the
|
|
/// reviewed head (that IS the merge), so `push_config`'s `main` refspec is a
|
|
/// no-op — but it still mirrors the `deployed/<id>` / `failed/<id>` tag the
|
|
/// deploy tail plants onto the merged sha, giving the merged commit a
|
|
/// forge-visible deploy marker. A `MergeConfigPr` is never a first spawn (the
|
|
/// agent already exists).
|
|
pub async fn run_approval_merge_config_pr(
|
|
coord: &Arc<Coordinator>,
|
|
queue_entry_id: Option<u64>,
|
|
approval_id: i64,
|
|
) -> Result<()> {
|
|
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::MergeConfigPr)?;
|
|
let agent_dir = crate::paths::agent_runtime_dir(approval.agent.as_str());
|
|
let applied_dir = crate::paths::applied_dir(approval.agent.as_str());
|
|
// Captured up front to scope the failure-comment's build-log lookup to
|
|
// rows this deploy produced (see `post_merge_failure_to_pr`).
|
|
let since_ts = hive_sh4re::wire_time::now_unix();
|
|
coord.set_queue_step(queue_entry_id, "merge config pr");
|
|
let (result, terminal_tag) =
|
|
run_merge_config_pr(coord, &approval, &agent_dir, &applied_dir, queue_entry_id).await;
|
|
// Mirror the deploy bookkeeping tag (`deployed/<id>` or `failed/<id>`) the
|
|
// deploy tail planted onto the merged sha to the forge config repo, so the
|
|
// merged commit carries a forge-visible deploy marker. `main` is already
|
|
// ff'd by the merge, so only the tag refspec actually lands; best-effort,
|
|
// never fails the approval.
|
|
coord.set_queue_step(queue_entry_id, "forge push");
|
|
if let Err(e) = crate::forge::push_config(approval.agent.as_str()).await {
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after merge failed");
|
|
}
|
|
// On a failed deploy, surface the failing build log back onto the PR so
|
|
// the manager sees why it was rejected without leaving the forge.
|
|
if let Err(e) = &result {
|
|
post_merge_failure_to_pr(coord, &approval, since_ts, e).await;
|
|
}
|
|
finish_approval(coord, &approval, result, terminal_tag)
|
|
}
|
|
|
|
/// Max stderr bytes to inline in a PR failure comment. Keeps the comment
|
|
/// readable and under forge's size limits while still carrying the tail
|
|
/// where the nix/build error actually surfaces.
|
|
const PR_FAIL_LOG_TAIL_BYTES: usize = 4000;
|
|
|
|
/// On a failed `MergeConfigPr` deploy, post the failing build log back to the
|
|
/// config PR as a comment so the manager sees the rejection reason on the PR
|
|
/// itself. Best-effort: any error here is logged, never allowed to disturb the
|
|
/// approval-resolution path.
|
|
///
|
|
/// The failing `build_log` row is located heuristically: the most recent `fail`
|
|
/// row for this agent that started at/after `since_ts` (the caller's function
|
|
/// entry). Because deploys are serialised per agent through the queue, that is
|
|
/// the step which just failed — `verify`, `prepare-deploy`, `prebuild`, or the
|
|
/// container rebuild. Pre-build failures (drift gate, fetch) create no `build_log`
|
|
/// row, so the comment then carries only the error text.
|
|
async fn post_merge_failure_to_pr(
|
|
coord: &Arc<Coordinator>,
|
|
approval: &hive_sh4re::Approval,
|
|
since_ts: i64,
|
|
err: &anyhow::Error,
|
|
) {
|
|
let Ok(pr) = approval.commit_ref.parse::<u64>() else {
|
|
return;
|
|
};
|
|
let repo = crate::forge::config_repo(approval.agent.as_str());
|
|
|
|
let log_section = coord
|
|
.build_logs
|
|
.list_recent_for_agent(approval.agent.as_str(), 10)
|
|
.ok()
|
|
.and_then(|rows| {
|
|
rows.into_iter()
|
|
.find(|r| r.status.as_deref() == Some("fail") && r.started_at >= since_ts)
|
|
})
|
|
.and_then(|row| coord.build_logs.get_full(row.id).ok().flatten())
|
|
.map(|full| {
|
|
let tail = tail_bytes(full.stderr.trim_end(), PR_FAIL_LOG_TAIL_BYTES);
|
|
format!(
|
|
"\n\n**Failing step:** `{}` (build log #{})\n\n```\n{tail}\n```",
|
|
full.header.kind, full.header.id
|
|
)
|
|
})
|
|
.unwrap_or_default();
|
|
|
|
let body = format!(
|
|
"## ⚠️ config deploy failed\n\n\
|
|
Approval #{} to merge this PR could not be deployed:\n\n\
|
|
```\n{err:#}\n```{log_section}",
|
|
approval.id
|
|
);
|
|
|
|
if let Err(e) = crate::forge::post_pr_comment(&repo, pr, &body).await {
|
|
tracing::warn!(agent = %approval.agent, %pr, error = ?e, "post merge-failure comment to PR failed");
|
|
}
|
|
}
|
|
|
|
/// Return the last `max_bytes` of `s`, snapped to a char boundary, prefixed
|
|
/// with an elision marker when truncated.
|
|
fn tail_bytes(s: &str, max_bytes: usize) -> String {
|
|
if s.len() <= max_bytes {
|
|
return s.to_owned();
|
|
}
|
|
let mut start = s.len() - max_bytes;
|
|
while start < s.len() && !s.is_char_boundary(start) {
|
|
start += 1;
|
|
}
|
|
format!("[… truncated …]\n{}", &s[start..])
|
|
}
|
|
|
|
/// PR-merge config pipeline. `approval.commit_ref` is the PR number;
|
|
/// `approval.fetched_sha` is the PR head sha the operator reviewed. Steps:
|
|
/// 1. drift gate — re-read the live PR head; if it moved since review, abort
|
|
/// WITHOUT mutating anything (the operator must re-review the new head);
|
|
/// 2. fetch the reviewed head into the applied repo so later git ops resolve
|
|
/// it locally;
|
|
/// 3. eval-verify the reviewed commit against the meta flake BEFORE the
|
|
/// irreversible push;
|
|
/// 4. fast-forward the forge repo's `main` to the reviewed head — THE merge;
|
|
/// 5. mark the PR merged (best-effort: `main` is already at the head, so a
|
|
/// failure here is logged, not fatal);
|
|
/// 6. run the shared deploy tail (`deploy_applied_target`): ff applied/main,
|
|
/// meta deploy, container rebuild, finalize/rollback.
|
|
///
|
|
/// Returns `(build result, terminal tag)` like `deploy_applied_target`. Any
|
|
/// pre-merge abort returns `Err` with no mutation; the operator re-reviews.
|
|
async fn run_merge_config_pr(
|
|
coord: &Arc<Coordinator>,
|
|
approval: &hive_sh4re::Approval,
|
|
agent_dir: &std::path::Path,
|
|
applied_dir: &std::path::Path,
|
|
queue_entry_id: Option<u64>,
|
|
) -> (Result<()>, Option<String>) {
|
|
let id = approval.id;
|
|
let pr: u64 = match approval.commit_ref.parse() {
|
|
Ok(n) => n,
|
|
Err(e) => {
|
|
return (
|
|
Err(anyhow::anyhow!(
|
|
"parse PR number from commit_ref {:?}: {e}",
|
|
approval.commit_ref
|
|
)),
|
|
None,
|
|
);
|
|
}
|
|
};
|
|
let reviewed = match approval.fetched_sha.as_deref() {
|
|
Some(s) => s.to_owned(),
|
|
None => {
|
|
return (
|
|
Err(anyhow::anyhow!(
|
|
"merge config pr approval {id} has no reviewed head sha"
|
|
)),
|
|
None,
|
|
);
|
|
}
|
|
};
|
|
let repo = crate::forge::config_repo(approval.agent.as_str());
|
|
|
|
// 1. Drift gate: the live PR head must still equal what was reviewed.
|
|
coord.set_queue_step(queue_entry_id, "verify PR head");
|
|
let head = match crate::forge::pr_head_sha(&repo, pr).await {
|
|
Ok(h) => h,
|
|
Err(e) => return (Err(anyhow::anyhow!("read PR #{pr} head: {e}")), None),
|
|
};
|
|
if head != reviewed {
|
|
return (
|
|
Err(anyhow::anyhow!(
|
|
"PR #{pr} head drifted since review (reviewed {reviewed}, now {head}); re-review before merging"
|
|
)),
|
|
None,
|
|
);
|
|
}
|
|
|
|
// 2. Fetch the reviewed head into applied so ff/verify/deploy resolve it.
|
|
coord.set_queue_step(queue_entry_id, "fetch PR head");
|
|
if let Err(e) = crate::forge::fetch_pr_head_into_applied(&repo, pr).await {
|
|
return (
|
|
Err(anyhow::anyhow!("fetch PR #{pr} head into applied: {e}")),
|
|
None,
|
|
);
|
|
}
|
|
|
|
// 3. Eval-verify BEFORE the irreversible push (bad nix fails fast here).
|
|
coord.set_queue_step(queue_entry_id, "verify proposal (eval)");
|
|
if let Err(e) =
|
|
crate::meta::verify_commit(approval.agent.as_str(), applied_dir, &reviewed).await
|
|
{
|
|
return (
|
|
Err(anyhow::anyhow!("verify merge head {reviewed}: {e:#}")),
|
|
None,
|
|
);
|
|
}
|
|
|
|
// Capture the currently-deployed sha for the deploy tail's rollback.
|
|
let prev_main_sha = match lifecycle::git_rev_parse(applied_dir, "refs/heads/main").await {
|
|
Ok(s) => s,
|
|
Err(e) => return (Err(anyhow::anyhow!("read applied/main: {e:#}")), None),
|
|
};
|
|
|
|
// 4. THE merge: fast-forward-only merge the reviewed head to `main` via the
|
|
// forge API, pinned to the reviewed sha (`head_commit_id`). This one call
|
|
// both advances `main` to the reviewed head and marks the PR merged — no
|
|
// direct push to the protected branch. Unlike the old push-then-mark split,
|
|
// a failure here means `main` was NOT advanced, so it's fatal: we must not
|
|
// deploy a head the forge didn't merge.
|
|
coord.set_queue_step(queue_entry_id, "fast-forward-merge PR");
|
|
match crate::forge::merge_config_pr_ff(&repo, pr, &reviewed).await {
|
|
Ok(()) => {}
|
|
Err(crate::forge::ForgeMergeError::HeadDrift { expected, actual }) => {
|
|
return (
|
|
Err(anyhow::anyhow!(
|
|
"PR #{pr} head drifted before merge (reviewed {expected}, now {actual}); re-review before merging"
|
|
)),
|
|
None,
|
|
);
|
|
}
|
|
Err(e) => return (Err(anyhow::anyhow!("ff-merge PR #{pr}: {e}")), None),
|
|
}
|
|
|
|
// 5. Deploy tail. target == finalize == the reviewed head.
|
|
deploy_applied_target(
|
|
coord,
|
|
approval.agent.as_str(),
|
|
agent_dir,
|
|
applied_dir,
|
|
&reviewed,
|
|
&reviewed,
|
|
id,
|
|
&prev_main_sha,
|
|
queue_entry_id,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// Inline (non-queued) handler for `ApprovalKind::SchedulePrompt`.
|
|
/// On approve, decode the `SchedulePromptPayload` JSON from the
|
|
/// approval's `commit_ref`, insert a row into `scheduled_prompts`
|
|
/// (with `source = Approval { id }`), and fire `ApprovalResolved`.
|
|
/// The worker takes over from here — fan-out at fire time.
|
|
async fn run_approval_schedule_prompt(
|
|
coord: &Coordinator,
|
|
approval: hive_sh4re::Approval,
|
|
) -> Result<()> {
|
|
let result: Result<()> = async {
|
|
let payload: hive_sh4re::SchedulePromptPayload = serde_json::from_str(&approval.commit_ref)
|
|
.context("decode SchedulePromptPayload from approval.commit_ref")?;
|
|
coord
|
|
.scheduled_prompts
|
|
.submit(&crate::scheduled_prompts::NewSchedule {
|
|
owner: approval.agent.to_string(),
|
|
targets: payload.targets,
|
|
body: payload.body,
|
|
first_fire_at_unix: payload.first_fire_at_unix,
|
|
interval_seconds: payload.interval_seconds,
|
|
description: payload.description,
|
|
source: crate::scheduled_prompts::ScheduleSource::Approval { id: approval.id },
|
|
})
|
|
.map(|_| ())
|
|
.context("insert scheduled prompt")
|
|
}
|
|
.await;
|
|
finish_approval(coord, &approval, result, None)
|
|
}
|
|
|
|
/// Terminal hook for approval-carrying DAGs — the job queue's
|
|
/// scheduler calls this exactly once when such a DAG settles terminal.
|
|
/// `MetaUpdate` and `Spawn` approval DAGs resolve here (their work is
|
|
/// ordinary queue nodes); the opaque `ApprovalDeploy` pipeline resolves
|
|
/// *inside* its node, so its DAG is skipped — unless it was cancelled
|
|
/// while still queued, in which case the node never ran and the row
|
|
/// would otherwise dangle forever.
|
|
pub(crate) async fn resolve_approval_dag(
|
|
coord: &Arc<Coordinator>,
|
|
terminal: &crate::job_queue::TerminalDag,
|
|
) {
|
|
use crate::job_queue::{State, Template};
|
|
let Some(approval_id) = terminal.approval_id else {
|
|
return;
|
|
};
|
|
if terminal.template == Template::Rebuild && terminal.state != State::Cancelled {
|
|
return; // ApprovalDeploy resolved inside the node.
|
|
}
|
|
let approval = match coord.approvals.get(approval_id) {
|
|
Ok(Some(a)) => a,
|
|
Ok(None) => {
|
|
tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
|
|
return;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
|
|
return;
|
|
}
|
|
};
|
|
let result: Result<()> = match terminal.state {
|
|
State::Done => Ok(()),
|
|
State::Cancelled => Err(anyhow::anyhow!("cancelled before completion")),
|
|
_ => Err(anyhow::anyhow!(
|
|
"{}",
|
|
terminal
|
|
.error
|
|
.clone()
|
|
.unwrap_or_else(|| "job dag failed".to_owned())
|
|
)),
|
|
};
|
|
if approval.kind == ApprovalKind::Spawn {
|
|
// Post-spawn forge bookkeeping (user, config repo mirror, meta
|
|
// access) — warn-only, then the resolution events + a rescan so
|
|
// the dashboard reflects the post-spawn state either way.
|
|
if result.is_ok() {
|
|
forge_after_first_spawn(coord, approval.agent.as_str()).await;
|
|
} else {
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
}
|
|
}
|
|
if let Err(e) = finish_approval(coord, &approval, result, None) {
|
|
tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
|
|
}
|
|
}
|
|
|
|
/// Re-fetch an approval row from sqlite for a queue-worker dispatch.
|
|
/// Bails if the row is gone (deny race), if its kind doesn't match,
|
|
/// or if the lookup itself fails. The kind check is defensive — the
|
|
/// queue's `dispatch` already routes by `QueueKind`, but the approval
|
|
/// kind is the authoritative source of truth and a mismatch points
|
|
/// at a deeper bug we'd want to surface.
|
|
fn fetch_approval_for_worker(
|
|
coord: &Coordinator,
|
|
approval_id: i64,
|
|
expected_kind: ApprovalKind,
|
|
) -> Result<hive_sh4re::Approval> {
|
|
let approval = coord
|
|
.approvals
|
|
.get(approval_id)
|
|
.map_err(|e| anyhow::anyhow!("read approval {approval_id}: {e:#}"))?
|
|
.ok_or_else(|| anyhow::anyhow!("approval {approval_id} no longer exists"))?;
|
|
if approval.kind != expected_kind {
|
|
bail!(
|
|
"approval {approval_id} kind mismatch: queue expected {expected_kind:?}, row is {actual:?}",
|
|
actual = approval.kind
|
|
);
|
|
}
|
|
Ok(approval)
|
|
}
|
|
|
|
/// Forge bookkeeping run once after the very first container spawn:
|
|
/// create the per-agent forge user, mirror the applied repo, and grant
|
|
/// read access to core/meta. Also rescans containers so the dashboard
|
|
/// reflects the post-spawn state.
|
|
async fn forge_after_first_spawn(coord: &Arc<Coordinator>, agent: &str) {
|
|
if let Err(e) = crate::forge::ensure_user_for(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_user after first spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_config_repo(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_config_repo after first spawn failed");
|
|
}
|
|
if let Some(core_token) = crate::forge::core_token()
|
|
&& let Err(e) = crate::forge::meta_read_access(agent, &core_token).await
|
|
{
|
|
tracing::warn!(%agent, error = ?e, "forge: meta_read_access after first spawn failed");
|
|
}
|
|
if let Err(e) = crate::forge::ensure_meta_remote(agent).await {
|
|
tracing::warn!(%agent, error = ?e, "forge: ensure_meta_remote after first spawn failed");
|
|
}
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
}
|
|
|
|
/// Inline (non-queued) handler for `ApprovalKind::InitConfig`. Just
|
|
/// seeds the proposed git repo + the per-agent dirs — sub-second
|
|
/// work that doesn't justify a queue card.
|
|
async fn run_approval_init_config(
|
|
coord: &Coordinator,
|
|
approval: hive_sh4re::Approval,
|
|
proposed_dir: std::path::PathBuf,
|
|
claude_dir: std::path::PathBuf,
|
|
notes_dir: std::path::PathBuf,
|
|
) -> Result<()> {
|
|
let result: Result<()> = async {
|
|
// Place the new child under its requesting parent (carried in
|
|
// commit_ref by submit_init_config). An empty commit_ref means
|
|
// no explicit parent was named (privileged manager socket, or an
|
|
// approval queued before the new-child feature) — write no edge
|
|
// and let `topology::reconcile` assign the default position on
|
|
// first spawn, so this path never names a specific root agent.
|
|
if !approval.commit_ref.is_empty() {
|
|
crate::topology::add_child(approval.agent.as_str(), &approval.commit_ref)
|
|
.map_err(|e| anyhow::anyhow!("topology add_child: {e}"))?;
|
|
}
|
|
// Create the agent's state root as a btrfs subvolume FIRST, before
|
|
// any dir-seed touches it. `ensure_agent_state_subvolume` is
|
|
// progressive ("root exists → skip"), and `setup_proposed` does a
|
|
// `create_dir_all` on the proposed-config path which would
|
|
// materialise the state root as a plain directory — after which
|
|
// the subvolume create is silently skipped and the agent never
|
|
// lands on a subvolume (no quota, no snapshot). Order matters.
|
|
lifecycle::ensure_agent_state_subvolume(approval.agent.as_str()).await?;
|
|
lifecycle::setup_proposed(&proposed_dir, approval.agent.as_str()).await?;
|
|
lifecycle::ensure_claude_dir(&claude_dir)?;
|
|
lifecycle::ensure_state_dir(¬es_dir)?;
|
|
Ok(())
|
|
}
|
|
.await;
|
|
if result.is_ok()
|
|
&& let Err(e) = crate::forge::ensure_meta_remote(approval.agent.as_str()).await
|
|
{
|
|
tracing::warn!(agent = %approval.agent, error = ?e, "forge: ensure_meta_remote after init_config failed");
|
|
}
|
|
finish_approval(coord, &approval, result, None)
|
|
}
|
|
|
|
fn finish_approval(
|
|
coord: &Coordinator,
|
|
approval: &hive_sh4re::Approval,
|
|
result: Result<()>,
|
|
terminal_tag: Option<String>,
|
|
) -> Result<()> {
|
|
let (status, note, ok) = match &result {
|
|
Ok(()) => (ApprovalStatus::Approved, None, true),
|
|
Err(e) => {
|
|
let note = format!("{e:#}");
|
|
let _ = coord.approvals.mark_failed(approval.id, ¬e);
|
|
(ApprovalStatus::Failed, Some(note), false)
|
|
}
|
|
};
|
|
coord.notify_submitter(
|
|
approval.id,
|
|
&HelperEvent::ApprovalResolved {
|
|
id: approval.id,
|
|
agent: approval.agent.to_string(),
|
|
commit_ref: approval.commit_ref.clone(),
|
|
status,
|
|
note: note.clone(),
|
|
sha: approval.fetched_sha.clone(),
|
|
tag: terminal_tag.clone(),
|
|
},
|
|
);
|
|
// Phase 5b: also fire on the dashboard event channel so the
|
|
// browser moves the row out of pending into history without a
|
|
// snapshot refetch. `approved` rows that succeed get the
|
|
// approval's logged resolved_at indirectly via `now_unix()`;
|
|
// failures already wrote it via mark_failed above.
|
|
let approval_kind = approval.kind.as_str();
|
|
let sha_short = approval
|
|
.fetched_sha
|
|
.as_deref()
|
|
.map(|s| s[..s.len().min(12)].to_owned());
|
|
let status_str = if ok { "approved" } else { "failed" };
|
|
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
|
id: approval.id,
|
|
agent: approval.agent.as_str(),
|
|
approval_kind,
|
|
sha_short,
|
|
status: status_str,
|
|
note: note.clone(),
|
|
description: approval.description.clone(),
|
|
});
|
|
// For spawn/rebuild/init_config approvals, also surface the underlying
|
|
// action so the manager knows whether the lifecycle step succeeded.
|
|
// The ApprovalResolved event already carries the same `ok` signal but
|
|
// separating it lets the manager react to the lifecycle change
|
|
// without having to special-case approvals.
|
|
match approval.kind {
|
|
ApprovalKind::InitConfig => {
|
|
if ok {
|
|
coord.notify_submitter(
|
|
approval.id,
|
|
&HelperEvent::ConfigReady {
|
|
agent: approval.agent.to_string(),
|
|
},
|
|
);
|
|
}
|
|
}
|
|
ApprovalKind::Spawn => coord.notify_submitter(
|
|
approval.id,
|
|
&HelperEvent::Spawned {
|
|
agent: approval.agent.to_string(),
|
|
ok,
|
|
note,
|
|
},
|
|
),
|
|
// MergeConfigPr ends in a container rebuild — surface a Rebuilt
|
|
// lifecycle event. (It is never a first spawn — the agent already
|
|
// exists — so it never needs the Spawned arm above.)
|
|
ApprovalKind::MergeConfigPr => {
|
|
coord.notify_submitter(
|
|
approval.id,
|
|
&HelperEvent::Rebuilt {
|
|
agent: approval.agent.to_string(),
|
|
ok,
|
|
note,
|
|
sha: approval.fetched_sha.clone(),
|
|
tag: terminal_tag,
|
|
},
|
|
);
|
|
}
|
|
// UpdateMetaInputs / SchedulePrompt: ApprovalResolved already
|
|
// carries the result. No separate lifecycle event needed.
|
|
ApprovalKind::UpdateMetaInputs | ApprovalKind::SchedulePrompt => {}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// Deploy tail for the config-PR merge flow. Fast-forwards `applied/main` to
|
|
/// `target_ref`, syncs the working tree, runs the meta two-phase deploy +
|
|
/// container rebuild, and plants the `deployed/<tag_base>` /
|
|
/// `failed/<tag_base>` bookkeeping tags. On build failure it rolls
|
|
/// `applied/main` back to `prev_main_sha` and aborts the staged meta lock so
|
|
/// the agent stays on its last-good tree. Returns the build result + the
|
|
/// terminal tag name.
|
|
///
|
|
/// Caller-specific bits stay OUT of here: fetching the PR head, the
|
|
/// `verify_commit` gate, the ff-merge, and forge mark-merged. `finalize_sha`
|
|
/// is the sha recorded by `meta::finalize_deploy`; `target_ref` is what
|
|
/// `applied/main` fast-forwards to. The agent always already exists here (a
|
|
/// merge is never a first spawn), so there's no `sync_agents` step — the
|
|
/// operator `Spawn` flow owns first-time meta registration.
|
|
#[allow(
|
|
clippy::too_many_arguments,
|
|
clippy::too_many_lines,
|
|
reason = "one sequential ff/deploy/rebuild/finalize pipeline; splitting it \
|
|
would obscure the linear flow"
|
|
)]
|
|
async fn deploy_applied_target(
|
|
coord: &Arc<Coordinator>,
|
|
agent: &str,
|
|
agent_dir: &std::path::Path,
|
|
applied_dir: &std::path::Path,
|
|
target_ref: &str,
|
|
finalize_sha: &str,
|
|
tag_base: i64,
|
|
prev_main_sha: &str,
|
|
queue_entry_id: Option<u64>,
|
|
) -> (Result<()>, Option<String>) {
|
|
let id = tag_base;
|
|
|
|
coord.set_queue_step(queue_entry_id, "fast-forward applied/main");
|
|
// Fast-forward applied/main to target_ref + sync the working tree.
|
|
// Meta input pins `?ref=main`, so this is what makes nix re-lock to
|
|
// the target commit on the prepare_deploy step below. On build
|
|
// failure we roll main back to prev_main_sha so a crash leaves the
|
|
// agent on its last-good tree.
|
|
if let Err(e) = lifecycle::git_update_ref(applied_dir, "refs/heads/main", target_ref).await {
|
|
return (Err(anyhow::anyhow!("ff main to {target_ref}: {e:#}")), None);
|
|
}
|
|
if let Err(e) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
|
|
// main is ahead; working tree didn't sync. Roll main back to
|
|
// keep the two consistent before bailing.
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await;
|
|
return (Err(anyhow::anyhow!("read-tree to main: {e:#}")), None);
|
|
}
|
|
|
|
coord.set_queue_step(queue_entry_id, "meta prepare_deploy");
|
|
// Phase 1 of the meta two-phase deploy: relock without committing.
|
|
if let Err(e) = crate::meta::prepare_deploy(agent).await {
|
|
let _ = lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await;
|
|
let _ = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await;
|
|
return (Err(anyhow::anyhow!("meta prepare_deploy: {e:#}")), None);
|
|
}
|
|
|
|
// Container-level rebuild (or first-time create) against meta#<name>.
|
|
// Step labels are emitted inside rebuild_no_meta via the callback so
|
|
// the dashboard reflects actual phase progress rather than a static
|
|
// "nixos-container update" label for the whole multi-minute window.
|
|
let hive = coord.hive_env();
|
|
let paths = Coordinator::agent_paths(agent, agent_dir.to_path_buf());
|
|
let build_result = lifecycle::rebuild_no_meta(
|
|
agent,
|
|
&hive,
|
|
&paths,
|
|
// Inline start: the apply-commit flow verifies the agent comes
|
|
// back up before finalizing the deploy tag, so the start stays
|
|
// part of this entry rather than a deferred fast-lane follow-up.
|
|
false,
|
|
&|step| coord.set_queue_step(queue_entry_id, step),
|
|
&|log_id| coord.set_queue_build_log(queue_entry_id, log_id),
|
|
)
|
|
.await;
|
|
|
|
match build_result {
|
|
Ok(_) => {
|
|
coord.set_queue_step(queue_entry_id, "finalize deploy");
|
|
let tag = format!("deployed/{id}");
|
|
if let Err(e) = lifecycle::git_tag(applied_dir, &tag, target_ref).await {
|
|
tracing::warn!(%agent, %id, error = ?e, "plant deployed tag failed");
|
|
}
|
|
if let Err(e) = crate::meta::finalize_deploy(agent, finalize_sha, &tag).await {
|
|
// The build itself succeeded — meta lock landed but
|
|
// couldn't be committed. Surface as a soft warn so the
|
|
// operator can git-commit by hand if they care.
|
|
tracing::warn!(%agent, %id, error = ?e, "meta finalize_deploy failed");
|
|
}
|
|
// Wake the agent on its next turn so claude sees the
|
|
// config change took effect. Same hint pattern as
|
|
// auto_update::rebuild_agent — manager approved a
|
|
// proposal, agent picks up where it left off with the
|
|
// new env / packages.
|
|
coord.kick_agent(agent, "config update applied");
|
|
(Ok(()), Some(tag))
|
|
}
|
|
Err(e) => {
|
|
let tag = format!("failed/{id}");
|
|
let body = format!("{e:#}");
|
|
if let Err(te) =
|
|
lifecycle::git_tag_annotated(applied_dir, &tag, target_ref, &body).await
|
|
{
|
|
tracing::warn!(%agent, %id, error = ?te, "annotate failed tag failed");
|
|
}
|
|
// Roll main back to last known-good so the on-disk state
|
|
// matches what nixos-container last successfully built.
|
|
if let Err(re) =
|
|
lifecycle::git_update_ref(applied_dir, "refs/heads/main", prev_main_sha).await
|
|
{
|
|
tracing::warn!(%agent, %id, error = ?re, "main rollback failed");
|
|
}
|
|
if let Err(re) = lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main").await {
|
|
tracing::warn!(%agent, %id, error = ?re, "rollback read-tree failed");
|
|
}
|
|
// Drop the staged meta lock change so the deploy log
|
|
// only ever shows successes.
|
|
if let Err(ae) = crate::meta::abort_deploy().await {
|
|
tracing::warn!(%agent, %id, error = ?ae, "meta abort_deploy failed");
|
|
}
|
|
(Err(e), Some(tag))
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Tear down a sub-agent container. By default this is non-destructive to
|
|
/// persistent state: the proposed/applied config repos and the Claude
|
|
/// credentials dir under `/var/lib/hyperhive/{agents,applied}/<name>/` are
|
|
/// kept, so recreating an agent of the same name reuses prior config + creds
|
|
/// (no re-login). The ephemeral runtime dir under `/run/hyperhive/agents/`
|
|
/// is cleared because its contents (the mcp socket) don't survive restarts
|
|
/// anyway. With `purge=true` the persistent trees are also wiped — config
|
|
/// history, claude creds, notes — there is no undo.
|
|
/// The bootstrap/root container is destroyable like any other: it's
|
|
/// imperative infra that `auto_update::ensure_root_agent` recreates on the
|
|
/// next hive-c0re startup if absent, so destroying it is transient rather
|
|
/// than something to refuse at the API.
|
|
pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Result<()> {
|
|
tracing::info!(%name, purge, "destroy");
|
|
// Guard auto-clears on the success path's final scope exit and on
|
|
// every early-return / cancellation along the way.
|
|
let guard = coord.transient_guard(name, TransientKind::Destroying);
|
|
lifecycle::destroy(name).await?;
|
|
coord.unregister_agent(name);
|
|
let runtime = crate::paths::agent_runtime_dir(name);
|
|
if runtime.exists() {
|
|
let _ = std::fs::remove_dir_all(&runtime);
|
|
}
|
|
if purge {
|
|
// The state root may be a btrfs subvolume: a subvolume root
|
|
// can't be removed with rmdir/`remove_dir_all`, so delete it via
|
|
// hive-priv (root) first. No-op for plain-dir agents — the loop below
|
|
// then handles the plain-dir state root plus the applied dir.
|
|
if let Err(e) = crate::priv_client::delete_agent_subvolume(name).await {
|
|
tracing::warn!(error = ?e, %name, "purge: delete state subvolume failed");
|
|
}
|
|
// A malformed name can't have a persistent state tree (the state dir
|
|
// is only ever created under a validated Ident), so its removal is a
|
|
// no-op — skip the state-dir sweep and just clear the applied dir.
|
|
let state_dir = hive_types::Ident::parse(name)
|
|
.ok()
|
|
.map(|id| crate::paths::agent_state_dir(&id));
|
|
for dir in state_dir
|
|
.into_iter()
|
|
.chain([crate::paths::applied_dir(name)])
|
|
{
|
|
if dir.exists()
|
|
&& let Err(e) = std::fs::remove_dir_all(&dir)
|
|
{
|
|
tracing::warn!(error = ?e, dir = %dir.display(), "purge: remove failed");
|
|
}
|
|
}
|
|
}
|
|
// Meta flake: drop the agent's input + nixosConfiguration so a
|
|
// future spawn under the same name re-seeds cleanly, and so the
|
|
// meta lock doesn't reference a vanished applied repo. Log + keep
|
|
// going on failure — destroy already succeeded at the
|
|
// nixos-container level, the meta repo is just bookkeeping.
|
|
if let Err(e) = sync_meta_after_lifecycle(coord).await {
|
|
tracing::warn!(error = ?e, %name, "meta sync after destroy failed");
|
|
}
|
|
let _ = coord.approvals.fail_pending_for_agent(
|
|
name,
|
|
if purge {
|
|
"agent purged"
|
|
} else {
|
|
"agent destroyed"
|
|
},
|
|
);
|
|
// Drop the durable power intent — a future agent of the same name
|
|
// seeds fresh from its observed state.
|
|
if let Err(e) = coord.power.remove(name) {
|
|
tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed");
|
|
}
|
|
drop(guard);
|
|
coord.notify_manager(&HelperEvent::Destroyed {
|
|
agent: name.to_owned(),
|
|
});
|
|
// Container row disappeared — rescan so the dashboard fires
|
|
// `ContainerRemoved` for the gone row, then emit the
|
|
// tombstones snapshot (gained one on destroy, lost one on
|
|
// purge — recompute either way).
|
|
coord.rescan_containers_and_emit().await;
|
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
// Re-emit the schedules snapshot: the rescan above refreshed the live
|
|
// roster, so any schedule that still targets the just-destroyed agent
|
|
// now drops that ghost column live (no page reload needed).
|
|
coord.emit_schedules_snapshot();
|
|
// Update tmpfiles.d to remove the destroyed agent's dirs from the
|
|
// boot-time pre-creation list. Best-effort: failure is logged only.
|
|
tokio::spawn(lifecycle::sync_tmpfiles());
|
|
Ok(())
|
|
}
|
|
|
|
/// Rerender the meta flake from whatever containers still exist on
|
|
/// disk. Called after lifecycle ops that change the agent set (today:
|
|
/// destroy). Idempotent — a no-op when nothing changed.
|
|
async fn sync_meta_after_lifecycle(coord: &Coordinator) -> Result<()> {
|
|
let agents = lifecycle::agents_for_meta_listing().await?;
|
|
crate::meta::sync_agents(&coord.hive_env(), &agents).await
|
|
}
|
|
|
|
pub fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
|
|
let approval = coord.approvals.get(id)?;
|
|
coord.approvals.mark_denied(id, note)?;
|
|
tracing::info!(%id, note, "approval denied");
|
|
if let Some(a) = approval {
|
|
let sha = a.fetched_sha.clone();
|
|
let approval_kind = a.kind.as_str();
|
|
let sha_short = sha.as_deref().map(|s| s[..s.len().min(12)].to_owned());
|
|
let description = a.description.clone();
|
|
let agent_owned = a.agent.clone();
|
|
coord.notify_submitter(
|
|
a.id,
|
|
&HelperEvent::ApprovalResolved {
|
|
id: a.id,
|
|
agent: a.agent.to_string(),
|
|
commit_ref: a.commit_ref,
|
|
status: ApprovalStatus::Denied,
|
|
note: note.map(String::from),
|
|
sha,
|
|
// A denied config PR carries no git tag — it stays open on the forge.
|
|
tag: None,
|
|
},
|
|
);
|
|
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
|
id,
|
|
agent: agent_owned.as_str(),
|
|
approval_kind,
|
|
sha_short,
|
|
status: "denied",
|
|
note: note.map(String::from),
|
|
description,
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|