c3a54bbdd0
Scheduled prompts are swarm-level now (swarmctl schedule, #4936). The hive keeps the rows it has only so the operator can copy them over. - hive-c0re no longer fires anything: the scheduled-prompts worker is gone, and with it every create/edit/cancel/pause/resume/fire path (dashboard endpoints, agent-socket verbs, the SchedulePrompt approval kind and its approve handler). The store keeps open + list; GET /api/schedules stays. - A v6 approvals migration drops schedule_prompt rows. - Nothing submits approvals any more, so the submit path goes too: Approvals::submit_kind (now a test fixture), the ApprovalAdded dashboard event and its emitter. - Coordinator::push_todo, whose only caller was the worker, goes, and with it hive_host_sock::agent_todo_socket and hive-c0re's hive-agent-sock / hive-sock-client deps. - The schedules_changed dashboard event goes; nothing mutates the rows. - Agents lose request_schedule_prompt, cancel_schedule, fire_schedule_now, edit_schedule and list_schedules, the `scheduling` tool group, their system-prompt line and their transcript formatters. `remind` is unchanged.
417 lines
16 KiB
Rust
417 lines
16 KiB
Rust
//! Operations that are exposed through more than one surface (the host admin
|
|
//! socket *and* the dashboard's POST endpoints). Each function takes a
|
|
//! `&Coordinator` and the request parameters; callers stitch the response
|
|
//! shape they want (HTTP redirect vs JSON).
|
|
|
|
use std::sync::Arc;
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_sh4re::approvals::{ApprovalKind, ApprovalStatus};
|
|
use hive_sh4re::manager::HelperEvent;
|
|
|
|
use crate::coordinator::Coordinator;
|
|
use crate::lifecycle;
|
|
|
|
/// Approve a pending request. Marks the approval row durably, then submits
|
|
/// the work to the job queue so the dashboard POST returns immediately while
|
|
/// the long-running pipeline runs off-thread.
|
|
///
|
|
/// Dispatch:
|
|
/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
|
|
///
|
|
/// Every queued kind resolves its approval row via
|
|
/// [`resolve_approval_dag`] when the DAG settles terminal.
|
|
pub fn approve(coord: &Arc<Coordinator>, id: i64) -> Result<()> {
|
|
let approval = coord.approvals.mark_approved(id)?;
|
|
tracing::info!(
|
|
%approval.id,
|
|
%approval.agent,
|
|
kind = ?approval.kind,
|
|
%approval.commit_ref,
|
|
"approval: dispatching",
|
|
);
|
|
match approval.kind {
|
|
ApprovalKind::UpdateMetaInputs => {
|
|
// Inputs JSON-encoded into commit_ref by the manager's
|
|
// submit path — surface them on the DAG so the dashboard
|
|
// can show *which* inputs are about to bump. The cascade
|
|
// rebuilds fan out when the lock bump lands (so they build
|
|
// against the post-bump lock, and a failed bump fans out
|
|
// nothing).
|
|
let inputs: Vec<String> =
|
|
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
|
let inserted = coord.job_queue.insert_job(|b| {
|
|
crate::job_queue::templates::meta_update(b, inputs, Some(id));
|
|
Vec::new()
|
|
});
|
|
if let Err(e) = inserted {
|
|
return Err(e.context("insert meta-update dag"));
|
|
}
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `applied/main`, and whether `target` descends from it. Fast-forwarding
|
|
/// `main` to a `target` that does not is a rewind: it drops every commit on
|
|
/// `main` that `target` lacks.
|
|
async fn applied_main_under(applied_dir: &std::path::Path, target: &str) -> Result<(String, bool)> {
|
|
let current_main = lifecycle::git_rev_parse(applied_dir, "refs/heads/main")
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("read applied/main: {e:#}"))?;
|
|
let descends = lifecycle::git_is_ancestor(applied_dir, ¤t_main, target)
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("ancestry check {current_main}..{target}: {e:#}"))?;
|
|
Ok((current_main, descends))
|
|
}
|
|
|
|
/// What [`advance_applied_to_rev`] did to `applied/main`.
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
pub(crate) enum RevAdvance {
|
|
/// `applied/main` already was the rev, so there is nothing to deploy.
|
|
AlreadyApplied,
|
|
/// `applied/main` was fast-forwarded to the rev.
|
|
Advanced,
|
|
}
|
|
|
|
/// Fast-forward `agent`'s `applied/main` to `rev`, a commit the swarm asks
|
|
/// this hive to deploy, so the next relocking rebuild builds it.
|
|
///
|
|
/// No eval-verify on this path: a config that does not evaluate fails the
|
|
/// rebuild instead, with `applied/main` already at `rev`.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns an error if the forge fetch fails, if `rev` does not descend from
|
|
/// `applied/main`, or if `main` moved during the fast-forward. `main` does not
|
|
/// move in any of these.
|
|
pub(crate) async fn advance_applied_to_rev(agent: &str, rev: &str) -> Result<RevAdvance> {
|
|
advance_applied_main(
|
|
&crate::paths::applied_dir(agent),
|
|
rev,
|
|
crate::forge::fetch_forge_main(agent),
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// [`advance_applied_to_rev`] with the forge fetch passed in, so it is
|
|
/// testable against a local repo.
|
|
///
|
|
/// `applied/main` is compared to `rev` twice: before the fetch, so an applied
|
|
/// rev costs no forge round-trip, and after it, because a meta-update cascade
|
|
/// fast-forwards `main` to the forge's and may do so meanwhile.
|
|
async fn advance_applied_main<T>(
|
|
applied_dir: &std::path::Path,
|
|
rev: &str,
|
|
fetch: impl std::future::Future<Output = Result<T>>,
|
|
) -> Result<RevAdvance> {
|
|
let main = lifecycle::git_rev_parse(applied_dir, "refs/heads/main")
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("read applied/main: {e:#}"))?;
|
|
if main == rev {
|
|
return Ok(RevAdvance::AlreadyApplied);
|
|
}
|
|
fetch.await.context("fetch the forge config main")?;
|
|
let (main, descends) = applied_main_under(applied_dir, rev).await?;
|
|
if main == rev {
|
|
return Ok(RevAdvance::AlreadyApplied);
|
|
}
|
|
if !descends {
|
|
bail!(
|
|
"{rev} does not descend from applied/main ({main}); fast-forwarding to it would \
|
|
discard commits"
|
|
);
|
|
}
|
|
ff_applied_main(applied_dir, rev, &main).await?;
|
|
Ok(RevAdvance::Advanced)
|
|
}
|
|
|
|
/// Post why the merged commit `rev` of `agent`'s config repo was not deployed
|
|
/// onto the PR whose merge produced it. Best-effort: a failure here is logged
|
|
/// only.
|
|
pub(crate) async fn post_rev_deploy_failure_to_pr(agent: &str, rev: &str, err: &anyhow::Error) {
|
|
let repo = crate::forge::config_repo(agent);
|
|
let pr = match crate::forge::merged_pr_for_commit(&repo, rev).await {
|
|
Ok(pr) => pr,
|
|
Err(e) => {
|
|
tracing::warn!(%agent, %rev, error = %e, "find the PR that merged a failed deploy's rev failed");
|
|
return;
|
|
}
|
|
};
|
|
let body = format!(
|
|
"## ⚠️ config deploy failed\n\n\
|
|
The merged commit `{rev}` was not deployed:\n\n\
|
|
```\n{err:#}\n```"
|
|
);
|
|
if let Err(e) = crate::forge::post_pr_comment(&repo, pr, &body).await {
|
|
tracing::warn!(%agent, %pr, error = ?e, "post rev deploy-failure comment to PR failed");
|
|
}
|
|
}
|
|
|
|
/// Resolve an approval row from how its DAG's work ended — the body of the
|
|
/// [`NodeKind::ResolveApproval`] tail node. Every approval-carrying template
|
|
/// resolves here: the pipeline is ordinary queue nodes,
|
|
/// so the work's terminal state is the authoritative outcome and there's no
|
|
/// in-node resolution to skip around.
|
|
///
|
|
/// `outcome` is the one the calling node was built to report — a template emits
|
|
/// one tail per outcome, so success, failure and cancel each arrive here from
|
|
/// their own node rather than from one node branching.
|
|
///
|
|
/// [`NodeKind::ResolveApproval`]: crate::job_queue::NodeKind::ResolveApproval
|
|
pub(crate) fn resolve_approval_dag(
|
|
coord: &Arc<Coordinator>,
|
|
approval_id: i64,
|
|
outcome: crate::job_queue::TerminalState,
|
|
error: Option<&str>,
|
|
) {
|
|
use crate::job_queue::TerminalState;
|
|
let approval = match coord.approvals.get(approval_id) {
|
|
Ok(Some(a)) => a,
|
|
Ok(None) => {
|
|
tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
|
|
return;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
|
|
return;
|
|
}
|
|
};
|
|
let result: Result<()> = match outcome {
|
|
TerminalState::Done => Ok(()),
|
|
// `Skipped` never reaches here — no template emits a tail for it — but it
|
|
// reads the same to an operator either way: the work did not happen.
|
|
TerminalState::Cancelled | TerminalState::Skipped => {
|
|
Err(anyhow::anyhow!("cancelled before completion"))
|
|
}
|
|
TerminalState::Failed => Err(anyhow::anyhow!("{}", error.unwrap_or("job dag failed"))),
|
|
};
|
|
if let Err(e) = finish_approval(coord, &approval, result) {
|
|
tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
|
|
}
|
|
}
|
|
|
|
fn finish_approval(
|
|
coord: &Coordinator,
|
|
approval: &hive_sh4re::approvals::Approval,
|
|
result: Result<()>,
|
|
) -> Result<()> {
|
|
let (status, note, ok) = match &result {
|
|
Ok(()) => (ApprovalStatus::Approved, None, true),
|
|
Err(e) => {
|
|
let note = format!("{e:#}");
|
|
let _ = coord.approvals.mark_failed(approval.id, ¬e);
|
|
(ApprovalStatus::Failed, Some(note), false)
|
|
}
|
|
};
|
|
coord.notify_submitter(
|
|
approval.id,
|
|
&HelperEvent::ApprovalResolved {
|
|
id: approval.id,
|
|
agent: approval.agent.to_string(),
|
|
commit_ref: approval.commit_ref.clone(),
|
|
status,
|
|
note: note.clone(),
|
|
},
|
|
);
|
|
// Phase 5b: also fire on the dashboard event channel so the
|
|
// browser moves the row out of pending into history without a
|
|
// snapshot refetch. `approved` rows that succeed get the
|
|
// approval's logged resolved_at indirectly via `Utc::now()`;
|
|
// failures already wrote it via mark_failed above.
|
|
let approval_kind = <&str>::from(approval.kind);
|
|
let status_str = if ok { "approved" } else { "failed" };
|
|
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
|
id: approval.id,
|
|
agent: approval.agent.as_str(),
|
|
approval_kind,
|
|
status: status_str,
|
|
note: note.clone(),
|
|
description: approval.description.clone(),
|
|
});
|
|
result
|
|
}
|
|
|
|
/// Fast-forward `applied/main` to `target` and sync the working tree.
|
|
///
|
|
/// Compare-and-swap, not a bare set: `main` must still be `expected_main`, the
|
|
/// sha the caller's ancestry gate ([`applied_main_under`]) ran against. A plain
|
|
/// `update-ref` moves the branch to `target` whatever it currently points at,
|
|
/// which turns "fast-forward" into "discard anything that landed in the
|
|
/// meantime" — the gate only proves `target` safe against the `main` observed
|
|
/// *then*, so this is what makes that proof still true *now*.
|
|
async fn ff_applied_main(
|
|
applied_dir: &std::path::Path,
|
|
target: &str,
|
|
expected_main: &str,
|
|
) -> Result<()> {
|
|
lifecycle::git_update_ref_cas(applied_dir, "refs/heads/main", target, expected_main)
|
|
.await
|
|
.map_err(|e| {
|
|
anyhow::anyhow!("ff main {expected_main} -> {target} (concurrent move?): {e:#}")
|
|
})?;
|
|
lifecycle::git_read_tree_reset(applied_dir, "refs/heads/main")
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("read-tree to main: {e:#}"))
|
|
}
|
|
|
|
/// Tear down a sub-agent container. By default this is non-destructive to
|
|
/// persistent state: the proposed/applied config repos and the Claude
|
|
/// credentials dir under `/var/lib/hyperhive/{agents,applied}/<name>/` are
|
|
/// kept, so recreating an agent of the same name reuses prior config + creds
|
|
/// (no re-login). The ephemeral runtime dir under `/run/hyperhive/agents/`
|
|
/// is cleared because its contents (the mcp socket) don't survive restarts
|
|
/// anyway. With `purge=true` the persistent trees are also wiped — config
|
|
/// history, claude creds, notes — there is no undo.
|
|
/// The bootstrap/root container is destroyable like any other: it's
|
|
/// imperative infra that `auto_update::ensure_root_agent` recreates on the
|
|
/// next hive-c0re startup if absent, so destroying it is transient rather
|
|
/// than something to refuse at the API.
|
|
///
|
|
/// Submits the teardown DAG and returns — it does not wait for the container to
|
|
/// go away. Same contract as every other lifecycle op (`rebuild`, `kill`,
|
|
/// `restart`, `start`): the queue owns the work, the caller gets an
|
|
/// acknowledgement. Progress is visible as real nodes on the dashboard.
|
|
pub fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) {
|
|
tracing::info!(%name, purge, "destroy");
|
|
if let Err(e) = coord.job_queue.insert_job(|b| {
|
|
crate::job_queue::templates::destroy(b, name, purge);
|
|
Vec::new()
|
|
}) {
|
|
tracing::error!(agent = %name, error = ?e, "destroy: insert failed");
|
|
}
|
|
coord.emit_rebuild_queue_snapshot();
|
|
}
|
|
|
|
pub fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
|
|
let approval = coord.approvals.get(id)?;
|
|
coord.approvals.mark_denied(id, note)?;
|
|
tracing::info!(%id, note, "approval denied");
|
|
if let Some(a) = approval {
|
|
let approval_kind = <&str>::from(a.kind);
|
|
let description = a.description.clone();
|
|
let agent_owned = a.agent.clone();
|
|
coord.notify_submitter(
|
|
a.id,
|
|
&HelperEvent::ApprovalResolved {
|
|
id: a.id,
|
|
agent: a.agent.to_string(),
|
|
commit_ref: a.commit_ref,
|
|
status: ApprovalStatus::Denied,
|
|
note: note.map(String::from),
|
|
},
|
|
);
|
|
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
|
id,
|
|
agent: agent_owned.as_str(),
|
|
approval_kind,
|
|
status: "denied",
|
|
note: note.map(String::from),
|
|
description,
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{RevAdvance, advance_applied_main};
|
|
use crate::lifecycle;
|
|
|
|
/// A repo with `main` at `b`, child of `a`, and `c`, a sibling of `b` off
|
|
/// `a`. Returns the dir and the shas `[a, b, c]`.
|
|
async fn repo() -> (tempfile::TempDir, [String; 3]) {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let path = dir.path();
|
|
let commit = |msg: &'static str| async move {
|
|
lifecycle::git(
|
|
path,
|
|
&[
|
|
"-c",
|
|
"user.name=t",
|
|
"-c",
|
|
"user.email=t@t",
|
|
"commit",
|
|
"-q",
|
|
"--allow-empty",
|
|
"-m",
|
|
msg,
|
|
],
|
|
)
|
|
.await
|
|
.expect("commit");
|
|
lifecycle::git_rev_parse(path, "HEAD").await.expect("HEAD")
|
|
};
|
|
lifecycle::git(path, &["init", "-q", "--initial-branch=main"])
|
|
.await
|
|
.expect("init");
|
|
let a = commit("a").await;
|
|
let b = commit("b").await;
|
|
lifecycle::git(path, &["checkout", "-q", "-b", "side", &a])
|
|
.await
|
|
.expect("branch");
|
|
let c = commit("c").await;
|
|
lifecycle::git(path, &["checkout", "-q", "main"])
|
|
.await
|
|
.expect("checkout main");
|
|
(dir, [a, b, c])
|
|
}
|
|
|
|
async fn main_of(dir: &tempfile::TempDir) -> String {
|
|
lifecycle::git_rev_parse(dir.path(), "refs/heads/main")
|
|
.await
|
|
.expect("main")
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn the_applied_rev_is_a_no_op_without_a_fetch() {
|
|
let (dir, [_, b, _]) = repo().await;
|
|
let fetched = std::cell::Cell::new(false);
|
|
let fetch = async {
|
|
fetched.set(true);
|
|
anyhow::Ok(())
|
|
};
|
|
let outcome = advance_applied_main(dir.path(), &b, fetch).await;
|
|
assert_eq!(outcome.expect("no-op"), RevAdvance::AlreadyApplied);
|
|
assert!(!fetched.get(), "an applied rev must not reach the forge");
|
|
assert_eq!(main_of(&dir).await, b);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_rev_not_descending_from_main_is_refused_and_main_stays() {
|
|
let (dir, [_, b, c]) = repo().await;
|
|
let outcome = advance_applied_main(dir.path(), &c, async { anyhow::Ok(()) }).await;
|
|
let err = outcome.expect_err("a sibling of main is not a fast-forward");
|
|
assert!(
|
|
format!("{err:#}").contains("does not descend from applied/main"),
|
|
"{err:#}"
|
|
);
|
|
assert_eq!(main_of(&dir).await, b);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_descendant_rev_fast_forwards_main() {
|
|
let (dir, [a, b, _]) = repo().await;
|
|
lifecycle::git_update_ref(dir.path(), "refs/heads/main", &a)
|
|
.await
|
|
.expect("rewind main");
|
|
let outcome = advance_applied_main(dir.path(), &b, async { anyhow::Ok(()) }).await;
|
|
assert_eq!(outcome.expect("fast-forward"), RevAdvance::Advanced);
|
|
assert_eq!(main_of(&dir).await, b);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_failed_fetch_leaves_main_alone() {
|
|
let (dir, [a, b, _]) = repo().await;
|
|
lifecycle::git_update_ref(dir.path(), "refs/heads/main", &a)
|
|
.await
|
|
.expect("rewind main");
|
|
let outcome = advance_applied_main(dir.path(), &b, async {
|
|
Err::<(), _>(anyhow::anyhow!("forge down"))
|
|
})
|
|
.await;
|
|
assert!(outcome.is_err());
|
|
assert_eq!(main_of(&dir).await, a);
|
|
}
|
|
}
|