job_queue: submit boot-time forge/matrix/webhook/knowledge sweeps as DAG nodes
This commit is contained in:
parent
e15c499a31
commit
f367fa518e
3 changed files with 124 additions and 1 deletions
|
|
@ -148,10 +148,76 @@ pub(super) async fn run_node(
|
|||
// `Finishing` so the nodes under it start. What they declare stays held
|
||||
// until their whole subtree settles.
|
||||
NodeKind::DeployWindow { .. } | NodeKind::AgentWindow { .. } => Ok(()),
|
||||
NodeKind::ForgeSweep => run_forge_sweep().await,
|
||||
NodeKind::MatrixSweep => run_matrix_sweep().await,
|
||||
NodeKind::WebhookRegister => run_webhook_register().await,
|
||||
NodeKind::KnowledgePull => run_knowledge_pull(coord).await,
|
||||
};
|
||||
(builder, result)
|
||||
}
|
||||
|
||||
/// Boot-time forge user/token sweep as a DAG node — see
|
||||
/// [`NodeKind::ForgeSweep`]. `forge::ensure_all` already does its own
|
||||
/// per-step error handling and boot-warning banners internally (it's
|
||||
/// best-effort by design), so this wrapper has nothing left to report;
|
||||
/// it exists purely to make the sweep a visible unit of work.
|
||||
async fn run_forge_sweep() -> Result<()> {
|
||||
crate::forge::ensure_all().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Boot-time matrix user/space sweep as a DAG node — see
|
||||
/// [`NodeKind::MatrixSweep`]. Reports failure as the node's own error so a
|
||||
/// failed boot sweep is visible on the dashboard; the debounced
|
||||
/// `sweep_health`-driven warning banner is a separate concern owned by the
|
||||
/// periodic loop in `main.rs`, unaffected by this node's own outcome.
|
||||
async fn run_matrix_sweep() -> Result<()> {
|
||||
if crate::matrix::ensure_all().await {
|
||||
Ok(())
|
||||
} else {
|
||||
anyhow::bail!("matrix ensure_all: one or more agents failed sync (see logs)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Boot-time Forgejo webhook registration as a DAG node — see
|
||||
/// [`NodeKind::WebhookRegister`]. Mirrors the guard chain the
|
||||
/// `tokio::spawn` block it replaced used: no-op (not an error) when the
|
||||
/// HMAC secret, core token, or hive domain aren't available yet.
|
||||
async fn run_webhook_register() -> Result<()> {
|
||||
let Ok(webhook_secret) = crate::webhook_secret::load_or_generate() else {
|
||||
tracing::debug!("webhook secret unavailable; skipping hook registration");
|
||||
return Ok(());
|
||||
};
|
||||
let Some(token) = crate::forge::core_token() else {
|
||||
return Ok(());
|
||||
};
|
||||
let domain = std::env::var("HYPERHIVE_HIVE_DOMAIN")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty());
|
||||
let Some(domain) = domain else {
|
||||
tracing::debug!("HYPERHIVE_HIVE_DOMAIN unset; skipping webhook registration");
|
||||
return Ok(());
|
||||
};
|
||||
if let Err(e) =
|
||||
crate::workers::knowledge::ensure_webhook(&token, &domain, &webhook_secret).await
|
||||
{
|
||||
tracing::warn!(error = ?e, "knowledge: ensure_webhook failed");
|
||||
}
|
||||
if let Err(e) = crate::forge::ensure_config_pr_webhook(&token, &domain, &webhook_secret).await {
|
||||
tracing::warn!(error = ?e, "forge: ensure_config_pr_webhook failed");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Boot-time `/knowledge` pull as a DAG node — see
|
||||
/// [`NodeKind::KnowledgePull`]. Unlike the `main.rs` periodic loop's
|
||||
/// startup call, a failure here is *not* swallowed to debug level: the node
|
||||
/// exists so a failed boot pull is visible on the dashboard rather than
|
||||
/// only in the journal.
|
||||
async fn run_knowledge_pull(coord: &Arc<Coordinator>) -> Result<()> {
|
||||
crate::workers::knowledge::pull(coord).await
|
||||
}
|
||||
|
||||
/// Resolve the DAG's approval row the way this node's own `outcome` says.
|
||||
///
|
||||
/// Nothing is inspected: a template emits one of these per outcome, each edged to
|
||||
|
|
|
|||
|
|
@ -320,6 +320,31 @@ pub enum NodeKind {
|
|||
/// `Prebuild`, but that's a no-op there — the agent is down, so prebuild
|
||||
/// is skipped.)
|
||||
SetWanted { agent: String, up: bool },
|
||||
/// One-shot boot-time forge user/token sweep for every existing
|
||||
/// container (`forge::ensure_all`) — moved out of a bare `tokio::spawn`
|
||||
/// so it shows as real work on the dashboard instead of running
|
||||
/// invisibly until it fails. Agentless: it sweeps every container, not
|
||||
/// one. Store/network-I/O only — build-slot- and lease-exempt, same
|
||||
/// class as [`NodeKind::MetaSync`]/[`NodeKind::Provision`].
|
||||
ForgeSweep,
|
||||
/// One-shot boot-time matrix user/space sweep (`matrix::ensure_all`),
|
||||
/// for the same dashboard-visibility reason as [`NodeKind::ForgeSweep`].
|
||||
/// The *periodic* re-sweep (every 30 min, recovering token files
|
||||
/// `hive-matrix-daemon` deleted) stays a background loop in
|
||||
/// `main.rs` — only the boot-time instance is a DAG node.
|
||||
MatrixSweep,
|
||||
/// One-shot boot-time Forgejo webhook registration, for
|
||||
/// `internal/knowledge` (push → git pull) and the `agent-configs` org
|
||||
/// (`pull_request` → config-PR approval). No-op when the core token, hive
|
||||
/// domain, or HMAC secret aren't available yet — mirrors the guard the
|
||||
/// `tokio::spawn` block it replaced already used. Agentless, build-slot-
|
||||
/// and lease-exempt.
|
||||
WebhookRegister,
|
||||
/// One-shot boot-time `/knowledge` pull (`knowledge::pull`), reconciling
|
||||
/// any commits that landed while `hive-c0re` was down. Same rationale as
|
||||
/// [`NodeKind::MatrixSweep`]: the periodic hourly re-pull stays a
|
||||
/// background loop, only the boot-time instance is a DAG node.
|
||||
KnowledgePull,
|
||||
}
|
||||
|
||||
/// How a hive-c0re node describes itself to a generic graph viewer.
|
||||
|
|
@ -395,6 +420,10 @@ impl NodeKind {
|
|||
NodeKind::ResolveApproval { .. } => "resolve_approval",
|
||||
NodeKind::EmitRebuilt { .. } => "emit_rebuilt",
|
||||
NodeKind::SetWanted { .. } => "set_wanted",
|
||||
NodeKind::ForgeSweep => "forge_sweep",
|
||||
NodeKind::MatrixSweep => "matrix_sweep",
|
||||
NodeKind::WebhookRegister => "webhook_register",
|
||||
NodeKind::KnowledgePull => "knowledge_pull",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -434,7 +463,11 @@ impl NodeKind {
|
|||
| NodeKind::SetWanted { agent, .. } => agent,
|
||||
NodeKind::MetaLock { .. }
|
||||
| NodeKind::Reparent { .. }
|
||||
| NodeKind::ResolveApproval { .. } => "",
|
||||
| NodeKind::ResolveApproval { .. }
|
||||
| NodeKind::ForgeSweep
|
||||
| NodeKind::MatrixSweep
|
||||
| NodeKind::WebhookRegister
|
||||
| NodeKind::KnowledgePull => "",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -313,9 +313,33 @@ pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
|||
let fanout: Vec<String> = fanout.into_iter().map(|(name, _)| name).collect();
|
||||
|
||||
submit_boot_tree(&coord, any_stale, fanout, drifted, n_deferred, n_skipped);
|
||||
submit_startup_sweep_nodes(&coord);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Submit the boot-time forge/matrix/webhook/knowledge sweeps as DAG nodes —
|
||||
/// `ForgeSweep`, `MatrixSweep`, `WebhookRegister`, `KnowledgePull`. Unlike
|
||||
/// [`submit_boot_tree`] this runs on **every** boot, quiet or not: these
|
||||
/// aren't config-drift work, they're startup housekeeping that always needs
|
||||
/// to happen, and the point of moving them here is exactly so they show up
|
||||
/// as real work on the dashboard instead of an invisible `tokio::spawn` that
|
||||
/// only surfaces on failure. Four independent, build-slot- and lease-exempt
|
||||
/// roots — no dependency edges between them, matching the existing
|
||||
/// `Reconcile`-root pattern in [`boot_nodes`].
|
||||
fn submit_startup_sweep_nodes(coord: &Arc<Coordinator>) {
|
||||
use crate::job_queue::NodeKind;
|
||||
|
||||
if let Err(e) = coord.job_queue.insert_job(|b| {
|
||||
let _ = b.node(NodeKind::ForgeSweep);
|
||||
let _ = b.node(NodeKind::MatrixSweep);
|
||||
let _ = b.node(NodeKind::WebhookRegister);
|
||||
let _ = b.node(NodeKind::KnowledgePull);
|
||||
Vec::new()
|
||||
}) {
|
||||
tracing::warn!(error = ?e, "boot: startup sweep DAG insert failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// The boot DAG's node declarations, split out of [`submit_boot_tree`] so they
|
||||
/// can be exercised without a live [`Coordinator`].
|
||||
///
|
||||
|
|
|
|||
Loading…
Reference in a new issue