feat(#2289): debounced SweepHealth banner tracker, wire knowledge pull

This commit is contained in:
damocles 2026-07-10 14:11:41 +02:00 committed by mara
commit 28dbb529c0
4 changed files with 236 additions and 5 deletions

View file

@ -14,7 +14,7 @@ use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig};
use hive_c0re::{
agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge,
host_stats, job_queue, knowledge, matrix, mcp_sockets, migrate, reminder_scheduler,
scheduled_prompts_worker, server, socket_server, warnings,
scheduled_prompts_worker, server, socket_server, sweep_health, warnings,
};
#[derive(Parser)]
@ -321,16 +321,38 @@ async fn cmd_serve(
let mut knowledge_shutdown = coord.shutdown_rx();
tokio::spawn(async move {
// Initial pull — reconcile any commits that landed while c0re
// was offline.
// was offline. Not fed to the health tracker: a startup miss is
// expected (the clone may not exist yet) and is logged at debug.
if let Err(e) = knowledge::pull().await {
tracing::debug!(error = ?e, "knowledge: startup pull skipped (no clone yet?)");
}
// Persistent-failure → banner. An hourly sweep that keeps failing for
// several hours means the operator's `/knowledge` is drifting; raise a
// warn banner after 3 consecutive misses so a one-off network blip
// self-heals on the next tick without ever bannering. Cleared on the
// next successful pull.
let mut health = sweep_health::SweepHealth::new("knowledge_pull", "warn", 3);
let interval = std::time::Duration::from_hours(1);
loop {
tokio::select! {
() = tokio::time::sleep(interval) => {
if let Err(e) = knowledge::pull().await {
tracing::warn!(error = ?e, "knowledge: periodic pull failed");
match knowledge::pull().await {
Ok(()) => health.record_ok(),
Err(e) => {
tracing::warn!(error = ?e, "knowledge: periodic pull failed");
let err = format!("{e:#}");
health.record_err(|ctx| {
let age = ctx.since_last_ok.map_or_else(
|| "no success this session".to_owned(),
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
);
format!(
"knowledge repo pull failing ({} consecutive, {age}) \
/knowledge is stale until it recovers: {err}",
ctx.consecutive
)
});
}
}
}
_ = knowledge_shutdown.changed() => {