feat(#2289): debounced SweepHealth banner tracker, wire knowledge pull
This commit is contained in:
parent
8c908651bc
commit
28dbb529c0
4 changed files with 236 additions and 5 deletions
|
|
@ -14,7 +14,7 @@ use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig};
|
|||
use hive_c0re::{
|
||||
agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge,
|
||||
host_stats, job_queue, knowledge, matrix, mcp_sockets, migrate, reminder_scheduler,
|
||||
scheduled_prompts_worker, server, socket_server, warnings,
|
||||
scheduled_prompts_worker, server, socket_server, sweep_health, warnings,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -321,16 +321,38 @@ async fn cmd_serve(
|
|||
let mut knowledge_shutdown = coord.shutdown_rx();
|
||||
tokio::spawn(async move {
|
||||
// Initial pull — reconcile any commits that landed while c0re
|
||||
// was offline.
|
||||
// was offline. Not fed to the health tracker: a startup miss is
|
||||
// expected (the clone may not exist yet) and is logged at debug.
|
||||
if let Err(e) = knowledge::pull().await {
|
||||
tracing::debug!(error = ?e, "knowledge: startup pull skipped (no clone yet?)");
|
||||
}
|
||||
// Persistent-failure → banner. An hourly sweep that keeps failing for
|
||||
// several hours means the operator's `/knowledge` is drifting; raise a
|
||||
// warn banner after 3 consecutive misses so a one-off network blip
|
||||
// self-heals on the next tick without ever bannering. Cleared on the
|
||||
// next successful pull.
|
||||
let mut health = sweep_health::SweepHealth::new("knowledge_pull", "warn", 3);
|
||||
let interval = std::time::Duration::from_hours(1);
|
||||
loop {
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(interval) => {
|
||||
if let Err(e) = knowledge::pull().await {
|
||||
tracing::warn!(error = ?e, "knowledge: periodic pull failed");
|
||||
match knowledge::pull().await {
|
||||
Ok(()) => health.record_ok(),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "knowledge: periodic pull failed");
|
||||
let err = format!("{e:#}");
|
||||
health.record_err(|ctx| {
|
||||
let age = ctx.since_last_ok.map_or_else(
|
||||
|| "no success this session".to_owned(),
|
||||
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
|
||||
);
|
||||
format!(
|
||||
"knowledge repo pull failing ({} consecutive, {age}) \
|
||||
— /knowledge is stale until it recovers: {err}",
|
||||
ctx.consecutive
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = knowledge_shutdown.changed() => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue