sweep: wire matrix::ensure_all() into the warning-banner registry
Fast-follow for #2289 — matrix::ensure_all() (admin user, per-agent sync, hive Space + chat-room provisioning/invites) ran periodically every 30 minutes but only ever warn!'d to the journal on failure, so a persistent problem (missing invites, broken admin token, etc.) was invisible to the operator. ensure_all() now returns bool (aggregate ok/fail across every sub-step) instead of (), and both call sites in main.rs feed that into a debounced SweepHealth("matrix_ensure_all", warn, threshold=2) — matches the existing knowledge_pull pattern. A lone bad sweep self-heals silently; two consecutive failures raise a banner that clears on the next clean sweep. forge::ensure_all()'s remaining independent steps are still open — that sweep only runs once at startup (no periodic loop), so the debounced pattern doesn't map as directly; left for a follow-up.
This commit is contained in:
parent
8624da0057
commit
b889f403d5
2 changed files with 62 additions and 12 deletions
|
|
@ -414,11 +414,46 @@ async fn cmd_serve(
|
||||||
let mut matrix_shutdown = coord.shutdown_rx();
|
let mut matrix_shutdown = coord.shutdown_rx();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let interval = std::time::Duration::from_mins(30);
|
let interval = std::time::Duration::from_mins(30);
|
||||||
matrix::ensure_all().await;
|
// Debounced banner: a lone bad sweep (homeserver mid-restart, a
|
||||||
|
// transient HTTP blip) shouldn't flap the dashboard, but a sweep
|
||||||
|
// that's been failing for hours (missing agent invites, a broken
|
||||||
|
// admin token) should surface. Cleared the moment a sweep is clean.
|
||||||
|
let mut health = sweep_health::SweepHealth::new("matrix_ensure_all", "warn", 2);
|
||||||
|
if matrix::ensure_all().await {
|
||||||
|
health.record_ok();
|
||||||
|
} else {
|
||||||
|
health.record_err(|ctx| {
|
||||||
|
let age = ctx.since_last_ok.map_or_else(
|
||||||
|
|| "no success this session".to_owned(),
|
||||||
|
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
|
||||||
|
);
|
||||||
|
format!(
|
||||||
|
"matrix user/space sweep failing ({} consecutive, {age}) \
|
||||||
|
— some agents may be missing matrix accounts, space membership, \
|
||||||
|
or chat-room invites",
|
||||||
|
ctx.consecutive
|
||||||
|
)
|
||||||
|
});
|
||||||
|
}
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
() = tokio::time::sleep(interval) => {
|
() = tokio::time::sleep(interval) => {
|
||||||
matrix::ensure_all().await;
|
if matrix::ensure_all().await {
|
||||||
|
health.record_ok();
|
||||||
|
} else {
|
||||||
|
health.record_err(|ctx| {
|
||||||
|
let age = ctx.since_last_ok.map_or_else(
|
||||||
|
|| "no success this session".to_owned(),
|
||||||
|
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
|
||||||
|
);
|
||||||
|
format!(
|
||||||
|
"matrix user/space sweep failing ({} consecutive, {age}) \
|
||||||
|
— some agents may be missing matrix accounts, space membership, \
|
||||||
|
or chat-room invites",
|
||||||
|
ctx.consecutive
|
||||||
|
)
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
_ = matrix_shutdown.changed() => {
|
_ = matrix_shutdown.changed() => {
|
||||||
tracing::info!("matrix ensure_all: shutdown signal received");
|
tracing::info!("matrix ensure_all: shutdown signal received");
|
||||||
|
|
|
||||||
|
|
@ -1503,20 +1503,28 @@ async fn resolve_room_alias(
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sweep every existing container (manager + sub-agents) and ensure
|
/// Sweep every existing container (manager + sub-agents) and ensure
|
||||||
/// each has a matrix user + token on the local homeserver. Called once
|
/// each has a matrix user + token on the local homeserver. Called at
|
||||||
/// at hive-c0re startup, alongside `forge::ensure_all`. No-op when the
|
/// hive-c0re startup, alongside `forge::ensure_all`, and then
|
||||||
|
/// periodically (see the caller in `main.rs`). No-op when the
|
||||||
/// hive-matrix container isn't running. Per-step failures are logged
|
/// hive-matrix container isn't running. Per-step failures are logged
|
||||||
/// but don't abort the sweep.
|
/// but don't abort the sweep.
|
||||||
pub async fn ensure_all() {
|
///
|
||||||
|
/// Returns `true` when every step of the sweep succeeded, `false` when at
|
||||||
|
/// least one step failed — the caller feeds this into a
|
||||||
|
/// [`crate::stats::sweep_health::SweepHealth`] to raise a debounced
|
||||||
|
/// dashboard banner on persistent failure (this sweep re-runs every 30
|
||||||
|
/// minutes, so a one-off blip self-heals without ever bannering).
|
||||||
|
pub async fn ensure_all() -> bool {
|
||||||
if !is_present().await {
|
if !is_present().await {
|
||||||
tracing::debug!("matrix: hive-matrix container absent, skipping user sweep");
|
tracing::debug!("matrix: hive-matrix container absent, skipping user sweep");
|
||||||
return;
|
return true;
|
||||||
}
|
}
|
||||||
|
let mut ok = true;
|
||||||
let register_token = match ensure_register_token() {
|
let register_token = match ensure_register_token() {
|
||||||
Ok(t) => t,
|
Ok(t) => t,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = ?e, "matrix: ensure_register_token failed");
|
tracing::warn!(error = ?e, "matrix: ensure_register_token failed");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// One HTTP client for the whole sweep — connection pool is
|
// One HTTP client for the whole sweep — connection pool is
|
||||||
|
|
@ -1528,7 +1536,7 @@ pub async fn ensure_all() {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = ?e, "matrix: build HTTP client failed; skipping sweep");
|
tracing::warn!(error = ?e, "matrix: build HTTP client failed; skipping sweep");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// Provision hive admin user FIRST so it's the first registered
|
// Provision hive admin user FIRST so it's the first registered
|
||||||
|
|
@ -1536,10 +1544,11 @@ pub async fn ensure_all() {
|
||||||
// registered user admin automatically).
|
// registered user admin automatically).
|
||||||
if let Err(e) = ensure_admin_user(&client, ®ister_token).await {
|
if let Err(e) = ensure_admin_user(&client, ®ister_token).await {
|
||||||
tracing::warn!(error = ?e, "matrix: ensure_admin_user failed");
|
tracing::warn!(error = ?e, "matrix: ensure_admin_user failed");
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
let Ok(containers) = crate::lifecycle::list().await else {
|
let Ok(containers) = crate::lifecycle::list().await else {
|
||||||
tracing::warn!("matrix: nixos-container list failed; skipping user sweep");
|
tracing::warn!("matrix: nixos-container list failed; skipping user sweep");
|
||||||
return;
|
return false;
|
||||||
};
|
};
|
||||||
let mut agent_names: Vec<String> = Vec::new();
|
let mut agent_names: Vec<String> = Vec::new();
|
||||||
for c in &containers {
|
for c in &containers {
|
||||||
|
|
@ -1556,7 +1565,7 @@ pub async fn ensure_all() {
|
||||||
Ok(t) => t,
|
Ok(t) => t,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = ?e, "matrix: skipping hive space provisioning (no admin token)");
|
tracing::warn!(error = ?e, "matrix: skipping hive space provisioning (no admin token)");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// server_name first — the agent invites need it (fully-qualified user ids).
|
// server_name first — the agent invites need it (fully-qualified user ids).
|
||||||
|
|
@ -1564,14 +1573,14 @@ pub async fn ensure_all() {
|
||||||
Ok(s) => s,
|
Ok(s) => s,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = ?e, "matrix: discover_server_name failed; skipping space provisioning");
|
tracing::warn!(error = ?e, "matrix: discover_server_name failed; skipping space provisioning");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let room_id = match ensure_hive_space(&client, &admin_token).await {
|
let room_id = match ensure_hive_space(&client, &admin_token).await {
|
||||||
Ok(id) => id,
|
Ok(id) => id,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = ?e, "matrix: ensure_hive_space failed");
|
tracing::warn!(error = ?e, "matrix: ensure_hive_space failed");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// Invite @hive admin first, then all agents.
|
// Invite @hive admin first, then all agents.
|
||||||
|
|
@ -1585,10 +1594,12 @@ pub async fn ensure_all() {
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!(error = ?e, "matrix: invite @hive to space failed");
|
tracing::warn!(error = ?e, "matrix: invite @hive to space failed");
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
for name in &agent_names {
|
for name in &agent_names {
|
||||||
if let Err(e) = invite_to_room(&client, &admin_token, &room_id, name, &server_name).await {
|
if let Err(e) = invite_to_room(&client, &admin_token, &room_id, name, &server_name).await {
|
||||||
tracing::warn!(%name, error = ?e, "matrix: invite agent to space failed");
|
tracing::warn!(%name, error = ?e, "matrix: invite agent to space failed");
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1609,19 +1620,23 @@ pub async fn ensure_all() {
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!(error = ?e, "matrix: invite @hive to chat room failed");
|
tracing::warn!(error = ?e, "matrix: invite @hive to chat room failed");
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
for name in &agent_names {
|
for name in &agent_names {
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
invite_to_room(&client, &admin_token, &chat_room_id, name, &server_name).await
|
invite_to_room(&client, &admin_token, &chat_room_id, name, &server_name).await
|
||||||
{
|
{
|
||||||
tracing::warn!(%name, error = ?e, "matrix: invite agent to chat room failed");
|
tracing::warn!(%name, error = ?e, "matrix: invite agent to chat room failed");
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = ?e, "matrix: ensure_hive_chat_room failed");
|
tracing::warn!(error = ?e, "matrix: ensure_hive_chat_room failed");
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ok
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue