swarm-archive: check each agent's restic repository weekly
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
parent
c4919c5945
commit
5c30985a25
1 changed files with 503 additions and 18 deletions
|
|
@ -1724,6 +1724,16 @@ in
|
|||
for = "0s";
|
||||
summary = "the archive's daily prune failed for an agent's repository, so its retention is not applied; the unit's journal names the agent";
|
||||
})
|
||||
# `swarm-archive-check` prints `archive check failed: <agent>`; it runs
|
||||
# weekly, so the window spans a missed week.
|
||||
(logCountRule {
|
||||
uid = "hh-archive-check-failed";
|
||||
title = "archive check failed";
|
||||
windowSeconds = 700000;
|
||||
expr = ''_time:8d _SYSTEMD_UNIT:="swarm-archive-check.service" _msg:"archive check failed" | stats count() as failures'';
|
||||
for = "0s";
|
||||
summary = "the archive's weekly integrity check failed for an agent's repository, so its backups may not restore; the unit's journal names the agent and the restic error";
|
||||
})
|
||||
# Logged once per detected crash, by `emit_crash_transitions`
|
||||
# in hive-c0re's crash_watch.rs (10s poll interval). Replaces
|
||||
# the old per-hive dashboard `agents_crashing` banner — see
|
||||
|
|
|
|||
Loading…
Reference in a new issue