swarm-archive: back up every agent's state to an append-only restic repository per agent
swarm-archive serves restic repositories with rclone, append-only, one login and one repository per agent. swarm-controller writes each placed agent's login and restic password to the store, replaces them on a move and removes them when the agent is destroyed. Every hive backs its agents up hourly under its own store identity. The archive host prunes by retention.
This commit is contained in:
parent
409e7ee5a0
commit
306aae0378
1 changed files with 2214 additions and 5 deletions
|
|
@ -1684,6 +1684,29 @@ in
|
|||
for = "0s";
|
||||
summary = "swarm-controller failed to mint agent {{ $labels.F_AGENT }}'s store identity; the reason is the `F_ERROR` field of the swarm-controller log line";
|
||||
})
|
||||
# `hive-archive-backup` (nix/host-modules/hive-archive-backup.nix)
|
||||
# prints one `archive backup failed: <agent>` line per agent
|
||||
# it could not back up, and one for a hive swarm-controller
|
||||
# has granted nothing. The backup runs hourly, so `for`
|
||||
# rides out the minutes a newly placed agent waits for its
|
||||
# grant, and a second failed run keeps it firing.
|
||||
(logCountRule {
|
||||
uid = "hh-archive-backup-failed";
|
||||
title = "archive backup failed";
|
||||
windowSeconds = 7800;
|
||||
expr = ''_time:130m _SYSTEMD_UNIT:="hive-archive-backup.service" _msg:"archive backup failed" | stats by (_HOSTNAME) count() as failures'';
|
||||
for = "30m";
|
||||
summary = "{{ $labels._HOSTNAME }} failed to back up an agent to the archive; the unit's journal names the agent and the cause";
|
||||
})
|
||||
# `swarm-archive-prune` prints `archive prune failed: <agent>`.
|
||||
(logCountRule {
|
||||
uid = "hh-archive-prune-failed";
|
||||
title = "archive prune failed";
|
||||
windowSeconds = 90000;
|
||||
expr = ''_time:25h _SYSTEMD_UNIT:="swarm-archive-prune.service" _msg:"archive prune failed" | stats count() as failures'';
|
||||
for = "0s";
|
||||
summary = "the archive's daily prune failed for an agent's repository, so its retention is not applied; the unit's journal names the agent";
|
||||
})
|
||||
# Logged once per detected crash, by `emit_crash_transitions`
|
||||
# in hive-c0re's crash_watch.rs (10s poll interval). Replaces
|
||||
# the old per-hive dashboard `agents_crashing` banner — see
|
||||
|
|
|
|||
Loading…
Reference in a new issue