swarm-archive: check each agent's restic repository weekly
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
parent
c4919c5945
commit
5c30985a25
1 changed files with 503 additions and 18 deletions
|
|
@ -617,6 +617,7 @@ let
|
|||
a = atticHere.systemd.services.swarm-archive;
|
||||
users = atticHere.systemd.services.swarm-archive-users;
|
||||
prune = atticHere.systemd.services.swarm-archive-prune;
|
||||
check = atticHere.systemd.services.swarm-archive-check;
|
||||
vhost = atticHere.services.hyperhive.gateway.vhosts."archive.t.local";
|
||||
cfg = atticHere.services.hyperhive.deploy.archive;
|
||||
names = m: m.services.hyperhive.swarm.serviceDomains;
|
||||
|
|
@ -642,6 +643,14 @@ let
|
|||
&& !(lib.hasInfix "password" (lib.toLower (builtins.toJSON a.environment)))
|
||||
&& atticHere.systemd.timers ? swarm-archive-users
|
||||
&& atticHere.systemd.timers.swarm-archive-prune.timerConfig.OnCalendar == "daily"
|
||||
&& atticHere.systemd.timers.swarm-archive-check.timerConfig.OnCalendar == "weekly"
|
||||
&& atticHere.systemd.timers.swarm-archive-check.timerConfig.Persistent
|
||||
&& lib.hasInfix "--read-data-subset=5%" check.script
|
||||
&& check.serviceConfig.User == "swarm-archive"
|
||||
&& check.environment.ARCHIVE_MAINTENANCE_LOCK == prune.environment.ARCHIVE_MAINTENANCE_LOCK
|
||||
&& check.serviceConfig.RuntimeDirectory == prune.serviceConfig.RuntimeDirectory
|
||||
&& lib.hasPrefix "/run/${check.serviceConfig.RuntimeDirectory}/" check.environment.ARCHIVE_MAINTENANCE_LOCK
|
||||
&& lib.elem "swarm-bao-archive-policy.service" check.after
|
||||
&& lib.elem "swarm-bao-archive-policy.service" users.after
|
||||
&& !(bare.systemd.services ? swarm-archive)
|
||||
&& !(bare.systemd.services ? swarm-archive-users);
|
||||
|
|
|
|||
Loading…
Reference in a new issue