Watch
0
0
Fork
You've already forked hyperhive
0

swarm-archive: check each agent's restic repository weekly

A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
atlas 2026-11-14 11:51:25 +01:00
commit 5c30985a25

View file

@ -22,12 +22,12 @@
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
# therefore decrypt every backup, which is the cost of pruning where clients
# cannot delete.
# - `swarm-archive-check` (weekly) runs `restic check` on every repository.
#
# The units log in as the archive's own store principal (a leaf of the store's
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
# no secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
# ./lib/archive-server.nix); with `s3` the server also reads the S3 key from
# `swarm/services/archive/s3/…` as it starts and keeps it in rclone's environment.
# CA, minted by ./glue-bao-tls.nix). Plain host units, no container: no
# secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
# ./lib/archive-server.nix); `s3` also makes the server read the S3 key as it starts.
{
pkgs,
lib,
@ -56,6 +56,7 @@ let
server = import ./lib/archive-server.nix { inherit lib; } (
{
inherit (cfg) retention;
checkSubset = cfg.checkSubset;
inherit authFile;
rclone = cfg.package;
restic = cfg.resticPackage;
@ -90,6 +91,9 @@ let
BAO_CACERT = "%d/bao-ca";
};
# The lock file the prune and the check take, so they never run on a repository together.
maintenanceLock = "/run/swarm-archive-maintenance/lock";
serveScript = pkgs.writeShellScript "swarm-archive-serve" server.s3ServerScript;
haveClientIdentity =
@ -246,6 +250,25 @@ in
example = "*-*-* 04:00:00";
description = "systemd calendar expression for the prune: the retention rules are applied then.";
};
checkSchedule = lib.mkOption {
type = lib.types.str;
default = "weekly";
example = "Sun *-*-* 03:00:00";
description = "systemd calendar expression for the integrity check of every agent's repository.";
};
checkSubset = lib.mkOption {
type = lib.types.strMatching "[0-9]+(\\.[0-9]+)?%";
default = "5%";
example = "10%";
description = ''
The share of each repository's data the integrity check reads (restic's
`--read-data-subset`). The subset is random per run, so over many runs
the checks cover a repository without any one run reading all of it. The
repository's structure is checked in full on every run whatever this is.
'';
};
};
config = lib.mkIf cfg.enable {
@ -313,6 +336,7 @@ in
"swarm-archive.service"
"swarm-archive-users.service"
"swarm-archive-prune.service"
"swarm-archive-check.service"
];
systemd.tmpfiles.rules = [
@ -418,6 +442,7 @@ in
pkgs.curl
pkgs.gnugrep
pkgs.jq
pkgs.util-linux
]
# restic's rclone backend starts rclone.
++ lib.optional useS3 cfg.package;
@ -425,6 +450,7 @@ in
ownerStoreEnvironment
// {
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
}
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
serviceConfig = {
@ -433,6 +459,8 @@ in
Group = user;
SyslogIdentifier = "swarm-archive-prune";
CacheDirectory = "swarm-archive-prune";
RuntimeDirectory = "swarm-archive-maintenance";
RuntimeDirectoryPreserve = "yes";
LoadCredential = ownerStoreCredentials;
NoNewPrivileges = true;
ProtectSystem = "strict";
@ -454,5 +482,57 @@ in
Persistent = true;
};
};
systemd.services.swarm-archive-check = {
description = "check the integrity of every agent's restic repository";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [
baoDeploy.package
cfg.resticPackage
pkgs.coreutils
pkgs.curl
pkgs.gnugrep
pkgs.jq
pkgs.util-linux
]
# restic's rclone backend starts rclone.
++ lib.optional useS3 cfg.package;
environment =
ownerStoreEnvironment
// {
RESTIC_CACHE_DIR = "%C/swarm-archive-check";
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
}
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
serviceConfig = {
Type = "oneshot";
User = user;
Group = user;
SyslogIdentifier = "swarm-archive-check";
CacheDirectory = "swarm-archive-check";
RuntimeDirectory = "swarm-archive-maintenance";
RuntimeDirectoryPreserve = "yes";
LoadCredential = ownerStoreCredentials;
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
# Covers the wait for a running prune (up to its own six hours) and the check.
TimeoutStartSec = "24h";
}
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
script = server.checkScript;
};
systemd.timers.swarm-archive-check = {
description = "check the archive's repositories";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.checkSchedule;
RandomizedDelaySec = "6h";
Persistent = true;
};
};
};
}