swarm-archive: check each agent's restic repository weekly
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
parent
c4919c5945
commit
5c30985a25
1 changed files with 503 additions and 18 deletions
|
|
@ -22,12 +22,12 @@
|
|||
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
|
||||
# therefore decrypt every backup, which is the cost of pruning where clients
|
||||
# cannot delete.
|
||||
# - `swarm-archive-check` (weekly) runs `restic check` on every repository.
|
||||
#
|
||||
# The units log in as the archive's own store principal (a leaf of the store's
|
||||
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
|
||||
# no secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
|
||||
# ./lib/archive-server.nix); with `s3` the server also reads the S3 key from
|
||||
# `swarm/services/archive/s3/…` as it starts and keeps it in rclone's environment.
|
||||
# CA, minted by ./glue-bao-tls.nix). Plain host units, no container: no
|
||||
# secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
|
||||
# ./lib/archive-server.nix); `s3` also makes the server read the S3 key as it starts.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
|
|
@ -56,6 +56,7 @@ let
|
|||
server = import ./lib/archive-server.nix { inherit lib; } (
|
||||
{
|
||||
inherit (cfg) retention;
|
||||
checkSubset = cfg.checkSubset;
|
||||
inherit authFile;
|
||||
rclone = cfg.package;
|
||||
restic = cfg.resticPackage;
|
||||
|
|
@ -90,6 +91,9 @@ let
|
|||
BAO_CACERT = "%d/bao-ca";
|
||||
};
|
||||
|
||||
# The lock file the prune and the check take, so they never run on a repository together.
|
||||
maintenanceLock = "/run/swarm-archive-maintenance/lock";
|
||||
|
||||
serveScript = pkgs.writeShellScript "swarm-archive-serve" server.s3ServerScript;
|
||||
|
||||
haveClientIdentity =
|
||||
|
|
@ -246,6 +250,25 @@ in
|
|||
example = "*-*-* 04:00:00";
|
||||
description = "systemd calendar expression for the prune: the retention rules are applied then.";
|
||||
};
|
||||
|
||||
checkSchedule = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "weekly";
|
||||
example = "Sun *-*-* 03:00:00";
|
||||
description = "systemd calendar expression for the integrity check of every agent's repository.";
|
||||
};
|
||||
|
||||
checkSubset = lib.mkOption {
|
||||
type = lib.types.strMatching "[0-9]+(\\.[0-9]+)?%";
|
||||
default = "5%";
|
||||
example = "10%";
|
||||
description = ''
|
||||
The share of each repository's data the integrity check reads (restic's
|
||||
`--read-data-subset`). The subset is random per run, so over many runs
|
||||
the checks cover a repository without any one run reading all of it. The
|
||||
repository's structure is checked in full on every run whatever this is.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
|
|
@ -313,6 +336,7 @@ in
|
|||
"swarm-archive.service"
|
||||
"swarm-archive-users.service"
|
||||
"swarm-archive-prune.service"
|
||||
"swarm-archive-check.service"
|
||||
];
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
|
|
@ -418,6 +442,7 @@ in
|
|||
pkgs.curl
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
pkgs.util-linux
|
||||
]
|
||||
# restic's rclone backend starts rclone.
|
||||
++ lib.optional useS3 cfg.package;
|
||||
|
|
@ -425,6 +450,7 @@ in
|
|||
ownerStoreEnvironment
|
||||
// {
|
||||
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
||||
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
|
||||
}
|
||||
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
|
||||
serviceConfig = {
|
||||
|
|
@ -433,6 +459,8 @@ in
|
|||
Group = user;
|
||||
SyslogIdentifier = "swarm-archive-prune";
|
||||
CacheDirectory = "swarm-archive-prune";
|
||||
RuntimeDirectory = "swarm-archive-maintenance";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
LoadCredential = ownerStoreCredentials;
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
|
|
@ -454,5 +482,57 @@ in
|
|||
Persistent = true;
|
||||
};
|
||||
};
|
||||
systemd.services.swarm-archive-check = {
|
||||
description = "check the integrity of every agent's restic repository";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
cfg.resticPackage
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
pkgs.util-linux
|
||||
]
|
||||
# restic's rclone backend starts rclone.
|
||||
++ lib.optional useS3 cfg.package;
|
||||
environment =
|
||||
ownerStoreEnvironment
|
||||
// {
|
||||
RESTIC_CACHE_DIR = "%C/swarm-archive-check";
|
||||
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
|
||||
}
|
||||
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = user;
|
||||
Group = user;
|
||||
SyslogIdentifier = "swarm-archive-check";
|
||||
CacheDirectory = "swarm-archive-check";
|
||||
RuntimeDirectory = "swarm-archive-maintenance";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
LoadCredential = ownerStoreCredentials;
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
# Covers the wait for a running prune (up to its own six hours) and the check.
|
||||
TimeoutStartSec = "24h";
|
||||
}
|
||||
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
|
||||
script = server.checkScript;
|
||||
};
|
||||
|
||||
systemd.timers.swarm-archive-check = {
|
||||
description = "check the archive's repositories";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.checkSchedule;
|
||||
RandomizedDelaySec = "6h";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue