5c30985a25
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
90 lines
2.9 KiB
Nix
90 lines
2.9 KiB
Nix
# `checks.script-test-archive-metrics` — runs the metric functions of the archive's
|
|
# backup and prune scripts (../host-modules/lib/archive-backup.nix and
|
|
# archive-server.nix, including the check script's loop) against a stub `curl` and a fake `restic`. The cases are in
|
|
# ./archive-metrics.sh, which cuts each function out of the rendered script.
|
|
{
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
fakeRestic = pkgs.writeShellScriptBin "restic" ''
|
|
# Answers from the files named by FAKE_SNAPSHOTS and FAKE_STATS; an unset
|
|
# variable makes that subcommand fail, and so does a call without --no-lock:
|
|
# the metrics only read, and must not wait on a lock.
|
|
# `check` is the integrity check: it must hold the repository lock, and fails
|
|
# for the repositories named in FAKE_CHECK_FAIL (space separated agents).
|
|
case " $* " in
|
|
*" check "*)
|
|
echo "check $*" >>"$FAKE_LOG"
|
|
case " $* " in *" --no-lock "*) exit 1 ;; esac
|
|
for a in ''${FAKE_CHECK_FAIL:-}; do
|
|
case " $* " in *"/$a/ "*) exit 1 ;; esac
|
|
done
|
|
exit 0
|
|
;;
|
|
esac
|
|
case " $* " in *" --no-lock "*) ;; *) exit 1 ;; esac
|
|
case " $* " in
|
|
*" snapshots "*) [ -n "''${FAKE_SNAPSHOTS:-}" ] && cat "$FAKE_SNAPSHOTS" ;;
|
|
*" stats "*) [ -n "''${FAKE_STATS:-}" ] && cat "$FAKE_STATS" ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
'';
|
|
endpoint = "http://10.42.0.1:4318";
|
|
in
|
|
pkgs.runCommand "hyperhive-script-test-archive-metrics"
|
|
{
|
|
nativeBuildInputs = [
|
|
pkgs.coreutils
|
|
pkgs.jq
|
|
pkgs.util-linux
|
|
];
|
|
BACKUP_SCRIPT = pkgs.writeText "archive-backup.sh" (
|
|
import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
|
|
restic = fakeRestic;
|
|
hive = "h";
|
|
agentRoot = "/agents";
|
|
url = "https://archive.test";
|
|
scope = "paths";
|
|
paths = [ "state" ];
|
|
excludes = [ ];
|
|
caBundle = null;
|
|
otlpEndpoint = endpoint;
|
|
}
|
|
);
|
|
CHECK_SCRIPT = pkgs.writeText "archive-check.sh" (
|
|
(import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } {
|
|
rclone = pkgs.rclone;
|
|
restic = fakeRestic;
|
|
reposDir = "/repos";
|
|
addr = "127.0.0.1:1";
|
|
authFile = "/auth";
|
|
authGroup = "g";
|
|
retention = {
|
|
keepWithin = null;
|
|
keepLast = 1;
|
|
};
|
|
checkSubset = "7%";
|
|
otlpEndpoint = endpoint;
|
|
}).checkScript
|
|
);
|
|
PRUNE_SCRIPT = pkgs.writeText "archive-prune.sh" (
|
|
(import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } {
|
|
rclone = pkgs.rclone;
|
|
restic = fakeRestic;
|
|
reposDir = "/repos";
|
|
addr = "127.0.0.1:1";
|
|
authFile = "/auth";
|
|
authGroup = "g";
|
|
retention = {
|
|
keepWithin = null;
|
|
keepLast = 1;
|
|
};
|
|
otlpEndpoint = endpoint;
|
|
}).pruneScript
|
|
);
|
|
}
|
|
''
|
|
${pkgs.bash}/bin/bash ${./archive-metrics.sh}
|
|
touch "$out"
|
|
''
|