swarm-archive: check each agent's restic repository weekly
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
atlas
parent
c4919c5945
commit
5c30985a25
2 changed files with 503 additions and 18 deletions
|
|
@ -1,6 +1,6 @@
|
|||
# `checks.script-test-archive-metrics` — runs the metric functions of the archive's
|
||||
# backup and prune scripts (../host-modules/lib/archive-backup.nix and
|
||||
# archive-server.nix) against a stub `curl` and a fake `restic`. The cases are in
|
||||
# archive-server.nix, including the check script's loop) against a stub `curl` and a fake `restic`. The cases are in
|
||||
# ./archive-metrics.sh, which cuts each function out of the rendered script.
|
||||
{
|
||||
pkgs,
|
||||
|
|
@ -11,6 +11,18 @@ let
|
|||
# Answers from the files named by FAKE_SNAPSHOTS and FAKE_STATS; an unset
|
||||
# variable makes that subcommand fail, and so does a call without --no-lock:
|
||||
# the metrics only read, and must not wait on a lock.
|
||||
# `check` is the integrity check: it must hold the repository lock, and fails
|
||||
# for the repositories named in FAKE_CHECK_FAIL (space separated agents).
|
||||
case " $* " in
|
||||
*" check "*)
|
||||
echo "check $*" >>"$FAKE_LOG"
|
||||
case " $* " in *" --no-lock "*) exit 1 ;; esac
|
||||
for a in ''${FAKE_CHECK_FAIL:-}; do
|
||||
case " $* " in *"/$a/ "*) exit 1 ;; esac
|
||||
done
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
case " $* " in *" --no-lock "*) ;; *) exit 1 ;; esac
|
||||
case " $* " in
|
||||
*" snapshots "*) [ -n "''${FAKE_SNAPSHOTS:-}" ] && cat "$FAKE_SNAPSHOTS" ;;
|
||||
|
|
@ -25,6 +37,7 @@ pkgs.runCommand "hyperhive-script-test-archive-metrics"
|
|||
nativeBuildInputs = [
|
||||
pkgs.coreutils
|
||||
pkgs.jq
|
||||
pkgs.util-linux
|
||||
];
|
||||
BACKUP_SCRIPT = pkgs.writeText "archive-backup.sh" (
|
||||
import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
|
||||
|
|
@ -39,6 +52,22 @@ pkgs.runCommand "hyperhive-script-test-archive-metrics"
|
|||
otlpEndpoint = endpoint;
|
||||
}
|
||||
);
|
||||
CHECK_SCRIPT = pkgs.writeText "archive-check.sh" (
|
||||
(import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } {
|
||||
rclone = pkgs.rclone;
|
||||
restic = fakeRestic;
|
||||
reposDir = "/repos";
|
||||
addr = "127.0.0.1:1";
|
||||
authFile = "/auth";
|
||||
authGroup = "g";
|
||||
retention = {
|
||||
keepWithin = null;
|
||||
keepLast = 1;
|
||||
};
|
||||
checkSubset = "7%";
|
||||
otlpEndpoint = endpoint;
|
||||
}).checkScript
|
||||
);
|
||||
PRUNE_SCRIPT = pkgs.writeText "archive-prune.sh" (
|
||||
(import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } {
|
||||
rclone = pkgs.rclone;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ mkdir -p "$root/bin"
|
|||
cat >"$root/bin/curl" <<STUB
|
||||
#!$BASH
|
||||
cat >"$root/curl.payload"
|
||||
cat "$root/curl.payload" >>"$root/curl.all"
|
||||
echo >>"$root/curl.all"
|
||||
STUB
|
||||
chmod +x "$root/bin/curl"
|
||||
|
||||
|
|
@ -185,5 +187,99 @@ else
|
|||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
# --- the check side ---
|
||||
c=hyperhive.archive.check
|
||||
{
|
||||
cut_function "$CHECK_SCRIPT" otlp_gauges
|
||||
cut_function "$CHECK_SCRIPT" push_check_metrics
|
||||
cut_function "$CHECK_SCRIPT" check_agents
|
||||
} >"$root/check-fns.sh"
|
||||
grep -q 'check_agents()' "$root/check-fns.sh" || { echo "FAILED: check_agents not found"; exit 1; }
|
||||
grep -q 'push_check_metrics()' "$root/check-fns.sh" || { echo "FAILED: push_check_metrics not found"; exit 1; }
|
||||
|
||||
# The gauges pushed for agent $1 in the last run, as `name=value` lines.
|
||||
agent_gauges() {
|
||||
jq -rs --arg a "$1" '.[] | select(.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0].value.stringValue == $a)
|
||||
| .resourceMetrics[0].scopeMetrics[0].metrics[] | "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.all"
|
||||
}
|
||||
|
||||
case="the check loop goes on after a failure, exits non-zero and names the agent that failed"
|
||||
rm -f "$root/curl.all" "$root/fake.log"
|
||||
: >"$root/fake.log"
|
||||
FAKE_LOG="$root/fake.log" FAKE_CHECK_FAIL=beta run "$root/check-fns.sh" check_agents alpha beta gamma
|
||||
rc=$?
|
||||
[ "$rc" != 0 ] || fail "exit status was 0"
|
||||
grep -qx 'archive check failed: beta' "$root/err" || fail "no failure line: $(cat "$root/err")"
|
||||
grep -q 'archive check failed: alpha\|archive check failed: gamma' "$root/err" && fail "a passing agent was reported: $(cat "$root/err")"
|
||||
[ "$(wc -l <"$root/fake.log")" = 3 ] || fail "want a check of every agent: $(cat "$root/fake.log")"
|
||||
grep -q '/repos/gamma/ .*check --read-data-subset=7%' "$root/fake.log" || fail "gamma not checked with the subset: $(cat "$root/fake.log")"
|
||||
grep -qx 'checked alpha' "$root/out" || fail "no success line: $(cat "$root/out")"
|
||||
|
||||
case="the check pushes each agent's own result"
|
||||
[ "$(agent_gauges alpha | grep -c .)" = 3 ] || fail "alpha gauges: $(agent_gauges alpha)"
|
||||
agent_gauges alpha | grep -qx "$c.last_run_success=1" || fail "alpha result: $(agent_gauges alpha)"
|
||||
agent_gauges alpha | grep -q "^$c.last_run_timestamp_seconds=[0-9]" || fail "alpha time: $(agent_gauges alpha)"
|
||||
agent_gauges alpha | grep -q "^$c.duration_seconds=[0-9]" || fail "alpha duration: $(agent_gauges alpha)"
|
||||
agent_gauges beta | grep -qx "$c.last_run_success=0" || fail "beta result: $(agent_gauges beta)"
|
||||
agent_gauges gamma | grep -qx "$c.last_run_success=1" || fail "gamma result: $(agent_gauges gamma)"
|
||||
[ "$(jq -rs '.[0].resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.all")" = swarm-archive-check ] ||
|
||||
fail "service.name"
|
||||
|
||||
case="the check pushes the duration it measured"
|
||||
if run "$root/check-fns.sh" push_check_metrics alpha 1 100 190; then
|
||||
want_gauges "$c.last_run_success=1
|
||||
$c.last_run_timestamp_seconds=190
|
||||
$c.duration_seconds=90"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="the check exits zero when every repository passes"
|
||||
if FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents alpha beta; then
|
||||
grep -q 'archive check failed' "$root/err" && fail "failure line: $(cat "$root/err")"
|
||||
else
|
||||
fail "exit status was non-zero: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="the check refuses a name that is not an agent name and goes on"
|
||||
: >"$root/fake.log"
|
||||
FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents 'bad name;x' alpha
|
||||
rc=$?
|
||||
[ "$rc" != 0 ] || fail "exit status was 0"
|
||||
grep -qx 'archive check failed: bad name;x is not an agent name' "$root/err" || fail "no refusal: $(cat "$root/err")"
|
||||
[ "$(wc -l <"$root/fake.log")" = 1 ] || fail "alpha not checked: $(cat "$root/fake.log")"
|
||||
|
||||
# --- the lock the prune and the check share ---
|
||||
lock_prologue() {
|
||||
awk '/^if \[ -n "\$\{ARCHIVE_MAINTENANCE_LOCK:-\}" \]; then$/ {on=1} on {print} on && /^fi$/ {exit}' "$1"
|
||||
}
|
||||
for s in CHECK_SCRIPT PRUNE_SCRIPT; do
|
||||
lock_prologue "${!s}" >"$root/lock-$s.sh"
|
||||
grep -q flock "$root/lock-$s.sh" || { echo "FAILED: $s has no maintenance lock"; exit 1; }
|
||||
done
|
||||
|
||||
case="the check waits while the prune holds the lock, and runs once it is free"
|
||||
lockfile="$root/maintenance.lock"
|
||||
flock "$lockfile" sleep 5 &
|
||||
holder=$!
|
||||
sleep 0.5
|
||||
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-CHECK_SCRIPT.sh"
|
||||
[ "$?" = 124 ] || fail "did not wait for the held lock"
|
||||
kill "$holder" 2>/dev/null
|
||||
wait "$holder" 2>/dev/null
|
||||
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "did not run on a free lock"
|
||||
|
||||
case="the prune waits on the same lock"
|
||||
flock "$lockfile" sleep 5 &
|
||||
holder=$!
|
||||
sleep 0.5
|
||||
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-PRUNE_SCRIPT.sh"
|
||||
[ "$?" = 124 ] || fail "did not wait for the held lock"
|
||||
kill "$holder" 2>/dev/null
|
||||
wait "$holder" 2>/dev/null
|
||||
|
||||
case="without the lock variable the scripts take no lock"
|
||||
timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "failed without the variable"
|
||||
|
||||
[ "$failures" -eq 0 ] || exit 1
|
||||
echo "all cases passed"
|
||||
|
|
|
|||
Loading…
Reference in a new issue