swarm-archive: check each agent's restic repository weekly
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
atlas
parent
c4919c5945
commit
5c30985a25
1 changed files with 503 additions and 18 deletions
|
|
@ -16,6 +16,8 @@ mkdir -p "$root/bin"
|
|||
cat >"$root/bin/curl" <<STUB
|
||||
#!$BASH
|
||||
cat >"$root/curl.payload"
|
||||
cat "$root/curl.payload" >>"$root/curl.all"
|
||||
echo >>"$root/curl.all"
|
||||
STUB
|
||||
chmod +x "$root/bin/curl"
|
||||
|
||||
|
|
@ -185,5 +187,99 @@ else
|
|||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
# --- the check side ---
|
||||
c=hyperhive.archive.check
|
||||
{
|
||||
cut_function "$CHECK_SCRIPT" otlp_gauges
|
||||
cut_function "$CHECK_SCRIPT" push_check_metrics
|
||||
cut_function "$CHECK_SCRIPT" check_agents
|
||||
} >"$root/check-fns.sh"
|
||||
grep -q 'check_agents()' "$root/check-fns.sh" || { echo "FAILED: check_agents not found"; exit 1; }
|
||||
grep -q 'push_check_metrics()' "$root/check-fns.sh" || { echo "FAILED: push_check_metrics not found"; exit 1; }
|
||||
|
||||
# The gauges pushed for agent $1 in the last run, as `name=value` lines.
|
||||
agent_gauges() {
|
||||
jq -rs --arg a "$1" '.[] | select(.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0].value.stringValue == $a)
|
||||
| .resourceMetrics[0].scopeMetrics[0].metrics[] | "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.all"
|
||||
}
|
||||
|
||||
case="the check loop goes on after a failure, exits non-zero and names the agent that failed"
|
||||
rm -f "$root/curl.all" "$root/fake.log"
|
||||
: >"$root/fake.log"
|
||||
FAKE_LOG="$root/fake.log" FAKE_CHECK_FAIL=beta run "$root/check-fns.sh" check_agents alpha beta gamma
|
||||
rc=$?
|
||||
[ "$rc" != 0 ] || fail "exit status was 0"
|
||||
grep -qx 'archive check failed: beta' "$root/err" || fail "no failure line: $(cat "$root/err")"
|
||||
grep -q 'archive check failed: alpha\|archive check failed: gamma' "$root/err" && fail "a passing agent was reported: $(cat "$root/err")"
|
||||
[ "$(wc -l <"$root/fake.log")" = 3 ] || fail "want a check of every agent: $(cat "$root/fake.log")"
|
||||
grep -q '/repos/gamma/ .*check --read-data-subset=7%' "$root/fake.log" || fail "gamma not checked with the subset: $(cat "$root/fake.log")"
|
||||
grep -qx 'checked alpha' "$root/out" || fail "no success line: $(cat "$root/out")"
|
||||
|
||||
case="the check pushes each agent's own result"
|
||||
[ "$(agent_gauges alpha | grep -c .)" = 3 ] || fail "alpha gauges: $(agent_gauges alpha)"
|
||||
agent_gauges alpha | grep -qx "$c.last_run_success=1" || fail "alpha result: $(agent_gauges alpha)"
|
||||
agent_gauges alpha | grep -q "^$c.last_run_timestamp_seconds=[0-9]" || fail "alpha time: $(agent_gauges alpha)"
|
||||
agent_gauges alpha | grep -q "^$c.duration_seconds=[0-9]" || fail "alpha duration: $(agent_gauges alpha)"
|
||||
agent_gauges beta | grep -qx "$c.last_run_success=0" || fail "beta result: $(agent_gauges beta)"
|
||||
agent_gauges gamma | grep -qx "$c.last_run_success=1" || fail "gamma result: $(agent_gauges gamma)"
|
||||
[ "$(jq -rs '.[0].resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.all")" = swarm-archive-check ] ||
|
||||
fail "service.name"
|
||||
|
||||
case="the check pushes the duration it measured"
|
||||
if run "$root/check-fns.sh" push_check_metrics alpha 1 100 190; then
|
||||
want_gauges "$c.last_run_success=1
|
||||
$c.last_run_timestamp_seconds=190
|
||||
$c.duration_seconds=90"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="the check exits zero when every repository passes"
|
||||
if FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents alpha beta; then
|
||||
grep -q 'archive check failed' "$root/err" && fail "failure line: $(cat "$root/err")"
|
||||
else
|
||||
fail "exit status was non-zero: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="the check refuses a name that is not an agent name and goes on"
|
||||
: >"$root/fake.log"
|
||||
FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents 'bad name;x' alpha
|
||||
rc=$?
|
||||
[ "$rc" != 0 ] || fail "exit status was 0"
|
||||
grep -qx 'archive check failed: bad name;x is not an agent name' "$root/err" || fail "no refusal: $(cat "$root/err")"
|
||||
[ "$(wc -l <"$root/fake.log")" = 1 ] || fail "alpha not checked: $(cat "$root/fake.log")"
|
||||
|
||||
# --- the lock the prune and the check share ---
|
||||
lock_prologue() {
|
||||
awk '/^if \[ -n "\$\{ARCHIVE_MAINTENANCE_LOCK:-\}" \]; then$/ {on=1} on {print} on && /^fi$/ {exit}' "$1"
|
||||
}
|
||||
for s in CHECK_SCRIPT PRUNE_SCRIPT; do
|
||||
lock_prologue "${!s}" >"$root/lock-$s.sh"
|
||||
grep -q flock "$root/lock-$s.sh" || { echo "FAILED: $s has no maintenance lock"; exit 1; }
|
||||
done
|
||||
|
||||
case="the check waits while the prune holds the lock, and runs once it is free"
|
||||
lockfile="$root/maintenance.lock"
|
||||
flock "$lockfile" sleep 5 &
|
||||
holder=$!
|
||||
sleep 0.5
|
||||
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-CHECK_SCRIPT.sh"
|
||||
[ "$?" = 124 ] || fail "did not wait for the held lock"
|
||||
kill "$holder" 2>/dev/null
|
||||
wait "$holder" 2>/dev/null
|
||||
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "did not run on a free lock"
|
||||
|
||||
case="the prune waits on the same lock"
|
||||
flock "$lockfile" sleep 5 &
|
||||
holder=$!
|
||||
sleep 0.5
|
||||
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-PRUNE_SCRIPT.sh"
|
||||
[ "$?" = 124 ] || fail "did not wait for the held lock"
|
||||
kill "$holder" 2>/dev/null
|
||||
wait "$holder" 2>/dev/null
|
||||
|
||||
case="without the lock variable the scripts take no lock"
|
||||
timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "failed without the variable"
|
||||
|
||||
[ "$failures" -eq 0 ] || exit 1
|
||||
echo "all cases passed"
|
||||
|
|
|
|||
Loading…
Reference in a new issue