5c30985a25
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
285 lines
12 KiB
Shell
285 lines
12 KiB
Shell
# Cases for the archive scripts' metric functions. Each function is cut out of
|
|
# the rendered script by name (they start and end at column 0), so the test runs
|
|
# the text the units run. `curl` is a stub that records the payload.
|
|
set -uo pipefail
|
|
|
|
failures=0
|
|
|
|
fail() {
|
|
echo "FAILED: $case: $*" >&2
|
|
failures=$((failures + 1))
|
|
}
|
|
|
|
root=$(mktemp -d)
|
|
mkdir -p "$root/bin"
|
|
|
|
cat >"$root/bin/curl" <<STUB
|
|
#!$BASH
|
|
cat >"$root/curl.payload"
|
|
cat "$root/curl.payload" >>"$root/curl.all"
|
|
echo >>"$root/curl.all"
|
|
STUB
|
|
chmod +x "$root/bin/curl"
|
|
|
|
# Prints function $2 of script $1: from `$2() {` to the first closing brace at column 0.
|
|
cut_function() {
|
|
awk -v name="$2" '$0 ~ "^"name"\\(\\) \\{$" {on=1} on {print} on && /^}$/ {exit}' "$1"
|
|
}
|
|
|
|
{
|
|
cut_function "$BACKUP_SCRIPT" otlp_gauges
|
|
cut_function "$BACKUP_SCRIPT" push_backup_metrics
|
|
cut_function "$BACKUP_SCRIPT" restic_output_filter
|
|
} >"$root/backup-fns.sh"
|
|
{
|
|
cut_function "$PRUNE_SCRIPT" otlp_gauges
|
|
cut_function "$PRUNE_SCRIPT" push_repo_metrics
|
|
} >"$root/prune-fns.sh"
|
|
for f in backup-fns prune-fns; do
|
|
grep -q 'otlp_gauges()' "$root/$f.sh" || { echo "FAILED: $f has no otlp_gauges"; exit 1; }
|
|
done
|
|
grep -q 'push_backup_metrics()' "$root/backup-fns.sh" || { echo "FAILED: push_backup_metrics not found"; exit 1; }
|
|
grep -q 'restic_output_filter()' "$root/backup-fns.sh" || { echo "FAILED: restic_output_filter not found"; exit 1; }
|
|
grep -q 'push_repo_metrics()' "$root/prune-fns.sh" || { echo "FAILED: push_repo_metrics not found"; exit 1; }
|
|
|
|
# Runs "$2 ..." with the functions of file $1 loaded; payload lands in curl.payload.
|
|
run() {
|
|
local fns="$1"
|
|
shift
|
|
rm -f "$root/curl.payload"
|
|
PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; "$@"' _ "$fns" "$@" >"$root/out" 2>"$root/err"
|
|
}
|
|
|
|
# The pushed gauges as `name=value` lines, in order.
|
|
gauges() {
|
|
jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[]
|
|
| "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.payload"
|
|
}
|
|
|
|
want_gauges() {
|
|
local got
|
|
got="$(gauges 2>&1)"
|
|
[ "$got" = "$1" ] || fail "gauges were:
|
|
$got
|
|
want:
|
|
$1"
|
|
}
|
|
|
|
summary='{"message_type":"summary","files_new":3,"files_changed":2,"files_unmodified":9,"data_added":1234,"total_duration":1.5}'
|
|
status='{"message_type":"status","percent_done":0.5,"total_files":14}'
|
|
error='{"message_type":"error","error":{"message":"read denied"}}'
|
|
b=hyperhive.archive.backup
|
|
|
|
case="a successful run with a summary pushes every gauge"
|
|
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 "$status
|
|
$summary"; then
|
|
want_gauges "$b.last_run_timestamp_seconds=130
|
|
$b.last_run_success=1
|
|
$b.duration_seconds=30
|
|
$b.last_success_timestamp_seconds=130
|
|
$b.data_added_bytes=1234
|
|
$b.files_new=3
|
|
$b.files_changed=2"
|
|
[ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = hive-archive-backup ] ||
|
|
fail "service.name"
|
|
[ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0] | "\(.key)=\(.value.stringValue)"' "$root/curl.payload")" = agent=updo ] ||
|
|
fail "agent attribute"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="a failed run pushes the result and no success time or counts"
|
|
if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$error"; then
|
|
want_gauges "$b.last_run_timestamp_seconds=190
|
|
$b.last_run_success=0
|
|
$b.duration_seconds=90"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="a failed run ignores a summary in its output"
|
|
if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$summary"; then
|
|
want_gauges "$b.last_run_timestamp_seconds=190
|
|
$b.last_run_success=0
|
|
$b.duration_seconds=90"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="a success without a summary (nothing to back up) pushes no counts"
|
|
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 105 "updo has none of its backup paths yet; nothing to back up."; then
|
|
want_gauges "$b.last_run_timestamp_seconds=105
|
|
$b.last_run_success=1
|
|
$b.duration_seconds=5
|
|
$b.last_success_timestamp_seconds=105"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="a summary missing fields pushes the fields it has"
|
|
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 '{"message_type":"summary","data_added":77}'; then
|
|
want_gauges "$b.last_run_timestamp_seconds=130
|
|
$b.last_run_success=1
|
|
$b.duration_seconds=30
|
|
$b.last_success_timestamp_seconds=130
|
|
$b.data_added_bytes=77"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="the output filter keeps summary and error lines and drops progress"
|
|
got="$(printf '%s\n' "$status" "$error" "$status" "$summary" "open repository" 5 |
|
|
PATH="$root/bin:$PATH" bash -c "source $root/backup-fns.sh; restic_output_filter")"
|
|
want="$error
|
|
$summary
|
|
open repository
|
|
5"
|
|
[ "$got" = "$want" ] || fail "kept:
|
|
$got
|
|
want:
|
|
$want"
|
|
|
|
# --- the prune side ---
|
|
r=hyperhive.archive
|
|
printf '%s' '[{"id":"a","time":"2026-11-13T23:02:28.123456789+01:00"},{"id":"b","time":"2026-11-14T08:00:00.5Z"},{"id":"c","time":"2026-11-10T00:00:00Z"}]' >"$root/snapshots.json"
|
|
printf '%s' '{"total_size":98765,"total_blob_count":3}' >"$root/stats.json"
|
|
newest=$(date -d 2026-11-14T08:00:00.5Z +%s)
|
|
|
|
case="a prune that worked pushes the result, snapshot count, newest snapshot and size"
|
|
if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \
|
|
run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then
|
|
got="$(gauges)"
|
|
[ "$(printf '%s\n' "$got" | sed -n 1p)" = "$r.prune.last_run_success=1" ] || fail "first gauge: $got"
|
|
printf '%s\n' "$got" | grep -q "^$r.prune.last_run_timestamp_seconds=[0-9]" || fail "no prune timestamp: $got"
|
|
printf '%s\n' "$got" | grep -qx "$r.repo.snapshots=3" || fail "snapshot count: $got"
|
|
printf '%s\n' "$got" | grep -qx "$r.repo.latest_snapshot_timestamp_seconds=$newest" || fail "newest snapshot, want $newest: $got"
|
|
printf '%s\n' "$got" | grep -qx "$r.repo.size_bytes=98765" || fail "size: $got"
|
|
[ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = swarm-archive-prune ] ||
|
|
fail "service.name"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="a failed prune reports 0 and still reports the repository it can read"
|
|
if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \
|
|
run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then
|
|
[ "$(gauges | sed -n 1p)" = "$r.prune.last_run_success=0" ] || fail "result: $(gauges)"
|
|
gauges | grep -qx "$r.repo.snapshots=3" || fail "count: $(gauges)"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="an unreadable repository pushes only the prune result"
|
|
if run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then
|
|
got="$(gauges)"
|
|
[ "$(printf '%s\n' "$got" | wc -l)" = 2 ] || fail "want two gauges: $got"
|
|
printf '%s\n' "$got" | grep -q 'repo\.' && fail "repo gauge pushed: $got"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="an empty repository has no newest snapshot"
|
|
printf '%s' '[]' >"$root/empty.json"
|
|
if FAKE_SNAPSHOTS="$root/empty.json" run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then
|
|
gauges | grep -qx "$r.repo.snapshots=0" || fail "count: $(gauges)"
|
|
gauges | grep -q latest_snapshot && fail "newest pushed: $(gauges)"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
# --- the check side ---
|
|
c=hyperhive.archive.check
|
|
{
|
|
cut_function "$CHECK_SCRIPT" otlp_gauges
|
|
cut_function "$CHECK_SCRIPT" push_check_metrics
|
|
cut_function "$CHECK_SCRIPT" check_agents
|
|
} >"$root/check-fns.sh"
|
|
grep -q 'check_agents()' "$root/check-fns.sh" || { echo "FAILED: check_agents not found"; exit 1; }
|
|
grep -q 'push_check_metrics()' "$root/check-fns.sh" || { echo "FAILED: push_check_metrics not found"; exit 1; }
|
|
|
|
# The gauges pushed for agent $1 in the last run, as `name=value` lines.
|
|
agent_gauges() {
|
|
jq -rs --arg a "$1" '.[] | select(.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0].value.stringValue == $a)
|
|
| .resourceMetrics[0].scopeMetrics[0].metrics[] | "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.all"
|
|
}
|
|
|
|
case="the check loop goes on after a failure, exits non-zero and names the agent that failed"
|
|
rm -f "$root/curl.all" "$root/fake.log"
|
|
: >"$root/fake.log"
|
|
FAKE_LOG="$root/fake.log" FAKE_CHECK_FAIL=beta run "$root/check-fns.sh" check_agents alpha beta gamma
|
|
rc=$?
|
|
[ "$rc" != 0 ] || fail "exit status was 0"
|
|
grep -qx 'archive check failed: beta' "$root/err" || fail "no failure line: $(cat "$root/err")"
|
|
grep -q 'archive check failed: alpha\|archive check failed: gamma' "$root/err" && fail "a passing agent was reported: $(cat "$root/err")"
|
|
[ "$(wc -l <"$root/fake.log")" = 3 ] || fail "want a check of every agent: $(cat "$root/fake.log")"
|
|
grep -q '/repos/gamma/ .*check --read-data-subset=7%' "$root/fake.log" || fail "gamma not checked with the subset: $(cat "$root/fake.log")"
|
|
grep -qx 'checked alpha' "$root/out" || fail "no success line: $(cat "$root/out")"
|
|
|
|
case="the check pushes each agent's own result"
|
|
[ "$(agent_gauges alpha | grep -c .)" = 3 ] || fail "alpha gauges: $(agent_gauges alpha)"
|
|
agent_gauges alpha | grep -qx "$c.last_run_success=1" || fail "alpha result: $(agent_gauges alpha)"
|
|
agent_gauges alpha | grep -q "^$c.last_run_timestamp_seconds=[0-9]" || fail "alpha time: $(agent_gauges alpha)"
|
|
agent_gauges alpha | grep -q "^$c.duration_seconds=[0-9]" || fail "alpha duration: $(agent_gauges alpha)"
|
|
agent_gauges beta | grep -qx "$c.last_run_success=0" || fail "beta result: $(agent_gauges beta)"
|
|
agent_gauges gamma | grep -qx "$c.last_run_success=1" || fail "gamma result: $(agent_gauges gamma)"
|
|
[ "$(jq -rs '.[0].resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.all")" = swarm-archive-check ] ||
|
|
fail "service.name"
|
|
|
|
case="the check pushes the duration it measured"
|
|
if run "$root/check-fns.sh" push_check_metrics alpha 1 100 190; then
|
|
want_gauges "$c.last_run_success=1
|
|
$c.last_run_timestamp_seconds=190
|
|
$c.duration_seconds=90"
|
|
else
|
|
fail "failed: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="the check exits zero when every repository passes"
|
|
if FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents alpha beta; then
|
|
grep -q 'archive check failed' "$root/err" && fail "failure line: $(cat "$root/err")"
|
|
else
|
|
fail "exit status was non-zero: $(cat "$root/err")"
|
|
fi
|
|
|
|
case="the check refuses a name that is not an agent name and goes on"
|
|
: >"$root/fake.log"
|
|
FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents 'bad name;x' alpha
|
|
rc=$?
|
|
[ "$rc" != 0 ] || fail "exit status was 0"
|
|
grep -qx 'archive check failed: bad name;x is not an agent name' "$root/err" || fail "no refusal: $(cat "$root/err")"
|
|
[ "$(wc -l <"$root/fake.log")" = 1 ] || fail "alpha not checked: $(cat "$root/fake.log")"
|
|
|
|
# --- the lock the prune and the check share ---
|
|
lock_prologue() {
|
|
awk '/^if \[ -n "\$\{ARCHIVE_MAINTENANCE_LOCK:-\}" \]; then$/ {on=1} on {print} on && /^fi$/ {exit}' "$1"
|
|
}
|
|
for s in CHECK_SCRIPT PRUNE_SCRIPT; do
|
|
lock_prologue "${!s}" >"$root/lock-$s.sh"
|
|
grep -q flock "$root/lock-$s.sh" || { echo "FAILED: $s has no maintenance lock"; exit 1; }
|
|
done
|
|
|
|
case="the check waits while the prune holds the lock, and runs once it is free"
|
|
lockfile="$root/maintenance.lock"
|
|
flock "$lockfile" sleep 5 &
|
|
holder=$!
|
|
sleep 0.5
|
|
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-CHECK_SCRIPT.sh"
|
|
[ "$?" = 124 ] || fail "did not wait for the held lock"
|
|
kill "$holder" 2>/dev/null
|
|
wait "$holder" 2>/dev/null
|
|
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "did not run on a free lock"
|
|
|
|
case="the prune waits on the same lock"
|
|
flock "$lockfile" sleep 5 &
|
|
holder=$!
|
|
sleep 0.5
|
|
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-PRUNE_SCRIPT.sh"
|
|
[ "$?" = 124 ] || fail "did not wait for the held lock"
|
|
kill "$holder" 2>/dev/null
|
|
wait "$holder" 2>/dev/null
|
|
|
|
case="without the lock variable the scripts take no lock"
|
|
timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "failed without the variable"
|
|
|
|
[ "$failures" -eq 0 ] || exit 1
|
|
echo "all cases passed"
|