archive: test the metric functions and the backup output filter, and skip absent summary fields
This commit is contained in:
atlas
parent
723ce866f1
commit
b60779093e
1 changed files with 263 additions and 6 deletions
189
nix/script-tests/archive-metrics.sh
Normal file
189
nix/script-tests/archive-metrics.sh
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
# Cases for the archive scripts' metric functions. Each function is cut out of
|
||||
# the rendered script by name (they start and end at column 0), so the test runs
|
||||
# the text the units run. `curl` is a stub that records the payload.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
|
||||
fail() {
|
||||
echo "FAILED: $case: $*" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
root=$(mktemp -d)
|
||||
mkdir -p "$root/bin"
|
||||
|
||||
cat >"$root/bin/curl" <<STUB
|
||||
#!$BASH
|
||||
cat >"$root/curl.payload"
|
||||
STUB
|
||||
chmod +x "$root/bin/curl"
|
||||
|
||||
# Prints function $2 of script $1: from `$2() {` to the first closing brace at column 0.
|
||||
cut_function() {
|
||||
awk -v name="$2" '$0 ~ "^"name"\\(\\) \\{$" {on=1} on {print} on && /^}$/ {exit}' "$1"
|
||||
}
|
||||
|
||||
{
|
||||
cut_function "$BACKUP_SCRIPT" otlp_gauges
|
||||
cut_function "$BACKUP_SCRIPT" push_backup_metrics
|
||||
cut_function "$BACKUP_SCRIPT" restic_output_filter
|
||||
} >"$root/backup-fns.sh"
|
||||
{
|
||||
cut_function "$PRUNE_SCRIPT" otlp_gauges
|
||||
cut_function "$PRUNE_SCRIPT" push_repo_metrics
|
||||
} >"$root/prune-fns.sh"
|
||||
for f in backup-fns prune-fns; do
|
||||
grep -q 'otlp_gauges()' "$root/$f.sh" || { echo "FAILED: $f has no otlp_gauges"; exit 1; }
|
||||
done
|
||||
grep -q 'push_backup_metrics()' "$root/backup-fns.sh" || { echo "FAILED: push_backup_metrics not found"; exit 1; }
|
||||
grep -q 'restic_output_filter()' "$root/backup-fns.sh" || { echo "FAILED: restic_output_filter not found"; exit 1; }
|
||||
grep -q 'push_repo_metrics()' "$root/prune-fns.sh" || { echo "FAILED: push_repo_metrics not found"; exit 1; }
|
||||
|
||||
# Runs "$2 ..." with the functions of file $1 loaded; payload lands in curl.payload.
|
||||
run() {
|
||||
local fns="$1"
|
||||
shift
|
||||
rm -f "$root/curl.payload"
|
||||
PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; "$@"' _ "$fns" "$@" >"$root/out" 2>"$root/err"
|
||||
}
|
||||
|
||||
# The pushed gauges as `name=value` lines, in order.
|
||||
gauges() {
|
||||
jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[]
|
||||
| "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.payload"
|
||||
}
|
||||
|
||||
want_gauges() {
|
||||
local got
|
||||
got="$(gauges 2>&1)"
|
||||
[ "$got" = "$1" ] || fail "gauges were:
|
||||
$got
|
||||
want:
|
||||
$1"
|
||||
}
|
||||
|
||||
summary='{"message_type":"summary","files_new":3,"files_changed":2,"files_unmodified":9,"data_added":1234,"total_duration":1.5}'
|
||||
status='{"message_type":"status","percent_done":0.5,"total_files":14}'
|
||||
error='{"message_type":"error","error":{"message":"read denied"}}'
|
||||
b=hyperhive.archive.backup
|
||||
|
||||
case="a successful run with a summary pushes every gauge"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 "$status
|
||||
$summary"; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=130
|
||||
$b.last_run_success=1
|
||||
$b.duration_seconds=30
|
||||
$b.last_success_timestamp_seconds=130
|
||||
$b.data_added_bytes=1234
|
||||
$b.files_new=3
|
||||
$b.files_changed=2"
|
||||
[ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = hive-archive-backup ] ||
|
||||
fail "service.name"
|
||||
[ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0] | "\(.key)=\(.value.stringValue)"' "$root/curl.payload")" = agent=updo ] ||
|
||||
fail "agent attribute"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a failed run pushes the result and no success time or counts"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$error"; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=190
|
||||
$b.last_run_success=0
|
||||
$b.duration_seconds=90"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a failed run ignores a summary in its output"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$summary"; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=190
|
||||
$b.last_run_success=0
|
||||
$b.duration_seconds=90"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a success without a summary (nothing to back up) pushes no counts"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 105 "updo has none of its backup paths yet; nothing to back up."; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=105
|
||||
$b.last_run_success=1
|
||||
$b.duration_seconds=5
|
||||
$b.last_success_timestamp_seconds=105"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a summary missing fields pushes the fields it has"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 '{"message_type":"summary","data_added":77}'; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=130
|
||||
$b.last_run_success=1
|
||||
$b.duration_seconds=30
|
||||
$b.last_success_timestamp_seconds=130
|
||||
$b.data_added_bytes=77"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="the output filter keeps summary and error lines and drops progress"
|
||||
got="$(printf '%s\n' "$status" "$error" "$status" "$summary" "open repository" 5 |
|
||||
PATH="$root/bin:$PATH" bash -c "source $root/backup-fns.sh; restic_output_filter")"
|
||||
want="$error
|
||||
$summary
|
||||
open repository
|
||||
5"
|
||||
[ "$got" = "$want" ] || fail "kept:
|
||||
$got
|
||||
want:
|
||||
$want"
|
||||
|
||||
# --- the prune side ---
|
||||
r=hyperhive.archive
|
||||
printf '%s' '[{"id":"a","time":"2026-11-13T23:02:28.123456789+01:00"},{"id":"b","time":"2026-11-14T08:00:00.5Z"},{"id":"c","time":"2026-11-10T00:00:00Z"}]' >"$root/snapshots.json"
|
||||
printf '%s' '{"total_size":98765,"total_blob_count":3}' >"$root/stats.json"
|
||||
newest=$(date -d 2026-11-14T08:00:00.5Z +%s)
|
||||
|
||||
case="a prune that worked pushes the result, snapshot count, newest snapshot and size"
|
||||
if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \
|
||||
run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then
|
||||
got="$(gauges)"
|
||||
[ "$(printf '%s\n' "$got" | sed -n 1p)" = "$r.prune.last_run_success=1" ] || fail "first gauge: $got"
|
||||
printf '%s\n' "$got" | grep -q "^$r.prune.last_run_timestamp_seconds=[0-9]" || fail "no prune timestamp: $got"
|
||||
printf '%s\n' "$got" | grep -qx "$r.repo.snapshots=3" || fail "snapshot count: $got"
|
||||
printf '%s\n' "$got" | grep -qx "$r.repo.latest_snapshot_timestamp_seconds=$newest" || fail "newest snapshot, want $newest: $got"
|
||||
printf '%s\n' "$got" | grep -qx "$r.repo.size_bytes=98765" || fail "size: $got"
|
||||
[ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = swarm-archive-prune ] ||
|
||||
fail "service.name"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a failed prune reports 0 and still reports the repository it can read"
|
||||
if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \
|
||||
run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then
|
||||
[ "$(gauges | sed -n 1p)" = "$r.prune.last_run_success=0" ] || fail "result: $(gauges)"
|
||||
gauges | grep -qx "$r.repo.snapshots=3" || fail "count: $(gauges)"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="an unreadable repository pushes only the prune result"
|
||||
if run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then
|
||||
got="$(gauges)"
|
||||
[ "$(printf '%s\n' "$got" | wc -l)" = 2 ] || fail "want two gauges: $got"
|
||||
printf '%s\n' "$got" | grep -q 'repo\.' && fail "repo gauge pushed: $got"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="an empty repository has no newest snapshot"
|
||||
printf '%s' '[]' >"$root/empty.json"
|
||||
if FAKE_SNAPSHOTS="$root/empty.json" run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then
|
||||
gauges | grep -qx "$r.repo.snapshots=0" || fail "count: $(gauges)"
|
||||
gauges | grep -q latest_snapshot && fail "newest pushed: $(gauges)"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
[ "$failures" -eq 0 ] || exit 1
|
||||
echo "all cases passed"
|
||||
Loading…
Reference in a new issue