# Cases for the archive scripts' metric functions. Each function is cut out of # the rendered script by name (they start and end at column 0), so the test runs # the text the units run. `curl` is a stub that records the payload. set -uo pipefail failures=0 fail() { echo "FAILED: $case: $*" >&2 failures=$((failures + 1)) } root=$(mktemp -d) mkdir -p "$root/bin" cat >"$root/bin/curl" <"$root/curl.payload" cat "$root/curl.payload" >>"$root/curl.all" echo >>"$root/curl.all" STUB chmod +x "$root/bin/curl" # Prints function $2 of script $1: from `$2() {` to the first closing brace at column 0. cut_function() { awk -v name="$2" '$0 ~ "^"name"\\(\\) \\{$" {on=1} on {print} on && /^}$/ {exit}' "$1" } { cut_function "$BACKUP_SCRIPT" otlp_gauges cut_function "$BACKUP_SCRIPT" push_backup_metrics cut_function "$BACKUP_SCRIPT" restic_output_filter } >"$root/backup-fns.sh" { cut_function "$PRUNE_SCRIPT" otlp_gauges cut_function "$PRUNE_SCRIPT" push_repo_metrics } >"$root/prune-fns.sh" for f in backup-fns prune-fns; do grep -q 'otlp_gauges()' "$root/$f.sh" || { echo "FAILED: $f has no otlp_gauges"; exit 1; } done grep -q 'push_backup_metrics()' "$root/backup-fns.sh" || { echo "FAILED: push_backup_metrics not found"; exit 1; } grep -q 'restic_output_filter()' "$root/backup-fns.sh" || { echo "FAILED: restic_output_filter not found"; exit 1; } grep -q 'push_repo_metrics()' "$root/prune-fns.sh" || { echo "FAILED: push_repo_metrics not found"; exit 1; } # Runs "$2 ..." with the functions of file $1 loaded; payload lands in curl.payload. run() { local fns="$1" shift rm -f "$root/curl.payload" PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; "$@"' _ "$fns" "$@" >"$root/out" 2>"$root/err" } # The pushed gauges as `name=value` lines, in order. gauges() { jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[] | "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.payload" } want_gauges() { local got got="$(gauges 2>&1)" [ "$got" = "$1" ] || fail "gauges were: $got want: $1" } summary='{"message_type":"summary","files_new":3,"files_changed":2,"files_unmodified":9,"data_added":1234,"total_duration":1.5}' status='{"message_type":"status","percent_done":0.5,"total_files":14}' error='{"message_type":"error","error":{"message":"read denied"}}' b=hyperhive.archive.backup case="a successful run with a summary pushes every gauge" if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 "$status $summary"; then want_gauges "$b.last_run_timestamp_seconds=130 $b.last_run_success=1 $b.duration_seconds=30 $b.last_success_timestamp_seconds=130 $b.data_added_bytes=1234 $b.files_new=3 $b.files_changed=2" [ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = hive-archive-backup ] || fail "service.name" [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0] | "\(.key)=\(.value.stringValue)"' "$root/curl.payload")" = agent=updo ] || fail "agent attribute" else fail "failed: $(cat "$root/err")" fi case="a failed run pushes the result and no success time or counts" if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$error"; then want_gauges "$b.last_run_timestamp_seconds=190 $b.last_run_success=0 $b.duration_seconds=90" else fail "failed: $(cat "$root/err")" fi case="a failed run ignores a summary in its output" if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$summary"; then want_gauges "$b.last_run_timestamp_seconds=190 $b.last_run_success=0 $b.duration_seconds=90" else fail "failed: $(cat "$root/err")" fi case="a success without a summary (nothing to back up) pushes no counts" if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 105 "updo has none of its backup paths yet; nothing to back up."; then want_gauges "$b.last_run_timestamp_seconds=105 $b.last_run_success=1 $b.duration_seconds=5 $b.last_success_timestamp_seconds=105" else fail "failed: $(cat "$root/err")" fi case="a summary missing fields pushes the fields it has" if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 '{"message_type":"summary","data_added":77}'; then want_gauges "$b.last_run_timestamp_seconds=130 $b.last_run_success=1 $b.duration_seconds=30 $b.last_success_timestamp_seconds=130 $b.data_added_bytes=77" else fail "failed: $(cat "$root/err")" fi case="the output filter keeps summary and error lines and drops progress" got="$(printf '%s\n' "$status" "$error" "$status" "$summary" "open repository" 5 | PATH="$root/bin:$PATH" bash -c "source $root/backup-fns.sh; restic_output_filter")" want="$error $summary open repository 5" [ "$got" = "$want" ] || fail "kept: $got want: $want" # --- the prune side --- r=hyperhive.archive printf '%s' '[{"id":"a","time":"2026-11-13T23:02:28.123456789+01:00"},{"id":"b","time":"2026-11-14T08:00:00.5Z"},{"id":"c","time":"2026-11-10T00:00:00Z"}]' >"$root/snapshots.json" printf '%s' '{"total_size":98765,"total_blob_count":3}' >"$root/stats.json" newest=$(date -d 2026-11-14T08:00:00.5Z +%s) case="a prune that worked pushes the result, snapshot count, newest snapshot and size" if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \ run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then got="$(gauges)" [ "$(printf '%s\n' "$got" | sed -n 1p)" = "$r.prune.last_run_success=1" ] || fail "first gauge: $got" printf '%s\n' "$got" | grep -q "^$r.prune.last_run_timestamp_seconds=[0-9]" || fail "no prune timestamp: $got" printf '%s\n' "$got" | grep -qx "$r.repo.snapshots=3" || fail "snapshot count: $got" printf '%s\n' "$got" | grep -qx "$r.repo.latest_snapshot_timestamp_seconds=$newest" || fail "newest snapshot, want $newest: $got" printf '%s\n' "$got" | grep -qx "$r.repo.size_bytes=98765" || fail "size: $got" [ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = swarm-archive-prune ] || fail "service.name" else fail "failed: $(cat "$root/err")" fi case="a failed prune reports 0 and still reports the repository it can read" if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \ run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then [ "$(gauges | sed -n 1p)" = "$r.prune.last_run_success=0" ] || fail "result: $(gauges)" gauges | grep -qx "$r.repo.snapshots=3" || fail "count: $(gauges)" else fail "failed: $(cat "$root/err")" fi case="an unreadable repository pushes only the prune result" if run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then got="$(gauges)" [ "$(printf '%s\n' "$got" | wc -l)" = 2 ] || fail "want two gauges: $got" printf '%s\n' "$got" | grep -q 'repo\.' && fail "repo gauge pushed: $got" else fail "failed: $(cat "$root/err")" fi case="an empty repository has no newest snapshot" printf '%s' '[]' >"$root/empty.json" if FAKE_SNAPSHOTS="$root/empty.json" run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then gauges | grep -qx "$r.repo.snapshots=0" || fail "count: $(gauges)" gauges | grep -q latest_snapshot && fail "newest pushed: $(gauges)" else fail "failed: $(cat "$root/err")" fi # --- the check side --- c=hyperhive.archive.check { cut_function "$CHECK_SCRIPT" otlp_gauges cut_function "$CHECK_SCRIPT" push_check_metrics cut_function "$CHECK_SCRIPT" check_agents } >"$root/check-fns.sh" grep -q 'check_agents()' "$root/check-fns.sh" || { echo "FAILED: check_agents not found"; exit 1; } grep -q 'push_check_metrics()' "$root/check-fns.sh" || { echo "FAILED: push_check_metrics not found"; exit 1; } # The gauges pushed for agent $1 in the last run, as `name=value` lines. agent_gauges() { jq -rs --arg a "$1" '.[] | select(.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0].value.stringValue == $a) | .resourceMetrics[0].scopeMetrics[0].metrics[] | "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.all" } case="the check loop goes on after a failure, exits non-zero and names the agent that failed" rm -f "$root/curl.all" "$root/fake.log" : >"$root/fake.log" FAKE_LOG="$root/fake.log" FAKE_CHECK_FAIL=beta run "$root/check-fns.sh" check_agents alpha beta gamma rc=$? [ "$rc" != 0 ] || fail "exit status was 0" grep -qx 'archive check failed: beta' "$root/err" || fail "no failure line: $(cat "$root/err")" grep -q 'archive check failed: alpha\|archive check failed: gamma' "$root/err" && fail "a passing agent was reported: $(cat "$root/err")" [ "$(wc -l <"$root/fake.log")" = 3 ] || fail "want a check of every agent: $(cat "$root/fake.log")" grep -q '/repos/gamma/ .*check --read-data-subset=7%' "$root/fake.log" || fail "gamma not checked with the subset: $(cat "$root/fake.log")" grep -qx 'checked alpha' "$root/out" || fail "no success line: $(cat "$root/out")" case="the check pushes each agent's own result" [ "$(agent_gauges alpha | grep -c .)" = 3 ] || fail "alpha gauges: $(agent_gauges alpha)" agent_gauges alpha | grep -qx "$c.last_run_success=1" || fail "alpha result: $(agent_gauges alpha)" agent_gauges alpha | grep -q "^$c.last_run_timestamp_seconds=[0-9]" || fail "alpha time: $(agent_gauges alpha)" agent_gauges alpha | grep -q "^$c.duration_seconds=[0-9]" || fail "alpha duration: $(agent_gauges alpha)" agent_gauges beta | grep -qx "$c.last_run_success=0" || fail "beta result: $(agent_gauges beta)" agent_gauges gamma | grep -qx "$c.last_run_success=1" || fail "gamma result: $(agent_gauges gamma)" [ "$(jq -rs '.[0].resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.all")" = swarm-archive-check ] || fail "service.name" case="the check pushes the duration it measured" if run "$root/check-fns.sh" push_check_metrics alpha 1 100 190; then want_gauges "$c.last_run_success=1 $c.last_run_timestamp_seconds=190 $c.duration_seconds=90" else fail "failed: $(cat "$root/err")" fi case="the check exits zero when every repository passes" if FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents alpha beta; then grep -q 'archive check failed' "$root/err" && fail "failure line: $(cat "$root/err")" else fail "exit status was non-zero: $(cat "$root/err")" fi case="the check refuses a name that is not an agent name and goes on" : >"$root/fake.log" FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents 'bad name;x' alpha rc=$? [ "$rc" != 0 ] || fail "exit status was 0" grep -qx 'archive check failed: bad name;x is not an agent name' "$root/err" || fail "no refusal: $(cat "$root/err")" [ "$(wc -l <"$root/fake.log")" = 1 ] || fail "alpha not checked: $(cat "$root/fake.log")" # --- the lock the prune and the check share --- lock_prologue() { awk '/^if \[ -n "\$\{ARCHIVE_MAINTENANCE_LOCK:-\}" \]; then$/ {on=1} on {print} on && /^fi$/ {exit}' "$1" } for s in CHECK_SCRIPT PRUNE_SCRIPT; do lock_prologue "${!s}" >"$root/lock-$s.sh" grep -q flock "$root/lock-$s.sh" || { echo "FAILED: $s has no maintenance lock"; exit 1; } done case="the check waits while the prune holds the lock, and runs once it is free" lockfile="$root/maintenance.lock" flock "$lockfile" sleep 5 & holder=$! sleep 0.5 ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-CHECK_SCRIPT.sh" [ "$?" = 124 ] || fail "did not wait for the held lock" kill "$holder" 2>/dev/null wait "$holder" 2>/dev/null ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "did not run on a free lock" case="the prune waits on the same lock" flock "$lockfile" sleep 5 & holder=$! sleep 0.5 ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-PRUNE_SCRIPT.sh" [ "$?" = 124 ] || fail "did not wait for the held lock" kill "$holder" 2>/dev/null wait "$holder" 2>/dev/null case="without the lock variable the scripts take no lock" timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "failed without the variable" [ "$failures" -eq 0 ] || exit 1 echo "all cases passed"