archive: test the metric functions and the backup output filter, and skip absent summary fields
This commit is contained in:
atlas
parent
723ce866f1
commit
b60779093e
4 changed files with 263 additions and 6 deletions
|
|
@ -83,6 +83,9 @@ in
|
|||
# Runs ./host-modules/lib/otlp-gauges.nix's `otlp_gauges` against a stub `curl`: the
|
||||
# payload shape, a failed push and a bad value both return 0, no endpoint pushes nothing.
|
||||
script-test-otlp-gauges = import ./script-tests/otlp-gauges.nix { inherit pkgs; };
|
||||
# Runs the metric functions of the archive backup and prune scripts: the gauges for a
|
||||
# success, a failure and a partial summary, the newest snapshot, and the output filter.
|
||||
script-test-archive-metrics = import ./script-tests/archive-metrics.nix { inherit pkgs; };
|
||||
# Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent
|
||||
# trust-bundle fetch retries through inside its start job: success after two
|
||||
# failures, the wait schedule, and a return of 1 once the budget is spent.
|
||||
|
|
|
|||
|
|
@ -57,6 +57,14 @@ in
|
|||
root=${lib.escapeShellArg agentRoot}
|
||||
grants=${lib.escapeShellArg (grants hive)}
|
||||
|
||||
# restic's `--json` output on stdin, reduced to its error and summary lines;
|
||||
# lines that are not JSON objects pass through.
|
||||
restic_output_filter() {
|
||||
jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j
|
||||
| if $j == null or $j.message_type == "error" or $j.message_type == "summary"
|
||||
then $line else empty end'
|
||||
}
|
||||
|
||||
# One agent: a read-only snapshot of its subvolume when it is one, so the
|
||||
# databases are copied at one instant, else its directory in place.
|
||||
backup_agent() {
|
||||
|
|
@ -97,9 +105,7 @@ in
|
|||
${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \
|
||||
${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
|
||||
-- "''${targets[@]}" |
|
||||
jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j
|
||||
| if $j == null or $j.message_type == "error" or $j.message_type == "summary"
|
||||
then $line else empty end' || rc=$?
|
||||
restic_output_filter || rc=$?
|
||||
fi
|
||||
if [ -n "$snap" ]; then
|
||||
btrfs subvolume delete "$snap" >/dev/null || rc=1
|
||||
|
|
@ -120,9 +126,9 @@ in
|
|||
metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished")
|
||||
summary="$(printf '%s\n' "$out" | jq -R -r '
|
||||
fromjson? | select(type == "object" and .message_type == "summary")
|
||||
| "hyperhive.archive.backup.data_added_bytes=\(.data_added)",
|
||||
"hyperhive.archive.backup.files_new=\(.files_new)",
|
||||
"hyperhive.archive.backup.files_changed=\(.files_changed)"' 2>/dev/null)" || summary=""
|
||||
| {data_added_bytes: .data_added, files_new: .files_new, files_changed: .files_changed}
|
||||
| to_entries[] | select(.value | type == "number")
|
||||
| "hyperhive.archive.backup.\(.key)=\(.value)"' 2>/dev/null)" || summary=""
|
||||
while IFS= read -r line; do
|
||||
[ -z "$line" ] || metrics+=("$line")
|
||||
done <<<"$summary"
|
||||
|
|
|
|||
59
nix/script-tests/archive-metrics.nix
Normal file
59
nix/script-tests/archive-metrics.nix
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
# `checks.script-test-archive-metrics` — runs the metric functions of the archive's
|
||||
# backup and prune scripts (../host-modules/lib/archive-backup.nix and
|
||||
# archive-server.nix) against a stub `curl` and a fake `restic`. The cases are in
|
||||
# ./archive-metrics.sh, which cuts each function out of the rendered script.
|
||||
{
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
fakeRestic = pkgs.writeShellScriptBin "restic" ''
|
||||
# Answers from the files named by FAKE_SNAPSHOTS and FAKE_STATS; an unset
|
||||
# variable makes that subcommand fail.
|
||||
case " $* " in
|
||||
*" snapshots "*) [ -n "''${FAKE_SNAPSHOTS:-}" ] && cat "$FAKE_SNAPSHOTS" ;;
|
||||
*" stats "*) [ -n "''${FAKE_STATS:-}" ] && cat "$FAKE_STATS" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
'';
|
||||
endpoint = "http://10.42.0.1:4318";
|
||||
in
|
||||
pkgs.runCommand "hyperhive-script-test-archive-metrics"
|
||||
{
|
||||
nativeBuildInputs = [
|
||||
pkgs.coreutils
|
||||
pkgs.jq
|
||||
];
|
||||
BACKUP_SCRIPT = pkgs.writeText "archive-backup.sh" (
|
||||
import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
|
||||
restic = fakeRestic;
|
||||
hive = "h";
|
||||
agentRoot = "/agents";
|
||||
url = "https://archive.test";
|
||||
scope = "paths";
|
||||
paths = [ "state" ];
|
||||
excludes = [ ];
|
||||
caBundle = null;
|
||||
otlpEndpoint = endpoint;
|
||||
}
|
||||
);
|
||||
PRUNE_SCRIPT = pkgs.writeText "archive-prune.sh" (
|
||||
(import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } {
|
||||
rclone = pkgs.rclone;
|
||||
restic = fakeRestic;
|
||||
reposDir = "/repos";
|
||||
addr = "127.0.0.1:1";
|
||||
authFile = "/auth";
|
||||
authGroup = "g";
|
||||
retention = {
|
||||
keepWithin = null;
|
||||
keepLast = 1;
|
||||
};
|
||||
otlpEndpoint = endpoint;
|
||||
}).pruneScript
|
||||
);
|
||||
}
|
||||
''
|
||||
${pkgs.bash}/bin/bash ${./archive-metrics.sh}
|
||||
touch "$out"
|
||||
''
|
||||
189
nix/script-tests/archive-metrics.sh
Normal file
189
nix/script-tests/archive-metrics.sh
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
# Cases for the archive scripts' metric functions. Each function is cut out of
|
||||
# the rendered script by name (they start and end at column 0), so the test runs
|
||||
# the text the units run. `curl` is a stub that records the payload.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
|
||||
fail() {
|
||||
echo "FAILED: $case: $*" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
root=$(mktemp -d)
|
||||
mkdir -p "$root/bin"
|
||||
|
||||
cat >"$root/bin/curl" <<STUB
|
||||
#!$BASH
|
||||
cat >"$root/curl.payload"
|
||||
STUB
|
||||
chmod +x "$root/bin/curl"
|
||||
|
||||
# Prints function $2 of script $1: from `$2() {` to the first closing brace at column 0.
|
||||
cut_function() {
|
||||
awk -v name="$2" '$0 ~ "^"name"\\(\\) \\{$" {on=1} on {print} on && /^}$/ {exit}' "$1"
|
||||
}
|
||||
|
||||
{
|
||||
cut_function "$BACKUP_SCRIPT" otlp_gauges
|
||||
cut_function "$BACKUP_SCRIPT" push_backup_metrics
|
||||
cut_function "$BACKUP_SCRIPT" restic_output_filter
|
||||
} >"$root/backup-fns.sh"
|
||||
{
|
||||
cut_function "$PRUNE_SCRIPT" otlp_gauges
|
||||
cut_function "$PRUNE_SCRIPT" push_repo_metrics
|
||||
} >"$root/prune-fns.sh"
|
||||
for f in backup-fns prune-fns; do
|
||||
grep -q 'otlp_gauges()' "$root/$f.sh" || { echo "FAILED: $f has no otlp_gauges"; exit 1; }
|
||||
done
|
||||
grep -q 'push_backup_metrics()' "$root/backup-fns.sh" || { echo "FAILED: push_backup_metrics not found"; exit 1; }
|
||||
grep -q 'restic_output_filter()' "$root/backup-fns.sh" || { echo "FAILED: restic_output_filter not found"; exit 1; }
|
||||
grep -q 'push_repo_metrics()' "$root/prune-fns.sh" || { echo "FAILED: push_repo_metrics not found"; exit 1; }
|
||||
|
||||
# Runs "$2 ..." with the functions of file $1 loaded; payload lands in curl.payload.
|
||||
run() {
|
||||
local fns="$1"
|
||||
shift
|
||||
rm -f "$root/curl.payload"
|
||||
PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; "$@"' _ "$fns" "$@" >"$root/out" 2>"$root/err"
|
||||
}
|
||||
|
||||
# The pushed gauges as `name=value` lines, in order.
|
||||
gauges() {
|
||||
jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[]
|
||||
| "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.payload"
|
||||
}
|
||||
|
||||
want_gauges() {
|
||||
local got
|
||||
got="$(gauges 2>&1)"
|
||||
[ "$got" = "$1" ] || fail "gauges were:
|
||||
$got
|
||||
want:
|
||||
$1"
|
||||
}
|
||||
|
||||
summary='{"message_type":"summary","files_new":3,"files_changed":2,"files_unmodified":9,"data_added":1234,"total_duration":1.5}'
|
||||
status='{"message_type":"status","percent_done":0.5,"total_files":14}'
|
||||
error='{"message_type":"error","error":{"message":"read denied"}}'
|
||||
b=hyperhive.archive.backup
|
||||
|
||||
case="a successful run with a summary pushes every gauge"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 "$status
|
||||
$summary"; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=130
|
||||
$b.last_run_success=1
|
||||
$b.duration_seconds=30
|
||||
$b.last_success_timestamp_seconds=130
|
||||
$b.data_added_bytes=1234
|
||||
$b.files_new=3
|
||||
$b.files_changed=2"
|
||||
[ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = hive-archive-backup ] ||
|
||||
fail "service.name"
|
||||
[ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0] | "\(.key)=\(.value.stringValue)"' "$root/curl.payload")" = agent=updo ] ||
|
||||
fail "agent attribute"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a failed run pushes the result and no success time or counts"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$error"; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=190
|
||||
$b.last_run_success=0
|
||||
$b.duration_seconds=90"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a failed run ignores a summary in its output"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$summary"; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=190
|
||||
$b.last_run_success=0
|
||||
$b.duration_seconds=90"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a success without a summary (nothing to back up) pushes no counts"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 105 "updo has none of its backup paths yet; nothing to back up."; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=105
|
||||
$b.last_run_success=1
|
||||
$b.duration_seconds=5
|
||||
$b.last_success_timestamp_seconds=105"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a summary missing fields pushes the fields it has"
|
||||
if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 '{"message_type":"summary","data_added":77}'; then
|
||||
want_gauges "$b.last_run_timestamp_seconds=130
|
||||
$b.last_run_success=1
|
||||
$b.duration_seconds=30
|
||||
$b.last_success_timestamp_seconds=130
|
||||
$b.data_added_bytes=77"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="the output filter keeps summary and error lines and drops progress"
|
||||
got="$(printf '%s\n' "$status" "$error" "$status" "$summary" "open repository" 5 |
|
||||
PATH="$root/bin:$PATH" bash -c "source $root/backup-fns.sh; restic_output_filter")"
|
||||
want="$error
|
||||
$summary
|
||||
open repository
|
||||
5"
|
||||
[ "$got" = "$want" ] || fail "kept:
|
||||
$got
|
||||
want:
|
||||
$want"
|
||||
|
||||
# --- the prune side ---
|
||||
r=hyperhive.archive
|
||||
printf '%s' '[{"id":"a","time":"2026-11-13T23:02:28.123456789+01:00"},{"id":"b","time":"2026-11-14T08:00:00.5Z"},{"id":"c","time":"2026-11-10T00:00:00Z"}]' >"$root/snapshots.json"
|
||||
printf '%s' '{"total_size":98765,"total_blob_count":3}' >"$root/stats.json"
|
||||
newest=$(date -d 2026-11-14T08:00:00.5Z +%s)
|
||||
|
||||
case="a prune that worked pushes the result, snapshot count, newest snapshot and size"
|
||||
if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \
|
||||
run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then
|
||||
got="$(gauges)"
|
||||
[ "$(printf '%s\n' "$got" | sed -n 1p)" = "$r.prune.last_run_success=1" ] || fail "first gauge: $got"
|
||||
printf '%s\n' "$got" | grep -q "^$r.prune.last_run_timestamp_seconds=[0-9]" || fail "no prune timestamp: $got"
|
||||
printf '%s\n' "$got" | grep -qx "$r.repo.snapshots=3" || fail "snapshot count: $got"
|
||||
printf '%s\n' "$got" | grep -qx "$r.repo.latest_snapshot_timestamp_seconds=$newest" || fail "newest snapshot, want $newest: $got"
|
||||
printf '%s\n' "$got" | grep -qx "$r.repo.size_bytes=98765" || fail "size: $got"
|
||||
[ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = swarm-archive-prune ] ||
|
||||
fail "service.name"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="a failed prune reports 0 and still reports the repository it can read"
|
||||
if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \
|
||||
run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then
|
||||
[ "$(gauges | sed -n 1p)" = "$r.prune.last_run_success=0" ] || fail "result: $(gauges)"
|
||||
gauges | grep -qx "$r.repo.snapshots=3" || fail "count: $(gauges)"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="an unreadable repository pushes only the prune result"
|
||||
if run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then
|
||||
got="$(gauges)"
|
||||
[ "$(printf '%s\n' "$got" | wc -l)" = 2 ] || fail "want two gauges: $got"
|
||||
printf '%s\n' "$got" | grep -q 'repo\.' && fail "repo gauge pushed: $got"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
case="an empty repository has no newest snapshot"
|
||||
printf '%s' '[]' >"$root/empty.json"
|
||||
if FAKE_SNAPSHOTS="$root/empty.json" run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then
|
||||
gauges | grep -qx "$r.repo.snapshots=0" || fail "count: $(gauges)"
|
||||
gauges | grep -q latest_snapshot && fail "newest pushed: $(gauges)"
|
||||
else
|
||||
fail "failed: $(cat "$root/err")"
|
||||
fi
|
||||
|
||||
[ "$failures" -eq 0 ] || exit 1
|
||||
echo "all cases passed"
|
||||
Loading…
Reference in a new issue