diff --git a/nix/checks.nix b/nix/checks.nix index 046ad8e3e..496dfa496 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -83,6 +83,9 @@ in # Runs ./host-modules/lib/otlp-gauges.nix's `otlp_gauges` against a stub `curl`: the # payload shape, a failed push and a bad value both return 0, no endpoint pushes nothing. script-test-otlp-gauges = import ./script-tests/otlp-gauges.nix { inherit pkgs; }; + # Runs the metric functions of the archive backup and prune scripts: the gauges for a + # success, a failure and a partial summary, the newest snapshot, and the output filter. + script-test-archive-metrics = import ./script-tests/archive-metrics.nix { inherit pkgs; }; # Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent # trust-bundle fetch retries through inside its start job: success after two # failures, the wait schedule, and a return of 1 once the budget is spent. diff --git a/nix/host-modules/lib/archive-backup.nix b/nix/host-modules/lib/archive-backup.nix index f5abf1c0a..1b1b24572 100644 --- a/nix/host-modules/lib/archive-backup.nix +++ b/nix/host-modules/lib/archive-backup.nix @@ -57,6 +57,14 @@ in root=${lib.escapeShellArg agentRoot} grants=${lib.escapeShellArg (grants hive)} + # restic's `--json` output on stdin, reduced to its error and summary lines; + # lines that are not JSON objects pass through. + restic_output_filter() { + jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j + | if $j == null or $j.message_type == "error" or $j.message_type == "summary" + then $line else empty end' + } + # One agent: a read-only snapshot of its subvolume when it is one, so the # databases are copied at one instant, else its directory in place. backup_agent() { @@ -97,9 +105,7 @@ in ${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \ ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \ -- "''${targets[@]}" | - jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j - | if $j == null or $j.message_type == "error" or $j.message_type == "summary" - then $line else empty end' || rc=$? + restic_output_filter || rc=$? fi if [ -n "$snap" ]; then btrfs subvolume delete "$snap" >/dev/null || rc=1 @@ -120,9 +126,9 @@ in metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished") summary="$(printf '%s\n' "$out" | jq -R -r ' fromjson? | select(type == "object" and .message_type == "summary") - | "hyperhive.archive.backup.data_added_bytes=\(.data_added)", - "hyperhive.archive.backup.files_new=\(.files_new)", - "hyperhive.archive.backup.files_changed=\(.files_changed)"' 2>/dev/null)" || summary="" + | {data_added_bytes: .data_added, files_new: .files_new, files_changed: .files_changed} + | to_entries[] | select(.value | type == "number") + | "hyperhive.archive.backup.\(.key)=\(.value)"' 2>/dev/null)" || summary="" while IFS= read -r line; do [ -z "$line" ] || metrics+=("$line") done <<<"$summary" diff --git a/nix/script-tests/archive-metrics.nix b/nix/script-tests/archive-metrics.nix new file mode 100644 index 000000000..d2c6db25c --- /dev/null +++ b/nix/script-tests/archive-metrics.nix @@ -0,0 +1,59 @@ +# `checks.script-test-archive-metrics` — runs the metric functions of the archive's +# backup and prune scripts (../host-modules/lib/archive-backup.nix and +# archive-server.nix) against a stub `curl` and a fake `restic`. The cases are in +# ./archive-metrics.sh, which cuts each function out of the rendered script. +{ + pkgs, + ... +}: +let + fakeRestic = pkgs.writeShellScriptBin "restic" '' + # Answers from the files named by FAKE_SNAPSHOTS and FAKE_STATS; an unset + # variable makes that subcommand fail. + case " $* " in + *" snapshots "*) [ -n "''${FAKE_SNAPSHOTS:-}" ] && cat "$FAKE_SNAPSHOTS" ;; + *" stats "*) [ -n "''${FAKE_STATS:-}" ] && cat "$FAKE_STATS" ;; + *) exit 1 ;; + esac + ''; + endpoint = "http://10.42.0.1:4318"; +in +pkgs.runCommand "hyperhive-script-test-archive-metrics" + { + nativeBuildInputs = [ + pkgs.coreutils + pkgs.jq + ]; + BACKUP_SCRIPT = pkgs.writeText "archive-backup.sh" ( + import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } { + restic = fakeRestic; + hive = "h"; + agentRoot = "/agents"; + url = "https://archive.test"; + scope = "paths"; + paths = [ "state" ]; + excludes = [ ]; + caBundle = null; + otlpEndpoint = endpoint; + } + ); + PRUNE_SCRIPT = pkgs.writeText "archive-prune.sh" ( + (import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } { + rclone = pkgs.rclone; + restic = fakeRestic; + reposDir = "/repos"; + addr = "127.0.0.1:1"; + authFile = "/auth"; + authGroup = "g"; + retention = { + keepWithin = null; + keepLast = 1; + }; + otlpEndpoint = endpoint; + }).pruneScript + ); + } + '' + ${pkgs.bash}/bin/bash ${./archive-metrics.sh} + touch "$out" + '' diff --git a/nix/script-tests/archive-metrics.sh b/nix/script-tests/archive-metrics.sh new file mode 100644 index 000000000..8d7aaae67 --- /dev/null +++ b/nix/script-tests/archive-metrics.sh @@ -0,0 +1,189 @@ +# Cases for the archive scripts' metric functions. Each function is cut out of +# the rendered script by name (they start and end at column 0), so the test runs +# the text the units run. `curl` is a stub that records the payload. +set -uo pipefail + +failures=0 + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +root=$(mktemp -d) +mkdir -p "$root/bin" + +cat >"$root/bin/curl" <"$root/curl.payload" +STUB +chmod +x "$root/bin/curl" + +# Prints function $2 of script $1: from `$2() {` to the first closing brace at column 0. +cut_function() { + awk -v name="$2" '$0 ~ "^"name"\\(\\) \\{$" {on=1} on {print} on && /^}$/ {exit}' "$1" +} + +{ + cut_function "$BACKUP_SCRIPT" otlp_gauges + cut_function "$BACKUP_SCRIPT" push_backup_metrics + cut_function "$BACKUP_SCRIPT" restic_output_filter +} >"$root/backup-fns.sh" +{ + cut_function "$PRUNE_SCRIPT" otlp_gauges + cut_function "$PRUNE_SCRIPT" push_repo_metrics +} >"$root/prune-fns.sh" +for f in backup-fns prune-fns; do + grep -q 'otlp_gauges()' "$root/$f.sh" || { echo "FAILED: $f has no otlp_gauges"; exit 1; } +done +grep -q 'push_backup_metrics()' "$root/backup-fns.sh" || { echo "FAILED: push_backup_metrics not found"; exit 1; } +grep -q 'restic_output_filter()' "$root/backup-fns.sh" || { echo "FAILED: restic_output_filter not found"; exit 1; } +grep -q 'push_repo_metrics()' "$root/prune-fns.sh" || { echo "FAILED: push_repo_metrics not found"; exit 1; } + +# Runs "$2 ..." with the functions of file $1 loaded; payload lands in curl.payload. +run() { + local fns="$1" + shift + rm -f "$root/curl.payload" + PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; "$@"' _ "$fns" "$@" >"$root/out" 2>"$root/err" +} + +# The pushed gauges as `name=value` lines, in order. +gauges() { + jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[] + | "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.payload" +} + +want_gauges() { + local got + got="$(gauges 2>&1)" + [ "$got" = "$1" ] || fail "gauges were: +$got +want: +$1" +} + +summary='{"message_type":"summary","files_new":3,"files_changed":2,"files_unmodified":9,"data_added":1234,"total_duration":1.5}' +status='{"message_type":"status","percent_done":0.5,"total_files":14}' +error='{"message_type":"error","error":{"message":"read denied"}}' +b=hyperhive.archive.backup + +case="a successful run with a summary pushes every gauge" +if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 "$status +$summary"; then + want_gauges "$b.last_run_timestamp_seconds=130 +$b.last_run_success=1 +$b.duration_seconds=30 +$b.last_success_timestamp_seconds=130 +$b.data_added_bytes=1234 +$b.files_new=3 +$b.files_changed=2" + [ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = hive-archive-backup ] || + fail "service.name" + [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0] | "\(.key)=\(.value.stringValue)"' "$root/curl.payload")" = agent=updo ] || + fail "agent attribute" +else + fail "failed: $(cat "$root/err")" +fi + +case="a failed run pushes the result and no success time or counts" +if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$error"; then + want_gauges "$b.last_run_timestamp_seconds=190 +$b.last_run_success=0 +$b.duration_seconds=90" +else + fail "failed: $(cat "$root/err")" +fi + +case="a failed run ignores a summary in its output" +if run "$root/backup-fns.sh" push_backup_metrics updo 0 100 190 "$summary"; then + want_gauges "$b.last_run_timestamp_seconds=190 +$b.last_run_success=0 +$b.duration_seconds=90" +else + fail "failed: $(cat "$root/err")" +fi + +case="a success without a summary (nothing to back up) pushes no counts" +if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 105 "updo has none of its backup paths yet; nothing to back up."; then + want_gauges "$b.last_run_timestamp_seconds=105 +$b.last_run_success=1 +$b.duration_seconds=5 +$b.last_success_timestamp_seconds=105" +else + fail "failed: $(cat "$root/err")" +fi + +case="a summary missing fields pushes the fields it has" +if run "$root/backup-fns.sh" push_backup_metrics updo 1 100 130 '{"message_type":"summary","data_added":77}'; then + want_gauges "$b.last_run_timestamp_seconds=130 +$b.last_run_success=1 +$b.duration_seconds=30 +$b.last_success_timestamp_seconds=130 +$b.data_added_bytes=77" +else + fail "failed: $(cat "$root/err")" +fi + +case="the output filter keeps summary and error lines and drops progress" +got="$(printf '%s\n' "$status" "$error" "$status" "$summary" "open repository" 5 | + PATH="$root/bin:$PATH" bash -c "source $root/backup-fns.sh; restic_output_filter")" +want="$error +$summary +open repository +5" +[ "$got" = "$want" ] || fail "kept: +$got +want: +$want" + +# --- the prune side --- +r=hyperhive.archive +printf '%s' '[{"id":"a","time":"2026-11-13T23:02:28.123456789+01:00"},{"id":"b","time":"2026-11-14T08:00:00.5Z"},{"id":"c","time":"2026-11-10T00:00:00Z"}]' >"$root/snapshots.json" +printf '%s' '{"total_size":98765,"total_blob_count":3}' >"$root/stats.json" +newest=$(date -d 2026-11-14T08:00:00.5Z +%s) + +case="a prune that worked pushes the result, snapshot count, newest snapshot and size" +if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \ + run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then + got="$(gauges)" + [ "$(printf '%s\n' "$got" | sed -n 1p)" = "$r.prune.last_run_success=1" ] || fail "first gauge: $got" + printf '%s\n' "$got" | grep -q "^$r.prune.last_run_timestamp_seconds=[0-9]" || fail "no prune timestamp: $got" + printf '%s\n' "$got" | grep -qx "$r.repo.snapshots=3" || fail "snapshot count: $got" + printf '%s\n' "$got" | grep -qx "$r.repo.latest_snapshot_timestamp_seconds=$newest" || fail "newest snapshot, want $newest: $got" + printf '%s\n' "$got" | grep -qx "$r.repo.size_bytes=98765" || fail "size: $got" + [ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.payload")" = swarm-archive-prune ] || + fail "service.name" +else + fail "failed: $(cat "$root/err")" +fi + +case="a failed prune reports 0 and still reports the repository it can read" +if FAKE_SNAPSHOTS="$root/snapshots.json" FAKE_STATS="$root/stats.json" \ + run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then + [ "$(gauges | sed -n 1p)" = "$r.prune.last_run_success=0" ] || fail "result: $(gauges)" + gauges | grep -qx "$r.repo.snapshots=3" || fail "count: $(gauges)" +else + fail "failed: $(cat "$root/err")" +fi + +case="an unreadable repository pushes only the prune result" +if run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 0; then + got="$(gauges)" + [ "$(printf '%s\n' "$got" | wc -l)" = 2 ] || fail "want two gauges: $got" + printf '%s\n' "$got" | grep -q 'repo\.' && fail "repo gauge pushed: $got" +else + fail "failed: $(cat "$root/err")" +fi + +case="an empty repository has no newest snapshot" +printf '%s' '[]' >"$root/empty.json" +if FAKE_SNAPSHOTS="$root/empty.json" run "$root/prune-fns.sh" push_repo_metrics /repos/updo updo 1; then + gauges | grep -qx "$r.repo.snapshots=0" || fail "count: $(gauges)" + gauges | grep -q latest_snapshot && fail "newest pushed: $(gauges)" +else + fail "failed: $(cat "$root/err")" +fi + +[ "$failures" -eq 0 ] || exit 1 +echo "all cases passed"