archive: push restic backup and prune metrics, and add a backups dashboard

nix/script-tests/otlp-gauges.sh

@@ -0,0 +1,73 @@
+# Cases for ../host-modules/lib/otlp-gauges.nix. `curl` is a stub that records
+# the payload and the URL it is given.
+set -uo pipefail
+
+failures=0
+
+fail() {
+  echo "FAILED: $case: $*" >&2
+  failures=$((failures + 1))
+}
+
+root=$(mktemp -d)
+mkdir -p "$root/bin"
+
+cat >"$root/bin/curl" <<STUB
+#!$BASH
+printf '%s\n' "\$*" >"$root/curl.args"
+cat >"$root/curl.payload"
+exit "\${STUB_CURL_STATUS:-0}"
+STUB
+chmod +x "$root/bin/curl"
+
+# Runs \`otlp_gauges "\$@"\` from file \$1 and returns its status.
+run() {
+  local functions="$1"
+  shift
+  rm -f "$root/curl.args" "$root/curl.payload"
+  PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; otlp_gauges "$@"' _ "$functions" "$@" \
+    >"$root/out" 2>"$root/err"
+}
+
+case="two gauges share the resource and the attributes"
+if run "$FUNCTIONS" hive-archive-backup 'agent=updo,tier=a=b' \
+  hyperhive.archive.backup.last_run_success=1 hyperhive.archive.backup.duration_seconds=12.5; then
+  p="$root/curl.payload"
+  [ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$p")" = hive-archive-backup ] ||
+    fail "service.name: $(cat "$p")"
+  [ "$(jq -r '[.resourceMetrics[0].scopeMetrics[0].metrics[].name] | join(" ")' "$p")" = \
+    "hyperhive.archive.backup.last_run_success hyperhive.archive.backup.duration_seconds" ] ||
+    fail "names: $(cat "$p")"
+  [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[1].gauge.dataPoints[0].asDouble' "$p")" = 12.5 ] ||
+    fail "value: $(cat "$p")"
+  [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes | map("\(.key)=\(.value.stringValue)") | join(",")' "$p")" = \
+    "agent=updo,tier=a=b" ] || fail "attributes: $(cat "$p")"
+  grep -q 'http://10.42.0.1:4318/v1/metrics' "$root/curl.args" || fail "url: $(cat "$root/curl.args")"
+else
+  fail "the function failed: $(cat "$root/err")"
+fi
+
+case="a failing push warns and still returns 0"
+if STUB_CURL_STATUS=22 run "$FUNCTIONS" svc 'agent=x' m.a=1; then
+  grep -q 'warning: could not push' "$root/err" || fail "no warning: $(cat "$root/err")"
+else
+  fail "returned non-zero"
+fi
+
+case="a non-numeric value warns, pushes nothing and returns 0"
+if run "$FUNCTIONS" svc 'agent=x' m.a=oops; then
+  [ ! -e "$root/curl.payload" ] || fail "pushed $(cat "$root/curl.payload")"
+  grep -q 'warning: could not build' "$root/err" || fail "no warning: $(cat "$root/err")"
+else
+  fail "returned non-zero"
+fi
+
+case="without an endpoint nothing is pushed"
+if run "$FUNCTIONS_NO_ENDPOINT" svc 'agent=x' m.a=1; then
+  [ ! -e "$root/curl.payload" ] || fail "pushed $(cat "$root/curl.payload")"
+else
+  fail "returned non-zero"
+fi
+
+[ "$failures" -eq 0 ] || exit 1
+echo "all cases passed"