mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

archive: push restic backup and prune metrics, and add a backups dashboard

Browse source
This commit is contained in:
atlas 2026-11-14 11:06:38 +01:00
commit 723ce866f1
1 changed files with 1288 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

36
nix/host-modules/lib/archive-server.nix
View file

@ -31,11 +31,14 @@
authGroup,
# `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
retention,
# The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
otlpEndpoint ? null,
}:
let
atomicWriteSecret = import ./atomic-write-secret.nix { };
refreshConsumer = import ./refresh-consumer.nix { };
storeSeededSecret = import ./store-seeded-secret.nix { };
otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
usersPrefix = "secret/swarm/services/archive/users";
resticPrefix = "secret/swarm/services/archive/restic";
@ -242,6 +245,36 @@ in
fi
}
${otlpGauges}
# Pushes one repository's gauges: the outcome of this prune, the snapshot
# count and newest snapshot time, and the size of the data in the repository.
# Reads the repository after the prune. Never fails the run.
push_repo_metrics() {
local repo="$1" agent="$2" ok="$3" snaps stats time at latest="" count
local metrics=(
"hyperhive.archive.prune.last_run_success=$ok"
"hyperhive.archive.prune.last_run_timestamp_seconds=$(date +%s)"
)
if snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json 2>/dev/null)" \
&& count="$(jq 'length' <<<"$snaps" 2>/dev/null)"; then
metrics+=("hyperhive.archive.repo.snapshots=$count")
while read -r time; do
[ -n "$time" ] || continue
at="$(date -d "$time" +%s 2>/dev/null)" || continue
if [ -z "$latest" ] || [ "$at" -gt "$latest" ]; then latest="$at"; fi
done < <(jq -r '.[].time' <<<"$snaps")
if [ -n "$latest" ]; then
metrics+=("hyperhive.archive.repo.latest_snapshot_timestamp_seconds=$latest")
fi
fi
if stats="$(${restic}/bin/restic -r "$repo" --retry-lock 10m stats --mode raw-data --json 2>/dev/null)" \
&& stats="$(jq -r '.total_size' <<<"$stats" 2>/dev/null)"; then
metrics+=("hyperhive.archive.repo.size_bytes=$stats")
fi
otlp_gauges swarm-archive-prune "agent=$agent" "''${metrics[@]}"
}
store_init
store_login
if [ "$STORE_UP" != 1 ]; then
@ -265,13 +298,16 @@ in
fi
repo=${repoOf}
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
ok=0
if forget_future "$repo" "$agent" \
&& ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
echo "pruned $agent"
ok=1
else
echo "archive prune failed: $agent" >&2
failed=1
fi
push_repo_metrics "$repo" "$agent" "$ok"
done
exit "$failed"
'';