Watch
0
0
Fork
You've already forked hyperhive
0

archive: push restic backup and prune metrics, and add a backups dashboard

This commit is contained in:
atlas 2026-11-14 11:06:38 +01:00
commit 723ce866f1

View file

@ -4,11 +4,15 @@
# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
# same script against a stand-in `bao`.
#
# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
# Needs `bao`, `jq`, `curl`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
# and the `BAO_*` client set. A store that cannot be reached, or an agent whose
# backup fails, prints a line starting
# `archive backup failed`, which the swarm's alert reads, and ends the run with
# a non-zero status once the other agents have been tried.
#
# After each agent it pushes `hyperhive.archive.backup.*` gauges (`agent` attribute)
# through ./otlp-gauges.nix: last run and last success time, result, wall time,
# bytes added and files new and changed. A failed agent still reports its result.
{ lib }:
{
restic,
@ -25,11 +29,15 @@
excludes,
# A CA file to trust instead of restic's default, or null.
caBundle,
# The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
otlpEndpoint ? null,
}:
let
storeSeededSecret = import ./store-seeded-secret.nix { };
grants = hive: "secret/swarm/hives/${hive}/archive";
otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
in
''
set -euo pipefail
@ -45,6 +53,7 @@ in
${lib.optionalString (caBundle != null) ''
export RESTIC_CACERT=${lib.escapeShellArg caBundle}
''}
${otlpGauges}
root=${lib.escapeShellArg agentRoot}
grants=${lib.escapeShellArg (grants hive)}
@ -83,9 +92,14 @@ in
if [ "''${#targets[@]}" = 0 ]; then
echo "$agent has none of its backup paths yet; nothing to back up."
else
${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \
# `--json` for the summary line the metrics read. Only that line and the
# errors are printed; the progress lines are dropped.
${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \
${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
-- "''${targets[@]}" || rc=$?
-- "''${targets[@]}" |
jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j
| if $j == null or $j.message_type == "error" or $j.message_type == "summary"
then $line else empty end' || rc=$?
fi
if [ -n "$snap" ]; then
btrfs subvolume delete "$snap" >/dev/null || rc=1
@ -93,6 +107,29 @@ in
return "$rc"
}
# One agent's run as gauges: `$5` is its output, whose `summary` line has the
# counts when restic got that far. Never fails the run.
push_backup_metrics() {
local agent="$1" ok="$2" started="$3" finished="$4" out="$5" summary
local metrics=(
"hyperhive.archive.backup.last_run_timestamp_seconds=$finished"
"hyperhive.archive.backup.last_run_success=$ok"
"hyperhive.archive.backup.duration_seconds=$((finished - started))"
)
if [ "$ok" = 1 ]; then
metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished")
summary="$(printf '%s\n' "$out" | jq -R -r '
fromjson? | select(type == "object" and .message_type == "summary")
| "hyperhive.archive.backup.data_added_bytes=\(.data_added)",
"hyperhive.archive.backup.files_new=\(.files_new)",
"hyperhive.archive.backup.files_changed=\(.files_changed)"' 2>/dev/null)" || summary=""
while IFS= read -r line; do
[ -z "$line" ] || metrics+=("$line")
done <<<"$summary"
fi
otlp_gauges hive-archive-backup "agent=$agent" "''${metrics[@]}"
}
# The agents the controller granted this hive; none yet is "No value found".
rc=0
granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
@ -119,13 +156,19 @@ in
echo "$agent has no archive credentials on this hive; skipping."
continue
fi
if (backup_agent "$agent"); then
started="$(date +%s)"
ok=0
out="$(backup_agent "$agent")" && ok=1
[ -z "$out" ] || printf '%s\n' "$out"
finished="$(date +%s)"
if [ "$ok" = 1 ]; then
backed_up=$((backed_up + 1))
echo "backed up $agent"
else
echo "archive backup failed: $agent" >&2
failed=1
fi
push_backup_metrics "$agent" "$ok" "$started" "$finished" "$out"
done
# No grant for any agent here: the swarm has no archive, or its controller