186 lines
6.8 KiB
Nix
186 lines
6.8 KiB
Nix
# The hive's backup run: for each agent that has a state directory here and a
|
|
# grant from swarm-controller, one `restic backup` into that agent's repository
|
|
# on the archive. A pure function, NOT a NixOS module: ../hive-archive-backup.nix
|
|
# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
|
|
# same script against a stand-in `bao`.
|
|
#
|
|
# Needs `bao`, `jq`, `curl`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
|
|
# and the `BAO_*` client set. A store that cannot be reached, or an agent whose
|
|
# backup fails, prints a line starting
|
|
# `archive backup failed`, which the swarm's alert reads, and ends the run with
|
|
# a non-zero status once the other agents have been tried.
|
|
#
|
|
# After each agent it pushes `hyperhive.archive.backup.*` gauges (`agent` attribute)
|
|
# through ./otlp-gauges.nix: last run and last success time, result, wall time,
|
|
# bytes added and files new and changed. A failed agent still reports its result.
|
|
{ lib }:
|
|
{
|
|
restic,
|
|
# The hive's name in the store: grants are read from
|
|
# `secret/swarm/hives/<hive>/archive/`.
|
|
hive,
|
|
# The directory holding one directory per agent.
|
|
agentRoot,
|
|
# The repository server's base URL, no trailing slash.
|
|
url,
|
|
# "paths": `paths` under each agent's directory. "subvolume": the whole directory.
|
|
scope,
|
|
paths,
|
|
excludes,
|
|
# A CA file to trust instead of restic's default, or null.
|
|
caBundle,
|
|
# The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
|
|
otlpEndpoint ? null,
|
|
}:
|
|
let
|
|
storeSeededSecret = import ./store-seeded-secret.nix { };
|
|
|
|
grants = hive: "secret/swarm/hives/${hive}/archive";
|
|
|
|
otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
|
|
in
|
|
''
|
|
set -euo pipefail
|
|
${storeSeededSecret}
|
|
|
|
store_init
|
|
store_login
|
|
if [ "$STORE_UP" != 1 ]; then
|
|
echo "archive backup failed: could not log in to the swarm secret store" >&2
|
|
exit 1
|
|
fi
|
|
|
|
${lib.optionalString (caBundle != null) ''
|
|
export RESTIC_CACERT=${lib.escapeShellArg caBundle}
|
|
''}
|
|
${otlpGauges}
|
|
root=${lib.escapeShellArg agentRoot}
|
|
grants=${lib.escapeShellArg (grants hive)}
|
|
|
|
# restic's `--json` output on stdin, reduced to its error and summary lines;
|
|
# lines that are not JSON objects pass through.
|
|
restic_output_filter() {
|
|
jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j
|
|
| if $j == null or $j.message_type == "error" or $j.message_type == "summary"
|
|
then $line else empty end'
|
|
}
|
|
|
|
# One agent: a read-only snapshot of its subvolume when it is one, so the
|
|
# databases are copied at one instant, else its directory in place.
|
|
backup_agent() {
|
|
local agent="$1" leaf="$grants/$1" rest base snap="" target targets=()
|
|
rest="$(store_bao kv get -field=rest-password "$leaf")" || return 1
|
|
export RESTIC_REPOSITORY="rest:${url}/$agent/"
|
|
export RESTIC_REST_USERNAME="$agent"
|
|
export RESTIC_REST_PASSWORD="$rest"
|
|
export RESTIC_PASSWORD_COMMAND="bao kv get -field=restic-password $leaf"
|
|
|
|
if ! ${restic}/bin/restic cat config >/dev/null 2>&1; then
|
|
${restic}/bin/restic init || return 1
|
|
fi
|
|
|
|
base="$root/$agent"
|
|
if btrfs subvolume show "$base" >/dev/null 2>&1; then
|
|
snap="$root/.$agent.snapshot.archive"
|
|
btrfs subvolume delete "$snap" >/dev/null 2>&1 || true
|
|
btrfs subvolume snapshot -r "$base" "$snap" >/dev/null || return 1
|
|
base="$snap"
|
|
fi
|
|
|
|
${
|
|
if scope == "subvolume" then
|
|
''targets=("$base")''
|
|
else
|
|
''
|
|
for target in ${lib.escapeShellArgs paths}; do
|
|
[ -e "$base/$target" ] && targets+=("$base/$target")
|
|
done''
|
|
}
|
|
local rc=0
|
|
if [ "''${#targets[@]}" = 0 ]; then
|
|
echo "$agent has none of its backup paths yet; nothing to back up."
|
|
else
|
|
# `--json` for the summary line the metrics read. Only that line and the
|
|
# errors are printed; the progress lines are dropped.
|
|
${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \
|
|
${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
|
|
-- "''${targets[@]}" |
|
|
restic_output_filter || rc=$?
|
|
fi
|
|
if [ -n "$snap" ]; then
|
|
btrfs subvolume delete "$snap" >/dev/null || rc=1
|
|
fi
|
|
return "$rc"
|
|
}
|
|
|
|
# One agent's run as gauges: `$5` is its output, whose `summary` line has the
|
|
# counts when restic got that far. Never fails the run.
|
|
push_backup_metrics() {
|
|
local agent="$1" ok="$2" started="$3" finished="$4" out="$5" summary
|
|
local metrics=(
|
|
"hyperhive.archive.backup.last_run_timestamp_seconds=$finished"
|
|
"hyperhive.archive.backup.last_run_success=$ok"
|
|
"hyperhive.archive.backup.duration_seconds=$((finished - started))"
|
|
)
|
|
if [ "$ok" = 1 ]; then
|
|
metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished")
|
|
summary="$(printf '%s\n' "$out" | jq -R -r '
|
|
fromjson? | select(type == "object" and .message_type == "summary")
|
|
| {data_added_bytes: .data_added, files_new: .files_new, files_changed: .files_changed}
|
|
| to_entries[] | select(.value | type == "number")
|
|
| "hyperhive.archive.backup.\(.key)=\(.value)"' 2>/dev/null)" || summary=""
|
|
while IFS= read -r line; do
|
|
[ -z "$line" ] || metrics+=("$line")
|
|
done <<<"$summary"
|
|
fi
|
|
otlp_gauges hive-archive-backup "agent=$agent" "''${metrics[@]}"
|
|
}
|
|
|
|
# The agents the controller granted this hive; none yet is "No value found".
|
|
rc=0
|
|
granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
|
|
if [ "$rc" != 0 ]; then
|
|
if grep -q 'No value found' "$STORE_ERR"; then
|
|
granted='[]'
|
|
else
|
|
cat "$STORE_ERR" >&2
|
|
echo "archive backup failed: could not list $grants" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
granted="$(printf '%s' "$granted" | jq -r '.[]')"
|
|
|
|
failed=0
|
|
candidates=0
|
|
backed_up=0
|
|
for dir in "$root"/*/; do
|
|
agent="$(basename -- "$dir")"
|
|
case "$agent" in .*) continue ;; esac
|
|
[[ "$agent" =~ ^[A-Za-z0-9_-]+$ ]] || continue
|
|
candidates=$((candidates + 1))
|
|
if ! grep -qxF -- "$agent" <<<"$granted"; then
|
|
echo "$agent has no archive credentials on this hive; skipping."
|
|
continue
|
|
fi
|
|
started="$(date +%s)"
|
|
ok=0
|
|
out="$(backup_agent "$agent")" && ok=1
|
|
[ -z "$out" ] || printf '%s\n' "$out"
|
|
finished="$(date +%s)"
|
|
if [ "$ok" = 1 ]; then
|
|
backed_up=$((backed_up + 1))
|
|
echo "backed up $agent"
|
|
else
|
|
echo "archive backup failed: $agent" >&2
|
|
failed=1
|
|
fi
|
|
push_backup_metrics "$agent" "$ok" "$started" "$finished" "$out"
|
|
done
|
|
|
|
# No grant for any agent here: the swarm has no archive, or its controller
|
|
# has not granted this hive yet. Neither is a failure of this hive.
|
|
if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
|
|
echo "swarm-controller has granted this hive no archive credentials ($candidates agents here); nothing to back up."
|
|
fi
|
|
exit "$failed"
|
|
''
|