Watch
0
0
Fork
You've already forked hyperhive
0

swarm-archive: back up every agent's state to an append-only restic repository per agent

swarm-archive serves restic repositories with rclone, append-only, one login
and one repository per agent. swarm-controller writes each placed agent's
login and restic password to the store, replaces them on a move and removes
them when the agent is destroyed. Every hive backs its agents up hourly under
its own store identity. The archive host prunes by retention.
This commit is contained in:
atlas 2026-11-10 07:49:49 +01:00
commit 306aae0378

View file

@ -0,0 +1,137 @@
# The hive's backup run: for each agent that has a state directory here and a
# grant from swarm-controller, one `restic backup` into that agent's repository
# on the archive. A pure function, NOT a NixOS module: ../hive-archive-backup.nix
# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
# same script against a stand-in `bao`.
#
# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
# and the `BAO_*` client set. Every failure prints a line starting
# `archive backup failed`, which the swarm's alert reads, and ends the run with
# a non-zero status once the other agents have been tried.
{ lib }:
{
restic,
# The hive's name in the store: grants are read from
# `secret/swarm/hives/<hive>/archive/`.
hive,
# The directory holding one directory per agent.
agentRoot,
# The repository server's base URL, no trailing slash.
url,
# "paths": `paths` under each agent's directory. "subvolume": the whole directory.
scope,
paths,
excludes,
# A CA file to trust instead of restic's default, or null.
caBundle,
}:
let
storeSeededSecret = import ./store-seeded-secret.nix { };
grants = hive: "secret/swarm/hives/${hive}/archive";
in
''
set -euo pipefail
${storeSeededSecret}
store_init
store_login
if [ "$STORE_UP" != 1 ]; then
echo "archive backup failed: could not log in to the swarm secret store" >&2
exit 1
fi
${lib.optionalString (caBundle != null) ''
export RESTIC_CACERT=${lib.escapeShellArg caBundle}
''}
root=${lib.escapeShellArg agentRoot}
grants=${lib.escapeShellArg (grants hive)}
# One agent: a read-only snapshot of its subvolume when it is one, so the
# databases are copied at one instant, else its directory in place.
backup_agent() {
local agent="$1" leaf="$grants/$1" rest base snap="" target targets=()
rest="$(store_bao kv get -field=rest-password "$leaf")" || return 1
export RESTIC_REPOSITORY="rest:${url}/$agent/"
export RESTIC_REST_USERNAME="$agent"
export RESTIC_REST_PASSWORD="$rest"
export RESTIC_PASSWORD_COMMAND="bao kv get -field=restic-password $leaf"
if ! ${restic}/bin/restic cat config >/dev/null 2>&1; then
${restic}/bin/restic init || return 1
fi
base="$root/$agent"
if btrfs subvolume show "$base" >/dev/null 2>&1; then
snap="$root/.$agent.snapshot.archive"
btrfs subvolume delete "$snap" >/dev/null 2>&1 || true
btrfs subvolume snapshot -r "$base" "$snap" >/dev/null || return 1
base="$snap"
fi
${
if scope == "subvolume" then
''targets=("$base")''
else
''
for target in ${lib.escapeShellArgs paths}; do
[ -e "$base/$target" ] && targets+=("$base/$target")
done''
}
local rc=0
if [ "''${#targets[@]}" = 0 ]; then
echo "$agent has none of its backup paths yet; nothing to back up."
else
${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \
${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
-- "''${targets[@]}" || rc=$?
fi
if [ -n "$snap" ]; then
btrfs subvolume delete "$snap" >/dev/null || rc=1
fi
return "$rc"
}
# The agents the controller granted this hive; none yet is "No value found".
rc=0
granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
if [ "$rc" != 0 ]; then
if grep -q 'No value found' "$STORE_ERR"; then
granted='[]'
else
cat "$STORE_ERR" >&2
echo "archive backup failed: could not list $grants" >&2
exit 1
fi
fi
granted="$(printf '%s' "$granted" | jq -r '.[]')"
failed=0
candidates=0
backed_up=0
for dir in "$root"/*/; do
agent="$(basename -- "$dir")"
case "$agent" in .*) continue ;; esac
[[ "$agent" =~ ^[A-Za-z0-9_-]+$ ]] || continue
candidates=$((candidates + 1))
if ! grep -qxF -- "$agent" <<<"$granted"; then
echo "$agent has no archive credentials on this hive; skipping."
continue
fi
if (backup_agent "$agent"); then
backed_up=$((backed_up + 1))
echo "backed up $agent"
else
echo "archive backup failed: $agent" >&2
failed=1
fi
done
# Agents here and not one of them granted: swarm-controller has not written
# this hive's grants, which a controller without the archive never does.
if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
echo "archive backup failed: swarm-controller has granted this hive no archive credentials ($candidates agents here)" >&2
failed=1
fi
exit "$failed"
''