swarm-archive: back up every agent's state to an append-only restic repository per agent
swarm-archive serves restic repositories with rclone, append-only, one login and one repository per agent. swarm-controller writes each placed agent's login and restic password to the store, replaces them on a move and removes them when the agent is destroyed. Every hive backs its agents up hourly under its own store identity. The archive host prunes by retention.
This commit is contained in:
atlas
parent
409e7ee5a0
commit
306aae0378
2 changed files with 2214 additions and 5 deletions
137
nix/host-modules/lib/archive-backup.nix
Normal file
137
nix/host-modules/lib/archive-backup.nix
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
# The hive's backup run: for each agent that has a state directory here and a
|
||||
# grant from swarm-controller, one `restic backup` into that agent's repository
|
||||
# on the archive. A pure function, NOT a NixOS module: ../hive-archive-backup.nix
|
||||
# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
|
||||
# same script against a stand-in `bao`.
|
||||
#
|
||||
# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
|
||||
# and the `BAO_*` client set. Every failure prints a line starting
|
||||
# `archive backup failed`, which the swarm's alert reads, and ends the run with
|
||||
# a non-zero status once the other agents have been tried.
|
||||
{ lib }:
|
||||
{
|
||||
restic,
|
||||
# The hive's name in the store: grants are read from
|
||||
# `secret/swarm/hives/<hive>/archive/`.
|
||||
hive,
|
||||
# The directory holding one directory per agent.
|
||||
agentRoot,
|
||||
# The repository server's base URL, no trailing slash.
|
||||
url,
|
||||
# "paths": `paths` under each agent's directory. "subvolume": the whole directory.
|
||||
scope,
|
||||
paths,
|
||||
excludes,
|
||||
# A CA file to trust instead of restic's default, or null.
|
||||
caBundle,
|
||||
}:
|
||||
let
|
||||
storeSeededSecret = import ./store-seeded-secret.nix { };
|
||||
|
||||
grants = hive: "secret/swarm/hives/${hive}/archive";
|
||||
in
|
||||
''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "archive backup failed: could not log in to the swarm secret store" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
${lib.optionalString (caBundle != null) ''
|
||||
export RESTIC_CACERT=${lib.escapeShellArg caBundle}
|
||||
''}
|
||||
root=${lib.escapeShellArg agentRoot}
|
||||
grants=${lib.escapeShellArg (grants hive)}
|
||||
|
||||
# One agent: a read-only snapshot of its subvolume when it is one, so the
|
||||
# databases are copied at one instant, else its directory in place.
|
||||
backup_agent() {
|
||||
local agent="$1" leaf="$grants/$1" rest base snap="" target targets=()
|
||||
rest="$(store_bao kv get -field=rest-password "$leaf")" || return 1
|
||||
export RESTIC_REPOSITORY="rest:${url}/$agent/"
|
||||
export RESTIC_REST_USERNAME="$agent"
|
||||
export RESTIC_REST_PASSWORD="$rest"
|
||||
export RESTIC_PASSWORD_COMMAND="bao kv get -field=restic-password $leaf"
|
||||
|
||||
if ! ${restic}/bin/restic cat config >/dev/null 2>&1; then
|
||||
${restic}/bin/restic init || return 1
|
||||
fi
|
||||
|
||||
base="$root/$agent"
|
||||
if btrfs subvolume show "$base" >/dev/null 2>&1; then
|
||||
snap="$root/.$agent.snapshot.archive"
|
||||
btrfs subvolume delete "$snap" >/dev/null 2>&1 || true
|
||||
btrfs subvolume snapshot -r "$base" "$snap" >/dev/null || return 1
|
||||
base="$snap"
|
||||
fi
|
||||
|
||||
${
|
||||
if scope == "subvolume" then
|
||||
''targets=("$base")''
|
||||
else
|
||||
''
|
||||
for target in ${lib.escapeShellArgs paths}; do
|
||||
[ -e "$base/$target" ] && targets+=("$base/$target")
|
||||
done''
|
||||
}
|
||||
local rc=0
|
||||
if [ "''${#targets[@]}" = 0 ]; then
|
||||
echo "$agent has none of its backup paths yet; nothing to back up."
|
||||
else
|
||||
${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \
|
||||
${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
|
||||
-- "''${targets[@]}" || rc=$?
|
||||
fi
|
||||
if [ -n "$snap" ]; then
|
||||
btrfs subvolume delete "$snap" >/dev/null || rc=1
|
||||
fi
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# The agents the controller granted this hive; none yet is "No value found".
|
||||
rc=0
|
||||
granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
|
||||
if [ "$rc" != 0 ]; then
|
||||
if grep -q 'No value found' "$STORE_ERR"; then
|
||||
granted='[]'
|
||||
else
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "archive backup failed: could not list $grants" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
granted="$(printf '%s' "$granted" | jq -r '.[]')"
|
||||
|
||||
failed=0
|
||||
candidates=0
|
||||
backed_up=0
|
||||
for dir in "$root"/*/; do
|
||||
agent="$(basename -- "$dir")"
|
||||
case "$agent" in .*) continue ;; esac
|
||||
[[ "$agent" =~ ^[A-Za-z0-9_-]+$ ]] || continue
|
||||
candidates=$((candidates + 1))
|
||||
if ! grep -qxF -- "$agent" <<<"$granted"; then
|
||||
echo "$agent has no archive credentials on this hive; skipping."
|
||||
continue
|
||||
fi
|
||||
if (backup_agent "$agent"); then
|
||||
backed_up=$((backed_up + 1))
|
||||
echo "backed up $agent"
|
||||
else
|
||||
echo "archive backup failed: $agent" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Agents here and not one of them granted: swarm-controller has not written
|
||||
# this hive's grants, which a controller without the archive never does.
|
||||
if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
|
||||
echo "archive backup failed: swarm-controller has granted this hive no archive credentials ($candidates agents here)" >&2
|
||||
failed=1
|
||||
fi
|
||||
exit "$failed"
|
||||
''
|
||||
156
nix/host-modules/lib/archive-server.nix
Normal file
156
nix/host-modules/lib/archive-server.nix
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
# The archive server's moving parts: the rclone command that serves the
|
||||
# repositories, the script that renders its users file from the secret store,
|
||||
# and the script that prunes each repository. A pure function, NOT a NixOS
|
||||
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
|
||||
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
|
||||
# `bao`.
|
||||
#
|
||||
# Both scripts use only `bao login -method=cert -token-only`,
|
||||
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
|
||||
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
|
||||
{ lib }:
|
||||
{
|
||||
rclone,
|
||||
restic,
|
||||
# Where the repositories live: `<reposDir>/<agent>/`. The only place that
|
||||
# names the backend; `rclone serve restic` takes any remote here.
|
||||
reposDir,
|
||||
# The address rclone listens on.
|
||||
addr,
|
||||
# The users file rclone checks logins against, its group and the user the
|
||||
# server runs as.
|
||||
authFile,
|
||||
authGroup,
|
||||
# `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
|
||||
retention,
|
||||
}:
|
||||
let
|
||||
atomicWriteSecret = import ./atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./refresh-consumer.nix { };
|
||||
storeSeededSecret = import ./store-seeded-secret.nix { };
|
||||
|
||||
usersPrefix = "secret/swarm/services/archive/users";
|
||||
resticPrefix = "secret/swarm/services/archive/restic";
|
||||
|
||||
keepFlags = lib.concatStringsSep " " (
|
||||
[ "--keep-last ${toString retention.keepLast}" ]
|
||||
++ lib.optional (
|
||||
retention.keepWithin != null
|
||||
) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit usersPrefix resticPrefix;
|
||||
|
||||
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
||||
# index, so a hive that is compromised cannot erase history. Private repos:
|
||||
# a login reaches `/<its own name>/` and nothing else.
|
||||
serverCommand = lib.escapeShellArgs [
|
||||
"${rclone}/bin/rclone"
|
||||
"serve"
|
||||
"restic"
|
||||
"--addr"
|
||||
addr
|
||||
"--append-only"
|
||||
"--private-repos"
|
||||
"--htpasswd"
|
||||
authFile
|
||||
reposDir
|
||||
];
|
||||
|
||||
# Renders the users file from `usersPrefix`, one leaf per agent holding that
|
||||
# agent's htpasswd line. The file is written only when the content changed,
|
||||
# and never from a partial read: any leaf that cannot be read leaves the file
|
||||
# as it is.
|
||||
usersScript = ''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
file=${lib.escapeShellArg authFile}
|
||||
# The server will not start without the file; an empty one admits nobody.
|
||||
if [ ! -e "$file" ]; then
|
||||
atomic_write_secret 0640 root:${authGroup} "$file" ""
|
||||
fi
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "could not log in to the swarm secret store; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A prefix that holds nothing is "No value found"; it means no agent has a
|
||||
# user yet, not a failure.
|
||||
rc=0
|
||||
names="$(store_bao kv list -format=json ${lib.escapeShellArg usersPrefix} 2>"$STORE_ERR")" || rc=$?
|
||||
if [ "$rc" != 0 ]; then
|
||||
if grep -q 'No value found' "$STORE_ERR"; then
|
||||
names='[]'
|
||||
else
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "could not list ${usersPrefix}; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
content=""
|
||||
while IFS= read -r name; do
|
||||
[ -n "$name" ] || continue
|
||||
case "$name" in */) continue ;; esac
|
||||
if ! line="$(store_bao kv get -field=htpasswd ${lib.escapeShellArg usersPrefix}/"$name" 2>"$STORE_ERR")"; then
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "could not read the user $name; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
# The line must be this agent's own: a leaf cannot add another user.
|
||||
if [[ ! "$name" =~ ^[A-Za-z0-9_-]+$ ]] || [[ "$line" != "$name:{SHA}"* ]] \
|
||||
|| [[ ! "$line" =~ ^[A-Za-z0-9_-]+:\{SHA\}[A-Za-z0-9+/=]+$ ]]; then
|
||||
echo "the user leaf $name does not hold a {SHA} line for $name; leaving it out." >&2
|
||||
continue
|
||||
fi
|
||||
content+="$line"$'\n'
|
||||
done < <(printf '%s' "$names" | jq -r '.[]')
|
||||
content="''${content%$'\n'}"
|
||||
|
||||
if secret_differs "$file" "$content"; then
|
||||
atomic_write_secret 0640 root:${authGroup} "$file" "$content"
|
||||
echo "users file updated."
|
||||
fi
|
||||
'';
|
||||
|
||||
# `restic forget --prune` on every repository, with its password read from
|
||||
# the store at run time. The server refuses deletes from clients, so this is
|
||||
# the only place history is dropped.
|
||||
pruneScript = ''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "archive prune failed: could not log in to the swarm secret store" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
failed=0
|
||||
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
||||
[ -e "$repo/config" ] || continue
|
||||
agent="$(basename -- "$repo")"
|
||||
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
||||
echo "archive prune failed: $agent is not an agent name" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
|
||||
if ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
|
||||
echo "pruned $agent"
|
||||
else
|
||||
echo "archive prune failed: $agent" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
exit "$failed"
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue