swarm-archive: back up every agent's state to an append-only restic repository per agent
swarm-archive serves restic repositories with rclone, append-only, one login and one repository per agent. swarm-controller writes each placed agent's login and restic password to the store, replaces them on a move and removes them when the agent is destroyed. Every hive backs its agents up hourly under its own store identity. The archive host prunes by retention.
This commit is contained in:
atlas
parent
409e7ee5a0
commit
306aae0378
1 changed files with 2214 additions and 5 deletions
156
nix/host-modules/lib/archive-server.nix
Normal file
156
nix/host-modules/lib/archive-server.nix
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
# The archive server's moving parts: the rclone command that serves the
|
||||
# repositories, the script that renders its users file from the secret store,
|
||||
# and the script that prunes each repository. A pure function, NOT a NixOS
|
||||
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
|
||||
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
|
||||
# `bao`.
|
||||
#
|
||||
# Both scripts use only `bao login -method=cert -token-only`,
|
||||
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
|
||||
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
|
||||
{ lib }:
|
||||
{
|
||||
rclone,
|
||||
restic,
|
||||
# Where the repositories live: `<reposDir>/<agent>/`. The only place that
|
||||
# names the backend; `rclone serve restic` takes any remote here.
|
||||
reposDir,
|
||||
# The address rclone listens on.
|
||||
addr,
|
||||
# The users file rclone checks logins against, its group and the user the
|
||||
# server runs as.
|
||||
authFile,
|
||||
authGroup,
|
||||
# `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
|
||||
retention,
|
||||
}:
|
||||
let
|
||||
atomicWriteSecret = import ./atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./refresh-consumer.nix { };
|
||||
storeSeededSecret = import ./store-seeded-secret.nix { };
|
||||
|
||||
usersPrefix = "secret/swarm/services/archive/users";
|
||||
resticPrefix = "secret/swarm/services/archive/restic";
|
||||
|
||||
keepFlags = lib.concatStringsSep " " (
|
||||
[ "--keep-last ${toString retention.keepLast}" ]
|
||||
++ lib.optional (
|
||||
retention.keepWithin != null
|
||||
) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit usersPrefix resticPrefix;
|
||||
|
||||
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
||||
# index, so a hive that is compromised cannot erase history. Private repos:
|
||||
# a login reaches `/<its own name>/` and nothing else.
|
||||
serverCommand = lib.escapeShellArgs [
|
||||
"${rclone}/bin/rclone"
|
||||
"serve"
|
||||
"restic"
|
||||
"--addr"
|
||||
addr
|
||||
"--append-only"
|
||||
"--private-repos"
|
||||
"--htpasswd"
|
||||
authFile
|
||||
reposDir
|
||||
];
|
||||
|
||||
# Renders the users file from `usersPrefix`, one leaf per agent holding that
|
||||
# agent's htpasswd line. The file is written only when the content changed,
|
||||
# and never from a partial read: any leaf that cannot be read leaves the file
|
||||
# as it is.
|
||||
usersScript = ''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
file=${lib.escapeShellArg authFile}
|
||||
# The server will not start without the file; an empty one admits nobody.
|
||||
if [ ! -e "$file" ]; then
|
||||
atomic_write_secret 0640 root:${authGroup} "$file" ""
|
||||
fi
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "could not log in to the swarm secret store; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A prefix that holds nothing is "No value found"; it means no agent has a
|
||||
# user yet, not a failure.
|
||||
rc=0
|
||||
names="$(store_bao kv list -format=json ${lib.escapeShellArg usersPrefix} 2>"$STORE_ERR")" || rc=$?
|
||||
if [ "$rc" != 0 ]; then
|
||||
if grep -q 'No value found' "$STORE_ERR"; then
|
||||
names='[]'
|
||||
else
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "could not list ${usersPrefix}; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
content=""
|
||||
while IFS= read -r name; do
|
||||
[ -n "$name" ] || continue
|
||||
case "$name" in */) continue ;; esac
|
||||
if ! line="$(store_bao kv get -field=htpasswd ${lib.escapeShellArg usersPrefix}/"$name" 2>"$STORE_ERR")"; then
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "could not read the user $name; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
# The line must be this agent's own: a leaf cannot add another user.
|
||||
if [[ ! "$name" =~ ^[A-Za-z0-9_-]+$ ]] || [[ "$line" != "$name:{SHA}"* ]] \
|
||||
|| [[ ! "$line" =~ ^[A-Za-z0-9_-]+:\{SHA\}[A-Za-z0-9+/=]+$ ]]; then
|
||||
echo "the user leaf $name does not hold a {SHA} line for $name; leaving it out." >&2
|
||||
continue
|
||||
fi
|
||||
content+="$line"$'\n'
|
||||
done < <(printf '%s' "$names" | jq -r '.[]')
|
||||
content="''${content%$'\n'}"
|
||||
|
||||
if secret_differs "$file" "$content"; then
|
||||
atomic_write_secret 0640 root:${authGroup} "$file" "$content"
|
||||
echo "users file updated."
|
||||
fi
|
||||
'';
|
||||
|
||||
# `restic forget --prune` on every repository, with its password read from
|
||||
# the store at run time. The server refuses deletes from clients, so this is
|
||||
# the only place history is dropped.
|
||||
pruneScript = ''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "archive prune failed: could not log in to the swarm secret store" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
failed=0
|
||||
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
||||
[ -e "$repo/config" ] || continue
|
||||
agent="$(basename -- "$repo")"
|
||||
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
||||
echo "archive prune failed: $agent is not an agent name" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
|
||||
if ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
|
||||
echo "pruned $agent"
|
||||
else
|
||||
echo "archive prune failed: $agent" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
exit "$failed"
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue