793d987a88
A client chooses a snapshot's time, so snapshots dated ahead could fill --keep-last and push the real ones out of a prune. The prune now forgets snapshots dated more than an hour after now first, and logs their ids. The VM test forges one and checks the real snapshot stays. The bao-grants case that read a + path as a glob now names the literal path, and two cases that pinned 'nothing else' list the archive grants. The ciRunner stanza is back at its original indent.
194 lines
6.9 KiB
Nix
194 lines
6.9 KiB
Nix
# The archive server's moving parts: the rclone command that serves the
|
|
# repositories, the script that renders its users file from the secret store,
|
|
# and the script that prunes each repository. A pure function, NOT a NixOS
|
|
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
|
|
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
|
|
# `bao`.
|
|
#
|
|
# Both scripts use only `bao login -method=cert -token-only`,
|
|
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
|
|
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
|
|
{ lib }:
|
|
{
|
|
rclone,
|
|
restic,
|
|
# Where the repositories live: `<reposDir>/<agent>/`. The only place that
|
|
# names the backend; `rclone serve restic` takes any remote here.
|
|
reposDir,
|
|
# The address rclone listens on.
|
|
addr,
|
|
# The users file rclone checks logins against, its group and the user the
|
|
# server runs as.
|
|
authFile,
|
|
authGroup,
|
|
# `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
|
|
retention,
|
|
}:
|
|
let
|
|
atomicWriteSecret = import ./atomic-write-secret.nix { };
|
|
refreshConsumer = import ./refresh-consumer.nix { };
|
|
storeSeededSecret = import ./store-seeded-secret.nix { };
|
|
|
|
usersPrefix = "secret/swarm/services/archive/users";
|
|
resticPrefix = "secret/swarm/services/archive/restic";
|
|
readyPath = "secret/swarm/services/archive/ready";
|
|
|
|
# Slack for the clocks of the hives that stamp snapshots.
|
|
futureSkewSeconds = 3600;
|
|
|
|
keepFlags = lib.concatStringsSep " " (
|
|
[ "--keep-last ${toString retention.keepLast}" ]
|
|
++ lib.optional (
|
|
retention.keepWithin != null
|
|
) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
|
|
);
|
|
in
|
|
{
|
|
inherit usersPrefix resticPrefix;
|
|
|
|
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
|
# index, so a hive that is compromised cannot erase history. Private repos:
|
|
# a login reaches `/<its own name>/` and nothing else.
|
|
serverCommand = lib.escapeShellArgs [
|
|
"${rclone}/bin/rclone"
|
|
"serve"
|
|
"restic"
|
|
"--addr"
|
|
addr
|
|
"--append-only"
|
|
"--private-repos"
|
|
"--htpasswd"
|
|
authFile
|
|
reposDir
|
|
];
|
|
|
|
# Renders the users file from `usersPrefix`, one leaf per agent holding that
|
|
# agent's htpasswd line. The file is written only when the content changed,
|
|
# and never from a partial read: any leaf that cannot be read leaves the file
|
|
# as it is.
|
|
usersScript = ''
|
|
set -euo pipefail
|
|
${storeSeededSecret}
|
|
${atomicWriteSecret}
|
|
${refreshConsumer}
|
|
|
|
file=${lib.escapeShellArg authFile}
|
|
# The server will not start without the file; an empty one admits nobody.
|
|
if [ ! -e "$file" ]; then
|
|
atomic_write_secret 0640 root:${authGroup} "$file" ""
|
|
fi
|
|
|
|
store_init
|
|
store_login
|
|
if [ "$STORE_UP" != 1 ]; then
|
|
echo "could not log in to the swarm secret store; $file stays as it is." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Tells swarm-controller an archive exists; it grants no credentials until
|
|
# then. Created once, never rewritten.
|
|
if ! store_bao kv get -field=ready ${lib.escapeShellArg readyPath} >/dev/null 2>&1; then
|
|
store_bao kv put -cas=0 ${lib.escapeShellArg readyPath} ready=1 >/dev/null
|
|
fi
|
|
|
|
# A prefix that holds nothing is "No value found"; it means no agent has a
|
|
# user yet, not a failure.
|
|
rc=0
|
|
names="$(store_bao kv list -format=json ${lib.escapeShellArg usersPrefix} 2>"$STORE_ERR")" || rc=$?
|
|
if [ "$rc" != 0 ]; then
|
|
if grep -q 'No value found' "$STORE_ERR"; then
|
|
names='[]'
|
|
else
|
|
cat "$STORE_ERR" >&2
|
|
echo "could not list ${usersPrefix}; $file stays as it is." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
content=""
|
|
while IFS= read -r name; do
|
|
[ -n "$name" ] || continue
|
|
case "$name" in */) continue ;; esac
|
|
if ! line="$(store_bao kv get -field=htpasswd ${lib.escapeShellArg usersPrefix}/"$name" 2>"$STORE_ERR")"; then
|
|
cat "$STORE_ERR" >&2
|
|
echo "could not read the user $name; $file stays as it is." >&2
|
|
exit 1
|
|
fi
|
|
# The line must be this agent's own: a leaf cannot add another user.
|
|
if [[ ! "$name" =~ ^[A-Za-z0-9_-]+$ ]] || [[ "$line" != "$name:{SHA}"* ]] \
|
|
|| [[ ! "$line" =~ ^[A-Za-z0-9_-]+:\{SHA\}[A-Za-z0-9+/=]+$ ]]; then
|
|
echo "the user leaf $name does not hold a {SHA} line for $name; leaving it out." >&2
|
|
continue
|
|
fi
|
|
content+="$line"$'\n'
|
|
done < <(printf '%s' "$names" | jq -r '.[]')
|
|
content="''${content%$'\n'}"
|
|
|
|
if secret_differs "$file" "$content"; then
|
|
atomic_write_secret 0640 root:${authGroup} "$file" "$content"
|
|
echo "users file updated."
|
|
fi
|
|
'';
|
|
|
|
# `restic forget --prune` on every repository, with its password read from
|
|
# the store at run time. The server refuses deletes from clients, so this is
|
|
# the only place history is dropped.
|
|
#
|
|
# restic ranks snapshots by their time, which the client chooses (`restic
|
|
# backup --time`). Snapshots dated years ahead would fill `--keep-last` and
|
|
# push the real ones out, so each prune first forgets the snapshots dated more
|
|
# than `futureSkewSeconds` after now. restic has no rule relative to the
|
|
# clock for `--keep-last`.
|
|
pruneScript = ''
|
|
set -euo pipefail
|
|
${storeSeededSecret}
|
|
|
|
# Forgets the snapshots of $repo dated after the limit, naming them in the
|
|
# log; returns non-zero when it cannot read the list or a time.
|
|
forget_future() {
|
|
local repo="$1" agent="$2" snaps id time at future="" limit
|
|
limit=$(( $(date +%s) + ${toString futureSkewSeconds} ))
|
|
snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json \
|
|
| jq -r '.[] | "\(.id) \(.time)"')" || return 1
|
|
while read -r id time; do
|
|
[ -n "$id" ] || continue
|
|
at="$(date -d "$time" +%s)" || return 1
|
|
if [ "$at" -gt "$limit" ]; then
|
|
future+=" $id"
|
|
fi
|
|
done <<<"$snaps"
|
|
if [ -n "$future" ]; then
|
|
echo "archive prune warning: $agent has snapshots dated after now, forgetting them:$future" >&2
|
|
# shellcheck disable=SC2086
|
|
${restic}/bin/restic -r "$repo" --retry-lock 10m forget $future || return 1
|
|
fi
|
|
}
|
|
|
|
store_init
|
|
store_login
|
|
if [ "$STORE_UP" != 1 ]; then
|
|
echo "archive prune failed: could not log in to the swarm secret store" >&2
|
|
exit 1
|
|
fi
|
|
|
|
failed=0
|
|
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
|
[ -e "$repo/config" ] || continue
|
|
agent="$(basename -- "$repo")"
|
|
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
|
echo "archive prune failed: $agent is not an agent name" >&2
|
|
failed=1
|
|
continue
|
|
fi
|
|
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
|
|
if forget_future "$repo" "$agent" \
|
|
&& ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
|
|
echo "pruned $agent"
|
|
else
|
|
echo "archive prune failed: $agent" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
exit "$failed"
|
|
'';
|
|
}
|