86c5dac89d
- swarm-archive.nix, lib/archive-server.nix: deploy.archive.backend and deploy.archive.s3.* options; the s3 server and the prune read the key from the store into rclone's environment; the prune lists and addresses repositories through rclone instead of a local directory - swarm-bao.nix: the archive reads swarm/services/archive/s3/* - docs: archive.md "External S3" section, two credentials-reference rows - module-eval: builder.nix cases for local and s3, the missing-setting assertion; bao-grants.nix gains the s3 read grant - vm test: second node with a fake rclone serve s3 whose key is generated at test time
278 lines
10 KiB
Nix
278 lines
10 KiB
Nix
# The archive server's moving parts: the rclone command that serves the
|
|
# repositories, the script that renders its users file from the secret store,
|
|
# and the script that prunes each repository. A pure function, NOT a NixOS
|
|
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
|
|
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
|
|
# `bao`.
|
|
#
|
|
# All scripts use only `bao login -method=cert -token-only`,
|
|
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
|
|
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
|
|
#
|
|
# The repositories live in a directory (`reposDir`) or in an S3 bucket (`s3`).
|
|
# Only the pieces that name the backend differ: the remote rclone serves, how
|
|
# the prune lists and addresses the repositories, and the S3 key, which is read
|
|
# from the secret store when a process starts.
|
|
{ lib }:
|
|
{
|
|
rclone,
|
|
restic,
|
|
# Local backend: the repositories live at `<reposDir>/<agent>/`. Null with `s3`.
|
|
reposDir ? null,
|
|
# S3 backend: `{ bucket, prefix, endpoint, region, provider }`, the
|
|
# repositories at `<bucket>/<prefix>/<agent>/`; `region` may be null and
|
|
# `prefix` empty. Null with `reposDir`.
|
|
s3 ? null,
|
|
# The address rclone listens on.
|
|
addr,
|
|
# The users file rclone checks logins against, its group and the user the
|
|
# server runs as.
|
|
authFile,
|
|
authGroup,
|
|
# `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
|
|
retention,
|
|
}:
|
|
let
|
|
atomicWriteSecret = import ./atomic-write-secret.nix { };
|
|
refreshConsumer = import ./refresh-consumer.nix { };
|
|
storeSeededSecret = import ./store-seeded-secret.nix { };
|
|
|
|
usersPrefix = "secret/swarm/services/archive/users";
|
|
resticPrefix = "secret/swarm/services/archive/restic";
|
|
readyPath = "secret/swarm/services/archive/ready";
|
|
s3KeyIdPath = "secret/swarm/services/archive/s3/access-key-id";
|
|
s3SecretKeyPath = "secret/swarm/services/archive/s3/secret-access-key";
|
|
|
|
# Slack for the clocks of the hives that stamp snapshots.
|
|
futureSkewSeconds = 3600;
|
|
|
|
keepFlags = lib.concatStringsSep " " (
|
|
[ "--keep-last ${toString retention.keepLast}" ]
|
|
++ lib.optional (
|
|
retention.keepWithin != null
|
|
) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
|
|
);
|
|
|
|
# An rclone connection string, so no rclone config exists; the connection
|
|
# settings are `environment` below.
|
|
s3Remote = ":s3:${s3.bucket}${lib.optionalString (s3.prefix != "") "/${s3.prefix}"}";
|
|
|
|
# What rclone is told to serve.
|
|
backendArg = if s3 == null then reposDir else s3Remote;
|
|
|
|
# Reads the S3 key from the store into rclone's environment. Needs the
|
|
# caller's store login.
|
|
s3Credentials = ''
|
|
RCLONE_S3_ACCESS_KEY_ID="$(timeout 15 bao kv get -field=value ${lib.escapeShellArg s3KeyIdPath})"
|
|
RCLONE_S3_SECRET_ACCESS_KEY="$(timeout 15 bao kv get -field=value ${lib.escapeShellArg s3SecretKeyPath})"
|
|
export RCLONE_S3_ACCESS_KEY_ID RCLONE_S3_SECRET_ACCESS_KEY
|
|
'';
|
|
|
|
# Prints the name of every agent that has a repository, one per line.
|
|
listAgents =
|
|
if s3 == null then
|
|
''
|
|
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
|
[ -e "$repo/config" ] || continue
|
|
basename -- "$repo"
|
|
done
|
|
''
|
|
else
|
|
''
|
|
rclone lsf --dirs-only ${lib.escapeShellArg "${s3Remote}/"} | while read -r dir; do
|
|
dir="''${dir%/}"
|
|
[ -n "$(rclone lsf ${lib.escapeShellArg s3Remote}/"$dir"/config)" ] || continue
|
|
printf '%s\n' "$dir"
|
|
done
|
|
'';
|
|
|
|
# restic's name for the repository of `$agent`: the directory, or the rclone
|
|
# backend over the same remote the server serves.
|
|
repoOf =
|
|
if s3 == null then
|
|
''${lib.escapeShellArg reposDir}/"$agent"/''
|
|
else
|
|
''${lib.escapeShellArg "rclone:${s3Remote}/"}"$agent"'';
|
|
|
|
serverCommand = lib.escapeShellArgs [
|
|
"${rclone}/bin/rclone"
|
|
"serve"
|
|
"restic"
|
|
"--addr"
|
|
addr
|
|
"--append-only"
|
|
"--private-repos"
|
|
"--htpasswd"
|
|
authFile
|
|
backendArg
|
|
];
|
|
in
|
|
{
|
|
inherit usersPrefix resticPrefix;
|
|
|
|
# The S3 remote's connection settings, everything but the key. Empty for the
|
|
# local backend.
|
|
environment = lib.optionalAttrs (s3 != null) (
|
|
{
|
|
RCLONE_S3_PROVIDER = s3.provider;
|
|
RCLONE_S3_ENDPOINT = s3.endpoint;
|
|
# A key scoped to the bucket cannot create buckets.
|
|
RCLONE_S3_NO_CHECK_BUCKET = "true";
|
|
RCLONE_S3_ENV_AUTH = "false";
|
|
}
|
|
// lib.optionalAttrs (s3.region != null) { RCLONE_S3_REGION = s3.region; }
|
|
);
|
|
|
|
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
|
# index, so a hive that is compromised cannot erase history. Private repos:
|
|
# a login reaches `/<its own name>/` and nothing else.
|
|
inherit serverCommand;
|
|
|
|
# Start of the S3 server: logs in with the unit's store identity, puts the key
|
|
# in rclone's environment and becomes rclone. The key is never written out. A
|
|
# store that cannot answer fails the start, and the unit's restart retries.
|
|
s3ServerScript = ''
|
|
set -euo pipefail
|
|
BAO_TOKEN="$(timeout 15 bao login -method=cert -token-only)"
|
|
export BAO_TOKEN
|
|
${s3Credentials}
|
|
unset BAO_TOKEN
|
|
exec ${serverCommand}
|
|
'';
|
|
|
|
# Renders the users file from `usersPrefix`, one leaf per agent holding that
|
|
# agent's htpasswd line. The file is written only when the content changed,
|
|
# and never from a partial read: any leaf that cannot be read leaves the file
|
|
# as it is.
|
|
usersScript = ''
|
|
set -euo pipefail
|
|
${storeSeededSecret}
|
|
${atomicWriteSecret}
|
|
${refreshConsumer}
|
|
|
|
file=${lib.escapeShellArg authFile}
|
|
# The server will not start without the file; an empty one admits nobody.
|
|
if [ ! -e "$file" ]; then
|
|
atomic_write_secret 0640 root:${authGroup} "$file" ""
|
|
fi
|
|
|
|
store_init
|
|
store_login
|
|
if [ "$STORE_UP" != 1 ]; then
|
|
echo "could not log in to the swarm secret store; $file stays as it is." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Tells swarm-controller an archive exists; it grants no credentials until
|
|
# then. Created once, never rewritten.
|
|
if ! store_bao kv get -field=ready ${lib.escapeShellArg readyPath} >/dev/null 2>&1; then
|
|
store_bao kv put -cas=0 ${lib.escapeShellArg readyPath} ready=1 >/dev/null
|
|
fi
|
|
|
|
# A prefix that holds nothing is "No value found"; it means no agent has a
|
|
# user yet, not a failure.
|
|
rc=0
|
|
names="$(store_bao kv list -format=json ${lib.escapeShellArg usersPrefix} 2>"$STORE_ERR")" || rc=$?
|
|
if [ "$rc" != 0 ]; then
|
|
if grep -q 'No value found' "$STORE_ERR"; then
|
|
names='[]'
|
|
else
|
|
cat "$STORE_ERR" >&2
|
|
echo "could not list ${usersPrefix}; $file stays as it is." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
content=""
|
|
while IFS= read -r name; do
|
|
[ -n "$name" ] || continue
|
|
case "$name" in */) continue ;; esac
|
|
if ! line="$(store_bao kv get -field=htpasswd ${lib.escapeShellArg usersPrefix}/"$name" 2>"$STORE_ERR")"; then
|
|
cat "$STORE_ERR" >&2
|
|
echo "could not read the user $name; $file stays as it is." >&2
|
|
exit 1
|
|
fi
|
|
# The line must be this agent's own: a leaf cannot add another user.
|
|
if [[ ! "$name" =~ ^[A-Za-z0-9_-]+$ ]] || [[ "$line" != "$name:{SHA}"* ]] \
|
|
|| [[ ! "$line" =~ ^[A-Za-z0-9_-]+:\{SHA\}[A-Za-z0-9+/=]+$ ]]; then
|
|
echo "the user leaf $name does not hold a {SHA} line for $name; leaving it out." >&2
|
|
continue
|
|
fi
|
|
content+="$line"$'\n'
|
|
done < <(printf '%s' "$names" | jq -r '.[]')
|
|
content="''${content%$'\n'}"
|
|
|
|
if secret_differs "$file" "$content"; then
|
|
atomic_write_secret 0640 root:${authGroup} "$file" "$content"
|
|
echo "users file updated."
|
|
fi
|
|
'';
|
|
|
|
# `restic forget --prune` on every repository, with its password read from
|
|
# the store at run time. The server refuses deletes from clients, so this is
|
|
# the only place history is dropped.
|
|
#
|
|
# restic ranks snapshots by their time, which the client chooses (`restic
|
|
# backup --time`). Snapshots dated years ahead would fill `--keep-last` and
|
|
# push the real ones out, so each prune first forgets the snapshots dated more
|
|
# than `futureSkewSeconds` after now. restic has no rule relative to the
|
|
# clock for `--keep-last`.
|
|
pruneScript = ''
|
|
set -euo pipefail
|
|
${storeSeededSecret}
|
|
|
|
# Forgets the snapshots of $repo dated after the limit, naming them in the
|
|
# log; returns non-zero when it cannot read the list or a time.
|
|
forget_future() {
|
|
local repo="$1" agent="$2" snaps id time at future="" limit
|
|
limit=$(( $(date +%s) + ${toString futureSkewSeconds} ))
|
|
snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json \
|
|
| jq -r '.[] | "\(.id) \(.time)"')" || return 1
|
|
while read -r id time; do
|
|
[ -n "$id" ] || continue
|
|
at="$(date -d "$time" +%s)" || return 1
|
|
if [ "$at" -gt "$limit" ]; then
|
|
future+=" $id"
|
|
fi
|
|
done <<<"$snaps"
|
|
if [ -n "$future" ]; then
|
|
echo "archive prune warning: $agent has snapshots dated after now, forgetting them:$future" >&2
|
|
# shellcheck disable=SC2086
|
|
${restic}/bin/restic -r "$repo" --retry-lock 10m forget $future || return 1
|
|
fi
|
|
}
|
|
|
|
store_init
|
|
store_login
|
|
if [ "$STORE_UP" != 1 ]; then
|
|
echo "archive prune failed: could not log in to the swarm secret store" >&2
|
|
exit 1
|
|
fi
|
|
|
|
${lib.optionalString (s3 != null) s3Credentials}
|
|
list_agents() {
|
|
${listAgents}
|
|
}
|
|
# Under `set -e` a failed listing ends the run; inside the `for` it would not.
|
|
agents="$(list_agents)"
|
|
|
|
failed=0
|
|
for agent in $agents; do
|
|
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
|
echo "archive prune failed: $agent is not an agent name" >&2
|
|
failed=1
|
|
continue
|
|
fi
|
|
repo=${repoOf}
|
|
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
|
|
if forget_future "$repo" "$agent" \
|
|
&& ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
|
|
echo "pruned $agent"
|
|
else
|
|
echo "archive prune failed: $agent" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
exit "$failed"
|
|
'';
|
|
}
|