swarm-archive: external s3 backend (options, prune path, docs, module-eval cases, vm fake s3)
- swarm-archive.nix, lib/archive-server.nix: deploy.archive.backend and deploy.archive.s3.* options; the s3 server and the prune read the key from the store into rclone's environment; the prune lists and addresses repositories through rclone instead of a local directory - swarm-bao.nix: the archive reads swarm/services/archive/s3/* - docs: archive.md "External S3" section, two credentials-reference rows - module-eval: builder.nix cases for local and s3, the missing-setting assertion; bao-grants.nix gains the s3 read grant - vm test: second node with a fake rclone serve s3 whose key is generated at test time
This commit is contained in:
atlas
mara
parent
ebbf4fa37b
commit
86c5dac89d
1 changed files with 605 additions and 156 deletions
|
|
@ -5,16 +5,24 @@
|
|||
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
|
||||
# `bao`.
|
||||
#
|
||||
# Both scripts use only `bao login -method=cert -token-only`,
|
||||
# All scripts use only `bao login -method=cert -token-only`,
|
||||
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
|
||||
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
|
||||
#
|
||||
# The repositories live in a directory (`reposDir`) or in an S3 bucket (`s3`).
|
||||
# Only the pieces that name the backend differ: the remote rclone serves, how
|
||||
# the prune lists and addresses the repositories, and the S3 key, which is read
|
||||
# from the secret store when a process starts.
|
||||
{ lib }:
|
||||
{
|
||||
rclone,
|
||||
restic,
|
||||
# Where the repositories live: `<reposDir>/<agent>/`. The only place that
|
||||
# names the backend; `rclone serve restic` takes any remote here.
|
||||
reposDir,
|
||||
# Local backend: the repositories live at `<reposDir>/<agent>/`. Null with `s3`.
|
||||
reposDir ? null,
|
||||
# S3 backend: `{ bucket, prefix, endpoint, region, provider }`, the
|
||||
# repositories at `<bucket>/<prefix>/<agent>/`; `region` may be null and
|
||||
# `prefix` empty. Null with `reposDir`.
|
||||
s3 ? null,
|
||||
# The address rclone listens on.
|
||||
addr,
|
||||
# The users file rclone checks logins against, its group and the user the
|
||||
|
|
@ -32,6 +40,8 @@ let
|
|||
usersPrefix = "secret/swarm/services/archive/users";
|
||||
resticPrefix = "secret/swarm/services/archive/restic";
|
||||
readyPath = "secret/swarm/services/archive/ready";
|
||||
s3KeyIdPath = "secret/swarm/services/archive/s3/access-key-id";
|
||||
s3SecretKeyPath = "secret/swarm/services/archive/s3/secret-access-key";
|
||||
|
||||
# Slack for the clocks of the hives that stamp snapshots.
|
||||
futureSkewSeconds = 3600;
|
||||
|
|
@ -42,13 +52,48 @@ let
|
|||
retention.keepWithin != null
|
||||
) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit usersPrefix resticPrefix;
|
||||
|
||||
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
||||
# index, so a hive that is compromised cannot erase history. Private repos:
|
||||
# a login reaches `/<its own name>/` and nothing else.
|
||||
# An rclone connection string, so no rclone config exists; the connection
|
||||
# settings are `environment` below.
|
||||
s3Remote = ":s3:${s3.bucket}${lib.optionalString (s3.prefix != "") "/${s3.prefix}"}";
|
||||
|
||||
# What rclone is told to serve.
|
||||
backendArg = if s3 == null then reposDir else s3Remote;
|
||||
|
||||
# Reads the S3 key from the store into rclone's environment. Needs the
|
||||
# caller's store login.
|
||||
s3Credentials = ''
|
||||
RCLONE_S3_ACCESS_KEY_ID="$(timeout 15 bao kv get -field=value ${lib.escapeShellArg s3KeyIdPath})"
|
||||
RCLONE_S3_SECRET_ACCESS_KEY="$(timeout 15 bao kv get -field=value ${lib.escapeShellArg s3SecretKeyPath})"
|
||||
export RCLONE_S3_ACCESS_KEY_ID RCLONE_S3_SECRET_ACCESS_KEY
|
||||
'';
|
||||
|
||||
# Prints the name of every agent that has a repository, one per line.
|
||||
listAgents =
|
||||
if s3 == null then
|
||||
''
|
||||
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
||||
[ -e "$repo/config" ] || continue
|
||||
basename -- "$repo"
|
||||
done
|
||||
''
|
||||
else
|
||||
''
|
||||
rclone lsf --dirs-only ${lib.escapeShellArg "${s3Remote}/"} | while read -r dir; do
|
||||
dir="''${dir%/}"
|
||||
[ -n "$(rclone lsf ${lib.escapeShellArg s3Remote}/"$dir"/config)" ] || continue
|
||||
printf '%s\n' "$dir"
|
||||
done
|
||||
'';
|
||||
|
||||
# restic's name for the repository of `$agent`: the directory, or the rclone
|
||||
# backend over the same remote the server serves.
|
||||
repoOf =
|
||||
if s3 == null then
|
||||
''${lib.escapeShellArg reposDir}/"$agent"/''
|
||||
else
|
||||
''${lib.escapeShellArg "rclone:${s3Remote}/"}"$agent"'';
|
||||
|
||||
serverCommand = lib.escapeShellArgs [
|
||||
"${rclone}/bin/rclone"
|
||||
"serve"
|
||||
|
|
@ -59,8 +104,41 @@ in
|
|||
"--private-repos"
|
||||
"--htpasswd"
|
||||
authFile
|
||||
reposDir
|
||||
backendArg
|
||||
];
|
||||
in
|
||||
{
|
||||
inherit usersPrefix resticPrefix;
|
||||
|
||||
# The S3 remote's connection settings, everything but the key. Empty for the
|
||||
# local backend.
|
||||
environment = lib.optionalAttrs (s3 != null) (
|
||||
{
|
||||
RCLONE_S3_PROVIDER = s3.provider;
|
||||
RCLONE_S3_ENDPOINT = s3.endpoint;
|
||||
# A key scoped to the bucket cannot create buckets.
|
||||
RCLONE_S3_NO_CHECK_BUCKET = "true";
|
||||
RCLONE_S3_ENV_AUTH = "false";
|
||||
}
|
||||
// lib.optionalAttrs (s3.region != null) { RCLONE_S3_REGION = s3.region; }
|
||||
);
|
||||
|
||||
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
||||
# index, so a hive that is compromised cannot erase history. Private repos:
|
||||
# a login reaches `/<its own name>/` and nothing else.
|
||||
inherit serverCommand;
|
||||
|
||||
# Start of the S3 server: logs in with the unit's store identity, puts the key
|
||||
# in rclone's environment and becomes rclone. The key is never written out. A
|
||||
# store that cannot answer fails the start, and the unit's restart retries.
|
||||
s3ServerScript = ''
|
||||
set -euo pipefail
|
||||
BAO_TOKEN="$(timeout 15 bao login -method=cert -token-only)"
|
||||
export BAO_TOKEN
|
||||
${s3Credentials}
|
||||
unset BAO_TOKEN
|
||||
exec ${serverCommand}
|
||||
'';
|
||||
|
||||
# Renders the users file from `usersPrefix`, one leaf per agent holding that
|
||||
# agent's htpasswd line. The file is written only when the content changed,
|
||||
|
|
@ -171,15 +249,21 @@ in
|
|||
exit 1
|
||||
fi
|
||||
|
||||
${lib.optionalString (s3 != null) s3Credentials}
|
||||
list_agents() {
|
||||
${listAgents}
|
||||
}
|
||||
# Under `set -e` a failed listing ends the run; inside the `for` it would not.
|
||||
agents="$(list_agents)"
|
||||
|
||||
failed=0
|
||||
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
||||
[ -e "$repo/config" ] || continue
|
||||
agent="$(basename -- "$repo")"
|
||||
for agent in $agents; do
|
||||
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
||||
echo "archive prune failed: $agent is not an agent name" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
repo=${repoOf}
|
||||
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
|
||||
if forget_future "$repo" "$agent" \
|
||||
&& ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
|
||||
|
|
|
|||
Loading…
Reference in a new issue