swarm-archive: external s3 backend (options, prune path, docs, module-eval cases, vm fake s3)
- swarm-archive.nix, lib/archive-server.nix: deploy.archive.backend and deploy.archive.s3.* options; the s3 server and the prune read the key from the store into rclone's environment; the prune lists and addresses repositories through rclone instead of a local directory - swarm-bao.nix: the archive reads swarm/services/archive/s3/* - docs: archive.md "External S3" section, two credentials-reference rows - module-eval: builder.nix cases for local and s3, the missing-setting assertion; bao-grants.nix gains the s3 read grant - vm test: second node with a fake rclone serve s3 whose key is generated at test time
This commit is contained in:
atlas
mara
parent
ebbf4fa37b
commit
86c5dac89d
3 changed files with 605 additions and 156 deletions
|
|
@ -5,16 +5,24 @@
|
|||
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
|
||||
# `bao`.
|
||||
#
|
||||
# Both scripts use only `bao login -method=cert -token-only`,
|
||||
# All scripts use only `bao login -method=cert -token-only`,
|
||||
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
|
||||
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
|
||||
#
|
||||
# The repositories live in a directory (`reposDir`) or in an S3 bucket (`s3`).
|
||||
# Only the pieces that name the backend differ: the remote rclone serves, how
|
||||
# the prune lists and addresses the repositories, and the S3 key, which is read
|
||||
# from the secret store when a process starts.
|
||||
{ lib }:
|
||||
{
|
||||
rclone,
|
||||
restic,
|
||||
# Where the repositories live: `<reposDir>/<agent>/`. The only place that
|
||||
# names the backend; `rclone serve restic` takes any remote here.
|
||||
reposDir,
|
||||
# Local backend: the repositories live at `<reposDir>/<agent>/`. Null with `s3`.
|
||||
reposDir ? null,
|
||||
# S3 backend: `{ bucket, prefix, endpoint, region, provider }`, the
|
||||
# repositories at `<bucket>/<prefix>/<agent>/`; `region` may be null and
|
||||
# `prefix` empty. Null with `reposDir`.
|
||||
s3 ? null,
|
||||
# The address rclone listens on.
|
||||
addr,
|
||||
# The users file rclone checks logins against, its group and the user the
|
||||
|
|
@ -32,6 +40,8 @@ let
|
|||
usersPrefix = "secret/swarm/services/archive/users";
|
||||
resticPrefix = "secret/swarm/services/archive/restic";
|
||||
readyPath = "secret/swarm/services/archive/ready";
|
||||
s3KeyIdPath = "secret/swarm/services/archive/s3/access-key-id";
|
||||
s3SecretKeyPath = "secret/swarm/services/archive/s3/secret-access-key";
|
||||
|
||||
# Slack for the clocks of the hives that stamp snapshots.
|
||||
futureSkewSeconds = 3600;
|
||||
|
|
@ -42,13 +52,48 @@ let
|
|||
retention.keepWithin != null
|
||||
) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit usersPrefix resticPrefix;
|
||||
|
||||
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
||||
# index, so a hive that is compromised cannot erase history. Private repos:
|
||||
# a login reaches `/<its own name>/` and nothing else.
|
||||
# An rclone connection string, so no rclone config exists; the connection
|
||||
# settings are `environment` below.
|
||||
s3Remote = ":s3:${s3.bucket}${lib.optionalString (s3.prefix != "") "/${s3.prefix}"}";
|
||||
|
||||
# What rclone is told to serve.
|
||||
backendArg = if s3 == null then reposDir else s3Remote;
|
||||
|
||||
# Reads the S3 key from the store into rclone's environment. Needs the
|
||||
# caller's store login.
|
||||
s3Credentials = ''
|
||||
RCLONE_S3_ACCESS_KEY_ID="$(timeout 15 bao kv get -field=value ${lib.escapeShellArg s3KeyIdPath})"
|
||||
RCLONE_S3_SECRET_ACCESS_KEY="$(timeout 15 bao kv get -field=value ${lib.escapeShellArg s3SecretKeyPath})"
|
||||
export RCLONE_S3_ACCESS_KEY_ID RCLONE_S3_SECRET_ACCESS_KEY
|
||||
'';
|
||||
|
||||
# Prints the name of every agent that has a repository, one per line.
|
||||
listAgents =
|
||||
if s3 == null then
|
||||
''
|
||||
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
||||
[ -e "$repo/config" ] || continue
|
||||
basename -- "$repo"
|
||||
done
|
||||
''
|
||||
else
|
||||
''
|
||||
rclone lsf --dirs-only ${lib.escapeShellArg "${s3Remote}/"} | while read -r dir; do
|
||||
dir="''${dir%/}"
|
||||
[ -n "$(rclone lsf ${lib.escapeShellArg s3Remote}/"$dir"/config)" ] || continue
|
||||
printf '%s\n' "$dir"
|
||||
done
|
||||
'';
|
||||
|
||||
# restic's name for the repository of `$agent`: the directory, or the rclone
|
||||
# backend over the same remote the server serves.
|
||||
repoOf =
|
||||
if s3 == null then
|
||||
''${lib.escapeShellArg reposDir}/"$agent"/''
|
||||
else
|
||||
''${lib.escapeShellArg "rclone:${s3Remote}/"}"$agent"'';
|
||||
|
||||
serverCommand = lib.escapeShellArgs [
|
||||
"${rclone}/bin/rclone"
|
||||
"serve"
|
||||
|
|
@ -59,8 +104,41 @@ in
|
|||
"--private-repos"
|
||||
"--htpasswd"
|
||||
authFile
|
||||
reposDir
|
||||
backendArg
|
||||
];
|
||||
in
|
||||
{
|
||||
inherit usersPrefix resticPrefix;
|
||||
|
||||
# The S3 remote's connection settings, everything but the key. Empty for the
|
||||
# local backend.
|
||||
environment = lib.optionalAttrs (s3 != null) (
|
||||
{
|
||||
RCLONE_S3_PROVIDER = s3.provider;
|
||||
RCLONE_S3_ENDPOINT = s3.endpoint;
|
||||
# A key scoped to the bucket cannot create buckets.
|
||||
RCLONE_S3_NO_CHECK_BUCKET = "true";
|
||||
RCLONE_S3_ENV_AUTH = "false";
|
||||
}
|
||||
// lib.optionalAttrs (s3.region != null) { RCLONE_S3_REGION = s3.region; }
|
||||
);
|
||||
|
||||
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
||||
# index, so a hive that is compromised cannot erase history. Private repos:
|
||||
# a login reaches `/<its own name>/` and nothing else.
|
||||
inherit serverCommand;
|
||||
|
||||
# Start of the S3 server: logs in with the unit's store identity, puts the key
|
||||
# in rclone's environment and becomes rclone. The key is never written out. A
|
||||
# store that cannot answer fails the start, and the unit's restart retries.
|
||||
s3ServerScript = ''
|
||||
set -euo pipefail
|
||||
BAO_TOKEN="$(timeout 15 bao login -method=cert -token-only)"
|
||||
export BAO_TOKEN
|
||||
${s3Credentials}
|
||||
unset BAO_TOKEN
|
||||
exec ${serverCommand}
|
||||
'';
|
||||
|
||||
# Renders the users file from `usersPrefix`, one leaf per agent holding that
|
||||
# agent's htpasswd line. The file is written only when the content changed,
|
||||
|
|
@ -171,15 +249,21 @@ in
|
|||
exit 1
|
||||
fi
|
||||
|
||||
${lib.optionalString (s3 != null) s3Credentials}
|
||||
list_agents() {
|
||||
${listAgents}
|
||||
}
|
||||
# Under `set -e` a failed listing ends the run; inside the `for` it would not.
|
||||
agents="$(list_agents)"
|
||||
|
||||
failed=0
|
||||
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
||||
[ -e "$repo/config" ] || continue
|
||||
agent="$(basename -- "$repo")"
|
||||
for agent in $agents; do
|
||||
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
||||
echo "archive prune failed: $agent is not an agent name" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
repo=${repoOf}
|
||||
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
|
||||
if forget_future "$repo" "$agent" \
|
||||
&& ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
|
||||
|
|
|
|||
|
|
@ -23,11 +23,11 @@
|
|||
# therefore decrypt every backup, which is the cost of pruning where clients
|
||||
# cannot delete.
|
||||
#
|
||||
# Both units log in as the archive's own store principal (a leaf of the store's
|
||||
# The units log in as the archive's own store principal (a leaf of the store's
|
||||
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
|
||||
# no secret crosses a mount, and the backend is one string
|
||||
# (`reposDir`, in ./lib/archive-server.nix) that names where the repositories
|
||||
# are.
|
||||
# no secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
|
||||
# ./lib/archive-server.nix); with `s3` the server also reads the S3 key from
|
||||
# `swarm/services/archive/s3/…` as it starts and keeps it in rclone's environment.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
|
|
@ -51,17 +51,40 @@ let
|
|||
authFile = "${authDir}/htpasswd";
|
||||
reposDir = "${cfg.dataDir}/repos";
|
||||
|
||||
server = import ./lib/archive-server.nix { inherit lib; } {
|
||||
inherit (cfg) retention;
|
||||
inherit authFile reposDir;
|
||||
rclone = cfg.package;
|
||||
restic = cfg.resticPackage;
|
||||
addr = "127.0.0.1:${toString cfg.port}";
|
||||
authGroup = user;
|
||||
};
|
||||
useS3 = cfg.backend == "s3";
|
||||
|
||||
server = import ./lib/archive-server.nix { inherit lib; } (
|
||||
{
|
||||
inherit (cfg) retention;
|
||||
inherit authFile;
|
||||
rclone = cfg.package;
|
||||
restic = cfg.resticPackage;
|
||||
addr = "127.0.0.1:${toString cfg.port}";
|
||||
authGroup = user;
|
||||
}
|
||||
// (if useS3 then { inherit (cfg) s3; } else { inherit reposDir; })
|
||||
);
|
||||
|
||||
storeRetry = import ./lib/store-retry.nix { };
|
||||
|
||||
# A unit that runs as the repositories' owner gets the store identity as
|
||||
# credentials, which only that user can read.
|
||||
ownerStoreCredentials = [
|
||||
"bao-cert:${baoDeploy.archiveClientCertFile}"
|
||||
"bao-key:${baoDeploy.archiveClientKeyFile}"
|
||||
]
|
||||
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
|
||||
ownerStoreEnvironment = {
|
||||
BAO_ADDR = storeEnvironment.BAO_ADDR;
|
||||
BAO_CLIENT_CERT = "%d/bao-cert";
|
||||
BAO_CLIENT_KEY = "%d/bao-key";
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = "%d/bao-ca";
|
||||
};
|
||||
|
||||
serveScript = pkgs.writeShellScript "swarm-archive-serve" server.s3ServerScript;
|
||||
|
||||
haveClientIdentity =
|
||||
baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null;
|
||||
|
||||
|
|
@ -101,6 +124,72 @@ in
|
|||
description = "The restic package the prune unit runs.";
|
||||
};
|
||||
|
||||
backend = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"local"
|
||||
"s3"
|
||||
];
|
||||
default = "local";
|
||||
description = ''
|
||||
Where the repositories are kept: `local` is a directory on this host
|
||||
({option}`services.hyperhive.deploy.archive.dataDir`), `s3` a bucket of an
|
||||
external S3 service (`services.hyperhive.deploy.archive.s3.*`).
|
||||
Hives always talk to this archive, whatever the backend, and never see the
|
||||
S3 key. The archive reads the key from the secret store, in the leaves
|
||||
`swarm/services/archive/s3/access-key-id` and `…/secret-access-key`
|
||||
(field `value`), which you write.
|
||||
|
||||
Switching does not move the repositories already stored. Copy them first
|
||||
(`rclone copy <dataDir>/repos :s3:<bucket>/<prefix>`) or start with
|
||||
empty repositories.
|
||||
'';
|
||||
};
|
||||
|
||||
s3 = {
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "https://s3.eu-central-1.amazonaws.com";
|
||||
description = "URL of the S3 service, scheme included. Required with `backend = \"s3\"`.";
|
||||
};
|
||||
|
||||
bucket = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.strMatching "[A-Za-z0-9._-]+");
|
||||
default = null;
|
||||
description = ''
|
||||
The bucket that holds the repositories. Required with `backend = "s3"`.
|
||||
It must exist already: the archive creates none.
|
||||
'';
|
||||
};
|
||||
|
||||
prefix = lib.mkOption {
|
||||
type = lib.types.strMatching "([A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*)?";
|
||||
default = "";
|
||||
example = "archive";
|
||||
description = ''
|
||||
A path in the bucket, without leading or trailing slash. The
|
||||
repositories sit at `<prefix>/<agent>/`; empty puts them at the top.
|
||||
'';
|
||||
};
|
||||
|
||||
region = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "eu-central-1";
|
||||
description = "The bucket's region, for services that require one. `null` leaves rclone's default.";
|
||||
};
|
||||
|
||||
provider = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "Other";
|
||||
example = "AWS";
|
||||
description = ''
|
||||
rclone's `--s3-provider` value, which selects the service's quirks.
|
||||
`Other` suits a generic S3-compatible service.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8911;
|
||||
|
|
@ -114,8 +203,8 @@ in
|
|||
type = lib.types.path;
|
||||
default = "/var/lib/swarm-archive";
|
||||
description = ''
|
||||
Where the repositories live, under `repos/<agent>/`. Holds restic
|
||||
ciphertext: every backup of every agent, so size it for that.
|
||||
Where the repositories live with `backend = "local"`, under `repos/<agent>/`.
|
||||
Holds restic ciphertext: every backup of every agent, so size it for that.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -172,6 +261,14 @@ in
|
|||
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = !useS3 || (cfg.s3.endpoint != null && cfg.s3.bucket != null);
|
||||
message = ''
|
||||
services.hyperhive.deploy.archive.backend = "s3" requires both
|
||||
services.hyperhive.deploy.archive.s3.endpoint and
|
||||
services.hyperhive.deploy.archive.s3.bucket.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
users.users.${user} = {
|
||||
|
|
@ -212,9 +309,11 @@ in
|
|||
];
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${authDir} 0750 root ${user} -"
|
||||
]
|
||||
++ lib.optionals (!useS3) [
|
||||
"d ${cfg.dataDir} 0750 ${user} ${user} -"
|
||||
"d ${reposDir} 0750 ${user} ${user} -"
|
||||
"d ${authDir} 0750 root ${user} -"
|
||||
];
|
||||
|
||||
systemd.services.swarm-archive = {
|
||||
|
|
@ -227,9 +326,16 @@ in
|
|||
"systemd-tmpfiles-setup.service"
|
||||
];
|
||||
# rclone writes no config, and reads none.
|
||||
environment.RCLONE_CONFIG = "/dev/null";
|
||||
environment = {
|
||||
RCLONE_CONFIG = "/dev/null";
|
||||
}
|
||||
// lib.optionalAttrs useS3 (ownerStoreEnvironment // server.environment);
|
||||
path = lib.optionals useS3 [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
];
|
||||
serviceConfig = {
|
||||
ExecStart = server.serverCommand;
|
||||
ExecStart = if useS3 then serveScript else server.serverCommand;
|
||||
User = user;
|
||||
Group = user;
|
||||
Restart = "on-failure";
|
||||
|
|
@ -246,9 +352,10 @@ in
|
|||
"AF_INET"
|
||||
"AF_INET6"
|
||||
];
|
||||
ReadWritePaths = [ reposDir ];
|
||||
ReadOnlyPaths = [ authDir ];
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs useS3 { LoadCredential = ownerStoreCredentials; }
|
||||
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
|
||||
};
|
||||
|
||||
# The users file, from the store. A failed run leaves the file as it is;
|
||||
|
|
@ -303,37 +410,30 @@ in
|
|||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
];
|
||||
# The unit runs as the repositories' owner, so the store identity comes
|
||||
# in as credentials, which only it can read.
|
||||
environment = {
|
||||
BAO_ADDR = storeEnvironment.BAO_ADDR;
|
||||
BAO_CLIENT_CERT = "%d/bao-cert";
|
||||
BAO_CLIENT_KEY = "%d/bao-key";
|
||||
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = "%d/bao-ca";
|
||||
};
|
||||
]
|
||||
# restic's rclone backend starts rclone.
|
||||
++ lib.optional useS3 cfg.package;
|
||||
environment =
|
||||
ownerStoreEnvironment
|
||||
// {
|
||||
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
||||
}
|
||||
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = user;
|
||||
Group = user;
|
||||
SyslogIdentifier = "swarm-archive-prune";
|
||||
CacheDirectory = "swarm-archive-prune";
|
||||
LoadCredential = [
|
||||
"bao-cert:${baoDeploy.archiveClientCertFile}"
|
||||
"bao-key:${baoDeploy.archiveClientKeyFile}"
|
||||
]
|
||||
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
|
||||
LoadCredential = ownerStoreCredentials;
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
ReadWritePaths = [ reposDir ];
|
||||
TimeoutStartSec = "6h";
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
|
||||
script = server.pruneScript;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1394,7 +1394,8 @@ let
|
|||
# The backup archive: the users file is rendered from one leaf per agent, and
|
||||
# the prune reads each agent's restic password. Read and list only; the
|
||||
# controller writes both. It also creates the marker that tells the
|
||||
# controller an archive exists.
|
||||
# controller an archive exists. With an external S3 backend it reads the
|
||||
# provider key, which the operator writes.
|
||||
archiveReaders = [
|
||||
{
|
||||
name = "swarm-archive";
|
||||
|
|
@ -1402,6 +1403,7 @@ let
|
|||
policyText =
|
||||
readStanza "${credentialMountPath}/data/swarm/services/archive/users/*"
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/archive/restic/*"
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/archive/s3/*"
|
||||
+ listStanza "${credentialMountPath}/metadata/swarm/services/archive/users/"
|
||||
+ createReadStanza "${credentialMountPath}/data/swarm/services/archive/ready";
|
||||
}
|
||||
|
|
@ -2685,7 +2687,7 @@ in
|
|||
description = ''
|
||||
Subject the store's `swarm-archive` cert-auth role accepts: the identity
|
||||
the backup archive's host units present. Its grant is read on
|
||||
`swarm/services/archive/users/*` and `restic/*`, and list on `users`.
|
||||
`swarm/services/archive/users/*`, `restic/*` and `s3/*`, and list on `users`.
|
||||
|
||||
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
|
||||
'';
|
||||
|
|
|
|||
Loading…
Reference in a new issue