swarm-archive: external s3 backend (options, prune path, docs, module-eval cases, vm fake s3)
- swarm-archive.nix, lib/archive-server.nix: deploy.archive.backend and deploy.archive.s3.* options; the s3 server and the prune read the key from the store into rclone's environment; the prune lists and addresses repositories through rclone instead of a local directory - swarm-bao.nix: the archive reads swarm/services/archive/s3/* - docs: archive.md "External S3" section, two credentials-reference rows - module-eval: builder.nix cases for local and s3, the missing-setting assertion; bao-grants.nix gains the s3 read grant - vm test: second node with a fake rclone serve s3 whose key is generated at test time
This commit is contained in:
atlas
mara
parent
ebbf4fa37b
commit
86c5dac89d
1 changed files with 605 additions and 156 deletions
|
|
@ -23,11 +23,11 @@
|
|||
# therefore decrypt every backup, which is the cost of pruning where clients
|
||||
# cannot delete.
|
||||
#
|
||||
# Both units log in as the archive's own store principal (a leaf of the store's
|
||||
# The units log in as the archive's own store principal (a leaf of the store's
|
||||
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
|
||||
# no secret crosses a mount, and the backend is one string
|
||||
# (`reposDir`, in ./lib/archive-server.nix) that names where the repositories
|
||||
# are.
|
||||
# no secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
|
||||
# ./lib/archive-server.nix); with `s3` the server also reads the S3 key from
|
||||
# `swarm/services/archive/s3/…` as it starts and keeps it in rclone's environment.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
|
|
@ -51,17 +51,40 @@ let
|
|||
authFile = "${authDir}/htpasswd";
|
||||
reposDir = "${cfg.dataDir}/repos";
|
||||
|
||||
server = import ./lib/archive-server.nix { inherit lib; } {
|
||||
inherit (cfg) retention;
|
||||
inherit authFile reposDir;
|
||||
rclone = cfg.package;
|
||||
restic = cfg.resticPackage;
|
||||
addr = "127.0.0.1:${toString cfg.port}";
|
||||
authGroup = user;
|
||||
};
|
||||
useS3 = cfg.backend == "s3";
|
||||
|
||||
server = import ./lib/archive-server.nix { inherit lib; } (
|
||||
{
|
||||
inherit (cfg) retention;
|
||||
inherit authFile;
|
||||
rclone = cfg.package;
|
||||
restic = cfg.resticPackage;
|
||||
addr = "127.0.0.1:${toString cfg.port}";
|
||||
authGroup = user;
|
||||
}
|
||||
// (if useS3 then { inherit (cfg) s3; } else { inherit reposDir; })
|
||||
);
|
||||
|
||||
storeRetry = import ./lib/store-retry.nix { };
|
||||
|
||||
# A unit that runs as the repositories' owner gets the store identity as
|
||||
# credentials, which only that user can read.
|
||||
ownerStoreCredentials = [
|
||||
"bao-cert:${baoDeploy.archiveClientCertFile}"
|
||||
"bao-key:${baoDeploy.archiveClientKeyFile}"
|
||||
]
|
||||
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
|
||||
ownerStoreEnvironment = {
|
||||
BAO_ADDR = storeEnvironment.BAO_ADDR;
|
||||
BAO_CLIENT_CERT = "%d/bao-cert";
|
||||
BAO_CLIENT_KEY = "%d/bao-key";
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = "%d/bao-ca";
|
||||
};
|
||||
|
||||
serveScript = pkgs.writeShellScript "swarm-archive-serve" server.s3ServerScript;
|
||||
|
||||
haveClientIdentity =
|
||||
baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null;
|
||||
|
||||
|
|
@ -101,6 +124,72 @@ in
|
|||
description = "The restic package the prune unit runs.";
|
||||
};
|
||||
|
||||
backend = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"local"
|
||||
"s3"
|
||||
];
|
||||
default = "local";
|
||||
description = ''
|
||||
Where the repositories are kept: `local` is a directory on this host
|
||||
({option}`services.hyperhive.deploy.archive.dataDir`), `s3` a bucket of an
|
||||
external S3 service (`services.hyperhive.deploy.archive.s3.*`).
|
||||
Hives always talk to this archive, whatever the backend, and never see the
|
||||
S3 key. The archive reads the key from the secret store, in the leaves
|
||||
`swarm/services/archive/s3/access-key-id` and `…/secret-access-key`
|
||||
(field `value`), which you write.
|
||||
|
||||
Switching does not move the repositories already stored. Copy them first
|
||||
(`rclone copy <dataDir>/repos :s3:<bucket>/<prefix>`) or start with
|
||||
empty repositories.
|
||||
'';
|
||||
};
|
||||
|
||||
s3 = {
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "https://s3.eu-central-1.amazonaws.com";
|
||||
description = "URL of the S3 service, scheme included. Required with `backend = \"s3\"`.";
|
||||
};
|
||||
|
||||
bucket = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.strMatching "[A-Za-z0-9._-]+");
|
||||
default = null;
|
||||
description = ''
|
||||
The bucket that holds the repositories. Required with `backend = "s3"`.
|
||||
It must exist already: the archive creates none.
|
||||
'';
|
||||
};
|
||||
|
||||
prefix = lib.mkOption {
|
||||
type = lib.types.strMatching "([A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*)?";
|
||||
default = "";
|
||||
example = "archive";
|
||||
description = ''
|
||||
A path in the bucket, without leading or trailing slash. The
|
||||
repositories sit at `<prefix>/<agent>/`; empty puts them at the top.
|
||||
'';
|
||||
};
|
||||
|
||||
region = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "eu-central-1";
|
||||
description = "The bucket's region, for services that require one. `null` leaves rclone's default.";
|
||||
};
|
||||
|
||||
provider = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "Other";
|
||||
example = "AWS";
|
||||
description = ''
|
||||
rclone's `--s3-provider` value, which selects the service's quirks.
|
||||
`Other` suits a generic S3-compatible service.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8911;
|
||||
|
|
@ -114,8 +203,8 @@ in
|
|||
type = lib.types.path;
|
||||
default = "/var/lib/swarm-archive";
|
||||
description = ''
|
||||
Where the repositories live, under `repos/<agent>/`. Holds restic
|
||||
ciphertext: every backup of every agent, so size it for that.
|
||||
Where the repositories live with `backend = "local"`, under `repos/<agent>/`.
|
||||
Holds restic ciphertext: every backup of every agent, so size it for that.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -172,6 +261,14 @@ in
|
|||
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = !useS3 || (cfg.s3.endpoint != null && cfg.s3.bucket != null);
|
||||
message = ''
|
||||
services.hyperhive.deploy.archive.backend = "s3" requires both
|
||||
services.hyperhive.deploy.archive.s3.endpoint and
|
||||
services.hyperhive.deploy.archive.s3.bucket.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
users.users.${user} = {
|
||||
|
|
@ -212,9 +309,11 @@ in
|
|||
];
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${authDir} 0750 root ${user} -"
|
||||
]
|
||||
++ lib.optionals (!useS3) [
|
||||
"d ${cfg.dataDir} 0750 ${user} ${user} -"
|
||||
"d ${reposDir} 0750 ${user} ${user} -"
|
||||
"d ${authDir} 0750 root ${user} -"
|
||||
];
|
||||
|
||||
systemd.services.swarm-archive = {
|
||||
|
|
@ -227,9 +326,16 @@ in
|
|||
"systemd-tmpfiles-setup.service"
|
||||
];
|
||||
# rclone writes no config, and reads none.
|
||||
environment.RCLONE_CONFIG = "/dev/null";
|
||||
environment = {
|
||||
RCLONE_CONFIG = "/dev/null";
|
||||
}
|
||||
// lib.optionalAttrs useS3 (ownerStoreEnvironment // server.environment);
|
||||
path = lib.optionals useS3 [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
];
|
||||
serviceConfig = {
|
||||
ExecStart = server.serverCommand;
|
||||
ExecStart = if useS3 then serveScript else server.serverCommand;
|
||||
User = user;
|
||||
Group = user;
|
||||
Restart = "on-failure";
|
||||
|
|
@ -246,9 +352,10 @@ in
|
|||
"AF_INET"
|
||||
"AF_INET6"
|
||||
];
|
||||
ReadWritePaths = [ reposDir ];
|
||||
ReadOnlyPaths = [ authDir ];
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs useS3 { LoadCredential = ownerStoreCredentials; }
|
||||
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
|
||||
};
|
||||
|
||||
# The users file, from the store. A failed run leaves the file as it is;
|
||||
|
|
@ -303,37 +410,30 @@ in
|
|||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
];
|
||||
# The unit runs as the repositories' owner, so the store identity comes
|
||||
# in as credentials, which only it can read.
|
||||
environment = {
|
||||
BAO_ADDR = storeEnvironment.BAO_ADDR;
|
||||
BAO_CLIENT_CERT = "%d/bao-cert";
|
||||
BAO_CLIENT_KEY = "%d/bao-key";
|
||||
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = "%d/bao-ca";
|
||||
};
|
||||
]
|
||||
# restic's rclone backend starts rclone.
|
||||
++ lib.optional useS3 cfg.package;
|
||||
environment =
|
||||
ownerStoreEnvironment
|
||||
// {
|
||||
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
||||
}
|
||||
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = user;
|
||||
Group = user;
|
||||
SyslogIdentifier = "swarm-archive-prune";
|
||||
CacheDirectory = "swarm-archive-prune";
|
||||
LoadCredential = [
|
||||
"bao-cert:${baoDeploy.archiveClientCertFile}"
|
||||
"bao-key:${baoDeploy.archiveClientKeyFile}"
|
||||
]
|
||||
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
|
||||
LoadCredential = ownerStoreCredentials;
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
ReadWritePaths = [ reposDir ];
|
||||
TimeoutStartSec = "6h";
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
|
||||
script = server.pruneScript;
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue