86c5dac89d
- swarm-archive.nix, lib/archive-server.nix: deploy.archive.backend and deploy.archive.s3.* options; the s3 server and the prune read the key from the store into rclone's environment; the prune lists and addresses repositories through rclone instead of a local directory - swarm-bao.nix: the archive reads swarm/services/archive/s3/* - docs: archive.md "External S3" section, two credentials-reference rows - module-eval: builder.nix cases for local and s3, the missing-setting assertion; bao-grants.nix gains the s3 read grant - vm test: second node with a fake rclone serve s3 whose key is generated at test time
450 lines
16 KiB
Nix
450 lines
16 KiB
Nix
# The swarm's backup archive: `rclone serve restic` on the host that sets
|
|
# `deploy.archive.enable`, behind the gateway at `archive.<swarm domain>`.
|
|
# Every hive's backup unit (./hive-archive-backup.nix) writes one restic
|
|
# repository per agent to it; nothing else uses it.
|
|
#
|
|
# - `--append-only`: a client cannot delete or overwrite a snapshot, a pack or
|
|
# the index, so a compromised hive cannot erase history. Dropping history is
|
|
# the prune unit's job, on this host, by retention.
|
|
# - `--private-repos`: a login reaches `/<its own name>/` only. There is one
|
|
# login per agent, created by swarm-controller (../../swarm-controller/src/archive_access.rs)
|
|
# when it places the agent, replaced when it moves and removed when it is
|
|
# destroyed. An agent's hive can write that agent's repository and no other.
|
|
#
|
|
# Everything secret lives in the swarm secret store and is read when needed:
|
|
#
|
|
# - `swarm-archive-users` (root, every two minutes) renders the htpasswd file
|
|
# from the users leaves into `/run`, which is tmpfs. rclone re-reads the file
|
|
# when it changes. The leaves hold password hashes only.
|
|
# The unit also creates `swarm/services/archive/ready`, which tells
|
|
# swarm-controller an archive exists: it grants no credentials before that.
|
|
# - `swarm-archive-prune` (daily) reads each agent's restic password through
|
|
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
|
|
# therefore decrypt every backup, which is the cost of pruning where clients
|
|
# cannot delete.
|
|
#
|
|
# The units log in as the archive's own store principal (a leaf of the store's
|
|
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
|
|
# no secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
|
|
# ./lib/archive-server.nix); with `s3` the server also reads the S3 key from
|
|
# `swarm/services/archive/s3/…` as it starts and keeps it in rclone's environment.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
cfg = hyperhiveCfg.deploy.archive;
|
|
archiveCfg = hyperhiveCfg.swarm.archive;
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
baoDeploy = hyperhiveCfg.deploy.bao;
|
|
|
|
swarmDomain = hyperhiveCfg.swarm.domain;
|
|
# Total on a null swarm domain, as the sibling options modules are: the
|
|
# required-domain assertion in hive-network.nix is what an operator sees.
|
|
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
|
|
|
user = "swarm-archive";
|
|
authDir = "/run/swarm-archive-auth";
|
|
authFile = "${authDir}/htpasswd";
|
|
reposDir = "${cfg.dataDir}/repos";
|
|
|
|
useS3 = cfg.backend == "s3";
|
|
|
|
server = import ./lib/archive-server.nix { inherit lib; } (
|
|
{
|
|
inherit (cfg) retention;
|
|
inherit authFile;
|
|
rclone = cfg.package;
|
|
restic = cfg.resticPackage;
|
|
addr = "127.0.0.1:${toString cfg.port}";
|
|
authGroup = user;
|
|
}
|
|
// (if useS3 then { inherit (cfg) s3; } else { inherit reposDir; })
|
|
);
|
|
|
|
storeRetry = import ./lib/store-retry.nix { };
|
|
|
|
# A unit that runs as the repositories' owner gets the store identity as
|
|
# credentials, which only that user can read.
|
|
ownerStoreCredentials = [
|
|
"bao-cert:${baoDeploy.archiveClientCertFile}"
|
|
"bao-key:${baoDeploy.archiveClientKeyFile}"
|
|
]
|
|
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
|
|
ownerStoreEnvironment = {
|
|
BAO_ADDR = storeEnvironment.BAO_ADDR;
|
|
BAO_CLIENT_CERT = "%d/bao-cert";
|
|
BAO_CLIENT_KEY = "%d/bao-key";
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = "%d/bao-ca";
|
|
};
|
|
|
|
serveScript = pkgs.writeShellScript "swarm-archive-serve" server.s3ServerScript;
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null;
|
|
|
|
storeEnvironment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = baoDeploy.serverCaFile;
|
|
};
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.archive.ownDomain = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "archive.${domainBase}";
|
|
defaultText = lib.literalExpression ''"archive.''${services.hyperhive.swarm.domain}"'';
|
|
description = ''
|
|
Read-only: the name the swarm's backup archive is served under and every
|
|
hive's backup unit dials. A sibling of the swarm's other service names, so
|
|
the swarm-services sub-CA can issue for it, and in `swarm.serviceDomains`
|
|
on every host whether or not one runs the archive.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.archive = {
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.rclone;
|
|
defaultText = lib.literalExpression "pkgs.rclone";
|
|
description = "The rclone package: `rclone serve restic`.";
|
|
};
|
|
|
|
resticPackage = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.restic;
|
|
defaultText = lib.literalExpression "pkgs.restic";
|
|
description = "The restic package the prune unit runs.";
|
|
};
|
|
|
|
backend = lib.mkOption {
|
|
type = lib.types.enum [
|
|
"local"
|
|
"s3"
|
|
];
|
|
default = "local";
|
|
description = ''
|
|
Where the repositories are kept: `local` is a directory on this host
|
|
({option}`services.hyperhive.deploy.archive.dataDir`), `s3` a bucket of an
|
|
external S3 service (`services.hyperhive.deploy.archive.s3.*`).
|
|
Hives always talk to this archive, whatever the backend, and never see the
|
|
S3 key. The archive reads the key from the secret store, in the leaves
|
|
`swarm/services/archive/s3/access-key-id` and `…/secret-access-key`
|
|
(field `value`), which you write.
|
|
|
|
Switching does not move the repositories already stored. Copy them first
|
|
(`rclone copy <dataDir>/repos :s3:<bucket>/<prefix>`) or start with
|
|
empty repositories.
|
|
'';
|
|
};
|
|
|
|
s3 = {
|
|
endpoint = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "https://s3.eu-central-1.amazonaws.com";
|
|
description = "URL of the S3 service, scheme included. Required with `backend = \"s3\"`.";
|
|
};
|
|
|
|
bucket = lib.mkOption {
|
|
type = lib.types.nullOr (lib.types.strMatching "[A-Za-z0-9._-]+");
|
|
default = null;
|
|
description = ''
|
|
The bucket that holds the repositories. Required with `backend = "s3"`.
|
|
It must exist already: the archive creates none.
|
|
'';
|
|
};
|
|
|
|
prefix = lib.mkOption {
|
|
type = lib.types.strMatching "([A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*)?";
|
|
default = "";
|
|
example = "archive";
|
|
description = ''
|
|
A path in the bucket, without leading or trailing slash. The
|
|
repositories sit at `<prefix>/<agent>/`; empty puts them at the top.
|
|
'';
|
|
};
|
|
|
|
region = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "eu-central-1";
|
|
description = "The bucket's region, for services that require one. `null` leaves rclone's default.";
|
|
};
|
|
|
|
provider = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "Other";
|
|
example = "AWS";
|
|
description = ''
|
|
rclone's `--s3-provider` value, which selects the service's quirks.
|
|
`Other` suits a generic S3-compatible service.
|
|
'';
|
|
};
|
|
};
|
|
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 8911;
|
|
description = ''
|
|
Port rclone listens on, on the host's loopback. The gateway's vhost
|
|
proxies to it; every login is checked by rclone itself.
|
|
'';
|
|
};
|
|
|
|
dataDir = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "/var/lib/swarm-archive";
|
|
description = ''
|
|
Where the repositories live with `backend = "local"`, under `repos/<agent>/`.
|
|
Holds restic ciphertext: every backup of every agent, so size it for that.
|
|
'';
|
|
};
|
|
|
|
retention = {
|
|
keepWithin = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = "3d";
|
|
example = "14d";
|
|
description = ''
|
|
Every snapshot younger than this is kept (restic's `--keep-within`,
|
|
written `1y2m3d4h`). It bounds how fast history can be evicted: a
|
|
hive that appends many snapshots cannot push the last few days out.
|
|
`null` leaves the flag off.
|
|
'';
|
|
};
|
|
|
|
keepLast = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 3;
|
|
description = ''
|
|
The newest this many snapshots of each agent are kept, however old
|
|
(restic's `--keep-last`). Together with
|
|
{option}`services.hyperhive.deploy.archive.retention.keepWithin` the
|
|
prune keeps a snapshot either rule keeps.
|
|
'';
|
|
};
|
|
};
|
|
|
|
pruneSchedule = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "daily";
|
|
example = "*-*-* 04:00:00";
|
|
description = "systemd calendar expression for the prune: the retention rules are applied then.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.archive.enable requires the archive's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.archiveClientCertFile
|
|
services.hyperhive.deploy.bao.archiveClientKeyFile
|
|
|
|
The archive reads its users and the repositories' passwords in the
|
|
store with it.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
|
store). Otherwise copy archive.pem and archive-key.pem from the store
|
|
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
|
'';
|
|
}
|
|
{
|
|
assertion = !useS3 || (cfg.s3.endpoint != null && cfg.s3.bucket != null);
|
|
message = ''
|
|
services.hyperhive.deploy.archive.backend = "s3" requires both
|
|
services.hyperhive.deploy.archive.s3.endpoint and
|
|
services.hyperhive.deploy.archive.s3.bucket.
|
|
'';
|
|
}
|
|
];
|
|
|
|
users.users.${user} = {
|
|
isSystemUser = true;
|
|
group = user;
|
|
description = "swarm backup archive";
|
|
};
|
|
users.groups.${user} = { };
|
|
|
|
# This host serves the vhost, and the host's own resolver finds the name.
|
|
services.hyperhive.gateway.localNames = [ archiveCfg.ownDomain ];
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# `forceSSL`: the logins ride in a header. No `auth_request`: rclone checks
|
|
# them, and restic cannot follow a login redirect. The vhost keeps the
|
|
# gateway's default `public`: `public = false` needs the gateway's
|
|
# `privateListenAddresses` set, and every request is authenticated either
|
|
# way. A backup is one request that can run for minutes, so the body is
|
|
# neither size-limited nor buffered.
|
|
services.hyperhive.gateway.vhosts."${archiveCfg.ownDomain}" = {
|
|
forceSSL = true;
|
|
locations."/" = {
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
|
extraConfig = ''
|
|
client_max_body_size 0;
|
|
proxy_request_buffering off;
|
|
proxy_read_timeout 1h;
|
|
proxy_send_timeout 1h;
|
|
'';
|
|
};
|
|
};
|
|
|
|
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
|
|
"swarm-archive.service"
|
|
"swarm-archive-users.service"
|
|
"swarm-archive-prune.service"
|
|
];
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d ${authDir} 0750 root ${user} -"
|
|
]
|
|
++ lib.optionals (!useS3) [
|
|
"d ${cfg.dataDir} 0750 ${user} ${user} -"
|
|
"d ${reposDir} 0750 ${user} ${user} -"
|
|
];
|
|
|
|
systemd.services.swarm-archive = {
|
|
description = "swarm backup archive (rclone serve restic, append-only, one repository per agent)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "swarm-archive-users.service" ];
|
|
after = [
|
|
"network.target"
|
|
"swarm-archive-users.service"
|
|
"systemd-tmpfiles-setup.service"
|
|
];
|
|
# rclone writes no config, and reads none.
|
|
environment = {
|
|
RCLONE_CONFIG = "/dev/null";
|
|
}
|
|
// lib.optionalAttrs useS3 (ownerStoreEnvironment // server.environment);
|
|
path = lib.optionals useS3 [
|
|
baoDeploy.package
|
|
pkgs.coreutils
|
|
];
|
|
serviceConfig = {
|
|
ExecStart = if useS3 then serveScript else server.serverCommand;
|
|
User = user;
|
|
Group = user;
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
SyslogIdentifier = "swarm-archive";
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
PrivateDevices = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectControlGroups = true;
|
|
RestrictAddressFamilies = [
|
|
"AF_INET"
|
|
"AF_INET6"
|
|
];
|
|
ReadOnlyPaths = [ authDir ];
|
|
}
|
|
// lib.optionalAttrs useS3 { LoadCredential = ownerStoreCredentials; }
|
|
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
|
|
};
|
|
|
|
# The users file, from the store. A failed run leaves the file as it is;
|
|
# the timer and the retry bring a store that comes back up to date.
|
|
systemd.services.swarm-archive-users = {
|
|
description = "render the archive's users file from the swarm secret store";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-online.target" ];
|
|
after = [
|
|
"network-online.target"
|
|
"systemd-tmpfiles-setup.service"
|
|
];
|
|
path = [
|
|
baoDeploy.package
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
pkgs.jq
|
|
pkgs.systemd
|
|
];
|
|
environment = storeEnvironment // {
|
|
BAO_CLIENT_CERT = baoDeploy.archiveClientCertFile;
|
|
BAO_CLIENT_KEY = baoDeploy.archiveClientKeyFile;
|
|
};
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
Type = "oneshot";
|
|
UMask = "0027";
|
|
SyslogIdentifier = "swarm-archive-users";
|
|
TimeoutStartSec = "5min";
|
|
};
|
|
script = server.usersScript;
|
|
};
|
|
|
|
# A new grant reaches the server within the controller's pass (five
|
|
# minutes) plus this interval.
|
|
systemd.timers.swarm-archive-users = {
|
|
description = "re-render the archive's users file";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnUnitInactiveSec = "2min";
|
|
RandomizedDelaySec = "20s";
|
|
};
|
|
};
|
|
|
|
systemd.services.swarm-archive-prune = {
|
|
description = "apply the archive's retention to every agent's repository";
|
|
after = [ "network-online.target" ];
|
|
wants = [ "network-online.target" ];
|
|
path = [
|
|
baoDeploy.package
|
|
cfg.resticPackage
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
pkgs.jq
|
|
]
|
|
# restic's rclone backend starts rclone.
|
|
++ lib.optional useS3 cfg.package;
|
|
environment =
|
|
ownerStoreEnvironment
|
|
// {
|
|
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
|
}
|
|
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
User = user;
|
|
Group = user;
|
|
SyslogIdentifier = "swarm-archive-prune";
|
|
CacheDirectory = "swarm-archive-prune";
|
|
LoadCredential = ownerStoreCredentials;
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
PrivateDevices = true;
|
|
TimeoutStartSec = "6h";
|
|
}
|
|
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
|
|
script = server.pruneScript;
|
|
};
|
|
|
|
systemd.timers.swarm-archive-prune = {
|
|
description = "apply the archive's retention";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = cfg.pruneSchedule;
|
|
RandomizedDelaySec = "30min";
|
|
Persistent = true;
|
|
};
|
|
};
|
|
};
|
|
}
|