306aae0378
swarm-archive serves restic repositories with rclone, append-only, one login and one repository per agent. swarm-controller writes each placed agent's login and restic password to the store, replaces them on a move and removes them when the agent is destroyed. Every hive backs its agents up hourly under its own store identity. The archive host prunes by retention.
348 lines
12 KiB
Nix
348 lines
12 KiB
Nix
# The swarm's backup archive: `rclone serve restic` on the host that sets
|
|
# `deploy.archive.enable`, behind the gateway at `archive.<swarm domain>`.
|
|
# Every hive's backup unit (./hive-archive-backup.nix) writes one restic
|
|
# repository per agent to it; nothing else uses it.
|
|
#
|
|
# - `--append-only`: a client cannot delete or overwrite a snapshot, a pack or
|
|
# the index, so a compromised hive cannot erase history. Dropping history is
|
|
# the prune unit's job, on this host, by retention.
|
|
# - `--private-repos`: a login reaches `/<its own name>/` only. There is one
|
|
# login per agent, created by swarm-controller (../../swarm-controller/src/archive_access.rs)
|
|
# when it places the agent, replaced when it moves and removed when it is
|
|
# destroyed. An agent's hive can write that agent's repository and no other.
|
|
#
|
|
# Everything secret lives in the swarm secret store and is read when needed:
|
|
#
|
|
# - `swarm-archive-users` (root, every two minutes) renders the htpasswd file
|
|
# from the users leaves into `/run`, which is tmpfs. rclone re-reads the file
|
|
# when it changes. The leaves hold password hashes only.
|
|
# - `swarm-archive-prune` (daily) reads each agent's restic password through
|
|
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
|
|
# therefore decrypt every backup, which is the cost of pruning where clients
|
|
# cannot delete.
|
|
#
|
|
# Both units log in as the archive's own store principal (a leaf of the store's
|
|
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
|
|
# no secret crosses a mount, and the backend is one string
|
|
# (`reposDir`, in ./lib/archive-server.nix) that names where the repositories
|
|
# are.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
cfg = hyperhiveCfg.deploy.archive;
|
|
archiveCfg = hyperhiveCfg.swarm.archive;
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
baoDeploy = hyperhiveCfg.deploy.bao;
|
|
|
|
swarmDomain = hyperhiveCfg.swarm.domain;
|
|
# Total on a null swarm domain, as the sibling options modules are: the
|
|
# required-domain assertion in hive-network.nix is what an operator sees.
|
|
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
|
|
|
user = "swarm-archive";
|
|
authDir = "/run/swarm-archive-auth";
|
|
authFile = "${authDir}/htpasswd";
|
|
reposDir = "${cfg.dataDir}/repos";
|
|
|
|
server = import ./lib/archive-server.nix { inherit lib; } {
|
|
inherit (cfg) retention;
|
|
inherit authFile reposDir;
|
|
rclone = cfg.package;
|
|
restic = cfg.resticPackage;
|
|
addr = "127.0.0.1:${toString cfg.port}";
|
|
authGroup = user;
|
|
};
|
|
|
|
storeRetry = import ./lib/store-retry.nix { };
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null;
|
|
|
|
storeEnvironment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = baoDeploy.serverCaFile;
|
|
};
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.archive.ownDomain = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "archive.${domainBase}";
|
|
defaultText = lib.literalExpression ''"archive.''${services.hyperhive.swarm.domain}"'';
|
|
description = ''
|
|
Read-only: the name the swarm's backup archive is served under and every
|
|
hive's backup unit dials. A sibling of the swarm's other service names, so
|
|
the swarm-services sub-CA can issue for it, and in `swarm.serviceDomains`
|
|
on every host whether or not one runs the archive.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.archive = {
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.rclone;
|
|
defaultText = lib.literalExpression "pkgs.rclone";
|
|
description = "The rclone package: `rclone serve restic`.";
|
|
};
|
|
|
|
resticPackage = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.restic;
|
|
defaultText = lib.literalExpression "pkgs.restic";
|
|
description = "The restic package the prune unit runs.";
|
|
};
|
|
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 8911;
|
|
description = ''
|
|
Port rclone listens on, on the host's loopback. The gateway's vhost
|
|
proxies to it; every login is checked by rclone itself.
|
|
'';
|
|
};
|
|
|
|
dataDir = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "/var/lib/swarm-archive";
|
|
description = ''
|
|
Where the repositories live, under `repos/<agent>/`. Holds restic
|
|
ciphertext: every backup of every agent, so size it for that.
|
|
'';
|
|
};
|
|
|
|
retention = {
|
|
keepWithin = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = "3d";
|
|
example = "14d";
|
|
description = ''
|
|
Every snapshot younger than this is kept (restic's `--keep-within`,
|
|
written `1y2m3d4h`). It bounds how fast history can be evicted: a
|
|
hive that appends many snapshots cannot push the last few days out.
|
|
`null` leaves the flag off.
|
|
'';
|
|
};
|
|
|
|
keepLast = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 3;
|
|
description = ''
|
|
The newest this many snapshots of each agent are kept, however old
|
|
(restic's `--keep-last`). Together with
|
|
{option}`services.hyperhive.deploy.archive.retention.keepWithin` the
|
|
prune keeps a snapshot either rule keeps.
|
|
'';
|
|
};
|
|
};
|
|
|
|
pruneSchedule = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "daily";
|
|
example = "*-*-* 04:00:00";
|
|
description = "systemd calendar expression for the prune: the retention rules are applied then.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.archive.enable requires the archive's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.archiveClientCertFile
|
|
services.hyperhive.deploy.bao.archiveClientKeyFile
|
|
|
|
The archive reads its users and the repositories' passwords in the
|
|
store with it.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
|
store). Otherwise copy archive.pem and archive-key.pem from the store
|
|
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
|
'';
|
|
}
|
|
];
|
|
|
|
users.users.${user} = {
|
|
isSystemUser = true;
|
|
group = user;
|
|
description = "swarm backup archive";
|
|
};
|
|
users.groups.${user} = { };
|
|
|
|
# This host serves the vhost, and the host's own resolver finds the name.
|
|
services.hyperhive.gateway.localNames = [ archiveCfg.ownDomain ];
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# `forceSSL`: the logins ride in a header. No `auth_request`: rclone checks
|
|
# them, and restic cannot follow a login redirect. The vhost keeps the
|
|
# gateway's default `public`: `public = false` needs the gateway's
|
|
# `privateListenAddresses` set, and every request is authenticated either
|
|
# way. A backup is one request that can run for minutes, so the body is
|
|
# neither size-limited nor buffered.
|
|
services.hyperhive.gateway.vhosts."${archiveCfg.ownDomain}" = {
|
|
forceSSL = true;
|
|
locations."/" = {
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
|
extraConfig = ''
|
|
client_max_body_size 0;
|
|
proxy_request_buffering off;
|
|
proxy_read_timeout 1h;
|
|
proxy_send_timeout 1h;
|
|
'';
|
|
};
|
|
};
|
|
|
|
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
|
|
"swarm-archive.service"
|
|
"swarm-archive-users.service"
|
|
"swarm-archive-prune.service"
|
|
];
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d ${cfg.dataDir} 0750 ${user} ${user} -"
|
|
"d ${reposDir} 0750 ${user} ${user} -"
|
|
"d ${authDir} 0750 root ${user} -"
|
|
];
|
|
|
|
systemd.services.swarm-archive = {
|
|
description = "swarm backup archive (rclone serve restic, append-only, one repository per agent)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "swarm-archive-users.service" ];
|
|
after = [
|
|
"network.target"
|
|
"swarm-archive-users.service"
|
|
"systemd-tmpfiles-setup.service"
|
|
];
|
|
# rclone writes no config, and reads none.
|
|
environment.RCLONE_CONFIG = "/dev/null";
|
|
serviceConfig = {
|
|
ExecStart = server.serverCommand;
|
|
User = user;
|
|
Group = user;
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
SyslogIdentifier = "swarm-archive";
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
PrivateDevices = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectControlGroups = true;
|
|
RestrictAddressFamilies = [
|
|
"AF_INET"
|
|
"AF_INET6"
|
|
];
|
|
ReadWritePaths = [ reposDir ];
|
|
ReadOnlyPaths = [ authDir ];
|
|
};
|
|
};
|
|
|
|
# The users file, from the store. A failed run leaves the file as it is;
|
|
# the timer and the retry bring a store that comes back up to date.
|
|
systemd.services.swarm-archive-users = {
|
|
description = "render the archive's users file from the swarm secret store";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-online.target" ];
|
|
after = [
|
|
"network-online.target"
|
|
"systemd-tmpfiles-setup.service"
|
|
];
|
|
path = [
|
|
baoDeploy.package
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
pkgs.jq
|
|
pkgs.systemd
|
|
];
|
|
environment = storeEnvironment // {
|
|
BAO_CLIENT_CERT = baoDeploy.archiveClientCertFile;
|
|
BAO_CLIENT_KEY = baoDeploy.archiveClientKeyFile;
|
|
};
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
Type = "oneshot";
|
|
UMask = "0027";
|
|
SyslogIdentifier = "swarm-archive-users";
|
|
TimeoutStartSec = "5min";
|
|
};
|
|
script = server.usersScript;
|
|
};
|
|
|
|
# A new grant reaches the server within the controller's pass (five
|
|
# minutes) plus this interval.
|
|
systemd.timers.swarm-archive-users = {
|
|
description = "re-render the archive's users file";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnUnitInactiveSec = "2min";
|
|
RandomizedDelaySec = "20s";
|
|
};
|
|
};
|
|
|
|
systemd.services.swarm-archive-prune = {
|
|
description = "apply the archive's retention to every agent's repository";
|
|
after = [ "network-online.target" ];
|
|
wants = [ "network-online.target" ];
|
|
path = [
|
|
baoDeploy.package
|
|
cfg.resticPackage
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
pkgs.jq
|
|
];
|
|
# The unit runs as the repositories' owner, so the store identity comes
|
|
# in as credentials, which only it can read.
|
|
environment = {
|
|
BAO_ADDR = storeEnvironment.BAO_ADDR;
|
|
BAO_CLIENT_CERT = "%d/bao-cert";
|
|
BAO_CLIENT_KEY = "%d/bao-key";
|
|
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = "%d/bao-ca";
|
|
};
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
User = user;
|
|
Group = user;
|
|
SyslogIdentifier = "swarm-archive-prune";
|
|
CacheDirectory = "swarm-archive-prune";
|
|
LoadCredential = [
|
|
"bao-cert:${baoDeploy.archiveClientCertFile}"
|
|
"bao-key:${baoDeploy.archiveClientKeyFile}"
|
|
]
|
|
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
PrivateDevices = true;
|
|
ReadWritePaths = [ reposDir ];
|
|
TimeoutStartSec = "6h";
|
|
};
|
|
script = server.pruneScript;
|
|
};
|
|
|
|
systemd.timers.swarm-archive-prune = {
|
|
description = "apply the archive's retention";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = cfg.pruneSchedule;
|
|
RandomizedDelaySec = "30min";
|
|
Persistent = true;
|
|
};
|
|
};
|
|
};
|
|
}
|