Watch
0
0
Fork
You've already forked hyperhive
0

swarm-archive: back up every agent's state to an append-only restic repository per agent

swarm-archive serves restic repositories with rclone, append-only, one login
and one repository per agent. swarm-controller writes each placed agent's
login and restic password to the store, replaces them on a move and removes
them when the agent is destroyed. Every hive backs its agents up hourly under
its own store identity. The archive host prunes by retention.
This commit is contained in:
atlas 2026-11-10 07:49:49 +01:00
commit 306aae0378

View file

@ -0,0 +1,348 @@
# The swarm's backup archive: `rclone serve restic` on the host that sets
# `deploy.archive.enable`, behind the gateway at `archive.<swarm domain>`.
# Every hive's backup unit (./hive-archive-backup.nix) writes one restic
# repository per agent to it; nothing else uses it.
#
# - `--append-only`: a client cannot delete or overwrite a snapshot, a pack or
# the index, so a compromised hive cannot erase history. Dropping history is
# the prune unit's job, on this host, by retention.
# - `--private-repos`: a login reaches `/<its own name>/` only. There is one
# login per agent, created by swarm-controller (../../swarm-controller/src/archive_access.rs)
# when it places the agent, replaced when it moves and removed when it is
# destroyed. An agent's hive can write that agent's repository and no other.
#
# Everything secret lives in the swarm secret store and is read when needed:
#
# - `swarm-archive-users` (root, every two minutes) renders the htpasswd file
# from the users leaves into `/run`, which is tmpfs. rclone re-reads the file
# when it changes. The leaves hold password hashes only.
# - `swarm-archive-prune` (daily) reads each agent's restic password through
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
# therefore decrypt every backup, which is the cost of pruning where clients
# cannot delete.
#
# Both units log in as the archive's own store principal (a leaf of the store's
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
# no secret crosses a mount, and the backend is one string
# (`reposDir`, in ./lib/archive-server.nix) that names where the repositories
# are.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.deploy.archive;
archiveCfg = hyperhiveCfg.swarm.archive;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
swarmDomain = hyperhiveCfg.swarm.domain;
# Total on a null swarm domain, as the sibling options modules are: the
# required-domain assertion in hive-network.nix is what an operator sees.
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
user = "swarm-archive";
authDir = "/run/swarm-archive-auth";
authFile = "${authDir}/htpasswd";
reposDir = "${cfg.dataDir}/repos";
server = import ./lib/archive-server.nix { inherit lib; } {
inherit (cfg) retention;
inherit authFile reposDir;
rclone = cfg.package;
restic = cfg.resticPackage;
addr = "127.0.0.1:${toString cfg.port}";
authGroup = user;
};
storeRetry = import ./lib/store-retry.nix { };
haveClientIdentity =
baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null;
storeEnvironment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
in
{
options.services.hyperhive.swarm.archive.ownDomain = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "archive.${domainBase}";
defaultText = lib.literalExpression ''"archive.''${services.hyperhive.swarm.domain}"'';
description = ''
Read-only: the name the swarm's backup archive is served under and every
hive's backup unit dials. A sibling of the swarm's other service names, so
the swarm-services sub-CA can issue for it, and in `swarm.serviceDomains`
on every host whether or not one runs the archive.
'';
};
options.services.hyperhive.deploy.archive = {
package = lib.mkOption {
type = lib.types.package;
default = pkgs.rclone;
defaultText = lib.literalExpression "pkgs.rclone";
description = "The rclone package: `rclone serve restic`.";
};
resticPackage = lib.mkOption {
type = lib.types.package;
default = pkgs.restic;
defaultText = lib.literalExpression "pkgs.restic";
description = "The restic package the prune unit runs.";
};
port = lib.mkOption {
type = lib.types.port;
default = 8911;
description = ''
Port rclone listens on, on the host's loopback. The gateway's vhost
proxies to it; every login is checked by rclone itself.
'';
};
dataDir = lib.mkOption {
type = lib.types.path;
default = "/var/lib/swarm-archive";
description = ''
Where the repositories live, under `repos/<agent>/`. Holds restic
ciphertext: every backup of every agent, so size it for that.
'';
};
retention = {
keepWithin = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "3d";
example = "14d";
description = ''
Every snapshot younger than this is kept (restic's `--keep-within`,
written `1y2m3d4h`). It bounds how fast history can be evicted: a
hive that appends many snapshots cannot push the last few days out.
`null` leaves the flag off.
'';
};
keepLast = lib.mkOption {
type = lib.types.ints.positive;
default = 3;
description = ''
The newest this many snapshots of each agent are kept, however old
(restic's `--keep-last`). Together with
{option}`services.hyperhive.deploy.archive.retention.keepWithin` the
prune keeps a snapshot either rule keeps.
'';
};
};
pruneSchedule = lib.mkOption {
type = lib.types.str;
default = "daily";
example = "*-*-* 04:00:00";
description = "systemd calendar expression for the prune: the retention rules are applied then.";
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.archive.enable requires the archive's
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.archiveClientCertFile
services.hyperhive.deploy.bao.archiveClientKeyFile
The archive reads its users and the repositories' passwords in the
store with it.
Both default to the pair glue-bao-tls.nix mints when
services.hyperhive.deploy.bao.enable is true (the host that runs the
store). Otherwise copy archive.pem and archive-key.pem from the store
host's /var/lib/swarm-bao-pki/ and name them in those two options.
'';
}
];
users.users.${user} = {
isSystemUser = true;
group = user;
description = "swarm backup archive";
};
users.groups.${user} = { };
# This host serves the vhost, and the host's own resolver finds the name.
services.hyperhive.gateway.localNames = [ archiveCfg.ownDomain ];
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# `forceSSL`: the logins ride in a header. No `auth_request`: rclone checks
# them, and restic cannot follow a login redirect. The vhost keeps the
# gateway's default `public`: `public = false` needs the gateway's
# `privateListenAddresses` set, and every request is authenticated either
# way. A backup is one request that can run for minutes, so the body is
# neither size-limited nor buffered.
services.hyperhive.gateway.vhosts."${archiveCfg.ownDomain}" = {
forceSSL = true;
locations."/" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}";
extraConfig = ''
client_max_body_size 0;
proxy_request_buffering off;
proxy_read_timeout 1h;
proxy_send_timeout 1h;
'';
};
};
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
"swarm-archive.service"
"swarm-archive-users.service"
"swarm-archive-prune.service"
];
systemd.tmpfiles.rules = [
"d ${cfg.dataDir} 0750 ${user} ${user} -"
"d ${reposDir} 0750 ${user} ${user} -"
"d ${authDir} 0750 root ${user} -"
];
systemd.services.swarm-archive = {
description = "swarm backup archive (rclone serve restic, append-only, one repository per agent)";
wantedBy = [ "multi-user.target" ];
wants = [ "swarm-archive-users.service" ];
after = [
"network.target"
"swarm-archive-users.service"
"systemd-tmpfiles-setup.service"
];
# rclone writes no config, and reads none.
environment.RCLONE_CONFIG = "/dev/null";
serviceConfig = {
ExecStart = server.serverCommand;
User = user;
Group = user;
Restart = "on-failure";
RestartSec = 5;
SyslogIdentifier = "swarm-archive";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
ProtectKernelTunables = true;
ProtectControlGroups = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
];
ReadWritePaths = [ reposDir ];
ReadOnlyPaths = [ authDir ];
};
};
# The users file, from the store. A failed run leaves the file as it is;
# the timer and the retry bring a store that comes back up to date.
systemd.services.swarm-archive-users = {
description = "render the archive's users file from the swarm secret store";
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" ];
after = [
"network-online.target"
"systemd-tmpfiles-setup.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.gnugrep
pkgs.jq
pkgs.systemd
];
environment = storeEnvironment // {
BAO_CLIENT_CERT = baoDeploy.archiveClientCertFile;
BAO_CLIENT_KEY = baoDeploy.archiveClientKeyFile;
};
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0027";
SyslogIdentifier = "swarm-archive-users";
TimeoutStartSec = "5min";
};
script = server.usersScript;
};
# A new grant reaches the server within the controller's pass (five
# minutes) plus this interval.
systemd.timers.swarm-archive-users = {
description = "re-render the archive's users file";
wantedBy = [ "timers.target" ];
timerConfig = {
OnUnitInactiveSec = "2min";
RandomizedDelaySec = "20s";
};
};
systemd.services.swarm-archive-prune = {
description = "apply the archive's retention to every agent's repository";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [
baoDeploy.package
cfg.resticPackage
pkgs.coreutils
pkgs.gnugrep
pkgs.jq
];
# The unit runs as the repositories' owner, so the store identity comes
# in as credentials, which only it can read.
environment = {
BAO_ADDR = storeEnvironment.BAO_ADDR;
BAO_CLIENT_CERT = "%d/bao-cert";
BAO_CLIENT_KEY = "%d/bao-key";
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = "%d/bao-ca";
};
serviceConfig = {
Type = "oneshot";
User = user;
Group = user;
SyslogIdentifier = "swarm-archive-prune";
CacheDirectory = "swarm-archive-prune";
LoadCredential = [
"bao-cert:${baoDeploy.archiveClientCertFile}"
"bao-key:${baoDeploy.archiveClientKeyFile}"
]
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
ReadWritePaths = [ reposDir ];
TimeoutStartSec = "6h";
};
script = server.pruneScript;
};
systemd.timers.swarm-archive-prune = {
description = "apply the archive's retention";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.pruneSchedule;
RandomizedDelaySec = "30min";
Persistent = true;
};
};
};
}