Watch
0
0
Fork
You've already forked hyperhive
0

swarm-archive: back up every agent's state to an append-only restic repository per agent

swarm-archive serves restic repositories with rclone, append-only, one login
and one repository per agent. swarm-controller writes each placed agent's
login and restic password to the store, replaces them on a move and removes
them when the agent is destroyed. Every hive backs its agents up hourly under
its own store identity. The archive host prunes by retention.
This commit is contained in:
atlas 2026-11-10 07:49:49 +01:00
commit 306aae0378

View file

@ -0,0 +1,185 @@
# The hive's backup of its agents' state to the swarm archive
# (./swarm-archive.nix): one restic repository per agent, at
# `https://archive.<swarm domain>/<agent>/`.
#
# A root oneshot on a timer, not hive-c0re and not a hive-priv verb. Root,
# because the agents' files belong to the agents (and, under user namespaces,
# to remapped ids) and hive-c0re cannot read them; a unit of its own, because
# no caller-supplied argument may reach root, and this one takes none.
#
# Each agent backs up under the credentials swarm-controller wrote for it at
# `swarm/hives/<this hive>/archive/<agent>`
# (../../swarm-controller/src/archive_access.rs), read with the hive's own
# store identity: the agent's login to the archive, and its restic password,
# which restic reads through `RESTIC_PASSWORD_COMMAND` and nothing writes to
# disk. An agent the controller has not granted this hive is skipped; a hive
# with agents and no grant at all fails, which is what a swarm-controller
# without the archive looks like.
#
# A failed agent prints `archive backup failed: <agent>` to the journal,
# which the swarm's `archive backup failed` alert reads, and fails the unit
# after the others have been tried.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
hiveController = hyperhiveCfg.deploy.hive-controller;
cfg = hiveController.archive;
tls = hiveController.tls;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
enabled =
cfg.enable
&& hiveController.enable
&& hiveController.hiveName != null
&& tls.baoHiveIdentityCertFile != null
&& tls.baoHiveIdentityKeyFile != null;
# The CA bundle restic trusts: the host's, then the swarm's root, which the
# archive's certificate chains to when the gateway is self-signed. Written by
# the unit at each run; `--cacert` replaces restic's own roots.
runtimeDir = "/run/hive-archive-backup";
caBundle = "${runtimeDir}/ca.pem";
swarmCa = "${tls.stateDir}/swarm-services-root.pem";
storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service";
backup = import ./lib/archive-backup.nix { inherit lib; } {
inherit (cfg) scope paths excludes;
restic = cfg.package;
hive = hiveController.hiveName;
agentRoot = "/var/lib/hyperhive/agents";
url = "https://${hyperhiveCfg.swarm.archive.ownDomain}";
inherit caBundle;
};
in
{
options.services.hyperhive.deploy.hive-controller.archive = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = ''
Back this hive's agents up to the swarm archive. On wherever the hive
has its store identity; a swarm that runs no archive
({option}`services.hyperhive.deploy.archive.enable` on one host) sees
every run fail, so turn it off there.
'';
};
schedule = lib.mkOption {
type = lib.types.str;
default = "hourly";
example = "*-*-* 00/6:00:00";
description = ''
systemd calendar expression for the backup. Each run sends only what
changed since the last, so an hourly run on an idle agent is small.
'';
};
scope = lib.mkOption {
type = lib.types.enum [
"paths"
"subvolume"
];
default = "paths";
description = ''
What of each agent is backed up. `paths`: the entries of
{option}`services.hyperhive.deploy.hive-controller.archive.paths`
inside the agent's directory. `subvolume`: the whole directory, which
adds `work/` and everything an agent keeps there.
'';
};
paths = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"state"
"claude"
"harness"
];
description = ''
Entries of an agent's directory backed up under `scope = "paths"`: its
state, its claude login and the harness. Caches and `work/` are left
out unless listed. An entry the agent does not have is skipped.
'';
};
excludes = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "*/target" ];
description = "restic `--exclude` patterns, on top of `--exclude-caches`.";
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.restic;
defaultText = lib.literalExpression "pkgs.restic";
description = "The restic package.";
};
};
config = lib.mkIf enabled {
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
"hive-archive-backup.service"
];
systemd.services.hive-archive-backup = {
description = "back up this hive's agents to the swarm archive";
wants = [ "network-online.target" ];
after = [ "network-online.target" ] ++ storeContainer;
path = [
baoDeploy.package
pkgs.btrfs-progs
pkgs.coreutils
pkgs.gnugrep
pkgs.jq
];
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile;
BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile;
RESTIC_CACHE_DIR = "/var/cache/hive-archive-backup";
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
serviceConfig = {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "hive-archive-backup";
RuntimeDirectory = "hive-archive-backup";
CacheDirectory = "hive-archive-backup";
# A first backup of a large agent sends all of it.
TimeoutStartSec = "6h";
Nice = 10;
IOSchedulingClass = "idle";
};
script = ''
set -euo pipefail
# A missing file is left out: the bundle is the host's roots alone
# until the swarm's root is there.
cat ${lib.escapeShellArg (toString config.security.pki.caBundle)} > ${caBundle}
if [ -s ${lib.escapeShellArg swarmCa} ]; then
cat ${lib.escapeShellArg swarmCa} >> ${caBundle}
fi
${backup}
'';
};
systemd.timers.hive-archive-backup = {
description = "back up this hive's agents to the swarm archive";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.schedule;
RandomizedDelaySec = "5min";
Persistent = true;
};
};
};
}