mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-archive: back up every agent's state to an append-only restic repository per agent

Browse source 
swarm-archive serves restic repositories with rclone, append-only, one login
and one repository per agent. swarm-controller writes each placed agent's
login and restic password to the store, replaces them on a move and removes
them when the agent is destroyed. Every hive backs its agents up hourly under
its own store identity. The archive host prunes by retention.
This commit is contained in:
atlas 2026-11-10 07:49:49 +01:00
commit 306aae0378
1 changed files with 2214 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

273
nix/vm-tests/swarm-archive.nix Normal file
View file

@ -0,0 +1,273 @@
# `vmTests.<system>.swarm-archive`: runs the archive server's real units
# (../host-modules/swarm-archive.nix) and the hive's real backup script
# (../host-modules/lib/archive-backup.nix) in a VM, with restic and rclone, and
# checks what the module-eval suite can only read:
#
# - a backup of two agents lands in one repository each, and `work/` is left out;
# - one agent's login cannot read another's repository;
# - a client cannot delete or overwrite anything, and the server-side prune
# applies the retention;
# - the users file follows the store: a user removed from it is refused without
# a restart of the server;
# - a hive that has agents and no grant fails its backup.
#
# One node, because the gateway hop is outside what is under test: the
# backup script dials `http://127.0.0.1:<port>` instead of
# `https://archive.<domain>`, and the swarm secret store is a stand-in `bao`
# that serves `kv list`, `kv get` and `login` from a directory. The module-eval
# suite covers the gateway vhost and the store grants.
{ pkgs }:
let
port = 8911;
store = "/var/lib/fake-bao";
hive = "h1";
agentRoot = "/var/lib/hyperhive/agents";
# A directory is a leaf, a file in it a field: `kv get -field=<f> <path>`.
fakeBao = pkgs.writeShellScriptBin "bao" ''
store=${store}
case "$1 $2" in
"login -method=cert") echo faketoken; exit 0 ;;
"kv list")
path="$4"
[ -d "$store/$path" ] || { echo "No value found at $path" >&2; exit 2; }
ls "$store/$path" | ${pkgs.jq}/bin/jq -R . | ${pkgs.jq}/bin/jq -s .
exit 0 ;;
"kv get")
field="''${3#-field=}"
[ -f "$store/$4/$field" ] || { echo "No value found at $4" >&2; exit 2; }
cat "$store/$4/$field"
exit 0 ;;
esac
echo "fake bao: unsupported: $*" >&2
exit 1
'';
# The stubs for the options the archive module reads from the rest of the
# hive modules.
hiveStubs =
{ lib, ... }:
{
options.services.hyperhive = {
swarm.domain = lib.mkOption {
type = lib.types.str;
default = "t.local";
};
swarm.bao = {
domain = lib.mkOption {
type = lib.types.str;
default = "bao.t.local";
};
port = lib.mkOption {
type = lib.types.port;
default = 8200;
};
};
deploy.bao = {
package = lib.mkOption {
type = lib.types.package;
default = fakeBao;
};
archiveClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "/etc/fake-archive-cert";
};
archiveClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "/etc/fake-archive-key";
};
serverCaFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
};
};
deploy.swarm-otel.enable = lib.mkOption {
type = lib.types.bool;
default = false;
};
deploy.archive.enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
otel.journalUnits = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
};
gateway = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
};
dns.enable = lib.mkOption {
type = lib.types.bool;
default = false;
};
localNames = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
};
vhosts = lib.mkOption {
type = lib.types.attrsOf lib.types.anything;
default = { };
};
};
};
config.environment.etc."fake-archive-cert".text = "";
config.environment.etc."fake-archive-key".text = "";
};
backupScript = import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
inherit hive agentRoot;
restic = pkgs.restic;
url = "http://127.0.0.1:${toString port}";
scope = "paths";
paths = [
"state"
"claude"
"harness"
];
excludes = [ ];
caBundle = null;
};
in
pkgs.testers.runNixOSTest {
name = "swarm-archive";
nodes.machine =
{ pkgs, ... }:
{
imports = [
hiveStubs
../host-modules/swarm-archive.nix
];
# Keep only the newest snapshot, so the prune is visible after two backups.
services.hyperhive.deploy.archive.retention = {
keepWithin = null;
keepLast = 1;
};
environment.systemPackages = [
pkgs.curl
pkgs.openssl
pkgs.restic
];
systemd.services.test-archive-backup = {
path = [
fakeBao
pkgs.btrfs-progs
pkgs.coreutils
pkgs.gnugrep
pkgs.jq
];
serviceConfig = {
Type = "oneshot";
Environment = "RESTIC_CACHE_DIR=/var/cache/test-archive-backup";
};
script = backupScript;
};
};
testScript = ''
store = "${store}"
url = "http://127.0.0.1:${toString port}"
agents = ["alpha", "beta"]
def sha_line(agent, password):
return machine.succeed(
f"printf '%s:{{SHA}}%s' {agent} "
f"\"$(printf %s {password} | openssl dgst -sha1 -binary | base64)\""
)
def put(path, field, value):
machine.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}")
def status(agent, repo, path="config", method="GET"):
return machine.succeed(
f"curl -s -o /dev/null -w '%{{http_code}}' -X {method} "
f"-u {agent}:restpw-{agent} {url}/{repo}/{path}"
)
def restic(agent, repo, args):
return (
f"RESTIC_PASSWORD=resticpw-{repo} restic "
f"-r rest:http://{agent}:restpw-{agent}@127.0.0.1:${toString port}/{repo}/ {args}"
)
machine.wait_for_unit("multi-user.target")
machine.wait_for_open_port(${toString port})
with subtest("the server starts with no users and admits nobody"):
machine.succeed("test -e /run/swarm-archive-auth/htpasswd")
assert status("alpha", "alpha") == "401", status("alpha", "alpha")
# What swarm-controller writes: a user line and a restic password per agent
# for the archive, a REST password and the restic password for the hive.
for agent in agents:
put(f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(agent, f"restpw-{agent}"))
put(f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}")
put(f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}")
put(f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}")
machine.succeed(
f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude "
f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work"
)
machine.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data")
machine.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch")
with subtest("the users file follows the store without a restart of the server"):
pid = machine.succeed("systemctl show -p MainPID --value swarm-archive").strip()
machine.succeed("systemctl start swarm-archive-users")
assert machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid
assert status("alpha", "alpha") in ("200", "404"), status("alpha", "alpha")
assert status("beta", "beta") in ("200", "404"), status("beta", "beta")
# Wrong password: the control that the 200 above is a login, not an open door.
machine.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401")
with subtest("the users file holds hashes, readable by the server only"):
machine.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd")
machine.fail("grep -q restpw /run/swarm-archive-auth/htpasswd")
machine.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'")
with subtest("the hive's backup puts each agent in its own repository, without work/"):
machine.succeed("systemctl start test-archive-backup")
for agent in agents:
snaps = machine.succeed(restic(agent, agent, "snapshots --json"))
assert snaps.count('"id"') == 1, snaps
listing = machine.succeed(restic(agent, agent, "ls latest"))
assert f"/{agent}/state/data" in listing, listing
assert "scratch" not in listing, listing
with subtest("one agent's login cannot reach another's repository"):
assert status("beta", "alpha") == "403", status("beta", "alpha")
machine.fail(restic("beta", "alpha", "snapshots"))
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
assert status("alpha", "beta") == "403", status("alpha", "beta")
with subtest("a client cannot delete or overwrite"):
snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
# The presence control: a lock, which a backup creates and removes, is deletable.
machine.succeed(restic("alpha", "alpha", "backup --host alpha ${agentRoot}/alpha/state"))
assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
assert status("alpha", "alpha", "config", "POST") == "403"
machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
with subtest("the server-side prune applies the retention"):
machine.succeed("systemctl start swarm-archive-prune")
for agent in agents:
assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
with subtest("a user removed from the store is refused without a restart"):
machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")
machine.succeed("systemctl start swarm-archive-users")
assert status("beta", "beta") == "401", status("beta", "beta")
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
with subtest("a hive with agents and no grant fails its backup"):
machine.succeed(f"rm -r {store}/secret/swarm/hives/${hive}")
machine.fail("systemctl start test-archive-backup")
machine.succeed("journalctl -u test-archive-backup | grep -q 'archive backup failed: swarm-controller has granted this hive no archive credentials'")
'';
}