91f36f39d2
The archive host creates a ready marker in the store and swarm-controller writes no credentials before it exists, so a swarm without an archive, and a mixed-version swarm, leave every hive backup a logged no-op. A hive with no declaration no longer counts as placing nothing: its copies and its agents' user lines stay. The restic password is pinned by tests across a revoke, a re-placement and a move.
283 lines
11 KiB
Nix
283 lines
11 KiB
Nix
# `vmTests.<system>.swarm-archive`: runs the archive server's real units
|
|
# (../host-modules/swarm-archive.nix) and the hive's real backup script
|
|
# (../host-modules/lib/archive-backup.nix) in a VM, with restic and rclone, and
|
|
# checks what the module-eval suite can only read:
|
|
#
|
|
# - a backup of two agents lands in one repository each, and `work/` is left out;
|
|
# - one agent's login cannot read another's repository;
|
|
# - a client cannot delete or overwrite anything, and the server-side prune
|
|
# applies the retention;
|
|
# - the users file follows the store: a user removed from it is refused without
|
|
# a restart of the server;
|
|
# - a hive that has agents and no grant backs up nothing and succeeds.
|
|
#
|
|
# One node, because the gateway hop is outside what is under test: the
|
|
# backup script dials `http://127.0.0.1:<port>` instead of
|
|
# `https://archive.<domain>`, and the swarm secret store is a stand-in `bao`
|
|
# that serves `kv list`, `kv get` and `login` from a directory. The module-eval
|
|
# suite covers the gateway vhost and the store grants.
|
|
{ pkgs }:
|
|
let
|
|
port = 8911;
|
|
store = "/var/lib/fake-bao";
|
|
hive = "h1";
|
|
agentRoot = "/var/lib/hyperhive/agents";
|
|
|
|
# A directory is a leaf, a file in it a field: `kv get -field=<f> <path>`.
|
|
fakeBao = pkgs.writeShellScriptBin "bao" ''
|
|
store=${store}
|
|
case "$1 $2" in
|
|
"login -method=cert") echo faketoken; exit 0 ;;
|
|
"kv list")
|
|
path="$4"
|
|
[ -d "$store/$path" ] || { echo "No value found at $path" >&2; exit 2; }
|
|
ls "$store/$path" | ${pkgs.jq}/bin/jq -R . | ${pkgs.jq}/bin/jq -s .
|
|
exit 0 ;;
|
|
"kv put")
|
|
[ -e "$store/$4" ] && { echo "check-and-set parameter did not match" >&2; exit 2; }
|
|
mkdir -p "$store/$4"
|
|
printf %s "''${5#*=}" > "$store/$4/''${5%%=*}"
|
|
exit 0 ;;
|
|
"kv get")
|
|
field="''${3#-field=}"
|
|
[ -f "$store/$4/$field" ] || { echo "No value found at $4" >&2; exit 2; }
|
|
cat "$store/$4/$field"
|
|
exit 0 ;;
|
|
esac
|
|
echo "fake bao: unsupported: $*" >&2
|
|
exit 1
|
|
'';
|
|
|
|
# The stubs for the options the archive module reads from the rest of the
|
|
# hive modules.
|
|
hiveStubs =
|
|
{ lib, ... }:
|
|
{
|
|
options.services.hyperhive = {
|
|
swarm.domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "t.local";
|
|
};
|
|
swarm.bao = {
|
|
domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "bao.t.local";
|
|
};
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 8200;
|
|
};
|
|
};
|
|
deploy.bao = {
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = fakeBao;
|
|
};
|
|
archiveClientCertFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = "/etc/fake-archive-cert";
|
|
};
|
|
archiveClientKeyFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = "/etc/fake-archive-key";
|
|
};
|
|
serverCaFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
};
|
|
};
|
|
deploy.swarm-otel.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
deploy.archive.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
otel.journalUnits = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
};
|
|
gateway = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
dns.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
localNames = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
};
|
|
vhosts = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.anything;
|
|
default = { };
|
|
};
|
|
};
|
|
};
|
|
config.environment.etc."fake-archive-cert".text = "";
|
|
config.environment.etc."fake-archive-key".text = "";
|
|
};
|
|
|
|
backupScript = import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
|
|
inherit hive agentRoot;
|
|
restic = pkgs.restic;
|
|
url = "http://127.0.0.1:${toString port}";
|
|
scope = "paths";
|
|
paths = [
|
|
"state"
|
|
"claude"
|
|
"harness"
|
|
];
|
|
excludes = [ ];
|
|
caBundle = null;
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "swarm-archive";
|
|
|
|
nodes.machine =
|
|
{ pkgs, ... }:
|
|
{
|
|
imports = [
|
|
hiveStubs
|
|
../host-modules/swarm-archive.nix
|
|
];
|
|
|
|
# Keep only the newest snapshot, so the prune is visible after two backups.
|
|
services.hyperhive.deploy.archive.retention = {
|
|
keepWithin = null;
|
|
keepLast = 1;
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.curl
|
|
pkgs.openssl
|
|
pkgs.restic
|
|
];
|
|
|
|
systemd.services.test-archive-backup = {
|
|
path = [
|
|
fakeBao
|
|
pkgs.btrfs-progs
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
pkgs.jq
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
Environment = "RESTIC_CACHE_DIR=/var/cache/test-archive-backup";
|
|
};
|
|
script = backupScript;
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
store = "${store}"
|
|
url = "http://127.0.0.1:${toString port}"
|
|
agents = ["alpha", "beta"]
|
|
|
|
def sha_line(agent, password):
|
|
return machine.succeed(
|
|
f"printf '%s:{{SHA}}%s' {agent} "
|
|
f"\"$(printf %s {password} | openssl dgst -sha1 -binary | base64)\""
|
|
)
|
|
|
|
def put(path, field, value):
|
|
machine.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}")
|
|
|
|
def status(agent, repo, path="config", method="GET"):
|
|
return machine.succeed(
|
|
f"curl -s -o /dev/null -w '%{{http_code}}' -X {method} "
|
|
f"-u {agent}:restpw-{agent} {url}/{repo}/{path}"
|
|
)
|
|
|
|
def restic(agent, repo, args):
|
|
return (
|
|
f"RESTIC_PASSWORD=resticpw-{repo} restic "
|
|
f"-r rest:http://{agent}:restpw-{agent}@127.0.0.1:${toString port}/{repo}/ {args}"
|
|
)
|
|
|
|
machine.wait_for_unit("multi-user.target")
|
|
machine.wait_for_open_port(${toString port})
|
|
|
|
with subtest("the server starts with no users and admits nobody"):
|
|
machine.succeed("test -e /run/swarm-archive-auth/htpasswd")
|
|
assert status("alpha", "alpha") == "401", status("alpha", "alpha")
|
|
|
|
# What swarm-controller writes: a user line and a restic password per agent
|
|
# for the archive, a REST password and the restic password for the hive.
|
|
for agent in agents:
|
|
put(f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(agent, f"restpw-{agent}"))
|
|
put(f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}")
|
|
put(f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}")
|
|
put(f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}")
|
|
machine.succeed(
|
|
f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude "
|
|
f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work"
|
|
)
|
|
machine.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data")
|
|
machine.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch")
|
|
|
|
with subtest("the users file follows the store without a restart of the server"):
|
|
pid = machine.succeed("systemctl show -p MainPID --value swarm-archive").strip()
|
|
machine.succeed("systemctl start swarm-archive-users")
|
|
assert machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid
|
|
assert status("alpha", "alpha") in ("200", "404"), status("alpha", "alpha")
|
|
assert status("beta", "beta") in ("200", "404"), status("beta", "beta")
|
|
# Wrong password: the control that the 200 above is a login, not an open door.
|
|
machine.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401")
|
|
|
|
with subtest("the users file holds hashes, readable by the server only"):
|
|
machine.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd")
|
|
machine.fail("grep -q restpw /run/swarm-archive-auth/htpasswd")
|
|
machine.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'")
|
|
|
|
with subtest("the hive's backup puts each agent in its own repository, without work/"):
|
|
machine.succeed("systemctl start test-archive-backup")
|
|
for agent in agents:
|
|
snaps = machine.succeed(restic(agent, agent, "snapshots --json"))
|
|
assert snaps.count('"id"') == 1, snaps
|
|
listing = machine.succeed(restic(agent, agent, "ls latest"))
|
|
assert f"/{agent}/state/data" in listing, listing
|
|
assert "scratch" not in listing, listing
|
|
|
|
with subtest("one agent's login cannot reach another's repository"):
|
|
assert status("beta", "alpha") == "403", status("beta", "alpha")
|
|
machine.fail(restic("beta", "alpha", "snapshots"))
|
|
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
|
assert status("alpha", "beta") == "403", status("alpha", "beta")
|
|
|
|
with subtest("a client cannot delete or overwrite"):
|
|
snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
|
|
# The presence control: a lock, which a backup creates and removes, is deletable.
|
|
machine.succeed(restic("alpha", "alpha", "backup --host alpha ${agentRoot}/alpha/state"))
|
|
assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
|
|
assert status("alpha", "alpha", "config", "POST") == "403"
|
|
machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
|
|
assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
|
|
|
with subtest("the server-side prune applies the retention"):
|
|
machine.succeed("systemctl start swarm-archive-prune")
|
|
for agent in agents:
|
|
assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
|
|
|
|
with subtest("a user removed from the store is refused without a restart"):
|
|
machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")
|
|
machine.succeed("systemctl start swarm-archive-users")
|
|
assert status("beta", "beta") == "401", status("beta", "beta")
|
|
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
|
|
|
with subtest("the archive announces itself once, so swarm-controller starts granting"):
|
|
machine.succeed(f"test \"$(cat {store}/secret/swarm/services/archive/ready/ready)\" = 1")
|
|
machine.succeed("systemctl start swarm-archive-users")
|
|
|
|
with subtest("a hive with agents and no grant backs up nothing and does not fail"):
|
|
machine.succeed(f"rm -r {store}/secret/swarm/hives/${hive}")
|
|
machine.succeed("systemctl start test-archive-backup")
|
|
machine.succeed("journalctl -u test-archive-backup | grep -q 'granted this hive no archive credentials'")
|
|
machine.fail("journalctl -u test-archive-backup | grep -q 'archive backup failed'")
|
|
'';
|
|
}
|