793d987a88
A client chooses a snapshot's time, so snapshots dated ahead could fill --keep-last and push the real ones out of a prune. The prune now forgets snapshots dated more than an hour after now first, and logs their ids. The VM test forges one and checks the real snapshot stays. The bao-grants case that read a + path as a glob now names the literal path, and two cases that pinned 'nothing else' list the archive grants. The ciRunner stanza is back at its original indent.
297 lines
12 KiB
Nix
297 lines
12 KiB
Nix
# `vmTests.<system>.swarm-archive`: runs the archive server's real units
|
|
# (../host-modules/swarm-archive.nix) and the hive's real backup script
|
|
# (../host-modules/lib/archive-backup.nix) in a VM, with restic and rclone, and
|
|
# checks what the module-eval suite can only read:
|
|
#
|
|
# - a backup of two agents lands in one repository each, and `work/` is left out;
|
|
# - one agent's login cannot read another's repository;
|
|
# - a client cannot delete or overwrite anything, and the server-side prune
|
|
# applies the retention, also when a client has forged a snapshot dated in
|
|
# the future;
|
|
# - the users file follows the store: a user removed from it is refused without
|
|
# a restart of the server;
|
|
# - a hive that has agents and no grant backs up nothing and succeeds.
|
|
#
|
|
# One node, because the gateway hop is outside what is under test: the
|
|
# backup script dials `http://127.0.0.1:<port>` instead of
|
|
# `https://archive.<domain>`, and the swarm secret store is a stand-in `bao`
|
|
# that serves `kv list`, `kv get` and `login` from a directory. The module-eval
|
|
# suite covers the gateway vhost and the store grants.
|
|
{ pkgs }:
|
|
let
|
|
port = 8911;
|
|
store = "/var/lib/fake-bao";
|
|
hive = "h1";
|
|
agentRoot = "/var/lib/hyperhive/agents";
|
|
|
|
# A directory is a leaf, a file in it a field: `kv get -field=<f> <path>`.
|
|
fakeBao = pkgs.writeShellScriptBin "bao" ''
|
|
store=${store}
|
|
case "$1 $2" in
|
|
"login -method=cert") echo faketoken; exit 0 ;;
|
|
"kv list")
|
|
path="$4"
|
|
[ -d "$store/$path" ] || { echo "No value found at $path" >&2; exit 2; }
|
|
ls "$store/$path" | ${pkgs.jq}/bin/jq -R . | ${pkgs.jq}/bin/jq -s .
|
|
exit 0 ;;
|
|
"kv put")
|
|
[ -e "$store/$4" ] && { echo "check-and-set parameter did not match" >&2; exit 2; }
|
|
mkdir -p "$store/$4"
|
|
printf %s "''${5#*=}" > "$store/$4/''${5%%=*}"
|
|
exit 0 ;;
|
|
"kv get")
|
|
field="''${3#-field=}"
|
|
[ -f "$store/$4/$field" ] || { echo "No value found at $4" >&2; exit 2; }
|
|
cat "$store/$4/$field"
|
|
exit 0 ;;
|
|
esac
|
|
echo "fake bao: unsupported: $*" >&2
|
|
exit 1
|
|
'';
|
|
|
|
# The stubs for the options the archive module reads from the rest of the
|
|
# hive modules.
|
|
hiveStubs =
|
|
{ lib, ... }:
|
|
{
|
|
options.services.hyperhive = {
|
|
swarm.domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "t.local";
|
|
};
|
|
swarm.bao = {
|
|
domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "bao.t.local";
|
|
};
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 8200;
|
|
};
|
|
};
|
|
deploy.bao = {
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = fakeBao;
|
|
};
|
|
archiveClientCertFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = "/etc/fake-archive-cert";
|
|
};
|
|
archiveClientKeyFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = "/etc/fake-archive-key";
|
|
};
|
|
serverCaFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
};
|
|
};
|
|
deploy.swarm-otel.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
deploy.archive.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
otel.journalUnits = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
};
|
|
gateway = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
dns.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
localNames = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
};
|
|
vhosts = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.anything;
|
|
default = { };
|
|
};
|
|
};
|
|
};
|
|
config.environment.etc."fake-archive-cert".text = "";
|
|
config.environment.etc."fake-archive-key".text = "";
|
|
};
|
|
|
|
backupScript = import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
|
|
inherit hive agentRoot;
|
|
restic = pkgs.restic;
|
|
url = "http://127.0.0.1:${toString port}";
|
|
scope = "paths";
|
|
paths = [
|
|
"state"
|
|
"claude"
|
|
"harness"
|
|
];
|
|
excludes = [ ];
|
|
caBundle = null;
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "swarm-archive";
|
|
|
|
nodes.machine =
|
|
{ pkgs, ... }:
|
|
{
|
|
imports = [
|
|
hiveStubs
|
|
../host-modules/swarm-archive.nix
|
|
];
|
|
|
|
# The newest snapshot, plus whatever is younger than three days.
|
|
services.hyperhive.deploy.archive.retention = {
|
|
keepWithin = "3d";
|
|
keepLast = 1;
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.curl
|
|
pkgs.openssl
|
|
pkgs.restic
|
|
];
|
|
|
|
systemd.services.test-archive-backup = {
|
|
path = [
|
|
fakeBao
|
|
pkgs.btrfs-progs
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
pkgs.jq
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
Environment = "RESTIC_CACHE_DIR=/var/cache/test-archive-backup";
|
|
};
|
|
script = backupScript;
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
store = "${store}"
|
|
url = "http://127.0.0.1:${toString port}"
|
|
agents = ["alpha", "beta"]
|
|
|
|
def sha_line(agent, password):
|
|
return machine.succeed(
|
|
f"printf '%s:{{SHA}}%s' {agent} "
|
|
f"\"$(printf %s {password} | openssl dgst -sha1 -binary | base64)\""
|
|
)
|
|
|
|
def put(path, field, value):
|
|
machine.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}")
|
|
|
|
def status(agent, repo, path="config", method="GET"):
|
|
return machine.succeed(
|
|
f"curl -s -o /dev/null -w '%{{http_code}}' -X {method} "
|
|
f"-u {agent}:restpw-{agent} {url}/{repo}/{path}"
|
|
)
|
|
|
|
def restic(agent, repo, args):
|
|
return (
|
|
f"RESTIC_PASSWORD=resticpw-{repo} restic "
|
|
f"-r rest:http://{agent}:restpw-{agent}@127.0.0.1:${toString port}/{repo}/ {args}"
|
|
)
|
|
|
|
machine.wait_for_unit("multi-user.target")
|
|
machine.wait_for_open_port(${toString port})
|
|
|
|
with subtest("the server starts with no users and admits nobody"):
|
|
machine.succeed("test -e /run/swarm-archive-auth/htpasswd")
|
|
assert status("alpha", "alpha") == "401", status("alpha", "alpha")
|
|
|
|
# What swarm-controller writes: a user line and a restic password per agent
|
|
# for the archive, a REST password and the restic password for the hive.
|
|
for agent in agents:
|
|
put(f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(agent, f"restpw-{agent}"))
|
|
put(f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}")
|
|
put(f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}")
|
|
put(f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}")
|
|
machine.succeed(
|
|
f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude "
|
|
f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work"
|
|
)
|
|
machine.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data")
|
|
machine.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch")
|
|
|
|
with subtest("the users file follows the store without a restart of the server"):
|
|
pid = machine.succeed("systemctl show -p MainPID --value swarm-archive").strip()
|
|
machine.succeed("systemctl start swarm-archive-users")
|
|
assert machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid
|
|
assert status("alpha", "alpha") in ("200", "404"), status("alpha", "alpha")
|
|
assert status("beta", "beta") in ("200", "404"), status("beta", "beta")
|
|
# Wrong password: the control that the 200 above is a login, not an open door.
|
|
machine.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401")
|
|
|
|
with subtest("the users file holds hashes, readable by the server only"):
|
|
machine.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd")
|
|
machine.fail("grep -q restpw /run/swarm-archive-auth/htpasswd")
|
|
machine.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'")
|
|
|
|
with subtest("the hive's backup puts each agent in its own repository, without work/"):
|
|
machine.succeed("systemctl start test-archive-backup")
|
|
for agent in agents:
|
|
snaps = machine.succeed(restic(agent, agent, "snapshots --json"))
|
|
assert snaps.count('"id"') == 1, snaps
|
|
listing = machine.succeed(restic(agent, agent, "ls latest"))
|
|
assert f"/{agent}/state/data" in listing, listing
|
|
assert "scratch" not in listing, listing
|
|
|
|
with subtest("one agent's login cannot reach another's repository"):
|
|
assert status("beta", "alpha") == "403", status("beta", "alpha")
|
|
machine.fail(restic("beta", "alpha", "snapshots"))
|
|
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
|
assert status("alpha", "beta") == "403", status("alpha", "beta")
|
|
|
|
with subtest("a client cannot delete or overwrite"):
|
|
snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
|
|
# An old snapshot for the prune to drop. The backup is also the presence control: a lock, which a backup creates and removes, is deletable.
|
|
machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2000-01-01 00:00:00' ${agentRoot}/alpha/state"))
|
|
assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
|
|
assert status("alpha", "alpha", "config", "POST") == "403"
|
|
machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
|
|
assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
|
|
|
with subtest("the server-side prune applies the retention"):
|
|
machine.succeed("systemctl start swarm-archive-prune")
|
|
for agent in agents:
|
|
assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
|
|
# The old snapshot went, the one from the hive's backup stayed.
|
|
assert snap in machine.succeed(restic("alpha", "alpha", "snapshots --json"))
|
|
|
|
with subtest("a snapshot dated in the future cannot push the real ones out of the retention"):
|
|
# restic counts --keep-within from the newest snapshot, and the client sets its time.
|
|
machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2100-01-01 00:00:00' ${agentRoot}/alpha/state"))
|
|
assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
|
machine.succeed("systemctl start swarm-archive-prune")
|
|
left = machine.succeed(restic("alpha", "alpha", "snapshots --json"))
|
|
assert left.count('"id"') == 1 and snap in left, left
|
|
machine.succeed("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: alpha has snapshots dated after now'")
|
|
# beta's repository had nothing forged, and says nothing.
|
|
machine.fail("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: beta'")
|
|
|
|
with subtest("a user removed from the store is refused without a restart"):
|
|
machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")
|
|
machine.succeed("systemctl start swarm-archive-users")
|
|
assert status("beta", "beta") == "401", status("beta", "beta")
|
|
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
|
|
|
with subtest("the archive announces itself once, so swarm-controller starts granting"):
|
|
machine.succeed(f"test \"$(cat {store}/secret/swarm/services/archive/ready/ready)\" = 1")
|
|
machine.succeed("systemctl start swarm-archive-users")
|
|
|
|
with subtest("a hive with agents and no grant backs up nothing and does not fail"):
|
|
machine.succeed(f"rm -r {store}/secret/swarm/hives/${hive}")
|
|
machine.succeed("systemctl start test-archive-backup")
|
|
machine.succeed("journalctl -u test-archive-backup | grep -q 'granted this hive no archive credentials'")
|
|
machine.fail("journalctl -u test-archive-backup | grep -q 'archive backup failed'")
|
|
'';
|
|
}
|