swarm-archive: external s3 backend (options, prune path, docs, module-eval cases, vm fake s3)
- swarm-archive.nix, lib/archive-server.nix: deploy.archive.backend and deploy.archive.s3.* options; the s3 server and the prune read the key from the store into rclone's environment; the prune lists and addresses repositories through rclone instead of a local directory - swarm-bao.nix: the archive reads swarm/services/archive/s3/* - docs: archive.md "External S3" section, two credentials-reference rows - module-eval: builder.nix cases for local and s3, the missing-setting assertion; bao-grants.nix gains the s3 read grant - vm test: second node with a fake rclone serve s3 whose key is generated at test time
This commit is contained in:
atlas
mara
parent
ebbf4fa37b
commit
86c5dac89d
1 changed files with 605 additions and 156 deletions
|
|
@ -1,7 +1,9 @@
|
|||
# `vmTests.<system>.swarm-archive`: runs the archive server's real units
|
||||
# (../host-modules/swarm-archive.nix) and the hive's real backup script
|
||||
# (../host-modules/lib/archive-backup.nix) in a VM, with restic and rclone, and
|
||||
# checks what the module-eval suite can only read:
|
||||
# checks what the module-eval suite can only read, on two nodes: `machine` keeps
|
||||
# the repositories in a directory, `s3archive` in a bucket of a fake S3
|
||||
# (`rclone serve s3`). On both:
|
||||
#
|
||||
# - a backup of two agents lands in one repository each, and `work/` is left out;
|
||||
# - one agent's login cannot read another's repository;
|
||||
|
|
@ -9,14 +11,18 @@
|
|||
# applies the retention, also when a client has forged a snapshot dated in
|
||||
# the future;
|
||||
# - the users file follows the store: a user removed from it is refused without
|
||||
# a restart of the server;
|
||||
# - a hive that has agents and no grant backs up nothing and succeeds.
|
||||
# a restart of the server.
|
||||
#
|
||||
# One node, because the gateway hop is outside what is under test: the
|
||||
# backup script dials `http://127.0.0.1:<port>` instead of
|
||||
# `https://archive.<domain>`, and the swarm secret store is a stand-in `bao`
|
||||
# that serves `kv list`, `kv get` and `login` from a directory. The module-eval
|
||||
# suite covers the gateway vhost and the store grants.
|
||||
# Only on `machine`: a hive that has agents and no grant backs up nothing and
|
||||
# succeeds. Only on `s3archive`: the server does not start while the store
|
||||
# holds no S3 key; the repositories are objects in the bucket and not on the
|
||||
# host; the key is in rclone's environment and in no unit or file.
|
||||
#
|
||||
# The gateway hop is outside what is under test: the backup script dials
|
||||
# `http://127.0.0.1:<port>` instead of `https://archive.<domain>`, and the swarm
|
||||
# secret store is a stand-in `bao` that serves `kv list`, `kv get` and `login`
|
||||
# from a directory. The module-eval suite covers the gateway vhost and the store
|
||||
# grants.
|
||||
{ pkgs }:
|
||||
let
|
||||
port = 8911;
|
||||
|
|
@ -135,16 +141,26 @@ let
|
|||
excludes = [ ];
|
||||
caBundle = null;
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "swarm-archive";
|
||||
|
||||
nodes.machine =
|
||||
# The external S3 of the second node: `rclone serve s3` over a directory,
|
||||
# whose sub-directories are the buckets. The archive reaches it over HTTP
|
||||
# with a key the unit generates at start, as it would reach a provider. rclone
|
||||
# refuses short keys.
|
||||
s3Port = 9000;
|
||||
s3Root = "/var/lib/fake-s3";
|
||||
s3Bucket = "archive-bucket";
|
||||
s3KeyDir = "/var/lib/fake-s3-key";
|
||||
|
||||
# What both nodes share: the archive module, the stubs for the hive options it
|
||||
# reads, and the hive's backup unit.
|
||||
archiveNode =
|
||||
extra:
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
imports = [
|
||||
hiveStubs
|
||||
../host-modules/swarm-archive.nix
|
||||
extra
|
||||
];
|
||||
|
||||
# The newest snapshot, plus whatever is younger than three days.
|
||||
|
|
@ -174,23 +190,56 @@ pkgs.testers.runNixOSTest {
|
|||
script = backupScript;
|
||||
};
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "swarm-archive";
|
||||
|
||||
nodes.machine = archiveNode { };
|
||||
|
||||
nodes.s3archive = archiveNode {
|
||||
services.hyperhive.deploy.archive = {
|
||||
backend = "s3";
|
||||
s3 = {
|
||||
endpoint = "http://127.0.0.1:${toString s3Port}";
|
||||
bucket = s3Bucket;
|
||||
prefix = "pfx";
|
||||
region = "us-east-1";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.fake-s3 = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "swarm-archive.service" ];
|
||||
environment.RCLONE_CONFIG = "/dev/null";
|
||||
script = ''
|
||||
mkdir -p ${s3Root}/${s3Bucket}
|
||||
install -d -m 0700 ${s3KeyDir}
|
||||
if [ ! -e ${s3KeyDir}/id ]; then
|
||||
head -c 15 /dev/urandom | base32 | tr -d = > ${s3KeyDir}/id
|
||||
head -c 24 /dev/urandom | base32 | tr -d = > ${s3KeyDir}/secret
|
||||
fi
|
||||
exec ${pkgs.rclone}/bin/rclone serve s3 --addr 127.0.0.1:${toString s3Port} \
|
||||
--auth-key "$(cat ${s3KeyDir}/id),$(cat ${s3KeyDir}/secret)" ${s3Root}
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
store = "${store}"
|
||||
url = "http://127.0.0.1:${toString port}"
|
||||
agents = ["alpha", "beta"]
|
||||
|
||||
def sha_line(agent, password):
|
||||
return machine.succeed(
|
||||
def sha_line(m, agent, password):
|
||||
return m.succeed(
|
||||
f"printf '%s:{{SHA}}%s' {agent} "
|
||||
f"\"$(printf %s {password} | openssl dgst -sha1 -binary | base64)\""
|
||||
)
|
||||
|
||||
def put(path, field, value):
|
||||
machine.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}")
|
||||
def put(m, path, field, value):
|
||||
m.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}")
|
||||
|
||||
def status(agent, repo, path="config", method="GET"):
|
||||
return machine.succeed(
|
||||
def status(m, agent, repo, path="config", method="GET"):
|
||||
return m.succeed(
|
||||
f"curl -s -o /dev/null -w '%{{http_code}}' -X {method} "
|
||||
f"-u {agent}:restpw-{agent} {url}/{repo}/{path}"
|
||||
)
|
||||
|
|
@ -201,88 +250,99 @@ pkgs.testers.runNixOSTest {
|
|||
f"-r rest:http://{agent}:restpw-{agent}@127.0.0.1:${toString port}/{repo}/ {args}"
|
||||
)
|
||||
|
||||
# What swarm-controller writes: a user line and a restic password per agent
|
||||
# for the archive, a REST password and the restic password for the hive.
|
||||
def grant_agents(m):
|
||||
for agent in agents:
|
||||
put(m, f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(m, agent, f"restpw-{agent}"))
|
||||
put(m, f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}")
|
||||
put(m, f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}")
|
||||
put(m, f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}")
|
||||
m.succeed(
|
||||
f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude "
|
||||
f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work"
|
||||
)
|
||||
m.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data")
|
||||
m.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch")
|
||||
|
||||
# What the archive promises, whichever backend holds the repositories.
|
||||
def archive_suite(m, tag):
|
||||
with subtest(f"{tag}: the server starts with no users and admits nobody"):
|
||||
m.succeed("test -e /run/swarm-archive-auth/htpasswd")
|
||||
assert status(m, "alpha", "alpha") == "401", status(m, "alpha", "alpha")
|
||||
|
||||
grant_agents(m)
|
||||
|
||||
with subtest(f"{tag}: the users file follows the store without a restart of the server"):
|
||||
pid = m.succeed("systemctl show -p MainPID --value swarm-archive").strip()
|
||||
m.succeed("systemctl start swarm-archive-users")
|
||||
assert m.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid
|
||||
assert status(m, "alpha", "alpha") in ("200", "404"), status(m, "alpha", "alpha")
|
||||
assert status(m, "beta", "beta") in ("200", "404"), status(m, "beta", "beta")
|
||||
# Wrong password: the control that the 200 above is a login, not an open door.
|
||||
m.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401")
|
||||
|
||||
with subtest(f"{tag}: the users file holds hashes, readable by the server only"):
|
||||
m.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd")
|
||||
m.fail("grep -q restpw /run/swarm-archive-auth/htpasswd")
|
||||
m.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'")
|
||||
|
||||
with subtest(f"{tag}: the hive's backup puts each agent in its own repository, without work/"):
|
||||
m.succeed("systemctl start test-archive-backup")
|
||||
for agent in agents:
|
||||
snaps = m.succeed(restic(agent, agent, "snapshots --json"))
|
||||
assert snaps.count('"id"') == 1, snaps
|
||||
listing = m.succeed(restic(agent, agent, "ls latest"))
|
||||
assert f"/{agent}/state/data" in listing, listing
|
||||
assert "scratch" not in listing, listing
|
||||
|
||||
with subtest(f"{tag}: one agent's login cannot reach another's repository"):
|
||||
assert status(m, "beta", "alpha") == "403", status(m, "beta", "alpha")
|
||||
m.fail(restic("beta", "alpha", "snapshots"))
|
||||
assert status(m, "alpha", "alpha") == "200", status(m, "alpha", "alpha")
|
||||
assert status(m, "alpha", "beta") == "403", status(m, "alpha", "beta")
|
||||
|
||||
with subtest(f"{tag}: a client cannot delete or overwrite"):
|
||||
snap = m.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
|
||||
# An old snapshot for the prune to drop. The backup is also the presence control: a lock, which a backup creates and removes, is deletable.
|
||||
m.succeed(restic("alpha", "alpha", "backup --host alpha --time '2000-01-01 00:00:00' ${agentRoot}/alpha/state"))
|
||||
assert status(m, "alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
|
||||
assert status(m, "alpha", "alpha", "config", "POST") == "403"
|
||||
m.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
|
||||
assert m.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
||||
|
||||
with subtest(f"{tag}: the server-side prune applies the retention"):
|
||||
m.succeed("systemctl start swarm-archive-prune")
|
||||
for agent in agents:
|
||||
assert m.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
|
||||
# The old snapshot went, the one from the hive's backup stayed.
|
||||
assert snap in m.succeed(restic("alpha", "alpha", "snapshots --json"))
|
||||
|
||||
with subtest(f"{tag}: a snapshot dated in the future cannot push the real ones out of the retention"):
|
||||
# restic counts --keep-within from the newest snapshot, and the client sets its time.
|
||||
m.succeed(restic("alpha", "alpha", "backup --host alpha --time '2100-01-01 00:00:00' ${agentRoot}/alpha/state"))
|
||||
assert m.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
||||
m.succeed("systemctl start swarm-archive-prune")
|
||||
left = m.succeed(restic("alpha", "alpha", "snapshots --json"))
|
||||
assert left.count('"id"') == 1 and snap in left, left
|
||||
m.succeed("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: alpha has snapshots dated after now'")
|
||||
# beta's repository had nothing forged, and says nothing.
|
||||
m.fail("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: beta'")
|
||||
|
||||
with subtest(f"{tag}: a user removed from the store is refused without a restart"):
|
||||
m.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")
|
||||
m.succeed("systemctl start swarm-archive-users")
|
||||
assert status(m, "beta", "beta") == "401", status(m, "beta", "beta")
|
||||
assert status(m, "alpha", "alpha") == "200", status(m, "alpha", "alpha")
|
||||
|
||||
start_all()
|
||||
machine.wait_for_unit("multi-user.target")
|
||||
machine.wait_for_open_port(${toString port})
|
||||
|
||||
with subtest("the server starts with no users and admits nobody"):
|
||||
machine.succeed("test -e /run/swarm-archive-auth/htpasswd")
|
||||
assert status("alpha", "alpha") == "401", status("alpha", "alpha")
|
||||
archive_suite(machine, "local")
|
||||
|
||||
# What swarm-controller writes: a user line and a restic password per agent
|
||||
# for the archive, a REST password and the restic password for the hive.
|
||||
for agent in agents:
|
||||
put(f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(agent, f"restpw-{agent}"))
|
||||
put(f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}")
|
||||
put(f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}")
|
||||
put(f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}")
|
||||
machine.succeed(
|
||||
f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude "
|
||||
f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work"
|
||||
)
|
||||
machine.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data")
|
||||
machine.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch")
|
||||
|
||||
with subtest("the users file follows the store without a restart of the server"):
|
||||
pid = machine.succeed("systemctl show -p MainPID --value swarm-archive").strip()
|
||||
machine.succeed("systemctl start swarm-archive-users")
|
||||
assert machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid
|
||||
assert status("alpha", "alpha") in ("200", "404"), status("alpha", "alpha")
|
||||
assert status("beta", "beta") in ("200", "404"), status("beta", "beta")
|
||||
# Wrong password: the control that the 200 above is a login, not an open door.
|
||||
machine.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401")
|
||||
|
||||
with subtest("the users file holds hashes, readable by the server only"):
|
||||
machine.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd")
|
||||
machine.fail("grep -q restpw /run/swarm-archive-auth/htpasswd")
|
||||
machine.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'")
|
||||
|
||||
with subtest("the hive's backup puts each agent in its own repository, without work/"):
|
||||
machine.succeed("systemctl start test-archive-backup")
|
||||
for agent in agents:
|
||||
snaps = machine.succeed(restic(agent, agent, "snapshots --json"))
|
||||
assert snaps.count('"id"') == 1, snaps
|
||||
listing = machine.succeed(restic(agent, agent, "ls latest"))
|
||||
assert f"/{agent}/state/data" in listing, listing
|
||||
assert "scratch" not in listing, listing
|
||||
|
||||
with subtest("one agent's login cannot reach another's repository"):
|
||||
assert status("beta", "alpha") == "403", status("beta", "alpha")
|
||||
machine.fail(restic("beta", "alpha", "snapshots"))
|
||||
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
||||
assert status("alpha", "beta") == "403", status("alpha", "beta")
|
||||
|
||||
with subtest("a client cannot delete or overwrite"):
|
||||
snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
|
||||
# An old snapshot for the prune to drop. The backup is also the presence control: a lock, which a backup creates and removes, is deletable.
|
||||
machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2000-01-01 00:00:00' ${agentRoot}/alpha/state"))
|
||||
assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
|
||||
assert status("alpha", "alpha", "config", "POST") == "403"
|
||||
machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
|
||||
assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
||||
|
||||
with subtest("the server-side prune applies the retention"):
|
||||
machine.succeed("systemctl start swarm-archive-prune")
|
||||
for agent in agents:
|
||||
assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
|
||||
# The old snapshot went, the one from the hive's backup stayed.
|
||||
assert snap in machine.succeed(restic("alpha", "alpha", "snapshots --json"))
|
||||
|
||||
with subtest("a snapshot dated in the future cannot push the real ones out of the retention"):
|
||||
# restic counts --keep-within from the newest snapshot, and the client sets its time.
|
||||
machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2100-01-01 00:00:00' ${agentRoot}/alpha/state"))
|
||||
assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
||||
machine.succeed("systemctl start swarm-archive-prune")
|
||||
left = machine.succeed(restic("alpha", "alpha", "snapshots --json"))
|
||||
assert left.count('"id"') == 1 and snap in left, left
|
||||
machine.succeed("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: alpha has snapshots dated after now'")
|
||||
# beta's repository had nothing forged, and says nothing.
|
||||
machine.fail("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: beta'")
|
||||
|
||||
with subtest("a user removed from the store is refused without a restart"):
|
||||
machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")
|
||||
machine.succeed("systemctl start swarm-archive-users")
|
||||
assert status("beta", "beta") == "401", status("beta", "beta")
|
||||
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
||||
with subtest("the repositories are under the data directory"):
|
||||
machine.succeed("test -d /var/lib/swarm-archive/repos/alpha")
|
||||
|
||||
with subtest("the archive announces itself once, so swarm-controller starts granting"):
|
||||
machine.succeed(f"test \"$(cat {store}/secret/swarm/services/archive/ready/ready)\" = 1")
|
||||
|
|
@ -293,5 +353,45 @@ pkgs.testers.runNixOSTest {
|
|||
machine.succeed("systemctl start test-archive-backup")
|
||||
machine.succeed("journalctl -u test-archive-backup | grep -q 'granted this hive no archive credentials'")
|
||||
machine.fail("journalctl -u test-archive-backup | grep -q 'archive backup failed'")
|
||||
|
||||
# The same archive on the second node, with its repositories on the fake S3.
|
||||
s3archive.wait_for_unit("fake-s3.service")
|
||||
s3_key_id = s3archive.succeed("cat ${s3KeyDir}/id").strip()
|
||||
s3_secret_key = s3archive.succeed("cat ${s3KeyDir}/secret").strip()
|
||||
|
||||
with subtest("s3: without the key in the store the server does not start"):
|
||||
s3archive.wait_until_succeeds("journalctl -u swarm-archive | grep -q 'No value found'")
|
||||
s3archive.fail("systemctl is-active --quiet swarm-archive")
|
||||
s3archive.fail(f"curl -sf --max-time 2 -o /dev/null {url}/")
|
||||
|
||||
with subtest("s3: the server starts once the store holds the key"):
|
||||
put(s3archive, "secret/swarm/services/archive/s3/access-key-id", "value", s3_key_id)
|
||||
put(s3archive, "secret/swarm/services/archive/s3/secret-access-key", "value", s3_secret_key)
|
||||
s3archive.wait_for_open_port(${toString port})
|
||||
|
||||
archive_suite(s3archive, "s3")
|
||||
|
||||
with subtest("s3: the repositories are in the bucket and not on this host"):
|
||||
for agent in agents:
|
||||
s3archive.succeed(f"test -f ${s3Root}/${s3Bucket}/pfx/{agent}/config")
|
||||
s3archive.succeed("find ${s3Root}/${s3Bucket}/pfx/alpha/snapshots -type f | grep -q .")
|
||||
s3archive.succeed("find ${s3Root}/${s3Bucket}/pfx/alpha/data -type f | grep -q .")
|
||||
s3archive.fail("test -e /var/lib/swarm-archive/repos")
|
||||
|
||||
with subtest("s3: the key is in rclone's environment and in no unit"):
|
||||
pid = s3archive.succeed("systemctl show -p MainPID --value swarm-archive").strip()
|
||||
env = s3archive.succeed(f"tr '\\0' '\\n' < /proc/{pid}/environ")
|
||||
assert f"RCLONE_S3_ACCESS_KEY_ID={s3_key_id}" in env, env
|
||||
assert f"RCLONE_S3_SECRET_ACCESS_KEY={s3_secret_key}" in env, env
|
||||
# The connection settings are in the unit text, the control that the search
|
||||
# for the key below looks at the units.
|
||||
units = "swarm-archive swarm-archive-prune swarm-archive-users"
|
||||
s3archive.succeed(f"systemctl cat {units} | grep -q RCLONE_S3_ENDPOINT")
|
||||
s3archive.fail(f"systemctl cat {units} | grep -q {s3_key_id}")
|
||||
s3archive.fail(f"systemctl cat {units} | grep -q {s3_secret_key}")
|
||||
wrapper = s3archive.succeed("systemctl show -p ExecStart --value swarm-archive | grep -o '/nix/store/[^ ;]*swarm-archive-serve' | head -1").strip()
|
||||
s3archive.succeed(f"grep -q 'kv get' {wrapper}")
|
||||
s3archive.fail(f"grep -q {s3_key_id} {wrapper}")
|
||||
s3archive.fail(f"grep -rq {s3_key_id} /run/swarm-archive-auth /var/cache/swarm-archive-prune")
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue