386 lines
18 KiB
Nix
386 lines
18 KiB
Nix
# `vmTests.<system>.full-setup`: one VM runs a whole single-host swarm
|
|
# (`deploy.singleHostSwarm`) with no network and no substituter, and a script
|
|
# walks docs/1-getting-started/setup.md until the controller has committed the
|
|
# agents' meta flake to the swarm's forge.
|
|
#
|
|
# Steps of setup.md, as the script runs them:
|
|
# 1 secret store `bao operator init`, the granter bootstrap with the root
|
|
# token, then the policy units reset and restarted.
|
|
# 4 unseal `bao operator unseal` (`deploy.bao.seal` is "shamir"; pkcs11
|
|
# needs a TPM). A pkcs11 store unseals itself at init, so here
|
|
# the unseal comes right after init, before the granter.
|
|
# 2 SSO account `swarmctl user add mara --group admins`.
|
|
# 8 ruth `POST /api/agents` on the controller socket, as the swarm
|
|
# UI's create form does, plus `manage_root_agent`.
|
|
# 9 first agent The same endpoint, for `iris`: `swarmctl agent create` has no
|
|
# archetype flag, and a custom agent's config repo would need
|
|
# inputs the VM cannot resolve offline.
|
|
# Not scripted yet; a later slice adds them: step 3, the first SSO login to the
|
|
# forge (an OIDC code flow against authelia), `swarmctl forge make-admin`, and
|
|
# the agents' deploy up to the point where each says it needs a login. Green
|
|
# when `internal/meta` on the forge holds the flake the toplevels were built from.
|
|
#
|
|
# Offline: the test seeds the swarm's forge with `hyperhive/hyperhive` (one
|
|
# commit of `self`, ./hyperhive-fixture-repo.nix). The controller renders
|
|
# `internal/meta` and the builder locks and builds it as in production.
|
|
# `hive-meta-flake/testdata/meta-flake-swarm.nix.in`, filled in with the
|
|
# controller's settings, is that flake: the agents' toplevels are built from it
|
|
# here, so the builder finds them in the store. The daemon runs with
|
|
# `max-jobs = 0`, so a missing toplevel fails the build instead of compiling.
|
|
{ pkgs, self }:
|
|
let
|
|
inherit (pkgs) lib;
|
|
|
|
hive = "pr1ma";
|
|
swarmDomain = "swarm.test";
|
|
agents = [
|
|
"iris"
|
|
"ruth"
|
|
];
|
|
|
|
inherit (import ./hyperhive-fixture-repo.nix { inherit pkgs self; }) hyperhiveRepo;
|
|
|
|
metaTemplate = ../../hive-meta-flake/testdata/meta-flake-swarm.nix.in;
|
|
|
|
# The values the fixture has a placeholder for. A swarm whose controller sends
|
|
# another set renders another flake, so evaluation stops there.
|
|
forwardedVars = [
|
|
"HIVE_AGENT_BAO_ADDR"
|
|
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT"
|
|
"HIVE_AGENT_LOGS_QUERY_URL"
|
|
"HIVE_AGENT_METRICS_URL"
|
|
"HIVE_FORGE_PUBLIC_URL"
|
|
"HIVE_FORGE_URL"
|
|
"HIVE_MATRIX_URL"
|
|
];
|
|
|
|
# The meta flake the controller renders for `agents`, from its settings JSON
|
|
# (`SWARM_CONTROLLER_META_FLAKE`). `hyperhive` is `self`: the agents' toplevels
|
|
# do not depend on how the input is pinned.
|
|
filled =
|
|
settingsJson: pin:
|
|
let
|
|
settings = builtins.fromJSON settingsJson;
|
|
names = [ "HYPERHIVE_URL" ] ++ forwardedVars;
|
|
values = [ pin ] ++ map (name: settings.vars.${name}) forwardedVars;
|
|
in
|
|
assert lib.assertMsg
|
|
(builtins.attrNames settings.vars == forwardedVars && !settings.github_disabled)
|
|
"the controller's meta flake settings changed: update hive-meta-flake/testdata/meta-flake-swarm.nix.in";
|
|
builtins.replaceStrings (map (name: "@${name}@") names) values (builtins.readFile metaTemplate);
|
|
|
|
toplevels =
|
|
settingsJson:
|
|
let
|
|
flake = import (builtins.toFile "meta-flake.nix" (filled settingsJson "unused"));
|
|
evaluated = flake.outputs {
|
|
self = null;
|
|
hyperhive = self;
|
|
};
|
|
in
|
|
map (agent: evaluated.nixosConfigurations.${agent}.config.system.build.toplevel) agents;
|
|
in
|
|
pkgs.testers.runNixOSTest (
|
|
{ config, ... }:
|
|
let
|
|
inherit (config.nodes.machine.services.hyperhive) swarm deploy;
|
|
settingsJson =
|
|
config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
|
controllerSocket = deploy.swarm-controller.socketPath;
|
|
|
|
# A derivation no store holds, to show the daemon refuses to build.
|
|
control = pkgs.writeText "control.nix" ''
|
|
derivation {
|
|
name = "build-control";
|
|
system = "${pkgs.stdenv.hostPlatform.system}";
|
|
builder = "/bin/sh";
|
|
args = [ "-c" "echo built > $out" ];
|
|
}
|
|
'';
|
|
in
|
|
{
|
|
name = "full-setup";
|
|
|
|
# Boot of fourteen containers and the setup steps, with the waits' own
|
|
# bounds well inside.
|
|
globalTimeout = 7200;
|
|
|
|
nodes.machine =
|
|
{ config, ... }:
|
|
{
|
|
imports = [ self.nixosModules.default ];
|
|
|
|
services.hyperhive = {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.hive-controller.enable = true;
|
|
deploy.hive-controller.hiveName = hive;
|
|
deploy.bao.seal = "shamir";
|
|
swarm.domain = swarmDomain;
|
|
swarm.hives.${hive} = { };
|
|
};
|
|
system.stateVersion = "25.11";
|
|
|
|
# 12 GB: fourteen service containers, the hive and a Rust-heavy agent
|
|
# closure through the controller, the builder and the cache. 4 cores: the
|
|
# containers boot in parallel. The disk holds the cache's copy of the
|
|
# closure; the store overlay is on disk, not in memory.
|
|
virtualisation.memorySize = 12288;
|
|
virtualisation.cores = 4;
|
|
virtualisation.diskSize = 40960;
|
|
virtualisation.writableStoreUseTmpfs = false;
|
|
|
|
# The toplevels, and every source tree the meta flake's lock names, keyed
|
|
# by narHash: nix takes a locked input from the store when its path is
|
|
# valid.
|
|
virtualisation.additionalPaths = [
|
|
self
|
|
self.inputs.nixpkgs
|
|
self.inputs.crane
|
|
self.inputs.treefmt-nix
|
|
hyperhiveRepo
|
|
]
|
|
++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
|
|
|
# No substituter, and no build.
|
|
nix.settings = {
|
|
substituters = lib.mkForce [ ];
|
|
max-jobs = 0;
|
|
};
|
|
|
|
# The store's raft data is on a tmpfs. Raft syncs every write, so on the
|
|
# VM's qcow2 root (`cache=writeback`, which passes each guest flush on to
|
|
# the host's disk) a login's lease write can outlast the 15 s the store's
|
|
# clients wait, and its calls then fail with `context canceled`. The
|
|
# state is under `/var/lib/private`, where the store's DynamicUser keeps
|
|
# it; the mount sits on that directory in the container's root, and
|
|
# nothing the test checks needs it to survive the VM.
|
|
fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
|
|
{
|
|
device = "tmpfs";
|
|
fsType = "tmpfs";
|
|
options = [
|
|
"mode=0700"
|
|
"size=1G"
|
|
];
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.jq
|
|
];
|
|
};
|
|
|
|
testScript = ''
|
|
import json
|
|
import shlex
|
|
|
|
API = "http://localhost"
|
|
SOCKET = "${controllerSocket}"
|
|
FORGE = "https://${swarm.forge.domain}"
|
|
HIVE = "${hive}"
|
|
TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))}
|
|
INIT = "/root/bao-init.json"
|
|
FORGE_AUTH = "/root/forge-auth.header"
|
|
NIX = "nix --extra-experimental-features nix-command"
|
|
ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
|
|
POLICY_UNITS_ACTIVE = (
|
|
"for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
|
|
"swarm-bao-agent-pki.service; do "
|
|
"[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done"
|
|
)
|
|
|
|
def controller(method, path, body=None, check="--fail-with-body"):
|
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
|
return machine.succeed(
|
|
f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}"
|
|
)
|
|
|
|
def forge(method, path, body=None):
|
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
|
return machine.succeed(
|
|
f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} "
|
|
f"-X {method} {FORGE}/api/v1{path} {data}"
|
|
)
|
|
|
|
def diagnose(unit=None, container=None):
|
|
# What a timed-out wait leaves behind: the failed units, and the status
|
|
# and the end of the journal of the unit that was waited on.
|
|
where = f"--machine={container} " if container else ""
|
|
units = [unit] if isinstance(unit, str) else list(unit or [])
|
|
flags = " ".join(f"-u '{u}'" for u in units)
|
|
for command in [
|
|
f"systemctl {where}--failed --no-legend --no-pager",
|
|
*([f"systemctl {where}status --no-pager -n 0 -l {' '.join(units)} | head -c 4000"] if units else []),
|
|
*([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []),
|
|
]:
|
|
print(f"--- {command}")
|
|
print(machine.execute(command)[1])
|
|
|
|
def settle(command, timeout, unit=None, container=None):
|
|
try:
|
|
machine.wait_until_succeeds(command, timeout=timeout)
|
|
except Exception:
|
|
diagnose(unit, container)
|
|
raise
|
|
|
|
def create_agent(body):
|
|
# The controller refuses a create until its dependencies answer (the
|
|
# archetype list comes from the builder's evaluation), so ask again.
|
|
for _ in range(60):
|
|
status, out = machine.execute(
|
|
f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' "
|
|
f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}"
|
|
)
|
|
if out.strip().startswith("2"):
|
|
return
|
|
machine.sleep(10)
|
|
diagnose("swarm-controller")
|
|
raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {machine.succeed('cat /root/create.out')}")
|
|
|
|
machine.start()
|
|
machine.wait_for_unit("multi-user.target")
|
|
|
|
with subtest("the agents' toplevels are in the store, and the daemon builds nothing"):
|
|
for toplevel in TOPLEVELS:
|
|
machine.succeed(f"{NIX} path-info {toplevel}")
|
|
# The control for the lookup: a path that is not there is not found.
|
|
machine.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent")
|
|
# The control for the missing build: a derivation no store holds is refused.
|
|
machine.fail("NIX_CONFIG='experimental-features = nix-command' nix build --no-link --impure -f ${control}")
|
|
|
|
# setup.md step 1: the secret store.
|
|
with subtest("step 1: bao operator init"):
|
|
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
|
machine.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")
|
|
|
|
with subtest("step 4: bao operator unseal, with the shamir seal"):
|
|
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
|
machine.succeed("bao status")
|
|
|
|
with subtest("step 1: the granter bootstrap"):
|
|
machine.succeed(
|
|
"set -o pipefail; "
|
|
f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
|
|
"bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
|
|
"bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
|
|
"| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
|
|
"unset BAO_TOKEN; "
|
|
"systemctl restart swarm-bao-granter-role"
|
|
)
|
|
machine.wait_until_succeeds(
|
|
"journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'",
|
|
timeout=300,
|
|
)
|
|
|
|
with subtest("step 1: the policy units that waited for the granter are restarted"):
|
|
# Some of these units retry every 30 seconds until something outside
|
|
# them exists: the bao UI's viewer policy waits for authelia's secrets
|
|
# unit, in its container, to publish the UI's OIDC client secret, and
|
|
# that unit waits for the secret publisher's policy written here. So
|
|
# the units are started without waiting, and polled until all are active.
|
|
machine.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true")
|
|
machine.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service")
|
|
machine.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=600)
|
|
# Under the shamir seal the PKI unit leaves a restart of the store to the
|
|
# operator when its listener's client CA changed, and says which command.
|
|
if machine.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
|
|
machine.succeed("systemctl --machine=swarm-bao restart openbao.service")
|
|
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
|
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
|
settle(POLICY_UNITS_ACTIVE, 1200, "swarm-bao-*-policy")
|
|
machine.succeed(
|
|
"journalctl -u swarm-bao-controller-policy --no-pager "
|
|
"| grep -F 'Data written to: auth/cert/certs/swarm-controller'"
|
|
)
|
|
|
|
with subtest("authelia's mint has published the clients' secrets"):
|
|
# The operator viewer policy is left out of the poll above: it waits for
|
|
# the UI client's secret, which authelia's secrets unit publishes from
|
|
# its container once the store and the trust bundle it needs exist. The
|
|
# controller's client secret is published by the same unit, and the
|
|
# controller, the hive's store logins and the UI policy all wait for it.
|
|
settle(
|
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
|
"secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
|
|
1200,
|
|
"authelia-swarm-secrets",
|
|
"swarm-authelia",
|
|
)
|
|
settle("[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 300, "swarm-bao-operator-viewer-policy")
|
|
|
|
with subtest("step 2: swarmctl user add"):
|
|
machine.wait_until_succeeds(
|
|
"swarmctl user add mara --display-name Mara --email mara@example.com --group admins "
|
|
">/root/user-add.out 2>&1",
|
|
timeout=900,
|
|
)
|
|
machine.succeed("[ -s /root/user-add.out ]")
|
|
machine.succeed("swarmctl user list | grep -w mara")
|
|
|
|
with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
|
|
# The controller's forge token, from the store as the controller reads it.
|
|
# Minted and stored by `forgejo-swarm-controller-account` in the forge
|
|
# container, which waits for forgejo and for a login to the store.
|
|
settle(
|
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
|
"secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
|
|
900,
|
|
["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],
|
|
"hive-forge",
|
|
)
|
|
machine.succeed(
|
|
f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}"
|
|
)
|
|
# The token being published says nothing about the gateway's vhost for
|
|
# the forge, which the calls below go through.
|
|
machine.wait_until_succeeds(
|
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600
|
|
)
|
|
status = machine.succeed(
|
|
f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive"
|
|
).strip()
|
|
if status != "200":
|
|
forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"})
|
|
forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"})
|
|
machine.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git")
|
|
machine.succeed(
|
|
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
|
f"push {FORGE}/hyperhive/hyperhive.git main"
|
|
)
|
|
pushed = machine.succeed(
|
|
f"GIT_SSL_NO_VERIFY=1 git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
|
f"ls-remote {FORGE}/hyperhive/hyperhive.git refs/heads/main"
|
|
).split()[0]
|
|
assert pushed == machine.succeed("cat ${hyperhiveRepo}/rev").strip(), pushed
|
|
|
|
with subtest("step 8: ruth, through the endpoint the swarm UI calls"):
|
|
machine.wait_until_succeeds(
|
|
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=900
|
|
)
|
|
default = json.loads(controller("GET", "/api/permissions"))["default"]
|
|
create_agent({"name": "ruth", "hive": HIVE, "permissions": default + ["manage_root_agent"], "archetype": ARCHETYPE})
|
|
|
|
with subtest("step 9: the first agent"):
|
|
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
|
|
|
|
with subtest("internal/meta holds the flake the toplevels were built from"):
|
|
settle(
|
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/internal/meta/raw/flake.nix "
|
|
">/root/meta-flake.nix",
|
|
1800,
|
|
"swarm-controller",
|
|
)
|
|
rev = machine.succeed("cat ${hyperhiveRepo}/rev").strip()
|
|
settings = json.loads(${builtins.toJSON settingsJson})
|
|
expected = open("${metaTemplate}").read().replace(
|
|
"@HYPERHIVE_URL@", f"git+{settings['git_base']}/hyperhive/hyperhive.git?rev={rev}"
|
|
)
|
|
for name, value in settings["vars"].items():
|
|
expected = expected.replace(f"@{name}@", value)
|
|
committed = machine.succeed("cat /root/meta-flake.nix")
|
|
assert committed == expected, "the controller's flake differs from the one the toplevels came from:\n" + committed
|
|
|
|
# The agents' deploy and the check that they need a login are not scripted yet.
|
|
'';
|
|
}
|
|
)
|