diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 7780893d7..0d63e35d5 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -99,6 +99,19 @@ jobs: fi echo "$files" | xargs nix develop -c shellcheck -S warning + ruff-vm-tests: + name: ruff (vm-tests) + if: vars.PUBLIC_FORGE != 'true' + runs-on: ${{ vars.PUBLIC_FORGE == 'true' && 'nixos' || 'hive-ci' }} + steps: + - uses: actions/checkout@v3 + - name: lint + # ruff from this flake's nixpkgs pin, as the jq job does. The config in + # nix/vm-tests/ruff.toml names the globals the test driver provides. + run: | + nix shell --inputs-from . nixpkgs#ruff --command sh -c \ + 'ruff check nix/vm-tests && ruff format --check nix/vm-tests' + prose-lint: name: prose lint (vale) if: vars.PUBLIC_FORGE != 'true' diff --git a/docs/4-implementation/scheduler/ci.md b/docs/4-implementation/scheduler/ci.md index b36c167b1..43299d379 100644 --- a/docs/4-implementation/scheduler/ci.md +++ b/docs/4-implementation/scheduler/ci.md @@ -124,6 +124,15 @@ root, through the real hive-priv unit), `hive-squid`, and `wg-hive` (two hives peered over the `wg-hive` mesh: handshake, traffic, and a wrong-key peer that must not handshake). +The `full-setup` script is plain Python in `nix/vm-tests/full-setup.py`, with no +Nix interpolation. Commands the script runs in the VM find their binaries on the +node's `environment.systemPackages`. Build-time values (socket path, forge +domain, store paths) arrive as `FULL_SETUP_*` environment variables, set in +`nix/vm-tests/full-setup.nix` through `overrideTestDerivation`. To add a value, +set one there and read it with `os.environ` in the script. The `ruff (vm-tests)` +job checks and format-checks `nix/vm-tests/*.py`; `nix/vm-tests/ruff.toml` names +the globals the test driver provides. + ## Configuration reference `services.hyperhive.deploy.ci-runner.enable = true` on the host that runs it diff --git a/nix/vm-tests/full-setup.nix b/nix/vm-tests/full-setup.nix index 9b9f0c1dd..6744c1fa2 100644 --- a/nix/vm-tests/full-setup.nix +++ b/nix/vm-tests/full-setup.nix @@ -45,332 +45,107 @@ let sourcePaths toplevels ; -in -pkgs.testers.runNixOSTest ( - { config, ... }: - let - inherit (config.nodes.machine.services.hyperhive) swarm deploy; - settingsJson = - config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; - controllerSocket = deploy.swarm-controller.socketPath; - # A derivation no store holds, to show the daemon refuses to build. - control = pkgs.writeText "control.nix" '' - derivation { - name = "build-control"; - system = "${pkgs.stdenv.hostPlatform.system}"; - builder = "/bin/sh"; - args = [ "-c" "echo built > $out" ]; - } - ''; - in - { - name = "full-setup"; + # A derivation no store holds, to show the daemon refuses to build. + control = pkgs.writeText "control.nix" '' + derivation { + name = "build-control"; + system = "${pkgs.stdenv.hostPlatform.system}"; + builder = "/bin/sh"; + args = [ "-c" "echo built > $out" ]; + } + ''; - # Boot of fourteen containers and the setup steps, with the waits' own - # bounds well inside. - globalTimeout = 7200; + test = pkgs.testers.runNixOSTest ( + { config, ... }: + { + name = "full-setup"; - nodes.machine = - { config, ... }: - { - imports = [ self.nixosModules.default ]; + # Boot of fourteen containers and the setup steps, with the waits' own + # bounds well inside. + globalTimeout = 7200; - services.hyperhive = { - deploy.singleHostSwarm = true; - deploy.hive-controller.enable = true; - deploy.hive-controller.hiveName = hive; - deploy.bao.seal = "shamir"; - # The test pushes its own hyperhive/hyperhive; a mirror of the - # upstream could not be created offline and would race the push. - deploy.forgejo.hyperhiveUpstream = null; - swarm.domain = swarmDomain; - swarm.hives.${hive} = { }; - }; - system.stateVersion = "25.11"; + nodes.machine = + { config, ... }: + { + imports = [ self.nixosModules.default ]; - # 12 GB: fourteen service containers, the hive and a Rust-heavy agent - # closure through the controller, the builder and the cache. 4 cores: the - # containers boot in parallel. The disk holds the cache's copy of the - # closure; the store overlay is on disk, not in memory. - virtualisation.memorySize = 12288; - virtualisation.cores = 4; - virtualisation.diskSize = 40960; - virtualisation.writableStoreUseTmpfs = false; + services.hyperhive = { + deploy.singleHostSwarm = true; + deploy.hive-controller.enable = true; + deploy.hive-controller.hiveName = hive; + deploy.bao.seal = "shamir"; + # The test pushes its own hyperhive/hyperhive; a mirror of the + # upstream could not be created offline and would race the push. + deploy.forgejo.hyperhiveUpstream = null; + swarm.domain = swarmDomain; + swarm.hives.${hive} = { }; + }; + system.stateVersion = "25.11"; - # The toplevels, and every source tree the meta flake's lock names. - virtualisation.additionalPaths = - sourcePaths - ++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; + # 12 GB: fourteen service containers, the hive and a Rust-heavy agent + # closure through the controller, the builder and the cache. 4 cores: the + # containers boot in parallel. The disk holds the cache's copy of the + # closure; the store overlay is on disk, not in memory. + virtualisation.memorySize = 12288; + virtualisation.cores = 4; + virtualisation.diskSize = 40960; + virtualisation.writableStoreUseTmpfs = false; - # No substituter, and no build. - nix.settings = { - substituters = lib.mkForce [ ]; - max-jobs = 0; - }; + # The toplevels, and every source tree the meta flake's lock names. + virtualisation.additionalPaths = + sourcePaths + ++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; - # The store's raft data is on a tmpfs. Raft syncs every write, so on the - # VM's qcow2 root (`cache=writeback`, which passes each guest flush on to - # the host's disk) a login's lease write can outlast the 15 s the store's - # clients wait, and its calls then fail with `context canceled`. The - # state is under `/var/lib/private`, where the store's DynamicUser keeps - # it; the mount sits on that directory in the container's root, and - # nothing the test checks needs it to survive the VM. - fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" = - { - device = "tmpfs"; - fsType = "tmpfs"; - options = [ - "mode=0700" - "size=1G" - ]; + # No substituter, and no build. + nix.settings = { + substituters = lib.mkForce [ ]; + max-jobs = 0; }; - environment.systemPackages = [ - pkgs.git - pkgs.jq - ]; - }; + # The store's raft data is on a tmpfs. Raft syncs every write, so on the + # VM's qcow2 root (`cache=writeback`, which passes each guest flush on to + # the host's disk) a login's lease write can outlast the 15 s the store's + # clients wait, and its calls then fail with `context canceled`. The + # state is under `/var/lib/private`, where the store's DynamicUser keeps + # it; the mount sits on that directory in the container's root, and + # nothing the test checks needs it to survive the VM. + fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" = + { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ + "mode=0700" + "size=1G" + ]; + }; - testScript = '' - import json - import shlex + environment.systemPackages = [ + pkgs.git + pkgs.jq + ]; + }; - API = "http://localhost" - SOCKET = "${controllerSocket}" - FORGE = "https://${swarm.forge.domain}" - HIVE = "${hive}" - TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))} - INIT = "/root/bao-init.json" - FORGE_AUTH = "/root/forge-auth.header" - NIX = "nix --extra-experimental-features nix-command" - ARCHETYPE = {"source": "hyperhive", "name": "reviewer"} - POLICY_UNITS_ACTIVE = ( - "for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) " - "swarm-bao-agent-pki.service; do " - "[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done" - ) + testScript = builtins.readFile ./full-setup.py; + } + ); - def controller(method, path, body=None, check="--fail-with-body"): - data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else "" - return machine.succeed( - f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}" - ) - - def forge(method, path, body=None): - data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else "" - return machine.succeed( - f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} " - f"-X {method} {FORGE}/api/v1{path} {data}" - ) - - def diagnose(unit=None, container=None): - # What a timed-out wait leaves behind: the failed units and the end of the - # journal of the units that were waited on. - where = f"--machine={container} " if container else "" - units = [unit] if isinstance(unit, str) else list(unit or []) - flags = " ".join(f"-u '{u}'" for u in units) - for command in [ - f"systemctl {where}--failed --no-legend --no-pager", - *([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []), - ]: - print(f"--- {command}") - print(machine.execute(command)[1]) - - def settle(command, timeout, unit=None, container=None): - try: - machine.wait_until_succeeds(command, timeout=timeout) - except Exception: - diagnose(unit, container) - raise - - def create_agent(body): - # The controller refuses a create until its dependencies answer (the - # archetype list comes from the builder's evaluation), so ask again. - for _ in range(60): - status, out = machine.execute( - f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' " - f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" - ) - if out.strip().startswith("2"): - return - machine.sleep(10) - diagnose("swarm-controller") - raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {machine.succeed('cat /root/create.out')}") - - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("the agents' toplevels are in the store, and the daemon builds nothing"): - for toplevel in TOPLEVELS: - machine.succeed(f"{NIX} path-info {toplevel}") - # The control for the lookup: a path that is not there is not found. - machine.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent") - # The control for the missing build: a derivation no store holds is refused. - machine.fail("NIX_CONFIG='experimental-features = nix-command' nix build --no-link --impure -f ${control}") - - # setup.md step 1: the secret store. - with subtest("step 1: bao operator init"): - machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600) - machine.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}") - - with subtest("step 4: bao operator unseal, with the shamir seal"): - machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") - machine.succeed("bao status") - - with subtest("step 1: the granter bootstrap"): - machine.succeed( - "set -o pipefail; " - f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; " - "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; " - "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token " - "| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; " - "unset BAO_TOKEN; " - "systemctl restart swarm-bao-granter-role" - ) - machine.wait_until_succeeds( - "journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'", - timeout=300, - ) - - with subtest("step 1: the policy units that waited for the granter are restarted"): - # Some of these units retry every 30 seconds until something outside - # them exists: the bao UI's viewer policy waits for authelia's secrets - # unit, in its container, to publish the UI's OIDC client secret, and - # that unit waits for the secret publisher's policy written here. So - # the units are started without waiting, and polled until all are active. - machine.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true") - machine.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service") - machine.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=600) - # Under the shamir seal the PKI unit leaves a restart of the store to the - # operator when its listener's client CA changed, and says which command. - if machine.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0: - machine.succeed("systemctl --machine=swarm-bao restart openbao.service") - machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600) - machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") - settle(POLICY_UNITS_ACTIVE, 1200, "swarm-bao-*-policy") - machine.succeed( - "journalctl -u swarm-bao-controller-policy --no-pager " - "| grep -F 'Data written to: auth/cert/certs/swarm-controller'" - ) - - with subtest("authelia's mint has published the clients' secrets"): - # The operator viewer policy is left out of the poll above: it waits for - # the UI client's secret, which authelia's secrets unit publishes from - # its container once the store and the trust bundle it needs exist. The - # controller's client secret is published by the same unit, and the - # controller, the hive's store logins and the UI policy all wait for it. - settle( - f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value " - "secret/swarm/controller/swarm-controller/oidc/client >/dev/null", - 1200, - "authelia-swarm-secrets", - "swarm-authelia", - ) - settle("[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 300, "swarm-bao-operator-viewer-policy") - - with subtest("step 2: swarmctl user add"): - machine.wait_until_succeeds( - "swarmctl user add mara --display-name Mara --email mara@example.com --group admins " - ">/root/user-add.out 2>&1", - timeout=900, - ) - machine.succeed("[ -s /root/user-add.out ]") - machine.succeed("swarmctl user list | grep -w mara") - - with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"): - # The controller's forge token, from the store as the controller reads it. - # Minted and stored by `forgejo-swarm-controller-account` in the forge - # container, which waits for forgejo and for a login to the store. - settle( - f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value " - "secret/swarm/controller/swarm-controller/forge/token > /root/forge-token", - 900, - ["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"], - "hive-forge", - ) - machine.succeed( - f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}" - ) - # The token being published says nothing about the gateway's vhost for - # the forge, which the calls below go through. - machine.wait_until_succeeds( - f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600 - ) - status = machine.succeed( - f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive" - ).strip() - if status != "200": - forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"}) - forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"}) - machine.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git") - machine.succeed( - f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" " - f"push {FORGE}/hyperhive/hyperhive.git main" - ) - pushed = machine.succeed( - f"GIT_SSL_NO_VERIFY=1 git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" " - f"ls-remote {FORGE}/hyperhive/hyperhive.git refs/heads/main" - ).split()[0] - assert pushed == machine.succeed("cat ${hyperhiveRepo}/rev").strip(), pushed - - with subtest("step 8: spawn, a reviewer-archetype agent, through the endpoint the swarm UI calls"): - machine.wait_until_succeeds( - f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=900 - ) - # A custom root agent (ruth, as the UI creates her) is a follow-up, not covered yet. - create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE}) - - with subtest("step 9: the first agent"): - create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE}) - - with subtest("internal/meta holds the flake the toplevels were built from"): - settle( - f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/internal/meta/raw/flake.nix " - ">/root/meta-flake.nix", - 1800, - "swarm-controller", - ) - rev = machine.succeed("cat ${hyperhiveRepo}/rev").strip() - settings = json.loads(${builtins.toJSON settingsJson}) - expected = open("${metaTemplate}").read().replace( - "@HYPERHIVE_URL@", f"git+{settings['git_base']}/hyperhive/hyperhive.git?rev={rev}" - ) - for name, value in settings["vars"].items(): - expected = expected.replace(f"@{name}@", value) - committed = machine.succeed("cat /root/meta-flake.nix") - assert committed == expected, "the controller's flake differs from the one the toplevels came from:\n" + committed - - with subtest("swarmctl setup status reports the steps done so far"): - # Step 9 names ruth, the root agent of the real setup; this test creates none, so the report - # ends on step 9 not done and the exit status is 1. - reported = [ - "✅ 1 secret store initialised", - "✅ 2 secret store unsealed", - "✅ 3 granter bootstrapped", - "✅ 4 granting units active", - "✅ 5 store unsealed after the granting units", - "✅ 6 SSO account in the admins group", - "👉 9 ruth created and up", - "✅ 10 another agent [optional]", - ] - for _ in range(60): - lines = machine.execute("swarmctl setup status")[1].splitlines() - if all(any(line.startswith(prefix) for line in lines) for prefix in reported): - break - machine.sleep(10) - else: - raise Exception("swarmctl setup status did not report the steps done so far:\n" + "\n".join(lines)) - assert machine.execute("swarmctl setup status >/dev/null")[0] == 1 - - with subtest("the agents are deployed and need a login"): - settle( - f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status " - "| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)" - " | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'", - 2400, - ["swarm-controller", "hive-c0re"], - ) - ''; - } -) + inherit (test.config.nodes.machine.services.hyperhive) swarm deploy; + settingsJson = + test.config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; +in +# The script is plain Python with no Nix in it: the build-time values reach it +# as environment variables of the test run, and the commands it runs in the VM +# find their binaries on the node's PATH. +test.overrideTestDerivation (old: { + env = (old.env or { }) // { + FULL_SETUP_SOCKET = deploy.swarm-controller.socketPath; + FULL_SETUP_FORGE_DOMAIN = swarm.forge.domain; + FULL_SETUP_HIVE = hive; + FULL_SETUP_TOPLEVELS = builtins.toJSON (map toString (toplevels settingsJson)); + FULL_SETUP_META_SETTINGS = settingsJson; + FULL_SETUP_META_TEMPLATE = "${metaTemplate}"; + FULL_SETUP_HYPERHIVE_REPO = hyperhiveRepo; + FULL_SETUP_CONTROL_NIX = control; + }; +}) diff --git a/nix/vm-tests/full-setup.py b/nix/vm-tests/full-setup.py new file mode 100644 index 000000000..8a30f94f6 --- /dev/null +++ b/nix/vm-tests/full-setup.py @@ -0,0 +1,309 @@ +import json +import os +import shlex + +API = "http://localhost" +SOCKET = os.environ["FULL_SETUP_SOCKET"] +FORGE = "https://" + os.environ["FULL_SETUP_FORGE_DOMAIN"] +HIVE = os.environ["FULL_SETUP_HIVE"] +TOPLEVELS = json.loads(os.environ["FULL_SETUP_TOPLEVELS"]) +REPO = os.environ["FULL_SETUP_HYPERHIVE_REPO"] +CONTROL_NIX = os.environ["FULL_SETUP_CONTROL_NIX"] +INIT = "/root/bao-init.json" +FORGE_AUTH = "/root/forge-auth.header" +NIX = "nix --extra-experimental-features nix-command" +ARCHETYPE = {"source": "hyperhive", "name": "reviewer"} +POLICY_UNITS_ACTIVE = ( + "for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) " + "swarm-bao-agent-pki.service; do " + '[ "$(systemctl is-active $u)" = active ] || { echo "$u is not active"; exit 1; }; done' +) + + +def controller(method, path, body=None, check="--fail-with-body"): + data = ( + f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" + if body + else "" + ) + return machine.succeed( + f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}" + ) + + +def forge(method, path, body=None): + data = ( + f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" + if body + else "" + ) + return machine.succeed( + f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} " + f"-X {method} {FORGE}/api/v1{path} {data}" + ) + + +def diagnose(unit=None, container=None): + # What a timed-out wait leaves behind: the failed units and the end of the + # journal of the units that were waited on. + where = f"--machine={container} " if container else "" + units = [unit] if isinstance(unit, str) else list(unit or []) + flags = " ".join(f"-u '{u}'" for u in units) + for command in [ + f"systemctl {where}--failed --no-legend --no-pager", + *( + [f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] + if units + else [] + ), + ]: + print(f"--- {command}") + print(machine.execute(command)[1]) + + +def settle(command, timeout, unit=None, container=None): + try: + machine.wait_until_succeeds(command, timeout=timeout) + except Exception: + diagnose(unit, container) + raise + + +def create_agent(body): + # The controller refuses a create until its dependencies answer (the + # archetype list comes from the builder's evaluation), so ask again. + for _ in range(60): + status, out = machine.execute( + f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' " + f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" + ) + if out.strip().startswith("2"): + return + machine.sleep(10) + diagnose("swarm-controller") + raise Exception( + f"the controller refused to create {body['name']}: HTTP {out}: {machine.succeed('cat /root/create.out')}" + ) + + +machine.start() +machine.wait_for_unit("multi-user.target") + +with subtest("the agents' toplevels are in the store, and the daemon builds nothing"): + for toplevel in TOPLEVELS: + machine.succeed(f"{NIX} path-info {toplevel}") + # The control for the lookup: a path that is not there is not found. + machine.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent") + # The control for the missing build: a derivation no store holds is refused. + machine.fail( + f"NIX_CONFIG='experimental-features = nix-command' nix build --no-link --impure -f {CONTROL_NIX}" + ) + +# setup.md step 1: the secret store. +with subtest("step 1: bao operator init"): + machine.wait_until_succeeds( + '[ "$(bao status >/dev/null 2>&1; echo $?)" = 2 ]', timeout=600 + ) + machine.succeed( + f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}" + ) + +with subtest("step 4: bao operator unseal, with the shamir seal"): + machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") + machine.succeed("bao status") + +with subtest("step 1: the granter bootstrap"): + machine.succeed( + "set -o pipefail; " + f'export BAO_TOKEN="$(jq -r .root_token {INIT})"; ' + "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; " + "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token " + "| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; " + "unset BAO_TOKEN; " + "systemctl restart swarm-bao-granter-role" + ) + machine.wait_until_succeeds( + "journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'", + timeout=300, + ) + +with subtest("step 1: the policy units that waited for the granter are restarted"): + # Some of these units retry every 30 seconds until something outside + # them exists: the bao UI's viewer policy waits for authelia's secrets + # unit, in its container, to publish the UI's OIDC client secret, and + # that unit waits for the secret publisher's policy written here. So + # the units are started without waiting, and polled until all are active. + machine.succeed( + "systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true" + ) + machine.succeed( + "systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service" + ) + machine.wait_until_succeeds( + '[ "$(systemctl is-active swarm-bao-agent-pki.service)" = active ]', timeout=600 + ) + # Under the shamir seal the PKI unit leaves a restart of the store to the + # operator when its listener's client CA changed, and says which command. + if ( + machine.execute( + "journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'" + )[0] + == 0 + ): + machine.succeed("systemctl --machine=swarm-bao restart openbao.service") + machine.wait_until_succeeds( + '[ "$(bao status >/dev/null 2>&1; echo $?)" = 2 ]', timeout=600 + ) + machine.succeed( + f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"" + ) + settle(POLICY_UNITS_ACTIVE, 1200, "swarm-bao-*-policy") + machine.succeed( + "journalctl -u swarm-bao-controller-policy --no-pager " + "| grep -F 'Data written to: auth/cert/certs/swarm-controller'" + ) + +with subtest("authelia's mint has published the clients' secrets"): + # The operator viewer policy is left out of the poll above: it waits for + # the UI client's secret, which authelia's secrets unit publishes from + # its container once the store and the trust bundle it needs exist. The + # controller's client secret is published by the same unit, and the + # controller, the hive's store logins and the UI policy all wait for it. + settle( + f'BAO_TOKEN="$(jq -r .root_token {INIT})" bao kv get -field=value ' + "secret/swarm/controller/swarm-controller/oidc/client >/dev/null", + 1200, + "authelia-swarm-secrets", + "swarm-authelia", + ) + settle( + '[ "$(systemctl is-active swarm-bao-operator-viewer-policy.service)" = active ]', + 300, + "swarm-bao-operator-viewer-policy", + ) + +with subtest("step 2: swarmctl user add"): + machine.wait_until_succeeds( + "swarmctl user add mara --display-name Mara --email mara@example.com --group admins " + ">/root/user-add.out 2>&1", + timeout=900, + ) + machine.succeed("[ -s /root/user-add.out ]") + machine.succeed("swarmctl user list | grep -w mara") + +with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"): + # The controller's forge token, from the store as the controller reads it. + # Minted and stored by `forgejo-swarm-controller-account` in the forge + # container, which waits for forgejo and for a login to the store. + settle( + f'BAO_TOKEN="$(jq -r .root_token {INIT})" bao kv get -field=value ' + "secret/swarm/controller/swarm-controller/forge/token > /root/forge-token", + 900, + ["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"], + "hive-forge", + ) + machine.succeed( + f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}" + ) + # The token being published says nothing about the gateway's vhost for + # the forge, which the calls below go through. + machine.wait_until_succeeds( + f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", + timeout=600, + ) + status = machine.succeed( + f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive" + ).strip() + if status != "200": + forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"}) + forge( + "POST", + "/orgs/hyperhive/repos", + {"name": "hyperhive", "private": False, "default_branch": "main"}, + ) + machine.succeed( + f"cp -r {REPO}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git" + ) + machine.succeed( + f'GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c "http.extraHeader=$(cat {FORGE_AUTH})" ' + f"push {FORGE}/hyperhive/hyperhive.git main" + ) + pushed = machine.succeed( + f'GIT_SSL_NO_VERIFY=1 git -c "http.extraHeader=$(cat {FORGE_AUTH})" ' + f"ls-remote {FORGE}/hyperhive/hyperhive.git refs/heads/main" + ).split()[0] + assert pushed == machine.succeed(f"cat {REPO}/rev").strip(), pushed + +with subtest( + "step 8: spawn, a reviewer-archetype agent, through the endpoint the swarm UI calls" +): + machine.wait_until_succeeds( + f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", + timeout=900, + ) + # A custom root agent (ruth, as the UI creates her) is a follow-up, not covered yet. + create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE}) + +with subtest("step 9: the first agent"): + create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE}) + +with subtest("internal/meta holds the flake the toplevels were built from"): + settle( + f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/internal/meta/raw/flake.nix " + ">/root/meta-flake.nix", + 1800, + "swarm-controller", + ) + rev = machine.succeed(f"cat {REPO}/rev").strip() + settings = json.loads(os.environ["FULL_SETUP_META_SETTINGS"]) + expected = ( + open(os.environ["FULL_SETUP_META_TEMPLATE"]) + .read() + .replace( + "@HYPERHIVE_URL@", + f"git+{settings['git_base']}/hyperhive/hyperhive.git?rev={rev}", + ) + ) + for name, value in settings["vars"].items(): + expected = expected.replace(f"@{name}@", value) + committed = machine.succeed("cat /root/meta-flake.nix") + assert committed == expected, ( + "the controller's flake differs from the one the toplevels came from:\n" + + committed + ) + +with subtest("swarmctl setup status reports the steps done so far"): + # Step 9 names ruth, the root agent of the real setup; this test creates none, so the report + # ends on step 9 not done and the exit status is 1. + reported = [ + "✅ 1 secret store initialised", + "✅ 2 secret store unsealed", + "✅ 3 granter bootstrapped", + "✅ 4 granting units active", + "✅ 5 store unsealed after the granting units", + "✅ 6 SSO account in the admins group", + "👉 9 ruth created and up", + "✅ 10 another agent [optional]", + ] + for _ in range(60): + lines = machine.execute("swarmctl setup status")[1].splitlines() + if all(any(line.startswith(prefix) for line in lines) for prefix in reported): + break + machine.sleep(10) + else: + raise Exception( + "swarmctl setup status did not report the steps done so far:\n" + + "\n".join(lines) + ) + assert machine.execute("swarmctl setup status >/dev/null")[0] == 1 + +with subtest("the agents are deployed and need a login"): + settle( + f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status " + "| jq -e '[.[] | select(.hive == \"" + + HIVE + + '" and .freshness == "fresh" and .needs_login == true)' + ' | .name] | map(select(. == "iris" or . == "spawn")) | unique | length == 2\'', + 2400, + ["swarm-controller", "hive-c0re"], + ) diff --git a/nix/vm-tests/ruff.toml b/nix/vm-tests/ruff.toml new file mode 100644 index 000000000..a02105bd9 --- /dev/null +++ b/nix/vm-tests/ruff.toml @@ -0,0 +1,2 @@ +# The names the NixOS test driver puts in scope of a test script. +builtins = ["machine", "subtest"]