diff --git a/nix/vm-tests.nix b/nix/vm-tests.nix index 28870ed92..bf7f90329 100644 --- a/nix/vm-tests.nix +++ b/nix/vm-tests.nix @@ -13,4 +13,5 @@ swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; }; swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; }; full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; }; + semi-spread = import ./vm-tests/semi-spread.nix { inherit pkgs self; }; } diff --git a/nix/vm-tests/semi-spread.nix b/nix/vm-tests/semi-spread.nix new file mode 100644 index 000000000..c02f1109d --- /dev/null +++ b/nix/vm-tests/semi-spread.nix @@ -0,0 +1,629 @@ +# `vmTests..semi-spread`: the semi-spread example +# (docs/examples/semi-spread/, docs/2-handbook/swarm/semi-spread-deploy.md) on +# three VMs and an outside client. +# +# swarm swarm-central.nix and swarm-services.nix on one machine +# heavyduty swarm-heavyduty.nix: the builder, the CI runner, the cache +# hive hive.nix, as `hive-one` +# outside a plain client on the public network, not on wg0 +# +# The example's files are imported from a copy of the directory with two +# changes: `mesh.nix` holds the VMs' endpoints and seed-derived keys and has no +# swarm-services entry (its machine is swarm-central here; `hive-two` stays as +# an entry nobody answers for), and `dns.nix` lists swarm-services' names under +# swarm-central, because the file leaves out the names its own machine serves. +# +# The script, in order: +# assert 1 wg0: every machine pings the other two over IPv4 and IPv6. +# assert 3 split DNS: `hive` resolves the internal names to wg0 addresses, +# A and AAAA; `outside` resolves the public names only. +# ceremony ./full-setup.nix's store steps on `swarm`, then the operator's +# copy of the store's CA and leaves to the other machines. +# assert 2 `outside` reaches forge, chat, auth and the swarm UI on the public +# address and nothing else; `hive` and `swarm` reach the private +# names over wg0. +# assert 4 two agents are created on the controller, built on `heavyduty`, +# pushed to its cache and pulled by `hive`, whose store holds +# neither toplevel before and which has no other substituter. +# Green when both are fresh and need a login. +{ pkgs, self }: +let + inherit (pkgs) lib; + + hiveName = "one"; + agents = [ + "iris" + "spawn" + ]; + + inherit (import ./agent-toplevels.nix { inherit pkgs self agents; }) + hyperhiveRepo + sourcePaths + toplevels + ; + + example = ../../docs/examples/semi-spread; + exampleMesh = import (example + "/mesh.nix"); + + # Test-only keys: base64 of sha256(seed); wg clamps the scalar itself. The + # same seeds on every store build the same bytes. + mkKey = + name: + pkgs.runCommand "semi-spread-test-key-${name}" + { + nativeBuildInputs = [ + pkgs.wireguard-tools + pkgs.openssl + ]; + } + '' + mkdir $out + printf '%s' 'hyperhive-semi-spread-test-${name}' | openssl dgst -sha256 -binary | base64 > $out/private + wg pubkey < $out/private > $out/public + ''; + keys = lib.genAttrs [ + "swarm" + "heavyduty" + "hive" + # Never installed anywhere: the key of the hive-two entry. + "absent" + ] mkKey; + pub = name: lib.removeSuffix "\n" (builtins.readFile "${keys.${name}}/public"); + + # The test driver numbers the nodes on vlan 1 in alphabetical order. + lanAddr = { + heavyduty = "192.168.1.1"; + hive = "192.168.1.2"; + outside = "192.168.1.3"; + swarm = "192.168.1.4"; + }; + + # The example's addresses, with the endpoints and keys of the VMs. + entry = + machine: node: + exampleMesh.${machine} + // { + endpoint = "${lanAddr.${node}}:51820"; + publicKey = pub node; + }; + testMesh = { + swarm-central = entry "swarm-central" "swarm"; + swarm-heavyduty = entry "swarm-heavyduty" "heavyduty"; + hive-one = entry "hive-one" "hive"; + hive-two = exampleMesh.hive-two // { + endpoint = "192.168.1.250:51820"; + publicKey = pub "absent"; + }; + }; + meshFile = pkgs.writeText "mesh.nix" ( + "builtins.fromJSON " + builtins.toJSON (builtins.toJSON testMesh) + ); + + exampleCopy = pkgs.runCommand "semi-spread-example" { nativeBuildInputs = [ pkgs.perl ]; } '' + cp -r ${example} $out + chmod -R u+w $out + cp ${meshFile} $out/mesh.nix + perl -0pi -e 's/(swarm\.bao\.ui\.domain\n)\s*\];\n\s*swarm-services = \[\n/$1/ or die "dns.nix: swarm-services entry not found"' $out/dns.nix + if grep -q 'swarm-services = \[' $out/dns.nix; then exit 1; fi + ''; + + wgKey = node: { + environment.etc."wireguard/wg0.key" = { + source = "${keys.${node}}/private"; + mode = "0400"; + }; + }; + + # No substituter and no build: a closure comes from the swarm's cache or not + # at all. + offline = { + nix.settings = { + substituters = lib.mkForce [ ]; + max-jobs = 0; + }; + }; + + tools = { + environment.systemPackages = [ + pkgs.curl + pkgs.jq + pkgs.wireguard-tools + ]; + }; +in +pkgs.testers.runNixOSTest ( + { config, ... }: + let + swarmCfg = config.nodes.swarm.services.hyperhive; + inherit (swarmCfg) swarm; + controllerSocket = swarmCfg.deploy.swarm-controller.socketPath; + settingsJson = + config.nodes.swarm.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; + + # What the public DNS holds (semi-spread-deploy.md, "DNS"), and the rest of + # the swarm's names, which only the VPN resolves. + publicNames = { + ui = swarm.domain; + forge = swarm.forge.domain; + chat = swarm.matrix.gatewayHost; + auth = swarm.authelia.domain; + }; + privateNames = { + grafana = swarm.grafana.domain; + metrics = swarm.victoriametrics.domain; + logs = swarm.victorialogs.domain; + otel = swarm.otel.domain; + bao-ui = swarm.bao.ui.domain; + attic = swarm.attic.ownDomain; + }; + # Every internal name, with the machine whose VPN address it resolves to. + internalNames = + lib.genAttrs [ + "ui" + "forge" + "chat" + "auth" + "grafana" + "metrics" + "logs" + "otel" + "bao-ui" + "bao" + "nats" + ] (_: "swarm-central") + // { + attic = "swarm-heavyduty"; + }; + nameOf = { + inherit (publicNames) + ui + forge + chat + auth + ; + inherit (privateNames) + grafana + metrics + logs + otel + bao-ui + attic + ; + bao = swarm.bao.domain; + nats = swarm.nats.domain; + }; + in + { + name = "semi-spread"; + + # Three hosts' boots, the setup ceremony and the agents' build and pull. + globalTimeout = 10800; + + nodes.swarm = + { config, ... }: + { + imports = [ + self.nixosModules.default + tools + (wgKey "swarm") + "${exampleCopy}/common.nix" + "${exampleCopy}/swarm-central.nix" + "${exampleCopy}/swarm-services.nix" + ]; + networking.hostName = "swarm-central"; + system.stateVersion = "25.11"; + + services.hyperhive = { + # swarm-services.nix says the store is elsewhere; here it is on this machine. + deploy.bao.enable = lib.mkForce true; + deploy.bao.seal = "shamir"; + deploy.bao.bootstrapTokenFile = "/var/lib/swarm-bao-bootstrap/grant.token"; + # Names this machine serves are not in the example's table, and a + # process on the machine reaches them on the machine. + gateway.localHostsEntry = true; + # The test pushes its own hyperhive/hyperhive; a mirror of the + # upstream could not be created offline and would race the push. + deploy.forgejo.hyperhiveUpstream = null; + }; + + # 8 GB: the store and the swarm's services in containers, the + # controller, and the forge's repositories. + virtualisation.memorySize = 8192; + virtualisation.cores = 4; + virtualisation.diskSize = 20480; + virtualisation.writableStoreUseTmpfs = false; + virtualisation.additionalPaths = [ hyperhiveRepo ]; + + # The store's raft data is on a tmpfs, for the reason in ./full-setup.nix. + fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" = + { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ + "mode=0700" + "size=1G" + ]; + }; + + environment.systemPackages = [ pkgs.git ]; + }; + + nodes.heavyduty = { + imports = [ + self.nixosModules.default + tools + offline + (wgKey "heavyduty") + "${exampleCopy}/common.nix" + "${exampleCopy}/swarm-heavyduty.nix" + ]; + networking.hostName = "swarm-heavyduty"; + system.stateVersion = "25.11"; + + # 5 GB: the builder evaluates and builds the agents' flake, beside the + # cache and the CI runner. The disk holds the toplevels and the cache's + # copy of them. + virtualisation.memorySize = 5120; + virtualisation.cores = 4; + virtualisation.diskSize = 40960; + virtualisation.writableStoreUseTmpfs = false; + # The toplevels, and the source trees the meta flake's lock names. + virtualisation.additionalPaths = sourcePaths ++ toplevels settingsJson; + }; + + nodes.hive = { + imports = [ + self.nixosModules.default + tools + offline + (wgKey "hive") + "${exampleCopy}/common.nix" + "${exampleCopy}/hive.nix" + ]; + networking.hostName = "hive-one"; + system.stateVersion = "25.11"; + + # 4 GB: the hive and two agent containers. The disk holds the pulled + # closures. + virtualisation.memorySize = 4096; + virtualisation.cores = 2; + virtualisation.diskSize = 30720; + virtualisation.writableStoreUseTmpfs = false; + }; + + nodes.outside = { + environment.systemPackages = [ pkgs.curl ]; + # The public DNS: the four names, at the swarm machine's public address. + networking.hosts.${lanAddr.swarm} = lib.attrValues publicNames; + virtualisation.memorySize = 512; + }; + + testScript = '' + import json + import shlex + + API = "http://localhost" + SOCKET = "${controllerSocket}" + FORGE = "https://${publicNames.forge}" + HIVE = "${hiveName}" + TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))} + INIT = "/root/bao-init.json" + FORGE_AUTH = "/root/forge-auth.header" + NIX = "nix --extra-experimental-features nix-command" + ARCHETYPE = {"source": "hyperhive", "name": "reviewer"} + POOL = "/var/lib/swarm-pki" + MINTED = "/var/lib/swarm-bao-pki" + POLICY_UNITS_ACTIVE = ( + "for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) " + "swarm-bao-agent-pki.service; do " + "[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done" + ) + + # Each machine's VPN addresses, from the example's mesh.nix. + WG = { + swarm: ("${exampleMesh.swarm-central.address}", "${exampleMesh.swarm-central.address6}"), + heavyduty: ("${exampleMesh.swarm-heavyduty.address}", "${exampleMesh.swarm-heavyduty.address6}"), + hive: ("${exampleMesh.hive-one.address}", "${exampleMesh.hive-one.address6}"), + } + LAN = {"swarm": "${lanAddr.swarm}", "heavyduty": "${lanAddr.heavyduty}", "hive": "${lanAddr.hive}", "outside": "${lanAddr.outside}"} + PUBLIC = {"ui": ("/", "23"), "forge": ("/api/healthz", "2"), "chat": ("/_matrix/client/versions", "2"), "auth": ("/api/health", "2")} + NAMES = ${builtins.toJSON nameOf} + INTERNAL = ${builtins.toJSON internalNames} + PRIVATE = ["grafana", "metrics", "logs", "otel", "bao-ui", "attic"] + + def diagnose(m, unit=None, container=None): + # What a timed-out wait leaves behind: the failed units and the end of the + # journal of the units that were waited on. + where = f"--machine={container} " if container else "" + units = [unit] if isinstance(unit, str) else list(unit or []) + flags = " ".join(f"-u '{u}'" for u in units) + for command in [ + f"systemctl {where}--failed --no-legend --no-pager", + *([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []), + ]: + print(f"--- {m.name}: {command}") + print(m.execute(command)[1]) + + def settle(m, command, timeout, unit=None, container=None): + try: + m.wait_until_succeeds(command, timeout=timeout) + except Exception: + diagnose(m, unit, container) + raise + + def controller(method, path, body=None, check="--fail-with-body"): + data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else "" + return swarm.succeed( + f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}" + ) + + def forge(method, path, body=None): + data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else "" + return swarm.succeed( + f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} " + f"-X {method} {FORGE}/api/v1{path} {data}" + ) + + def create_agent(body): + # The controller refuses a create until its dependencies answer (the + # archetype list comes from the builder's evaluation), so ask again. + for _ in range(60): + status, out = swarm.execute( + f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' " + f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" + ) + if out.strip().startswith("2"): + return + swarm.sleep(10) + diagnose(swarm, "swarm-controller") + raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {swarm.succeed('cat /root/create.out')}") + + def http(m, name, path="/", resolve=None): + # The status code of an HTTPS request, 000 when nothing answers: a + # refused connection, and the gateway's catch-all, which closes the + # connection without a response. + pin = f"--resolve {name}:443:{resolve} " if resolve else "" + return m.execute( + f"curl --silent --insecure --max-time 10 {pin}-o /dev/null -w '%{{http_code}}' https://{name}{path}" + )[1].strip() + + def answers(code): + return code not in ("000", "502", "503", "504") + + def wait_http(m, name, accept, path="/", resolve=None, tries=90): + for _ in range(tries): + code = http(m, name, path, resolve) + if accept(code): + return code + m.sleep(10) + diagnose(m) + raise Exception(f"{m.name}: https://{name}{path} answered {code}") + + def redeploy(m): + # What a second deploy does once the leaves are in place: start what + # was not started and restart what failed for want of them. + m.execute( + "failed=\"$(systemctl list-units --failed --plain --no-legend | cut -d' ' -f1)\"; " + "systemctl reset-failed; " + "[ -z \"$failed\" ] || systemctl restart --no-block $failed; " + "systemctl start --no-block multi-user.target" + ) + + def push_file(src_path, m, dest): + # One file from the store's host to another machine, through the + # driver: the operator's copy over the VPN. + payload = swarm.succeed(f"base64 -w0 {src_path}").strip() + m.succeed(f"install -d -m 0700 {POOL} && echo {payload} | base64 -d | install -m 0600 /dev/stdin {dest}") + + start_all() + for m in (swarm, heavyduty, hive, outside): + m.wait_for_unit("multi-user.target") + + with subtest("the VMs sit on the public network where the mesh expects them"): + for m, name in ((swarm, "swarm"), (heavyduty, "heavyduty"), (hive, "hive"), (outside, "outside")): + assert f"inet {LAN[name]}/" in m.succeed("ip -4 -o addr show"), name + + with subtest("the toplevels are in heavyduty's store only, and nobody builds"): + for toplevel in TOPLEVELS: + heavyduty.succeed(f"{NIX} path-info {toplevel}") + hive.fail(f"{NIX} path-info {toplevel}") + # The control for the lookup: a path that is not there is not found. + heavyduty.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent") + + with subtest("assert 1: wg0 is up between all three machines, over IPv4 and IPv6"): + for m in (swarm, heavyduty, hive): + m.wait_for_unit("wireguard-wg0.service") + for src in (swarm, heavyduty, hive): + own = WG[src] + assert own[0] in src.succeed("ip -4 -o addr show dev wg0"), own + assert own[1] in src.succeed("ip -6 -o addr show dev wg0"), own + for dst in (swarm, heavyduty, hive): + if dst is src: + continue + v4, v6 = WG[dst] + src.wait_until_succeeds(f"ping -c 1 -W 2 {v4}", timeout=180) + src.wait_until_succeeds(f"ping -6 -c 1 -W 2 {v6}", timeout=180) + src.succeed(f"ping -c 3 -W 5 {v4}") + src.succeed(f"ping -6 -c 3 -W 5 {v6}") + # The address is routed through the tunnel, the public one is not. + assert "dev wg0" in hive.succeed(f"ip route get {WG[swarm][0]}") + assert "dev wg0" not in hive.succeed(f"ip route get {LAN['swarm']}") + # Negative control: the hive-two entry has no machine behind it. + hive.fail("ping -c 2 -W 2 ${exampleMesh.hive-two.address}") + + with subtest("assert 3: split DNS"): + for key, machine in INTERNAL.items(): + v4, v6 = WG[{"swarm-central": swarm, "swarm-heavyduty": heavyduty}[machine]] + got4 = hive.succeed(f"getent ahostsv4 {NAMES[key]}").split()[0] + got6 = hive.succeed(f"getent ahostsv6 {NAMES[key]}").split()[0] + assert got4 == v4, f"{NAMES[key]}: A {got4}, want {v4}" + assert got6 == v6, f"{NAMES[key]}: AAAA {got6}, want {v6}" + # The public names, from outside, are the swarm machine's public address. + for key in PUBLIC: + got = outside.succeed(f"getent ahostsv4 {NAMES[key]}").split()[0] + assert got == LAN["swarm"], f"{NAMES[key]}: {got}" + # The rest of the names do not resolve there, and none resolves to the VPN. + for key in PRIVATE + ["bao", "nats"]: + outside.fail(f"getent ahosts {NAMES[key]}") + for key in PUBLIC: + assert "10.77." not in outside.succeed(f"getent ahosts {NAMES[key]}") + + with subtest("the example's leaves are copied to the machine that runs them"): + swarm.wait_until_succeeds(f"[ -s {MINTED}/hives/{HIVE}/hive-leaf.pem ] && [ -s {MINTED}/forge-publisher.pem ]", timeout=900) + # The store's host keeps the CA's and the granter's keys, and a hive's + # leaves are not another machine's. + swarm.succeed( + f"install -d -m 0700 {POOL} && cd {MINTED} && " + f"for f in *.pem; do case $f in ca-key.pem|granter*|server*) ;; *) install -m 0600 $f {POOL}/$f ;; esac; done" + ) + swarm.succeed(f"[ -s {POOL}/ca.pem ] && [ -s {POOL}/services-issuer.pem ] && [ ! -e {POOL}/ca-key.pem ]") + redeploy(swarm) + + with subtest("step 1: bao operator init"): + swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=900) + swarm.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}") + + with subtest("step 4: bao operator unseal, with the shamir seal"): + swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") + swarm.succeed("bao status") + + with subtest("step 1: the granter bootstrap"): + swarm.succeed( + "set -o pipefail; " + f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; " + "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; " + "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token " + "| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; " + "unset BAO_TOKEN; " + "systemctl restart swarm-bao-granter-role" + ) + swarm.wait_until_succeeds( + "journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'", + timeout=300, + ) + + with subtest("step 1: the policy units that waited for the granter are restarted"): + # See ./full-setup.nix for why these are started without waiting and polled. + swarm.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true") + swarm.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service") + swarm.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=900) + if swarm.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0: + swarm.succeed("systemctl --machine=swarm-bao restart openbao.service") + swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600) + swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") + settle(swarm, POLICY_UNITS_ACTIVE, 1800, "swarm-bao-*-policy") + swarm.succeed( + "journalctl -u swarm-bao-controller-policy --no-pager " + "| grep -F 'Data written to: auth/cert/certs/swarm-controller'" + ) + + with subtest("the leaves for heavyduty and the hive are copied over, and their units start"): + # The builder, the CI runner and the cache each read the store with a + # leaf of their own; the hive with its reader and gateway leaves. The + # CA is what lets either check the store's certificate. + files = swarm.succeed( + f"cd {MINTED} && for f in *.pem; do case $f in ca-key.pem|granter*|server*) ;; *) echo $f ;; esac; done" + ).split() + for name in files: + push_file(f"{MINTED}/{name}", heavyduty, f"{POOL}/{name}") + for name in ("client.pem", "client-key.pem", "hive-leaf.pem", "hive-leaf-key.pem"): + push_file(f"{MINTED}/hives/{HIVE}/{name}", hive, f"{POOL}/{name}") + push_file(f"{MINTED}/ca.pem", hive, f"{POOL}/ca.pem") + # Control: the lookup finds what was copied, and nothing of the CA's key. + heavyduty.succeed(f"[ -s {POOL}/builder.pem ] && [ -s {POOL}/ca.pem ] && [ ! -e {POOL}/ca-key.pem ]") + hive.succeed(f"[ -s {POOL}/hive-leaf.pem ] && [ ! -e {POOL}/builder.pem ]") + redeploy(heavyduty) + redeploy(hive) + + with subtest("authelia's mint has published the clients' secrets"): + settle( + swarm, + f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value " + "secret/swarm/controller/swarm-controller/oidc/client >/dev/null", + 1800, + "authelia-swarm-secrets", + "swarm-authelia", + ) + settle(swarm, "[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 600, "swarm-bao-operator-viewer-policy") + + with subtest("assert 2: from outside, the forge, chat, SSO and the swarm UI answer on the public address"): + for key, (path, accept) in PUBLIC.items(): + code = wait_http(outside, NAMES[key], lambda c, a=accept: c[:1] in a, path, tries=180) + print(f"{NAMES[key]}{path}: {code}") + + with subtest("assert 2: from hive, over wg0, the private names answer"): + for key in PRIVATE: + code = wait_http(hive, NAMES[key], answers, tries=180) + print(f"{NAMES[key]}: {code}") + # The hive's dashboard, from the store's machine, over wg0. + code = wait_http(swarm, "${swarm.dashboardDomains.${hiveName}}", answers, resolve=WG[hive][0]) + print(f"dashboard: {code}") + + with subtest("assert 2: from outside, the private names get nothing, on any machine's public address"): + # The positive controls are the two subtests above: the same + # requests from inside answered. Here nginx is up on the swarm + # machine, so 000 is the catch-all closing the connection. + for key in PRIVATE: + code = http(outside, NAMES[key], resolve=LAN["swarm"]) + assert code == "000", f"{NAMES[key]} at the swarm machine's public address: {code}" + code = http(outside, NAMES["attic"], resolve=LAN["heavyduty"]) + assert code == "000", f"the cache at heavyduty's public address: {code}" + code = http(outside, "${swarm.dashboardDomains.${hiveName}}", resolve=LAN["hive"]) + assert code == "000", f"the hive's dashboard at its public address: {code}" + # The public names are not on heavyduty or the hive either. + assert http(outside, NAMES["forge"], "/api/healthz", resolve=LAN["heavyduty"]) == "000" + + with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"): + settle( + swarm, + f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value " + "secret/swarm/controller/swarm-controller/forge/token > /root/forge-token", + 1800, + ["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"], + "hive-forge", + ) + swarm.succeed( + f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}" + ) + swarm.wait_until_succeeds( + f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600 + ) + status = swarm.succeed( + f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive" + ).strip() + if status != "200": + forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"}) + forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"}) + swarm.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git") + swarm.succeed( + f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" " + f"push {FORGE}/hyperhive/hyperhive.git main" + ) + + with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"): + swarm.wait_until_succeeds( + f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800 + ) + create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE}) + create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE}) + + with subtest("assert 4: the agents are deployed on the hive, from the cache on heavyduty, and need a login"): + settle( + swarm, + f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status " + "| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)" + " | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'", + 3600, + ["swarm-controller"], + ) + # The closures came over the VPN from the cache: the hive's nix + # names it as its only extra substituter and holds the toplevels now. + fragment = hive.succeed("cat /run/hyperhive-nix/cache.conf") + assert "extra-substituters" in fragment and "${swarm.attic.domain}" in fragment, fragment + for toplevel in TOPLEVELS: + hive.succeed(f"{NIX} path-info {toplevel}") + # Not built on the swarm machine, which has no copy. + swarm.fail(f"{NIX} path-info {toplevel}") + ''; + } +)