# `vmTests.<system>.semi-spread`: the semi-spread example
# (docs/examples/semi-spread/, docs/2-handbook/swarm/semi-spread-deploy.md) on
# three VMs and an outside client.
#
#   swarm      swarm-central.nix and swarm-services.nix on one machine
#   heavyduty  swarm-heavyduty.nix: the builder, the CI runner, the cache
#   hive       hive.nix, as `hive-one`
#   outside    a plain client on the public network, not on wg0
#
# The example's files are imported from a copy of the directory with two
# changes: `mesh.nix` holds the VMs' endpoints and seed-derived keys and has no
# swarm-services entry (its machine is swarm-central here; `hive-two` stays as
# an entry nobody answers for), and `dns.nix` lists swarm-services' names under
# swarm-central, because the file leaves out the names its own machine serves.
#
# The script, in order:
#   assert 1  wg0: every machine pings the other two over IPv4 and IPv6.
#   assert 3  split DNS: `hive` resolves the internal names to wg0 addresses,
#             A and AAAA; `outside` resolves the public names only.
#   ceremony  ./full-setup.nix's store steps on `swarm`, then the operator's
#             copy of the store's CA and leaves to the other machines.
#   assert 2  `outside` reaches forge, chat, auth and the swarm UI on the public
#             address and nothing else; `hive` and `swarm` reach the private
#             names over wg0.
#   assert 4  two agents are created on the controller, built on `heavyduty`,
#             pushed to its cache and pulled by `hive`, whose store holds
#             neither toplevel before and which has no other substituter.
#             Green when both are fresh and need a login.
{ pkgs, self }:
let
  inherit (pkgs) lib;

  hiveName = "one";
  agents = [
    "iris"
    "spawn"
  ];

  inherit (import ./agent-toplevels.nix { inherit pkgs self agents; })
    hyperhiveRepo
    sourcePaths
    toplevels
    ;

  example = ../../docs/examples/semi-spread;
  exampleMesh = import (example + "/mesh.nix");

  # Test-only keys: base64 of sha256(seed); wg clamps the scalar itself. The
  # same seeds on every store build the same bytes.
  mkKey =
    name:
    pkgs.runCommand "semi-spread-test-key-${name}"
      {
        nativeBuildInputs = [
          pkgs.wireguard-tools
          pkgs.openssl
        ];
      }
      ''
        mkdir $out
        printf '%s' 'hyperhive-semi-spread-test-${name}' | openssl dgst -sha256 -binary | base64 > $out/private
        wg pubkey < $out/private > $out/public
      '';
  keys = lib.genAttrs [
    "swarm"
    "heavyduty"
    "hive"
    # Never installed anywhere: the key of the hive-two entry.
    "absent"
  ] mkKey;
  pub = name: lib.removeSuffix "\n" (builtins.readFile "${keys.${name}}/public");

  # The test driver numbers the nodes on vlan 1 in alphabetical order.
  lanAddr = {
    heavyduty = "192.168.1.1";
    hive = "192.168.1.2";
    outside = "192.168.1.3";
    swarm = "192.168.1.4";
  };

  # The example's addresses, with the endpoints and keys of the VMs.
  entry =
    machine: node:
    exampleMesh.${machine}
    // {
      endpoint = "${lanAddr.${node}}:51820";
      publicKey = pub node;
    };
  testMesh = {
    swarm-central = entry "swarm-central" "swarm";
    swarm-heavyduty = entry "swarm-heavyduty" "heavyduty";
    hive-one = entry "hive-one" "hive";
    hive-two = exampleMesh.hive-two // {
      endpoint = "192.168.1.250:51820";
      publicKey = pub "absent";
    };
  };
  meshFile = pkgs.writeText "mesh.nix" (
    "builtins.fromJSON " + builtins.toJSON (builtins.toJSON testMesh)
  );

  exampleCopy = pkgs.runCommand "semi-spread-example" { nativeBuildInputs = [ pkgs.perl ]; } ''
    cp -r ${example} $out
    chmod -R u+w $out
    cp ${meshFile} $out/mesh.nix
    perl -0pi -e 's/(swarm\.bao\.ui\.domain\n)\s*\];\n\s*swarm-services = \[\n/$1/ or die "dns.nix: swarm-services entry not found"' $out/dns.nix
    if grep -q 'swarm-services = \[' $out/dns.nix; then exit 1; fi
  '';

  wgKey = node: {
    environment.etc."wireguard/wg0.key" = {
      source = "${keys.${node}}/private";
      mode = "0400";
    };
  };

  # No substituter and no build: a closure comes from the swarm's cache or not
  # at all.
  offline = {
    nix.settings = {
      substituters = lib.mkForce [ ];
      max-jobs = 0;
    };
  };

  tools = {
    environment.systemPackages = [
      pkgs.curl
      pkgs.jq
      pkgs.wireguard-tools
    ];
  };
in
pkgs.testers.runNixOSTest (
  { config, ... }:
  let
    swarmCfg = config.nodes.swarm.services.hyperhive;
    inherit (swarmCfg) swarm;
    controllerSocket = swarmCfg.deploy.swarm-controller.socketPath;
    settingsJson =
      config.nodes.swarm.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;

    # What the public DNS holds (semi-spread-deploy.md, "DNS"), and the rest of
    # the swarm's names, which only the VPN resolves.
    publicNames = {
      ui = swarm.domain;
      forge = swarm.forge.domain;
      chat = swarm.matrix.gatewayHost;
      auth = swarm.authelia.domain;
    };
    privateNames = {
      grafana = swarm.grafana.domain;
      metrics = swarm.victoriametrics.domain;
      logs = swarm.victorialogs.domain;
      otel = swarm.otel.domain;
      bao-ui = swarm.bao.ui.domain;
      attic = swarm.attic.ownDomain;
    };
    # Every internal name, with the machine whose VPN address it resolves to.
    internalNames =
      lib.genAttrs [
        "ui"
        "forge"
        "chat"
        "auth"
        "grafana"
        "metrics"
        "logs"
        "otel"
        "bao-ui"
        "bao"
        "nats"
      ] (_: "swarm-central")
      // {
        attic = "swarm-heavyduty";
      };
    nameOf = {
      inherit (publicNames)
        ui
        forge
        chat
        auth
        ;
      inherit (privateNames)
        grafana
        metrics
        logs
        otel
        bao-ui
        attic
        ;
      bao = swarm.bao.domain;
      nats = swarm.nats.domain;
    };
  in
  {
    name = "semi-spread";

    # Three hosts' boots, the setup ceremony and the agents' build and pull.
    globalTimeout = 10800;

    nodes.swarm =
      { config, ... }:
      {
        imports = [
          self.nixosModules.default
          tools
          (wgKey "swarm")
          "${exampleCopy}/common.nix"
          "${exampleCopy}/swarm-central.nix"
          "${exampleCopy}/swarm-services.nix"
        ];
        networking.hostName = "swarm-central";
        system.stateVersion = "25.11";

        services.hyperhive = {
          # swarm-services.nix says the store is elsewhere; here it is on this machine.
          deploy.bao.enable = lib.mkForce true;
          deploy.bao.seal = "shamir";
          deploy.bao.bootstrapTokenFile = "/var/lib/swarm-bao-bootstrap/grant.token";
          # Names this machine serves are not in the example's table, and a
          # process on the machine reaches them on the machine.
          gateway.localHostsEntry = true;
          # The test pushes its own hyperhive/hyperhive; a mirror of the
          # upstream could not be created offline and would race the push.
          deploy.forgejo.hyperhiveUpstream = null;
        };

        # 8 GB: the store and the swarm's services in containers, the
        # controller, and the forge's repositories.
        virtualisation.memorySize = 8192;
        virtualisation.cores = 4;
        virtualisation.diskSize = 20480;
        virtualisation.writableStoreUseTmpfs = false;
        virtualisation.additionalPaths = [ hyperhiveRepo ];

        # The store's raft data is on a tmpfs, for the reason in ./full-setup.nix.
        fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
          {
            device = "tmpfs";
            fsType = "tmpfs";
            options = [
              "mode=0700"
              "size=1G"
            ];
          };

        environment.systemPackages = [ pkgs.git ];
      };

    nodes.heavyduty = {
      imports = [
        self.nixosModules.default
        tools
        offline
        (wgKey "heavyduty")
        "${exampleCopy}/common.nix"
        "${exampleCopy}/swarm-heavyduty.nix"
      ];
      networking.hostName = "swarm-heavyduty";
      system.stateVersion = "25.11";

      # 5 GB: the builder evaluates and builds the agents' flake, beside the
      # cache and the CI runner. The disk holds the toplevels and the cache's
      # copy of them.
      virtualisation.memorySize = 5120;
      virtualisation.cores = 4;
      virtualisation.diskSize = 40960;
      virtualisation.writableStoreUseTmpfs = false;
      # The toplevels, and the source trees the meta flake's lock names.
      virtualisation.additionalPaths = sourcePaths ++ toplevels settingsJson;
    };

    nodes.hive = {
      imports = [
        self.nixosModules.default
        tools
        offline
        (wgKey "hive")
        "${exampleCopy}/common.nix"
        "${exampleCopy}/hive.nix"
      ];
      networking.hostName = "hive-one";
      system.stateVersion = "25.11";

      # 4 GB: the hive and two agent containers. The disk holds the pulled
      # closures.
      virtualisation.memorySize = 4096;
      virtualisation.cores = 2;
      virtualisation.diskSize = 30720;
      virtualisation.writableStoreUseTmpfs = false;
    };

    nodes.outside = {
      environment.systemPackages = [ pkgs.curl ];
      # The public DNS: the four names, at the swarm machine's public address.
      networking.hosts.${lanAddr.swarm} = lib.attrValues publicNames;
      virtualisation.memorySize = 512;
    };

    testScript = ''
      import json
      import shlex

      API = "http://localhost"
      SOCKET = "${controllerSocket}"
      FORGE = "https://${publicNames.forge}"
      HIVE = "${hiveName}"
      TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))}
      INIT = "/root/bao-init.json"
      FORGE_AUTH = "/root/forge-auth.header"
      NIX = "nix --extra-experimental-features nix-command"
      ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
      POOL = "/var/lib/swarm-pki"
      MINTED = "/var/lib/swarm-bao-pki"
      # The store's listeners refuse a connection that presents no certificate
      # from its CA, and this machine runs no hive, so its `bao` wrapper has
      # none. Any leaf does for the CLI; the services-issuer's is in POOL
      # before the first call.
      BAO = f"env BAO_CLIENT_CERT={POOL}/services-issuer.pem BAO_CLIENT_KEY={POOL}/services-issuer-key.pem bao"
      POLICY_UNITS_ACTIVE = (
          "for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
          "swarm-bao-agent-pki.service; do "
          "[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done"
      )

      # Each machine's VPN addresses, from the example's mesh.nix.
      WG = {
          swarm: ("${exampleMesh.swarm-central.address}", "${exampleMesh.swarm-central.address6}"),
          heavyduty: ("${exampleMesh.swarm-heavyduty.address}", "${exampleMesh.swarm-heavyduty.address6}"),
          hive: ("${exampleMesh.hive-one.address}", "${exampleMesh.hive-one.address6}"),
      }
      LAN = {"swarm": "${lanAddr.swarm}", "heavyduty": "${lanAddr.heavyduty}", "hive": "${lanAddr.hive}", "outside": "${lanAddr.outside}"}
      PUBLIC = {"ui": ("/", "23"), "forge": ("/api/healthz", "2"), "chat": ("/_matrix/client/versions", "2"), "auth": ("/api/health", "2")}
      NAMES = ${builtins.toJSON nameOf}
      INTERNAL = ${builtins.toJSON internalNames}
      PRIVATE = ["grafana", "metrics", "logs", "otel", "bao-ui", "attic"]

      def diagnose(m, unit=None, container=None):
          # What a timed-out wait leaves behind: the failed units and the end of the
          # journal of the units that were waited on.
          where = f"--machine={container} " if container else ""
          units = [unit] if isinstance(unit, str) else list(unit or [])
          flags = " ".join(f"-u '{u}'" for u in units)
          for command in [
              f"systemctl {where}--failed --no-legend --no-pager",
              *([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []),
          ]:
              print(f"--- {m.name}: {command}")
              print(m.execute(command)[1])

      def settle(m, command, timeout, unit=None, container=None):
          try:
              m.wait_until_succeeds(command, timeout=timeout)
          except Exception:
              diagnose(m, unit, container)
              raise

      def store_sealed(timeout):
          # `bao status` exits 2 when it reaches a sealed store and 1 for any
          # address, CA or client-certificate problem.
          try:
              swarm.wait_until_succeeds(f"[ \"$({BAO} status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=timeout)
          except Exception:
              out = swarm.execute(f"{BAO} status 2>&1; echo \"exit=$?\"; grep -h '^export BAO_' \"$(command -v bao)\" | tr '\\n' ' '")[1]
              print(f"bao status on swarm: {' '.join(out.split())}")
              diagnose(swarm, "openbao", "${swarm.bao.machine}")
              raise

      def controller(method, path, body=None, check="--fail-with-body"):
          data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
          return swarm.succeed(
              f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}"
          )

      def forge(method, path, body=None):
          data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
          return swarm.succeed(
              f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} "
              f"-X {method} {FORGE}/api/v1{path} {data}"
          )

      def create_agent(body):
          # The controller refuses a create until its dependencies answer (the
          # archetype list comes from the builder's evaluation), so ask again.
          for _ in range(60):
              status, out = swarm.execute(
                  f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' "
                  f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}"
              )
              if out.strip().startswith("2"):
                  return
              swarm.sleep(10)
          diagnose(swarm, "swarm-controller")
          raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {swarm.succeed('cat /root/create.out')}")

      def http(m, name, path="/", resolve=None):
          # The status code of an HTTPS request, 000 when nothing answers: a
          # refused connection, and the gateway's catch-all, which closes the
          # connection without a response.
          pin = f"--resolve {name}:443:{resolve} " if resolve else ""
          return m.execute(
              f"curl --silent --insecure --max-time 10 {pin}-o /dev/null -w '%{{http_code}}' https://{name}{path}"
          )[1].strip()

      def answers(code):
          return code not in ("000", "502", "503", "504")

      def wait_http(m, name, accept, path="/", resolve=None, tries=90):
          for _ in range(tries):
              code = http(m, name, path, resolve)
              if accept(code):
                  return code
              m.sleep(10)
          diagnose(m)
          raise Exception(f"{m.name}: https://{name}{path} answered {code}")

      def redeploy(m):
          # What a second deploy does once the leaves are in place: start what
          # was not started and restart what failed for want of them.
          m.execute(
              "failed=\"$(systemctl list-units --failed --plain --no-legend | cut -d' ' -f1)\"; "
              "systemctl reset-failed; "
              "[ -z \"$failed\" ] || systemctl restart --no-block $failed; "
              "systemctl start --no-block multi-user.target"
          )

      def push_file(src_path, m, dest):
          # One file from the store's host to another machine, through the
          # driver: the operator's copy over the VPN.
          payload = swarm.succeed(f"base64 -w0 {src_path}").strip()
          m.succeed(f"install -d -m 0700 {POOL} && echo {payload} | base64 -d | install -m 0600 /dev/stdin {dest}")

      start_all()
      for m in (swarm, heavyduty, hive, outside):
          m.wait_for_unit("multi-user.target")

      with subtest("the VMs sit on the public network where the mesh expects them"):
          for m, name in ((swarm, "swarm"), (heavyduty, "heavyduty"), (hive, "hive"), (outside, "outside")):
              assert f"inet {LAN[name]}/" in m.succeed("ip -4 -o addr show"), name

      with subtest("the toplevels are in heavyduty's store only, and nobody builds"):
          for toplevel in TOPLEVELS:
              heavyduty.succeed(f"{NIX} path-info {toplevel}")
              hive.fail(f"{NIX} path-info {toplevel}")
          # The control for the lookup: a path that is not there is not found.
          heavyduty.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent")

      with subtest("assert 1: wg0 is up between all three machines, over IPv4 and IPv6"):
          for m in (swarm, heavyduty, hive):
              m.wait_for_unit("wireguard-wg0.service")
          for src in (swarm, heavyduty, hive):
              own = WG[src]
              assert own[0] in src.succeed("ip -4 -o addr show dev wg0"), own
              assert own[1] in src.succeed("ip -6 -o addr show dev wg0"), own
              for dst in (swarm, heavyduty, hive):
                  if dst is src:
                      continue
                  v4, v6 = WG[dst]
                  src.wait_until_succeeds(f"ping -c 1 -W 2 {v4}", timeout=180)
                  src.wait_until_succeeds(f"ping -6 -c 1 -W 2 {v6}", timeout=180)
                  src.succeed(f"ping -c 3 -W 5 {v4}")
                  src.succeed(f"ping -6 -c 3 -W 5 {v6}")
          # The address is routed through the tunnel, the public one is not.
          assert "dev wg0" in hive.succeed(f"ip route get {WG[swarm][0]}")
          assert "dev wg0" not in hive.succeed(f"ip route get {LAN['swarm']}")
          # Negative control: the hive-two entry has no machine behind it.
          hive.fail("ping -c 2 -W 2 ${exampleMesh.hive-two.address}")

      with subtest("assert 3: split DNS"):
          for key, machine in INTERNAL.items():
              v4, v6 = WG[{"swarm-central": swarm, "swarm-heavyduty": heavyduty}[machine]]
              got4 = hive.succeed(f"getent ahostsv4 {NAMES[key]}").split()[0]
              got6 = hive.succeed(f"getent ahostsv6 {NAMES[key]}").split()[0]
              assert got4 == v4, f"{NAMES[key]}: A {got4}, want {v4}"
              assert got6 == v6, f"{NAMES[key]}: AAAA {got6}, want {v6}"
          # The public names, from outside, are the swarm machine's public address.
          for key in PUBLIC:
              got = outside.succeed(f"getent ahostsv4 {NAMES[key]}").split()[0]
              assert got == LAN["swarm"], f"{NAMES[key]}: {got}"
          # The rest of the names do not resolve there, and none resolves to the VPN.
          for key in PRIVATE + ["bao", "nats"]:
              outside.fail(f"getent ahosts {NAMES[key]}")
          for key in PUBLIC:
              assert "10.77." not in outside.succeed(f"getent ahosts {NAMES[key]}")

      with subtest("the example's leaves are copied to the machine that runs them"):
          swarm.wait_until_succeeds(f"[ -s {MINTED}/hives/{HIVE}/hive-leaf.pem ] && [ -s {MINTED}/forge-publisher.pem ]", timeout=900)
          # The store's host keeps the CA's and the granter's keys, and a hive's
          # leaves are not another machine's.
          swarm.succeed(
              f"install -d -m 0700 {POOL} && cd {MINTED} && "
              f"for f in *.pem; do case $f in ca-key.pem|granter*|server*) ;; *) install -m 0600 $f {POOL}/$f ;; esac; done"
          )
          swarm.succeed(f"[ -s {POOL}/ca.pem ] && [ -s {POOL}/services-issuer.pem ] && [ ! -e {POOL}/ca-key.pem ]")
          redeploy(swarm)

      with subtest("step 1: bao operator init"):
          store_sealed(900)
          swarm.succeed(f"umask 077 && {BAO} operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")

      with subtest("step 4: bao operator unseal, with the shamir seal"):
          swarm.succeed(f"{BAO} operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
          swarm.succeed(f"{BAO} status")

      with subtest("step 1: the granter bootstrap"):
          swarm.succeed(
              "set -o pipefail; "
              f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
              f"{BAO} policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
              f"{BAO} token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
              "| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
              "unset BAO_TOKEN; "
              "systemctl restart swarm-bao-granter-role"
          )
          swarm.wait_until_succeeds(
              "journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'",
              timeout=300,
          )

      with subtest("step 1: the policy units that waited for the granter are restarted"):
          # See ./full-setup.nix for why these are started without waiting and polled.
          swarm.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true")
          swarm.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service")
          swarm.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=900)
          if swarm.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
              swarm.succeed("systemctl --machine=swarm-bao restart openbao.service")
              store_sealed(600)
              swarm.succeed(f"{BAO} operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
          settle(swarm, POLICY_UNITS_ACTIVE, 1800, "swarm-bao-*-policy")
          swarm.succeed(
              "journalctl -u swarm-bao-controller-policy --no-pager "
              "| grep -F 'Data written to: auth/cert/certs/swarm-controller'"
          )

      with subtest("the leaves for heavyduty and the hive are copied over, and their units start"):
          # The builder, the CI runner and the cache each read the store with a
          # leaf of their own; the hive with its reader and gateway leaves. The
          # CA is what lets either check the store's certificate.
          files = swarm.succeed(
              f"cd {MINTED} && for f in *.pem; do case $f in ca-key.pem|granter*|server*) ;; *) echo $f ;; esac; done"
          ).split()
          for name in files:
              push_file(f"{MINTED}/{name}", heavyduty, f"{POOL}/{name}")
          for name in ("client.pem", "client-key.pem", "hive-leaf.pem", "hive-leaf-key.pem"):
              push_file(f"{MINTED}/hives/{HIVE}/{name}", hive, f"{POOL}/{name}")
          push_file(f"{MINTED}/ca.pem", hive, f"{POOL}/ca.pem")
          # Control: the lookup finds what was copied, and nothing of the CA's key.
          heavyduty.succeed(f"[ -s {POOL}/builder.pem ] && [ -s {POOL}/ca.pem ] && [ ! -e {POOL}/ca-key.pem ]")
          hive.succeed(f"[ -s {POOL}/hive-leaf.pem ] && [ ! -e {POOL}/builder.pem ]")
          redeploy(heavyduty)
          redeploy(hive)

      with subtest("authelia's mint has published the clients' secrets"):
          settle(
              swarm,
              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" {BAO} kv get -field=value "
              "secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
              1800,
              "authelia-swarm-secrets",
              "swarm-authelia",
          )
          settle(swarm, "[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 600, "swarm-bao-operator-viewer-policy")

      with subtest("assert 2: from outside, the forge, chat, SSO and the swarm UI answer on the public address"):
          for key, (path, accept) in PUBLIC.items():
              code = wait_http(outside, NAMES[key], lambda c, a=accept: c[:1] in a, path, tries=180)
              print(f"{NAMES[key]}{path}: {code}")

      with subtest("assert 2: from hive, over wg0, the private names answer"):
          for key in PRIVATE:
              code = wait_http(hive, NAMES[key], answers, tries=180)
              print(f"{NAMES[key]}: {code}")
          # The hive's dashboard, from the store's machine, over wg0.
          code = wait_http(swarm, "${swarm.dashboardDomains.${hiveName}}", answers, resolve=WG[hive][0])
          print(f"dashboard: {code}")

      with subtest("assert 2: from outside, the private names get nothing, on any machine's public address"):
          # The positive controls are the two subtests above: the same
          # requests from inside answered. Here nginx is up on the swarm
          # machine, so 000 is the catch-all closing the connection.
          for key in PRIVATE:
              code = http(outside, NAMES[key], resolve=LAN["swarm"])
              assert code == "000", f"{NAMES[key]} at the swarm machine's public address: {code}"
          code = http(outside, NAMES["attic"], resolve=LAN["heavyduty"])
          assert code == "000", f"the cache at heavyduty's public address: {code}"
          code = http(outside, "${swarm.dashboardDomains.${hiveName}}", resolve=LAN["hive"])
          assert code == "000", f"the hive's dashboard at its public address: {code}"
          # The public names are not on heavyduty or the hive either.
          assert http(outside, NAMES["forge"], "/api/healthz", resolve=LAN["heavyduty"]) == "000"

      with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
          settle(
              swarm,
              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" {BAO} kv get -field=value "
              "secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
              1800,
              ["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],
              "hive-forge",
          )
          swarm.succeed(
              f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}"
          )
          swarm.wait_until_succeeds(
              f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600
          )
          status = swarm.succeed(
              f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive"
          ).strip()
          if status != "200":
              forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"})
          forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"})
          swarm.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git")
          swarm.succeed(
              f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
              f"push {FORGE}/hyperhive/hyperhive.git main"
          )

      with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"):
          swarm.wait_until_succeeds(
              f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800
          )
          create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE})
          create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})

      with subtest("assert 4: the agents are deployed on the hive, from the cache on heavyduty, and need a login"):
          settle(
              swarm,
              f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status "
              "| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)"
              " | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'",
              3600,
              ["swarm-controller"],
          )
          # The closures came over the VPN from the cache: the hive's nix
          # names it as its only extra substituter and holds the toplevels now.
          fragment = hive.succeed("cat /run/hyperhive-nix/cache.conf")
          assert "extra-substituters" in fragment and "${swarm.attic.domain}" in fragment, fragment
          for toplevel in TOPLEVELS:
              hive.succeed(f"{NIX} path-info {toplevel}")
              # Not built on the swarm machine, which has no copy.
              swarm.fail(f"{NIX} path-info {toplevel}")
    '';
  }
)