# `vmTests.<system>.full-setup`: one VM runs a whole single-host swarm
# (`deploy.singleHostSwarm`) with no network and no substituter, and a script
# walks docs/1-getting-started/setup.md until the controller has committed the
# agents' meta flake to the swarm's forge.
#
# Steps of setup.md, as the script runs them:
#   1 secret store  `bao operator init`, the granter bootstrap with the root
#                   token, then the policy units reset and restarted.
#   4 unseal        `bao operator unseal` (`deploy.bao.seal` is "shamir"; pkcs11
#                   needs a TPM). A pkcs11 store unseals itself at init, so here
#                   the unseal comes right after init, before the granter.
#   2 SSO account   `swarmctl user add mara --group admins`.
#   8 ruth          `POST /api/agents` on the controller socket, as the swarm
#                   UI's create form does, plus `manage_root_agent`.
#   9 first agent   The same endpoint, for `iris`: `swarmctl agent create` has no
#                   archetype flag, and a custom agent's config repo would need
#                   inputs the VM cannot resolve offline.
# Not scripted yet; a later slice adds them: step 3, the first SSO login to the
# forge (an OIDC code flow against authelia), `swarmctl forge make-admin`, and
# the agents' deploy up to the point where each says it needs a login. Green
# when `internal/meta` on the forge holds the flake the toplevels were built from.
#
# Offline: the test seeds the swarm's forge with `hyperhive/hyperhive` (one
# commit of `self`, ./hyperhive-fixture-repo.nix). The controller renders
# `internal/meta` and the builder locks and builds it as in production.
# `hive-meta-flake/testdata/meta-flake-swarm.nix.in`, filled in with the
# controller's settings, is that flake: the agents' toplevels are built from it
# here, so the builder finds them in the store. The daemon runs with
# `max-jobs = 0`, so a missing toplevel fails the build instead of compiling.
{ pkgs, self }:
let
  inherit (pkgs) lib;

  hive = "pr1ma";
  swarmDomain = "swarm.test";
  agents = [
    "iris"
    "ruth"
  ];

  inherit (import ./hyperhive-fixture-repo.nix { inherit pkgs self; }) hyperhiveRepo;

  metaTemplate = ../../hive-meta-flake/testdata/meta-flake-swarm.nix.in;

  # The values the fixture has a placeholder for. A swarm whose controller sends
  # another set renders another flake, so evaluation stops there.
  forwardedVars = [
    "HIVE_AGENT_BAO_ADDR"
    "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT"
    "HIVE_AGENT_LOGS_QUERY_URL"
    "HIVE_AGENT_METRICS_URL"
    "HIVE_FORGE_PUBLIC_URL"
    "HIVE_FORGE_URL"
    "HIVE_MATRIX_URL"
  ];

  # The meta flake the controller renders for `agents`, from its settings JSON
  # (`SWARM_CONTROLLER_META_FLAKE`). `hyperhive` is `self`: the agents' toplevels
  # do not depend on how the input is pinned.
  filled =
    settingsJson: pin:
    let
      settings = builtins.fromJSON settingsJson;
      names = [ "HYPERHIVE_URL" ] ++ forwardedVars;
      values = [ pin ] ++ map (name: settings.vars.${name}) forwardedVars;
    in
    assert lib.assertMsg
      (builtins.attrNames settings.vars == forwardedVars && !settings.github_disabled)
      "the controller's meta flake settings changed: update hive-meta-flake/testdata/meta-flake-swarm.nix.in";
    builtins.replaceStrings (map (name: "@${name}@") names) values (builtins.readFile metaTemplate);

  toplevels =
    settingsJson:
    let
      flake = import (builtins.toFile "meta-flake.nix" (filled settingsJson "unused"));
      evaluated = flake.outputs {
        self = null;
        hyperhive = self;
      };
    in
    map (agent: evaluated.nixosConfigurations.${agent}.config.system.build.toplevel) agents;
in
pkgs.testers.runNixOSTest (
  { config, ... }:
  let
    inherit (config.nodes.machine.services.hyperhive) swarm deploy;
    settingsJson =
      config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
    controllerSocket = deploy.swarm-controller.socketPath;

    # A derivation no store holds, to show the daemon refuses to build.
    control = pkgs.writeText "control.nix" ''
      derivation {
        name = "build-control";
        system = "${pkgs.stdenv.hostPlatform.system}";
        builder = "/bin/sh";
        args = [ "-c" "echo built > $out" ];
      }
    '';
  in
  {
    name = "full-setup";

    # Boot of fourteen containers and the setup steps, with the waits' own
    # bounds well inside.
    globalTimeout = 7200;

    nodes.machine =
      { config, ... }:
      {
        imports = [ self.nixosModules.default ];

        services.hyperhive = {
          deploy.singleHostSwarm = true;
          deploy.hive-controller.enable = true;
          deploy.hive-controller.hiveName = hive;
          deploy.bao.seal = "shamir";
          swarm.domain = swarmDomain;
          swarm.hives.${hive} = { };
        };
        system.stateVersion = "25.11";

        # 12 GB: fourteen service containers, the hive and a Rust-heavy agent
        # closure through the controller, the builder and the cache. 4 cores: the
        # containers boot in parallel. The disk holds the cache's copy of the
        # closure; the store overlay is on disk, not in memory.
        virtualisation.memorySize = 12288;
        virtualisation.cores = 4;
        virtualisation.diskSize = 40960;
        virtualisation.writableStoreUseTmpfs = false;

        # The toplevels, and every source tree the meta flake's lock names, keyed
        # by narHash: nix takes a locked input from the store when its path is
        # valid.
        virtualisation.additionalPaths = [
          self
          self.inputs.nixpkgs
          self.inputs.crane
          self.inputs.treefmt-nix
          hyperhiveRepo
        ]
        ++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;

        # No substituter, and no build.
        nix.settings = {
          substituters = lib.mkForce [ ];
          max-jobs = 0;
        };

        # The store's raft data is on a tmpfs. Raft syncs every write, so on the
        # VM's qcow2 root (`cache=writeback`, which passes each guest flush on to
        # the host's disk) a login's lease write can outlast the 15 s the store's
        # clients wait, and its calls then fail with `context canceled`. The
        # state is under `/var/lib/private`, where the store's DynamicUser keeps
        # it; the mount sits on that directory in the container's root, and
        # nothing the test checks needs it to survive the VM.
        fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
          {
            device = "tmpfs";
            fsType = "tmpfs";
            options = [
              "mode=0700"
              "size=1G"
            ];
          };

        environment.systemPackages = [
          pkgs.git
          pkgs.jq
        ];
      };

    testScript = ''
      import json
      import shlex

      API = "http://localhost"
      SOCKET = "${controllerSocket}"
      FORGE = "https://${swarm.forge.domain}"
      HIVE = "${hive}"
      TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))}
      INIT = "/root/bao-init.json"
      FORGE_AUTH = "/root/forge-auth.header"
      NIX = "nix --extra-experimental-features nix-command"
      ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
      POLICY_UNITS_ACTIVE = (
          "for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
          "swarm-bao-agent-pki.service; do "
          "[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done"
      )

      def controller(method, path, body=None, check="--fail-with-body"):
          data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
          return machine.succeed(
              f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}"
          )

      def forge(method, path, body=None):
          data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
          return machine.succeed(
              f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} "
              f"-X {method} {FORGE}/api/v1{path} {data}"
          )

      def diagnose(unit=None, container=None):
          # What a timed-out wait leaves behind: the failed units, and the status
          # and the end of the journal of the unit that was waited on.
          where = f"--machine={container} " if container else ""
          units = [unit] if isinstance(unit, str) else list(unit or [])
          flags = " ".join(f"-u '{u}'" for u in units)
          for command in [
              f"systemctl {where}--failed --no-legend --no-pager",
              *([f"systemctl {where}status --no-pager -n 0 -l {' '.join(units)} | head -c 4000"] if units else []),
              *([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []),
          ]:
              print(f"--- {command}")
              print(machine.execute(command)[1])

      def settle(command, timeout, unit=None, container=None):
          try:
              machine.wait_until_succeeds(command, timeout=timeout)
          except Exception:
              diagnose(unit, container)
              raise

      def create_agent(body):
          # The controller refuses a create until its dependencies answer (the
          # archetype list comes from the builder's evaluation), so ask again.
          for _ in range(60):
              status, out = machine.execute(
                  f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' "
                  f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}"
              )
              if out.strip().startswith("2"):
                  return
              machine.sleep(10)
          diagnose("swarm-controller")
          raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {machine.succeed('cat /root/create.out')}")

      machine.start()
      machine.wait_for_unit("multi-user.target")

      with subtest("the agents' toplevels are in the store, and the daemon builds nothing"):
          for toplevel in TOPLEVELS:
              machine.succeed(f"{NIX} path-info {toplevel}")
          # The control for the lookup: a path that is not there is not found.
          machine.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent")
          # The control for the missing build: a derivation no store holds is refused.
          machine.fail("NIX_CONFIG='experimental-features = nix-command' nix build --no-link --impure -f ${control}")

      # setup.md step 1: the secret store.
      with subtest("step 1: bao operator init"):
          machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
          machine.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")

      with subtest("step 4: bao operator unseal, with the shamir seal"):
          machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
          machine.succeed("bao status")

      with subtest("step 1: the granter bootstrap"):
          machine.succeed(
              "set -o pipefail; "
              f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
              "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
              "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
              "| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
              "unset BAO_TOKEN; "
              "systemctl restart swarm-bao-granter-role"
          )
          machine.wait_until_succeeds(
              "journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'",
              timeout=300,
          )

      with subtest("step 1: the policy units that waited for the granter are restarted"):
          # Some of these units retry every 30 seconds until something outside
          # them exists: the bao UI's viewer policy waits for authelia's secrets
          # unit, in its container, to publish the UI's OIDC client secret, and
          # that unit waits for the secret publisher's policy written here. So
          # the units are started without waiting, and polled until all are active.
          machine.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true")
          machine.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service")
          machine.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=600)
          # Under the shamir seal the PKI unit leaves a restart of the store to the
          # operator when its listener's client CA changed, and says which command.
          if machine.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
              machine.succeed("systemctl --machine=swarm-bao restart openbao.service")
              machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
              machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
          settle(POLICY_UNITS_ACTIVE, 1200, "swarm-bao-*-policy")
          machine.succeed(
              "journalctl -u swarm-bao-controller-policy --no-pager "
              "| grep -F 'Data written to: auth/cert/certs/swarm-controller'"
          )

      with subtest("authelia's mint has published the clients' secrets"):
          # The operator viewer policy is left out of the poll above: it waits for
          # the UI client's secret, which authelia's secrets unit publishes from
          # its container once the store and the trust bundle it needs exist. The
          # controller's client secret is published by the same unit, and the
          # controller, the hive's store logins and the UI policy all wait for it.
          settle(
              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
              "secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
              1200,
              "authelia-swarm-secrets",
              "swarm-authelia",
          )
          settle("[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 300, "swarm-bao-operator-viewer-policy")

      with subtest("step 2: swarmctl user add"):
          machine.wait_until_succeeds(
              "swarmctl user add mara --display-name Mara --email mara@example.com --group admins "
              ">/root/user-add.out 2>&1",
              timeout=900,
          )
          machine.succeed("[ -s /root/user-add.out ]")
          machine.succeed("swarmctl user list | grep -w mara")

      with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
          # The controller's forge token, from the store as the controller reads it.
          # Minted and stored by `forgejo-swarm-controller-account` in the forge
          # container, which waits for forgejo and for a login to the store.
          settle(
              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
              "secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
              900,
              ["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],
              "hive-forge",
          )
          machine.succeed(
              f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}"
          )
          # The token being published says nothing about the gateway's vhost for
          # the forge, which the calls below go through.
          machine.wait_until_succeeds(
              f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600
          )
          status = machine.succeed(
              f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive"
          ).strip()
          if status != "200":
              forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"})
          forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"})
          machine.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git")
          machine.succeed(
              f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
              f"push {FORGE}/hyperhive/hyperhive.git main"
          )
          pushed = machine.succeed(
              f"GIT_SSL_NO_VERIFY=1 git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
              f"ls-remote {FORGE}/hyperhive/hyperhive.git refs/heads/main"
          ).split()[0]
          assert pushed == machine.succeed("cat ${hyperhiveRepo}/rev").strip(), pushed

      with subtest("step 8: ruth, through the endpoint the swarm UI calls"):
          machine.wait_until_succeeds(
              f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=900
          )
          default = json.loads(controller("GET", "/api/permissions"))["default"]
          create_agent({"name": "ruth", "hive": HIVE, "permissions": default + ["manage_root_agent"], "archetype": ARCHETYPE})

      with subtest("step 9: the first agent"):
          create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})

      with subtest("internal/meta holds the flake the toplevels were built from"):
          settle(
              f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/internal/meta/raw/flake.nix "
              ">/root/meta-flake.nix",
              1800,
              "swarm-controller",
          )
          rev = machine.succeed("cat ${hyperhiveRepo}/rev").strip()
          settings = json.loads(${builtins.toJSON settingsJson})
          expected = open("${metaTemplate}").read().replace(
              "@HYPERHIVE_URL@", f"git+{settings['git_base']}/hyperhive/hyperhive.git?rev={rev}"
          )
          for name, value in settings["vars"].items():
              expected = expected.replace(f"@{name}@", value)
          committed = machine.succeed("cat /root/meta-flake.nix")
          assert committed == expected, "the controller's flake differs from the one the toplevels came from:\n" + committed

      # The agents' deploy and the check that they need a login are not scripted yet.
    '';
  }
)