# `vmTests.<system>.swarm-archive`: runs the archive server's real units
# (../host-modules/swarm-archive.nix) and the hive's real backup script
# (../host-modules/lib/archive-backup.nix) in a VM, with restic and rclone, and
# checks what the module-eval suite can only read:
#
# - a backup of two agents lands in one repository each, and `work/` is left out;
# - one agent's login cannot read another's repository;
# - a client cannot delete or overwrite anything, and the server-side prune
#   applies the retention, also when a client has forged a snapshot dated in
#   the future;
# - the users file follows the store: a user removed from it is refused without
#   a restart of the server;
# - a hive that has agents and no grant backs up nothing and succeeds.
#
# One node, because the gateway hop is outside what is under test: the
# backup script dials `http://127.0.0.1:<port>` instead of
# `https://archive.<domain>`, and the swarm secret store is a stand-in `bao`
# that serves `kv list`, `kv get` and `login` from a directory. The module-eval
# suite covers the gateway vhost and the store grants.
{ pkgs }:
let
  port = 8911;
  store = "/var/lib/fake-bao";
  hive = "h1";
  agentRoot = "/var/lib/hyperhive/agents";

  # A directory is a leaf, a file in it a field: `kv get -field=<f> <path>`.
  fakeBao = pkgs.writeShellScriptBin "bao" ''
    store=${store}
    case "$1 $2" in
      "login -method=cert") echo faketoken; exit 0 ;;
      "kv list")
        path="$4"
        [ -d "$store/$path" ] || { echo "No value found at $path" >&2; exit 2; }
        ls "$store/$path" | ${pkgs.jq}/bin/jq -R . | ${pkgs.jq}/bin/jq -s .
        exit 0 ;;
      "kv put")
        [ -e "$store/$4" ] && { echo "check-and-set parameter did not match" >&2; exit 2; }
        mkdir -p "$store/$4"
        printf %s "''${5#*=}" > "$store/$4/''${5%%=*}"
        exit 0 ;;
      "kv get")
        field="''${3#-field=}"
        [ -f "$store/$4/$field" ] || { echo "No value found at $4" >&2; exit 2; }
        cat "$store/$4/$field"
        exit 0 ;;
    esac
    echo "fake bao: unsupported: $*" >&2
    exit 1
  '';

  # The stubs for the options the archive module reads from the rest of the
  # hive modules.
  hiveStubs =
    { lib, ... }:
    {
      options.services.hyperhive = {
        swarm.domain = lib.mkOption {
          type = lib.types.str;
          default = "t.local";
        };
        swarm.bao = {
          domain = lib.mkOption {
            type = lib.types.str;
            default = "bao.t.local";
          };
          port = lib.mkOption {
            type = lib.types.port;
            default = 8200;
          };
        };
        deploy.bao = {
          package = lib.mkOption {
            type = lib.types.package;
            default = fakeBao;
          };
          archiveClientCertFile = lib.mkOption {
            type = lib.types.nullOr lib.types.str;
            default = "/etc/fake-archive-cert";
          };
          archiveClientKeyFile = lib.mkOption {
            type = lib.types.nullOr lib.types.str;
            default = "/etc/fake-archive-key";
          };
          serverCaFile = lib.mkOption {
            type = lib.types.nullOr lib.types.str;
            default = null;
          };
        };
        deploy.swarm-otel.enable = lib.mkOption {
          type = lib.types.bool;
          default = false;
        };
        deploy.archive.enable = lib.mkOption {
          type = lib.types.bool;
          default = true;
        };
        otel.journalUnits = lib.mkOption {
          type = lib.types.listOf lib.types.str;
          default = [ ];
        };
        gateway = {
          enable = lib.mkOption {
            type = lib.types.bool;
            default = false;
          };
          dns.enable = lib.mkOption {
            type = lib.types.bool;
            default = false;
          };
          localNames = lib.mkOption {
            type = lib.types.listOf lib.types.str;
            default = [ ];
          };
          vhosts = lib.mkOption {
            type = lib.types.attrsOf lib.types.anything;
            default = { };
          };
        };
      };
      config.environment.etc."fake-archive-cert".text = "";
      config.environment.etc."fake-archive-key".text = "";
    };

  backupScript = import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
    inherit hive agentRoot;
    restic = pkgs.restic;
    url = "http://127.0.0.1:${toString port}";
    scope = "paths";
    paths = [
      "state"
      "claude"
      "harness"
    ];
    excludes = [ ];
    caBundle = null;
  };
in
pkgs.testers.runNixOSTest {
  name = "swarm-archive";

  nodes.machine =
    { pkgs, ... }:
    {
      imports = [
        hiveStubs
        ../host-modules/swarm-archive.nix
      ];

      # The newest snapshot, plus whatever is younger than three days.
      services.hyperhive.deploy.archive.retention = {
        keepWithin = "3d";
        keepLast = 1;
      };

      environment.systemPackages = [
        pkgs.curl
        pkgs.openssl
        pkgs.restic
      ];

      systemd.services.test-archive-backup = {
        path = [
          fakeBao
          pkgs.btrfs-progs
          pkgs.coreutils
          pkgs.gnugrep
          pkgs.jq
        ];
        serviceConfig = {
          Type = "oneshot";
          Environment = "RESTIC_CACHE_DIR=/var/cache/test-archive-backup";
        };
        script = backupScript;
      };
    };

  testScript = ''
    store = "${store}"
    url = "http://127.0.0.1:${toString port}"
    agents = ["alpha", "beta"]

    def sha_line(agent, password):
        return machine.succeed(
            f"printf '%s:{{SHA}}%s' {agent} "
            f"\"$(printf %s {password} | openssl dgst -sha1 -binary | base64)\""
        )

    def put(path, field, value):
        machine.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}")

    def status(agent, repo, path="config", method="GET"):
        return machine.succeed(
            f"curl -s -o /dev/null -w '%{{http_code}}' -X {method} "
            f"-u {agent}:restpw-{agent} {url}/{repo}/{path}"
        )

    def restic(agent, repo, args):
        return (
            f"RESTIC_PASSWORD=resticpw-{repo} restic "
            f"-r rest:http://{agent}:restpw-{agent}@127.0.0.1:${toString port}/{repo}/ {args}"
        )

    machine.wait_for_unit("multi-user.target")
    machine.wait_for_open_port(${toString port})

    with subtest("the server starts with no users and admits nobody"):
        machine.succeed("test -e /run/swarm-archive-auth/htpasswd")
        assert status("alpha", "alpha") == "401", status("alpha", "alpha")

    # What swarm-controller writes: a user line and a restic password per agent
    # for the archive, a REST password and the restic password for the hive.
    for agent in agents:
        put(f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(agent, f"restpw-{agent}"))
        put(f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}")
        put(f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}")
        put(f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}")
        machine.succeed(
            f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude "
            f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work"
        )
        machine.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data")
        machine.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch")

    with subtest("the users file follows the store without a restart of the server"):
        pid = machine.succeed("systemctl show -p MainPID --value swarm-archive").strip()
        machine.succeed("systemctl start swarm-archive-users")
        assert machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid
        assert status("alpha", "alpha") in ("200", "404"), status("alpha", "alpha")
        assert status("beta", "beta") in ("200", "404"), status("beta", "beta")
        # Wrong password: the control that the 200 above is a login, not an open door.
        machine.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401")

    with subtest("the users file holds hashes, readable by the server only"):
        machine.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd")
        machine.fail("grep -q restpw /run/swarm-archive-auth/htpasswd")
        machine.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'")

    with subtest("the hive's backup puts each agent in its own repository, without work/"):
        machine.succeed("systemctl start test-archive-backup")
        for agent in agents:
            snaps = machine.succeed(restic(agent, agent, "snapshots --json"))
            assert snaps.count('"id"') == 1, snaps
            listing = machine.succeed(restic(agent, agent, "ls latest"))
            assert f"/{agent}/state/data" in listing, listing
            assert "scratch" not in listing, listing

    with subtest("one agent's login cannot reach another's repository"):
        assert status("beta", "alpha") == "403", status("beta", "alpha")
        machine.fail(restic("beta", "alpha", "snapshots"))
        assert status("alpha", "alpha") == "200", status("alpha", "alpha")
        assert status("alpha", "beta") == "403", status("alpha", "beta")

    with subtest("a client cannot delete or overwrite"):
        snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
        # An old snapshot for the prune to drop. The backup is also the presence control: a lock, which a backup creates and removes, is deletable.
        machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2000-01-01 00:00:00' ${agentRoot}/alpha/state"))
        assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
        assert status("alpha", "alpha", "config", "POST") == "403"
        machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
        assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2

    with subtest("the server-side prune applies the retention"):
        machine.succeed("systemctl start swarm-archive-prune")
        for agent in agents:
            assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
        # The old snapshot went, the one from the hive's backup stayed.
        assert snap in machine.succeed(restic("alpha", "alpha", "snapshots --json"))

    with subtest("a snapshot dated in the future cannot push the real ones out of the retention"):
        # restic counts --keep-within from the newest snapshot, and the client sets its time.
        machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2100-01-01 00:00:00' ${agentRoot}/alpha/state"))
        assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
        machine.succeed("systemctl start swarm-archive-prune")
        left = machine.succeed(restic("alpha", "alpha", "snapshots --json"))
        assert left.count('"id"') == 1 and snap in left, left
        machine.succeed("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: alpha has snapshots dated after now'")
        # beta's repository had nothing forged, and says nothing.
        machine.fail("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: beta'")

    with subtest("a user removed from the store is refused without a restart"):
        machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")
        machine.succeed("systemctl start swarm-archive-users")
        assert status("beta", "beta") == "401", status("beta", "beta")
        assert status("alpha", "alpha") == "200", status("alpha", "alpha")

    with subtest("the archive announces itself once, so swarm-controller starts granting"):
        machine.succeed(f"test \"$(cat {store}/secret/swarm/services/archive/ready/ready)\" = 1")
        machine.succeed("systemctl start swarm-archive-users")

    with subtest("a hive with agents and no grant backs up nothing and does not fail"):
        machine.succeed(f"rm -r {store}/secret/swarm/hives/${hive}")
        machine.succeed("systemctl start test-archive-backup")
        machine.succeed("journalctl -u test-archive-backup | grep -q 'granted this hive no archive credentials'")
        machine.fail("journalctl -u test-archive-backup | grep -q 'archive backup failed'")
  '';
}