hyperhive/nix/vm-test-typecheck.nix

# `vm-test-typecheck-<name>` checks: run the type check and lint that nixpkgs'
# test driver applies to a VM test script (`ty check`, `ruff check --select F`)
# without building the driver. The driver derivation references every node's
# `system.build.vm`, so building or even evaluating it costs a full NixOS
# evaluation per node; the check itself needs only the script text and the
# machine names, and `nodes` is a lazy attribute set, so reading its names
# forces no node configuration.
{ pkgs, vmTests }:
let
  inherit (pkgs) lib;

  # nixos/lib/testing/driver.nix:24-31 (the pinned nixpkgs).
  pythonizeName =
    name:
    let
      head = lib.substring 0 1 name;
      tail = lib.substring 1 (-1) name;
    in
    (if builtins.match "[A-z_]" head == null then "_" else head)
    + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail;

  # The type hints of nixos/lib/testing/driver.nix:22-53: one stub machine per
  # node, one per container, and `machine` when it is the only node and no
  # machine already has that name. The `vlanN` hints are left out: they need
  # every node's `virtualisation.vlans`, and no script here uses a `vlanN` name.
  hintsFor =
    test:
    let
      nodeNames = lib.attrNames test.config.nodes;
      containerNames = lib.attrNames test.config.containers;
      alias = lib.optional (
        lib.length nodeNames == 1 && !lib.elem "machine" (nodeNames ++ containerNames)
      ) "machine";
    in
    lib.concatStringsSep "\n" (
      map (n: "${pythonizeName n} = create_fake_qemu_machine()") (nodeNames ++ alias)
      ++ map (n: "${pythonizeName n} = create_fake_nspawn_machine()") containerNames
    );

  # `test_driver` reaches ty through `buildInputs`, as in driver.nix:66.
  checkInputs = {
    nativeBuildInputs = [
      pkgs.ty
      pkgs.ruff
    ];
    buildInputs = [ pkgs.nixos-test-driver ];
  };

  # The text driver.nix:77-82 assembles: the prelude from the pinned nixpkgs,
  # the hints, then the script.
  assemble = ''
    cat ${pkgs.path}/nixos/lib/test-script-prepend.py > testScriptWithTypes
    echo "$hints" >> testScriptWithTypes
    echo -n "$testScript" >> testScriptWithTypes
  '';

  typecheck =
    name: test:
    pkgs.runCommand "vm-test-typecheck-${name}"
      (
        checkInputs
        // {
          testScript = test.config.testScriptString;
          hints = hintsFor test;
        }
      )
      ''
        ${assemble}
        ty check testScriptWithTypes
        # driver.nix:91-99
        ruff check --select F testScriptWithTypes
        touch $out
      '';

  # Succeeds only when `ty` rejects an `Optional` dereference and three calls on
  # the stub machines of `hintsFor`. If `test_driver` stops resolving, every
  # name is `Unknown`, the real checks pass on anything, and this one fails.
  control =
    pkgs.runCommand "vm-test-typecheck-control"
      (
        checkInputs
        // {
          testScript = ''
            import re

            def first_group(s: str) -> str:
                return re.search(r"x", s).group(1)

            my_node.no_such_node()
            machine.no_such_alias()
            c_1.no_such_container()
          '';
          hints = hintsFor {
            config = {
              nodes."my-node" = { };
              containers."c-1" = { };
            };
          };
        }
      )
      ''
        ${assemble}
        if ty check testScriptWithTypes > ty.out 2>&1; then
          cat ty.out
          echo "ty accepted re.search(...).group(1); the type check is not effective" >&2
          exit 1
        fi
        cat ty.out
        # `group` comes from the stdlib stubs; the other three are attributes of
        # the stub machines `hintsFor` declares, so they show the node, `machine`
        # alias and container hints each resolve to a typed machine.
        grep -q "Attribute .group. is not defined on .None." ty.out
        grep -q "has no attribute .no_such_node." ty.out
        grep -q "has no attribute .no_such_alias." ty.out
        grep -q "has no attribute .no_such_container." ty.out
        if grep -q unresolved-import ty.out; then
          echo "test_driver does not resolve" >&2
          exit 1
        fi
        touch $out
      '';

  # semi-spread and swarm-builder-offline are not listed: they interpolate node
  # config or an agent toplevel into their script, so reading the script text
  # evaluates a NixOS system. They need their script to be a function of those
  # values first.
  tests = {
    inherit (vmTests)
      hive-squid
      swarm-archive
      agent-ephemeral-rootfs
      full-setup
      wg-hive
      ;
  };
in
(lib.mapAttrs' (n: t: lib.nameValuePair "vm-test-typecheck-${n}" (typecheck n t)) tests)
// {
  vm-test-typecheck-control = control;
}