ci: type-check vm test scripts without building the vms

Adds checks.<system>.vm-test-typecheck-<name> for hive-squid, swarm-archive, agent-ephemeral-rootfs, full-setup and wg-hive: ty and ruff F over nixpkgs' test-script prelude, the machine hints from the node names, and the script text. Node configs are never evaluated. vm-test-typecheck-control passes only when ty rejects a known type error.

docs/4-implementation/scheduler/ci.md

@@ -114,7 +114,13 @@ slow); run those manually before pushing Rust changes.
 
 Tests that boot NixOS VMs are the `vmTests.<system>.<name>` flake output
 (`nix/vm-tests.nix`), not `checks`, so the per-PR **nix flake check** job never
-builds them. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml)
+builds them. It does build the `vm-test-typecheck-<name>` checks
+(`nix/vm-test-typecheck.nix`): the type check and lint the test driver runs over
+a test script, on the script text alone. They evaluate no node configuration
+and build no VM. They cover `hive-squid`,
+`swarm-archive`, `agent-ephemeral-rootfs`, `full-setup` and `wg-hive`;
+`vm-test-typecheck-control` passes only when the checker rejects a known type error, which shows the
+checker resolves `test_driver`. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml)
 builds all of them nightly at 02:30; a failing test fails that run. To run one
 by hand, start the workflow from the forge UI (Actions, vm-nightly, Run
 workflow, optional `test` input), or run

flake.nix

@@ -232,7 +232,8 @@
 
       # Tests that boot a NixOS VM and need KVM. Not `checks`, so `nix flake
       # check` and the per-PR CI job never build them;
-      # `.forgejo/workflows/vm-nightly.yml` does.
+      # `.forgejo/workflows/vm-nightly.yml` does. Their scripts are type-checked
+      # per PR by `checks.<system>.vm-test-typecheck-*` (nix/vm-test-typecheck.nix).
       vmTests = forAllSystems (
         { pkgs, system, ... }:
         import ./nix/vm-tests.nix {

nix/checks.nix

@@ -573,3 +573,10 @@ in
         touch "$out"
       '';
 }
+// import ./vm-test-typecheck.nix {
+  inherit pkgs;
+  vmTests = import ./vm-tests.nix {
+    inherit pkgs self;
+    hivePriv = self.packages.${system}.hive-priv;
+  };
+}

nix/vm-test-typecheck.nix

@@ -0,0 +1,140 @@
+# `vm-test-typecheck-<name>` checks: run the type check and lint that nixpkgs'
+# test driver applies to a VM test script (`ty check`, `ruff check --select F`)
+# without building the driver. The driver derivation references every node's
+# `system.build.vm`, so building or even evaluating it costs a full NixOS
+# evaluation per node; the check itself needs only the script text and the
+# machine names, and `nodes` is a lazy attribute set, so reading its names
+# forces no node configuration.
+{ pkgs, vmTests }:
+let
+  inherit (pkgs) lib;
+
+  # nixos/lib/testing/driver.nix:24-31 (the pinned nixpkgs).
+  pythonizeName =
+    name:
+    let
+      head = lib.substring 0 1 name;
+      tail = lib.substring 1 (-1) name;
+    in
+    (if builtins.match "[A-z_]" head == null then "_" else head)
+    + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail;
+
+  # The type hints of nixos/lib/testing/driver.nix:22-53: one stub machine per
+  # node, one per container, and `machine` when it is the only node and no
+  # machine already has that name. The `vlanN` hints are left out: they need
+  # every node's `virtualisation.vlans`, and no script here uses a `vlanN` name.
+  hintsFor =
+    test:
+    let
+      nodeNames = lib.attrNames test.config.nodes;
+      containerNames = lib.attrNames test.config.containers;
+      alias = lib.optional (
+        lib.length nodeNames == 1 && !lib.elem "machine" (nodeNames ++ containerNames)
+      ) "machine";
+    in
+    lib.concatStringsSep "\n" (
+      map (n: "${pythonizeName n} = create_fake_qemu_machine()") (nodeNames ++ alias)
+      ++ map (n: "${pythonizeName n} = create_fake_nspawn_machine()") containerNames
+    );
+
+  # `test_driver` reaches ty through `buildInputs`, as in driver.nix:66.
+  checkInputs = {
+    nativeBuildInputs = [
+      pkgs.ty
+      pkgs.ruff
+    ];
+    buildInputs = [ pkgs.nixos-test-driver ];
+  };
+
+  # The text driver.nix:77-82 assembles: the prelude from the pinned nixpkgs,
+  # the hints, then the script.
+  assemble = ''
+    cat ${pkgs.path}/nixos/lib/test-script-prepend.py > testScriptWithTypes
+    echo "$hints" >> testScriptWithTypes
+    echo -n "$testScript" >> testScriptWithTypes
+  '';
+
+  typecheck =
+    name: test:
+    pkgs.runCommand "vm-test-typecheck-${name}"
+      (
+        checkInputs
+        // {
+          testScript = test.config.testScriptString;
+          hints = hintsFor test;
+        }
+      )
+      ''
+        ${assemble}
+        ty check testScriptWithTypes
+        # driver.nix:91-99
+        ruff check --select F testScriptWithTypes
+        touch $out
+      '';
+
+  # Succeeds only when `ty` rejects an `Optional` dereference and three calls on
+  # the stub machines of `hintsFor`. If `test_driver` stops resolving, every
+  # name is `Unknown`, the real checks pass on anything, and this one fails.
+  control =
+    pkgs.runCommand "vm-test-typecheck-control"
+      (
+        checkInputs
+        // {
+          testScript = ''
+            import re
+
+            def first_group(s: str) -> str:
+                return re.search(r"x", s).group(1)
+
+            my_node.no_such_node()
+            machine.no_such_alias()
+            c_1.no_such_container()
+          '';
+          hints = hintsFor {
+            config = {
+              nodes."my-node" = { };
+              containers."c-1" = { };
+            };
+          };
+        }
+      )
+      ''
+        ${assemble}
+        if ty check testScriptWithTypes > ty.out 2>&1; then
+          cat ty.out
+          echo "ty accepted re.search(...).group(1); the type check is not effective" >&2
+          exit 1
+        fi
+        cat ty.out
+        # `group` comes from the stdlib stubs; the other three are attributes of
+        # the stub machines `hintsFor` declares, so they show the node, `machine`
+        # alias and container hints each resolve to a typed machine.
+        grep -q "Attribute .group. is not defined on .None." ty.out
+        grep -q "has no attribute .no_such_node." ty.out
+        grep -q "has no attribute .no_such_alias." ty.out
+        grep -q "has no attribute .no_such_container." ty.out
+        if grep -q unresolved-import ty.out; then
+          echo "test_driver does not resolve" >&2
+          exit 1
+        fi
+        touch $out
+      '';
+
+  # semi-spread and swarm-builder-offline are not listed: they interpolate node
+  # config or an agent toplevel into their script, so reading the script text
+  # evaluates a NixOS system. They need their script to be a function of those
+  # values first.
+  tests = {
+    inherit (vmTests)
+      hive-squid
+      swarm-archive
+      agent-ephemeral-rootfs
+      full-setup
+      wg-hive
+      ;
+  };
+in
+(lib.mapAttrs' (n: t: lib.nameValuePair "vm-test-typecheck-${n}" (typecheck n t)) tests)
+// {
+  vm-test-typecheck-control = control;
+}