From 32f5b74ba4bd79f3823ceab9563f3e6e95fa88fd Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 13 Nov 2026 08:07:04 +0100 Subject: [PATCH] ci: type-check vm test scripts without building the vms Adds checks..vm-test-typecheck- for hive-squid, swarm-archive, agent-ephemeral-rootfs, full-setup and wg-hive: ty and ruff F over nixpkgs' test-script prelude, the machine hints from the node names, and the script text. Node configs are never evaluated. vm-test-typecheck-control passes only when ty rejects a known type error. --- docs/4-implementation/scheduler/ci.md | 8 +- flake.nix | 3 +- nix/checks.nix | 7 ++ nix/vm-test-typecheck.nix | 140 ++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 nix/vm-test-typecheck.nix diff --git a/docs/4-implementation/scheduler/ci.md b/docs/4-implementation/scheduler/ci.md index dcc943203..f5f820d88 100644 --- a/docs/4-implementation/scheduler/ci.md +++ b/docs/4-implementation/scheduler/ci.md @@ -114,7 +114,13 @@ slow); run those manually before pushing Rust changes. Tests that boot NixOS VMs are the `vmTests..` flake output (`nix/vm-tests.nix`), not `checks`, so the per-PR **nix flake check** job never -builds them. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml) +builds them. It does build the `vm-test-typecheck-` checks +(`nix/vm-test-typecheck.nix`): the type check and lint the test driver runs over +a test script, on the script text alone. They evaluate no node configuration +and build no VM. They cover `hive-squid`, +`swarm-archive`, `agent-ephemeral-rootfs`, `full-setup` and `wg-hive`; +`vm-test-typecheck-control` passes only when the checker rejects a known type error, which shows the +checker resolves `test_driver`. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml) builds all of them nightly at 02:30; a failing test fails that run. To run one by hand, start the workflow from the forge UI (Actions, vm-nightly, Run workflow, optional `test` input), or run diff --git a/flake.nix b/flake.nix index 8eb4ac5f5..55266d23c 100644 --- a/flake.nix +++ b/flake.nix @@ -232,7 +232,8 @@ # Tests that boot a NixOS VM and need KVM. Not `checks`, so `nix flake # check` and the per-PR CI job never build them; - # `.forgejo/workflows/vm-nightly.yml` does. + # `.forgejo/workflows/vm-nightly.yml` does. Their scripts are type-checked + # per PR by `checks..vm-test-typecheck-*` (nix/vm-test-typecheck.nix). vmTests = forAllSystems ( { pkgs, system, ... }: import ./nix/vm-tests.nix { diff --git a/nix/checks.nix b/nix/checks.nix index 6f8226ad5..e569439e7 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -573,3 +573,10 @@ in touch "$out" ''; } +// import ./vm-test-typecheck.nix { + inherit pkgs; + vmTests = import ./vm-tests.nix { + inherit pkgs self; + hivePriv = self.packages.${system}.hive-priv; + }; +} diff --git a/nix/vm-test-typecheck.nix b/nix/vm-test-typecheck.nix new file mode 100644 index 000000000..c102ab0e4 --- /dev/null +++ b/nix/vm-test-typecheck.nix @@ -0,0 +1,140 @@ +# `vm-test-typecheck-` checks: run the type check and lint that nixpkgs' +# test driver applies to a VM test script (`ty check`, `ruff check --select F`) +# without building the driver. The driver derivation references every node's +# `system.build.vm`, so building or even evaluating it costs a full NixOS +# evaluation per node; the check itself needs only the script text and the +# machine names, and `nodes` is a lazy attribute set, so reading its names +# forces no node configuration. +{ pkgs, vmTests }: +let + inherit (pkgs) lib; + + # nixos/lib/testing/driver.nix:24-31 (the pinned nixpkgs). + pythonizeName = + name: + let + head = lib.substring 0 1 name; + tail = lib.substring 1 (-1) name; + in + (if builtins.match "[A-z_]" head == null then "_" else head) + + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail; + + # The type hints of nixos/lib/testing/driver.nix:22-53: one stub machine per + # node, one per container, and `machine` when it is the only node and no + # machine already has that name. The `vlanN` hints are left out: they need + # every node's `virtualisation.vlans`, and no script here uses a `vlanN` name. + hintsFor = + test: + let + nodeNames = lib.attrNames test.config.nodes; + containerNames = lib.attrNames test.config.containers; + alias = lib.optional ( + lib.length nodeNames == 1 && !lib.elem "machine" (nodeNames ++ containerNames) + ) "machine"; + in + lib.concatStringsSep "\n" ( + map (n: "${pythonizeName n} = create_fake_qemu_machine()") (nodeNames ++ alias) + ++ map (n: "${pythonizeName n} = create_fake_nspawn_machine()") containerNames + ); + + # `test_driver` reaches ty through `buildInputs`, as in driver.nix:66. + checkInputs = { + nativeBuildInputs = [ + pkgs.ty + pkgs.ruff + ]; + buildInputs = [ pkgs.nixos-test-driver ]; + }; + + # The text driver.nix:77-82 assembles: the prelude from the pinned nixpkgs, + # the hints, then the script. + assemble = '' + cat ${pkgs.path}/nixos/lib/test-script-prepend.py > testScriptWithTypes + echo "$hints" >> testScriptWithTypes + echo -n "$testScript" >> testScriptWithTypes + ''; + + typecheck = + name: test: + pkgs.runCommand "vm-test-typecheck-${name}" + ( + checkInputs + // { + testScript = test.config.testScriptString; + hints = hintsFor test; + } + ) + '' + ${assemble} + ty check testScriptWithTypes + # driver.nix:91-99 + ruff check --select F testScriptWithTypes + touch $out + ''; + + # Succeeds only when `ty` rejects an `Optional` dereference and three calls on + # the stub machines of `hintsFor`. If `test_driver` stops resolving, every + # name is `Unknown`, the real checks pass on anything, and this one fails. + control = + pkgs.runCommand "vm-test-typecheck-control" + ( + checkInputs + // { + testScript = '' + import re + + def first_group(s: str) -> str: + return re.search(r"x", s).group(1) + + my_node.no_such_node() + machine.no_such_alias() + c_1.no_such_container() + ''; + hints = hintsFor { + config = { + nodes."my-node" = { }; + containers."c-1" = { }; + }; + }; + } + ) + '' + ${assemble} + if ty check testScriptWithTypes > ty.out 2>&1; then + cat ty.out + echo "ty accepted re.search(...).group(1); the type check is not effective" >&2 + exit 1 + fi + cat ty.out + # `group` comes from the stdlib stubs; the other three are attributes of + # the stub machines `hintsFor` declares, so they show the node, `machine` + # alias and container hints each resolve to a typed machine. + grep -q "Attribute .group. is not defined on .None." ty.out + grep -q "has no attribute .no_such_node." ty.out + grep -q "has no attribute .no_such_alias." ty.out + grep -q "has no attribute .no_such_container." ty.out + if grep -q unresolved-import ty.out; then + echo "test_driver does not resolve" >&2 + exit 1 + fi + touch $out + ''; + + # semi-spread and swarm-builder-offline are not listed: they interpolate node + # config or an agent toplevel into their script, so reading the script text + # evaluates a NixOS system. They need their script to be a function of those + # values first. + tests = { + inherit (vmTests) + hive-squid + swarm-archive + agent-ephemeral-rootfs + full-setup + wg-hive + ; + }; +in +(lib.mapAttrs' (n: t: lib.nameValuePair "vm-test-typecheck-${n}" (typecheck n t)) tests) +// { + vm-test-typecheck-control = control; +}