ci: type-check vm test scripts without building the vms
Adds checks.<system>.vm-test-typecheck-<name> for hive-squid, swarm-archive, agent-ephemeral-rootfs, full-setup and wg-hive: ty and ruff F over nixpkgs' test-script prelude, the machine hints from the node names, and the script text. Node configs are never evaluated. vm-test-typecheck-control passes only when ty rejects a known type error.
This commit is contained in:
atlas
mara
parent
712dcf2b25
commit
32f5b74ba4
4 changed files with 156 additions and 2 deletions
|
|
@ -114,7 +114,13 @@ slow); run those manually before pushing Rust changes.
|
|||
|
||||
Tests that boot NixOS VMs are the `vmTests.<system>.<name>` flake output
|
||||
(`nix/vm-tests.nix`), not `checks`, so the per-PR **nix flake check** job never
|
||||
builds them. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml)
|
||||
builds them. It does build the `vm-test-typecheck-<name>` checks
|
||||
(`nix/vm-test-typecheck.nix`): the type check and lint the test driver runs over
|
||||
a test script, on the script text alone. They evaluate no node configuration
|
||||
and build no VM. They cover `hive-squid`,
|
||||
`swarm-archive`, `agent-ephemeral-rootfs`, `full-setup` and `wg-hive`;
|
||||
`vm-test-typecheck-control` passes only when the checker rejects a known type error, which shows the
|
||||
checker resolves `test_driver`. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml)
|
||||
builds all of them nightly at 02:30; a failing test fails that run. To run one
|
||||
by hand, start the workflow from the forge UI (Actions, vm-nightly, Run
|
||||
workflow, optional `test` input), or run
|
||||
|
|
|
|||
|
|
@ -232,7 +232,8 @@
|
|||
|
||||
# Tests that boot a NixOS VM and need KVM. Not `checks`, so `nix flake
|
||||
# check` and the per-PR CI job never build them;
|
||||
# `.forgejo/workflows/vm-nightly.yml` does.
|
||||
# `.forgejo/workflows/vm-nightly.yml` does. Their scripts are type-checked
|
||||
# per PR by `checks.<system>.vm-test-typecheck-*` (nix/vm-test-typecheck.nix).
|
||||
vmTests = forAllSystems (
|
||||
{ pkgs, system, ... }:
|
||||
import ./nix/vm-tests.nix {
|
||||
|
|
|
|||
|
|
@ -573,3 +573,10 @@ in
|
|||
touch "$out"
|
||||
'';
|
||||
}
|
||||
// import ./vm-test-typecheck.nix {
|
||||
inherit pkgs;
|
||||
vmTests = import ./vm-tests.nix {
|
||||
inherit pkgs self;
|
||||
hivePriv = self.packages.${system}.hive-priv;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
140
nix/vm-test-typecheck.nix
Normal file
140
nix/vm-test-typecheck.nix
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
# `vm-test-typecheck-<name>` checks: run the type check and lint that nixpkgs'
|
||||
# test driver applies to a VM test script (`ty check`, `ruff check --select F`)
|
||||
# without building the driver. The driver derivation references every node's
|
||||
# `system.build.vm`, so building or even evaluating it costs a full NixOS
|
||||
# evaluation per node; the check itself needs only the script text and the
|
||||
# machine names, and `nodes` is a lazy attribute set, so reading its names
|
||||
# forces no node configuration.
|
||||
{ pkgs, vmTests }:
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
|
||||
# nixos/lib/testing/driver.nix:24-31 (the pinned nixpkgs).
|
||||
pythonizeName =
|
||||
name:
|
||||
let
|
||||
head = lib.substring 0 1 name;
|
||||
tail = lib.substring 1 (-1) name;
|
||||
in
|
||||
(if builtins.match "[A-z_]" head == null then "_" else head)
|
||||
+ lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail;
|
||||
|
||||
# The type hints of nixos/lib/testing/driver.nix:22-53: one stub machine per
|
||||
# node, one per container, and `machine` when it is the only node and no
|
||||
# machine already has that name. The `vlanN` hints are left out: they need
|
||||
# every node's `virtualisation.vlans`, and no script here uses a `vlanN` name.
|
||||
hintsFor =
|
||||
test:
|
||||
let
|
||||
nodeNames = lib.attrNames test.config.nodes;
|
||||
containerNames = lib.attrNames test.config.containers;
|
||||
alias = lib.optional (
|
||||
lib.length nodeNames == 1 && !lib.elem "machine" (nodeNames ++ containerNames)
|
||||
) "machine";
|
||||
in
|
||||
lib.concatStringsSep "\n" (
|
||||
map (n: "${pythonizeName n} = create_fake_qemu_machine()") (nodeNames ++ alias)
|
||||
++ map (n: "${pythonizeName n} = create_fake_nspawn_machine()") containerNames
|
||||
);
|
||||
|
||||
# `test_driver` reaches ty through `buildInputs`, as in driver.nix:66.
|
||||
checkInputs = {
|
||||
nativeBuildInputs = [
|
||||
pkgs.ty
|
||||
pkgs.ruff
|
||||
];
|
||||
buildInputs = [ pkgs.nixos-test-driver ];
|
||||
};
|
||||
|
||||
# The text driver.nix:77-82 assembles: the prelude from the pinned nixpkgs,
|
||||
# the hints, then the script.
|
||||
assemble = ''
|
||||
cat ${pkgs.path}/nixos/lib/test-script-prepend.py > testScriptWithTypes
|
||||
echo "$hints" >> testScriptWithTypes
|
||||
echo -n "$testScript" >> testScriptWithTypes
|
||||
'';
|
||||
|
||||
typecheck =
|
||||
name: test:
|
||||
pkgs.runCommand "vm-test-typecheck-${name}"
|
||||
(
|
||||
checkInputs
|
||||
// {
|
||||
testScript = test.config.testScriptString;
|
||||
hints = hintsFor test;
|
||||
}
|
||||
)
|
||||
''
|
||||
${assemble}
|
||||
ty check testScriptWithTypes
|
||||
# driver.nix:91-99
|
||||
ruff check --select F testScriptWithTypes
|
||||
touch $out
|
||||
'';
|
||||
|
||||
# Succeeds only when `ty` rejects an `Optional` dereference and three calls on
|
||||
# the stub machines of `hintsFor`. If `test_driver` stops resolving, every
|
||||
# name is `Unknown`, the real checks pass on anything, and this one fails.
|
||||
control =
|
||||
pkgs.runCommand "vm-test-typecheck-control"
|
||||
(
|
||||
checkInputs
|
||||
// {
|
||||
testScript = ''
|
||||
import re
|
||||
|
||||
def first_group(s: str) -> str:
|
||||
return re.search(r"x", s).group(1)
|
||||
|
||||
my_node.no_such_node()
|
||||
machine.no_such_alias()
|
||||
c_1.no_such_container()
|
||||
'';
|
||||
hints = hintsFor {
|
||||
config = {
|
||||
nodes."my-node" = { };
|
||||
containers."c-1" = { };
|
||||
};
|
||||
};
|
||||
}
|
||||
)
|
||||
''
|
||||
${assemble}
|
||||
if ty check testScriptWithTypes > ty.out 2>&1; then
|
||||
cat ty.out
|
||||
echo "ty accepted re.search(...).group(1); the type check is not effective" >&2
|
||||
exit 1
|
||||
fi
|
||||
cat ty.out
|
||||
# `group` comes from the stdlib stubs; the other three are attributes of
|
||||
# the stub machines `hintsFor` declares, so they show the node, `machine`
|
||||
# alias and container hints each resolve to a typed machine.
|
||||
grep -q "Attribute .group. is not defined on .None." ty.out
|
||||
grep -q "has no attribute .no_such_node." ty.out
|
||||
grep -q "has no attribute .no_such_alias." ty.out
|
||||
grep -q "has no attribute .no_such_container." ty.out
|
||||
if grep -q unresolved-import ty.out; then
|
||||
echo "test_driver does not resolve" >&2
|
||||
exit 1
|
||||
fi
|
||||
touch $out
|
||||
'';
|
||||
|
||||
# semi-spread and swarm-builder-offline are not listed: they interpolate node
|
||||
# config or an agent toplevel into their script, so reading the script text
|
||||
# evaluates a NixOS system. They need their script to be a function of those
|
||||
# values first.
|
||||
tests = {
|
||||
inherit (vmTests)
|
||||
hive-squid
|
||||
swarm-archive
|
||||
agent-ephemeral-rootfs
|
||||
full-setup
|
||||
wg-hive
|
||||
;
|
||||
};
|
||||
in
|
||||
(lib.mapAttrs' (n: t: lib.nameValuePair "vm-test-typecheck-${n}" (typecheck n t)) tests)
|
||||
// {
|
||||
vm-test-typecheck-control = control;
|
||||
}
|
||||
Loading…
Reference in a new issue