From d3e15c7fbef26cadcf2d62dfc2ddb2981f02abd9 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 19 Nov 2026 11:39:38 +0100 Subject: [PATCH] vm-nightly: one job per test, with a lint that keeps the matrix list in step A stuck test in the shared job was cut at the runner's one-hour limit and hid the other tests' results. The matrix list is static; check-vm-test-list.sh fails CI when it differs from nix/vm-tests.nix. --- .forgejo/workflows/ci.yml | 9 +++++ .forgejo/workflows/vm-nightly.yml | 50 +++++++++++++++++---------- docs/4-implementation/scheduler/ci.md | 16 +++++---- nix/vm-tests.nix | 3 +- scripts/check-vm-test-list.sh | 38 ++++++++++++++++++++ scripts/pre-push | 6 ++++ 6 files changed, 96 insertions(+), 26 deletions(-) create mode 100755 scripts/check-vm-test-list.sh diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 7780893d7..4ee569b28 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -80,6 +80,15 @@ jobs: - name: lint run: sh scripts/check-dashboard-service-names.sh + vm-test-list: + name: vm-test-list lint + if: vars.PUBLIC_FORGE != 'true' + runs-on: ${{ vars.PUBLIC_FORGE == 'true' && 'nixos' || 'hive-ci' }} + steps: + - uses: actions/checkout@v3 + - name: lint + run: sh scripts/check-vm-test-list.sh + shellcheck: name: shellcheck if: vars.PUBLIC_FORGE != 'true' diff --git a/.forgejo/workflows/vm-nightly.yml b/.forgejo/workflows/vm-nightly.yml index 07e581eb0..043882b9d 100644 --- a/.forgejo/workflows/vm-nightly.yml +++ b/.forgejo/workflows/vm-nightly.yml @@ -2,16 +2,26 @@ name: vm-nightly # Builds the `vmTests` flake output: tests that boot NixOS VMs and need KVM. # They are not `checks`, so the per-PR `nix flake check` job never runs them. -# Nightly plus manual dispatch; a failing test fails the run, which the forge +# Nightly plus manual dispatch; a failing test fails its job, which the forge # shows as a failed scheduled run. # +# One job per test. The runner kills a job after its `jobTimeout` (one hour by +# default) and reports nothing for it, so a stuck test in a shared job would +# erase the others' results. Each test's `globalTimeout` stays below that hour. +# +# The matrix list is static: a matrix built from a `needs` output needs +# Forgejo 14 and a matching runner, and this runner is gitea-actions-runner. +# `scripts/check-vm-test-list.sh` fails CI when the list differs from +# `nix/vm-tests.nix`. +# # Run one by hand from the forge UI (Actions, vm-nightly, "Run workflow", # `test` input) or locally with `nix build .#vmTests.x86_64-linux.`. +# With `test` set, every other matrix job skips its steps and ends green. # # ⚠️ The hour is deliberate: at the default job capacity of 1 -# (`services.hyperhive.deploy.ci-runner.concurrency`) this job holds the -# runner for its whole run and everything else queues behind it. It sits -# between coverage (02:00) and flake-update (03:00). +# (`services.hyperhive.deploy.ci-runner.concurrency`) the matrix jobs run one +# after another and hold the runner until the last ends; everything else +# queues behind them. It sits between coverage (02:00) and flake-update (03:00). # # Separate from ci.yml because a `workflow_dispatch` trigger there fires every # job in that file, and this must not run per pull request. @@ -27,24 +37,26 @@ on: - cron: "30 2 * * *" jobs: - vm-tests: - name: nix build vmTests + vm-test: + name: vm test ${{ matrix.test }} if: vars.PUBLIC_FORGE != 'true' runs-on: ${{ vars.PUBLIC_FORGE == 'true' && 'nixos' || 'hive-ci' }} + strategy: + fail-fast: false + matrix: + test: + - agent-ephemeral-rootfs + - hive-squid + - hive-egress-antispoof + - swarm-archive + - swarm-builder-offline + - full-setup + - semi-spread steps: - uses: actions/checkout@v3 + if: inputs.test == '' || inputs.test == matrix.test - name: build + if: inputs.test == '' || inputs.test == matrix.test env: - TEST: ${{ inputs.test }} - run: | - set -eu - system=x86_64-linux - tests="$TEST" - if [ -z "$tests" ]; then - tests="$(nix eval --raw ".#vmTests.$system" --apply 'ts: builtins.concatStringsSep " " (builtins.attrNames ts)')" - fi - set -- - for t in $tests; do - set -- "$@" ".#vmTests.$system.$t" - done - nix build --no-link --keep-going --print-build-logs "$@" + TEST: ${{ matrix.test }} + run: nix build --no-link --print-build-logs ".#vmTests.x86_64-linux.$TEST" diff --git a/docs/4-implementation/scheduler/ci.md b/docs/4-implementation/scheduler/ci.md index 592ce687f..c1d8d6176 100644 --- a/docs/4-implementation/scheduler/ci.md +++ b/docs/4-implementation/scheduler/ci.md @@ -123,12 +123,16 @@ with stand-in values (`passthru.typecheckScript`), from the same function as the one. `vm-test-typecheck-control` passes only when the checker rejects a known type error, which shows the checker resolves `test_driver`. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml) -builds all of them nightly at 02:30; a failing test fails that run. To run one -by hand, start the workflow from the forge UI (Actions, vm-nightly, Run -workflow, then name one test in the `test` field), or run -`nix build .#vmTests.x86_64-linux.` on a host with KVM. An empty field builds all tests; together they can pass the runner's one-hour job limit, so name one test when you want one result. Current tests: -`agent-ephemeral-rootfs` (the switch of an agent container to an ephemeral -root, through the real hive-priv unit) and `hive-squid`. +builds them nightly at 02:30, one job per test, so the runner's one-hour job +limit applies to each test alone and a stuck test no longer hides the others' +results. The matrix in `vm-nightly.yml` is the list of tests; +`scripts/check-vm-test-list.sh` fails CI when it differs from `nix/vm-tests.nix`, +so a new test goes into both. +Each test sets `globalTimeout` below that hour. With the default job capacity of +1 the jobs run one after another. To run one by hand, start the workflow from +the forge UI (Actions, vm-nightly, Run workflow, then name one test in the +`test` field; the other jobs skip their steps), or run +`nix build .#vmTests.x86_64-linux.` on a host with KVM. The `full-setup` script is plain Python in `nix/vm-tests/full-setup.py`, with no Nix interpolation. Commands the script runs in the VM find their binaries on the diff --git a/nix/vm-tests.nix b/nix/vm-tests.nix index f45ac3e22..75d2cde69 100644 --- a/nix/vm-tests.nix +++ b/nix/vm-tests.nix @@ -1,6 +1,7 @@ # NixOS VM tests: one derivation per test, exposed as `vmTests..`. # Kept out of `checks` because each boots VMs and needs KVM; the nightly -# workflow `.forgejo/workflows/vm-nightly.yml` builds them. +# workflow `.forgejo/workflows/vm-nightly.yml` builds them, one job per test; a +# new test also needs an entry in that workflow's matrix. { pkgs, hivePriv, diff --git a/scripts/check-vm-test-list.sh b/scripts/check-vm-test-list.sh new file mode 100755 index 000000000..62bfacf33 --- /dev/null +++ b/scripts/check-vm-test-list.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# CI lint: the `test` matrix in vm-nightly.yml must list exactly the +# attributes of `nix/vm-tests.nix`. The matrix is static, so a test missing +# from it never runs nightly and a stale entry fails its job. +set -eu + +workflow=".forgejo/workflows/vm-nightly.yml" +tests_nix="nix/vm-tests.nix" + +declared="$(sed -n 's/^ \([A-Za-z0-9_-][A-Za-z0-9_-]*\) = import .*/\1/p' "$tests_nix" | sort)" +listed="$(awk ' + /^ test:$/ { inlist = 1; next } + inlist && /^ - / { sub(/^ - /, ""); print; next } + inlist { inlist = 0 } +' "$workflow" | sort)" + +if [ -z "$declared" ]; then + echo "::error file=$tests_nix::no test attributes found; the extraction pattern no longer matches the file" >&2 + exit 1 +fi +if [ -z "$listed" ]; then + echo "::error file=$workflow::no matrix entries found under 'test:'; the extraction pattern no longer matches the file" >&2 + exit 1 +fi + +if [ "$declared" != "$listed" ]; then + work="$(mktemp -d)" + trap 'rm -rf "$work"' EXIT + echo "$declared" >"$work/declared" + echo "$listed" >"$work/listed" + echo "::error file=$workflow::matrix test list differs from $tests_nix" >&2 + echo "only in $tests_nix:" >&2 + comm -23 "$work/declared" "$work/listed" >&2 + echo "only in $workflow:" >&2 + comm -13 "$work/declared" "$work/listed" >&2 + exit 1 +fi +echo "vm-nightly matrix lists all $(echo "$declared" | wc -l) tests of $tests_nix" diff --git a/scripts/pre-push b/scripts/pre-push index 7f36a492c..cb690b455 100755 --- a/scripts/pre-push +++ b/scripts/pre-push @@ -45,5 +45,11 @@ if ! sh "$repo_root/scripts/check-dashboard-service-names.sh"; then exit 1 fi +echo "pre-push: running vm-test-list lint..." >&2 +if ! sh "$repo_root/scripts/check-vm-test-list.sh"; then + echo "pre-push: vm-test-list lint FAILED — fix before pushing" >&2 + exit 1 +fi + echo "pre-push: lints passed ✓" >&2 exit 0