diff --git a/docs/4-implementation/scheduler/ci.md b/docs/4-implementation/scheduler/ci.md index f5f820d88..ae39445cc 100644 --- a/docs/4-implementation/scheduler/ci.md +++ b/docs/4-implementation/scheduler/ci.md @@ -117,8 +117,10 @@ Tests that boot NixOS VMs are the `vmTests..` flake output builds them. It does build the `vm-test-typecheck-` checks (`nix/vm-test-typecheck.nix`): the type check and lint the test driver runs over a test script, on the script text alone. They evaluate no node configuration -and build no VM. They cover `hive-squid`, -`swarm-archive`, `agent-ephemeral-rootfs`, `full-setup` and `wg-hive`; +and build no VM. They cover all seven tests. `semi-spread` and `swarm-builder-offline` +read node config or an agent toplevel into their script, so each builds the script +with stand-in values (`passthru.typecheckScript`), from the same function as the real +one. `vm-test-typecheck-control` passes only when the checker rejects a known type error, which shows the checker resolves `test_driver`. [`.forgejo/workflows/vm-nightly.yml`](../../../.forgejo/workflows/vm-nightly.yml) builds all of them nightly at 02:30; a failing test fails that run. To run one diff --git a/nix/vm-test-typecheck.nix b/nix/vm-test-typecheck.nix index c102ab0e4..506170333 100644 --- a/nix/vm-test-typecheck.nix +++ b/nix/vm-test-typecheck.nix @@ -54,13 +54,18 @@ let echo -n "$testScript" >> testScriptWithTypes ''; + # The tests that interpolate node config or an agent toplevel name a stand-in + # script in `passthru.typecheckScript`, built by the same function as their real + # `testScript`; reading `testScriptString` there would evaluate a NixOS system. + scriptOf = test: test.config.passthru.typecheckScript or test.config.testScriptString; + typecheck = name: test: pkgs.runCommand "vm-test-typecheck-${name}" ( checkInputs // { - testScript = test.config.testScriptString; + testScript = scriptOf test; hints = hintsFor test; } ) @@ -120,10 +125,6 @@ let touch $out ''; - # semi-spread and swarm-builder-offline are not listed: they interpolate node - # config or an agent toplevel into their script, so reading the script text - # evaluates a NixOS system. They need their script to be a function of those - # values first. tests = { inherit (vmTests) hive-squid @@ -131,6 +132,8 @@ let agent-ephemeral-rootfs full-setup wg-hive + semi-spread + swarm-builder-offline ; }; in diff --git a/nix/vm-tests/semi-spread.nix b/nix/vm-tests/semi-spread.nix index d83db83c1..37f72ee9f 100644 --- a/nix/vm-tests/semi-spread.nix +++ b/nix/vm-tests/semi-spread.nix @@ -130,178 +130,21 @@ let pkgs.wireguard-tools ]; }; -in -pkgs.testers.runNixOSTest ( - { config, ... }: - let - swarmCfg = config.nodes.swarm.services.hyperhive; - inherit (swarmCfg) swarm; - controllerSocket = swarmCfg.deploy.swarm-controller.socketPath; - settingsJson = - config.nodes.swarm.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; - # What the public DNS holds (semi-spread-deploy.md, "DNS"), and the rest of - # the swarm's names, which only the VPN resolves. - publicNames = { - ui = swarm.domain; - forge = swarm.forge.domain; - chat = swarm.matrix.gatewayHost; - auth = swarm.authelia.domain; - }; - privateNames = { - grafana = swarm.grafana.domain; - metrics = swarm.victoriametrics.domain; - logs = swarm.victorialogs.domain; - otel = swarm.otel.domain; - bao-ui = swarm.bao.ui.domain; - attic = swarm.attic.ownDomain; - }; - # Every internal name, with the machine whose VPN address it resolves to. - internalNames = - lib.genAttrs [ - "ui" - "forge" - "chat" - "auth" - "grafana" - "metrics" - "logs" - "otel" - "bao-ui" - "bao" - "nats" - ] (_: "swarm-central") - // { - attic = "swarm-heavyduty"; - }; - nameOf = { - inherit (publicNames) - ui - forge - chat - auth - ; - inherit (privateNames) - grafana - metrics - logs - otel - bao-ui - attic - ; - bao = swarm.bao.domain; - nats = swarm.nats.domain; - }; - in - { - name = "semi-spread"; - - # Three hosts' boots, the setup ceremony and the agents' build and pull. - globalTimeout = 10800; - - nodes.swarm = - { config, ... }: - { - imports = [ - self.nixosModules.default - tools - (wgKey "swarm") - "${exampleCopy}/common.nix" - "${exampleCopy}/swarm-central.nix" - "${exampleCopy}/swarm-services.nix" - ]; - networking.hostName = "swarm-central"; - system.stateVersion = "25.11"; - - services.hyperhive = { - # swarm-services.nix says the store is elsewhere; here it is on this machine. - deploy.bao.enable = lib.mkForce true; - deploy.bao.seal = "shamir"; - deploy.bao.bootstrapTokenFile = "/var/lib/swarm-bao-bootstrap/grant.token"; - # Names this machine serves are not in the example's table, and a - # process on the machine reaches them on the machine. - gateway.localHostsEntry = true; - # The test pushes its own hyperhive/hyperhive; a mirror of the - # upstream could not be created offline and would race the push. - deploy.forgejo.hyperhiveUpstream = null; - # The cache answers on the private addresses only, as on the - # example's heavyduty. - gateway.vhosts.${config.services.hyperhive.swarm.attic.ownDomain}.public = false; - }; - - # 8 GB: the store and the swarm's services in containers, the - # controller, and the forge's repositories. - virtualisation.memorySize = 8192; - virtualisation.cores = 4; - virtualisation.diskSize = 20480; - virtualisation.writableStoreUseTmpfs = false; - virtualisation.additionalPaths = [ hyperhiveRepo ]; - - # The store's raft data is on a tmpfs, for the reason in ./full-setup.nix. - fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" = - { - device = "tmpfs"; - fsType = "tmpfs"; - options = [ - "mode=0700" - "size=1G" - ]; - }; - - environment.systemPackages = [ pkgs.git ]; - }; - - nodes.heavyduty = { - imports = [ - self.nixosModules.default - tools - offline - (wgKey "heavyduty") - "${exampleCopy}/common.nix" - "${exampleCopy}/swarm-heavyduty.nix" - ]; - networking.hostName = "swarm-heavyduty"; - system.stateVersion = "25.11"; - - # 5 GB: the builder evaluates and builds the agents' flake, beside the - # cache and the CI runner. The disk holds the toplevels and the cache's - # copy of them. - virtualisation.memorySize = 5120; - virtualisation.cores = 4; - virtualisation.diskSize = 40960; - virtualisation.writableStoreUseTmpfs = false; - # The toplevels, and the source trees the meta flake's lock names. - virtualisation.additionalPaths = sourcePaths ++ toplevels settingsJson; - }; - - nodes.hive = { - imports = [ - self.nixosModules.default - tools - offline - (wgKey "hive") - "${exampleCopy}/common.nix" - "${exampleCopy}/hive.nix" - ]; - networking.hostName = "hive-one"; - system.stateVersion = "25.11"; - - # 4 GB: the hive and two agent containers. The disk holds the pulled - # closures. - virtualisation.memorySize = 4096; - virtualisation.cores = 2; - virtualisation.diskSize = 30720; - virtualisation.writableStoreUseTmpfs = false; - }; - - nodes.outside = { - environment.systemPackages = [ pkgs.curl ]; - # The public DNS: the four names, at the swarm machine's public address. - networking.hosts.${lanAddr.swarm} = lib.attrValues publicNames; - virtualisation.memorySize = 512; - }; - - testScript = '' + # The script is a function of the values it reads from node config and the + # agent toplevels, so the type check can build it with stand-in values and + # evaluate no node. + scriptWith = + { + controllerSocket, + publicNames, + swarm, + nameOf, + internalNames, + toplevelPaths, + hyperhiveRepo, + }: + '' import json import shlex import time @@ -310,7 +153,7 @@ pkgs.testers.runNixOSTest ( SOCKET = "${controllerSocket}" FORGE = "https://${publicNames.forge}" HIVE = "${hiveName}" - TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))} + TOPLEVELS = ${builtins.toJSON toplevelPaths} INIT = "/root/bao-init.json" FORGE_AUTH = "/root/forge-auth.header" NIX = "nix --extra-experimental-features nix-command" @@ -674,5 +517,202 @@ pkgs.testers.runNixOSTest ( # Not built on the swarm machine, which has no copy. swarm.fail(f"{NIX} path-info {toplevel}") ''; +in +pkgs.testers.runNixOSTest ( + { config, ... }: + let + swarmCfg = config.nodes.swarm.services.hyperhive; + inherit (swarmCfg) swarm; + controllerSocket = swarmCfg.deploy.swarm-controller.socketPath; + settingsJson = + config.nodes.swarm.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE; + + # What the public DNS holds (semi-spread-deploy.md, "DNS"), and the rest of + # the swarm's names, which only the VPN resolves. + publicNames = { + ui = swarm.domain; + forge = swarm.forge.domain; + chat = swarm.matrix.gatewayHost; + auth = swarm.authelia.domain; + }; + privateNames = { + grafana = swarm.grafana.domain; + metrics = swarm.victoriametrics.domain; + logs = swarm.victorialogs.domain; + otel = swarm.otel.domain; + bao-ui = swarm.bao.ui.domain; + attic = swarm.attic.ownDomain; + }; + # Every internal name, with the machine whose VPN address it resolves to. + internalNames = + lib.genAttrs [ + "ui" + "forge" + "chat" + "auth" + "grafana" + "metrics" + "logs" + "otel" + "bao-ui" + "bao" + "nats" + ] (_: "swarm-central") + // { + attic = "swarm-heavyduty"; + }; + nameOf = { + inherit (publicNames) + ui + forge + chat + auth + ; + inherit (privateNames) + grafana + metrics + logs + otel + bao-ui + attic + ; + bao = swarm.bao.domain; + nats = swarm.nats.domain; + }; + in + { + name = "semi-spread"; + + # Three hosts' boots, the setup ceremony and the agents' build and pull. + globalTimeout = 10800; + + nodes.swarm = + { config, ... }: + { + imports = [ + self.nixosModules.default + tools + (wgKey "swarm") + "${exampleCopy}/common.nix" + "${exampleCopy}/swarm-central.nix" + "${exampleCopy}/swarm-services.nix" + ]; + networking.hostName = "swarm-central"; + system.stateVersion = "25.11"; + + services.hyperhive = { + # swarm-services.nix says the store is elsewhere; here it is on this machine. + deploy.bao.enable = lib.mkForce true; + deploy.bao.seal = "shamir"; + deploy.bao.bootstrapTokenFile = "/var/lib/swarm-bao-bootstrap/grant.token"; + # Names this machine serves are not in the example's table, and a + # process on the machine reaches them on the machine. + gateway.localHostsEntry = true; + # The test pushes its own hyperhive/hyperhive; a mirror of the + # upstream could not be created offline and would race the push. + deploy.forgejo.hyperhiveUpstream = null; + # The cache answers on the private addresses only, as on the + # example's heavyduty. + gateway.vhosts.${config.services.hyperhive.swarm.attic.ownDomain}.public = false; + }; + + # 8 GB: the store and the swarm's services in containers, the + # controller, and the forge's repositories. + virtualisation.memorySize = 8192; + virtualisation.cores = 4; + virtualisation.diskSize = 20480; + virtualisation.writableStoreUseTmpfs = false; + virtualisation.additionalPaths = [ hyperhiveRepo ]; + + # The store's raft data is on a tmpfs, for the reason in ./full-setup.nix. + fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" = + { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ + "mode=0700" + "size=1G" + ]; + }; + + environment.systemPackages = [ pkgs.git ]; + }; + + nodes.heavyduty = { + imports = [ + self.nixosModules.default + tools + offline + (wgKey "heavyduty") + "${exampleCopy}/common.nix" + "${exampleCopy}/swarm-heavyduty.nix" + ]; + networking.hostName = "swarm-heavyduty"; + system.stateVersion = "25.11"; + + # 5 GB: the builder evaluates and builds the agents' flake, beside the + # cache and the CI runner. The disk holds the toplevels and the cache's + # copy of them. + virtualisation.memorySize = 5120; + virtualisation.cores = 4; + virtualisation.diskSize = 40960; + virtualisation.writableStoreUseTmpfs = false; + # The toplevels, and the source trees the meta flake's lock names. + virtualisation.additionalPaths = sourcePaths ++ toplevels settingsJson; + }; + + nodes.hive = { + imports = [ + self.nixosModules.default + tools + offline + (wgKey "hive") + "${exampleCopy}/common.nix" + "${exampleCopy}/hive.nix" + ]; + networking.hostName = "hive-one"; + system.stateVersion = "25.11"; + + # 4 GB: the hive and two agent containers. The disk holds the pulled + # closures. + virtualisation.memorySize = 4096; + virtualisation.cores = 2; + virtualisation.diskSize = 30720; + virtualisation.writableStoreUseTmpfs = false; + }; + + nodes.outside = { + environment.systemPackages = [ pkgs.curl ]; + # The public DNS: the four names, at the swarm machine's public address. + networking.hosts.${lanAddr.swarm} = lib.attrValues publicNames; + virtualisation.memorySize = 512; + }; + + testScript = scriptWith { + inherit + controllerSocket + publicNames + swarm + nameOf + internalNames + hyperhiveRepo + ; + toplevelPaths = map toString (toplevels settingsJson); + }; + + # What `nix/vm-test-typecheck.nix` checks: the same text with stand-in values. + passthru.typecheckScript = scriptWith { + controllerSocket = "/stub/socket"; + publicNames.forge = "forge.stub"; + swarm = { + bao.machine = "stub"; + dashboardDomains.${hiveName} = "dashboard.stub"; + attic.domain = "attic.stub"; + }; + nameOf.stub = "stub.stub"; + internalNames.stub = "stub"; + toplevelPaths = [ "/nix/store/stub" ]; + hyperhiveRepo = "/nix/store/stub"; + }; } ) diff --git a/nix/vm-tests/swarm-builder-offline.nix b/nix/vm-tests/swarm-builder-offline.nix index f81b9a863..b49f45684 100644 --- a/nix/vm-tests/swarm-builder-offline.nix +++ b/nix/vm-tests/swarm-builder-offline.nix @@ -74,6 +74,96 @@ let tokenFile = "/run/builder-offline/forge-token"; caBundle = "${tls}/ca.pem"; + + # The script is a function of the fixtures it names, so the type check can + # build it with stand-in paths and not evaluate the agent toplevel. + scriptWith = + { + toplevel, + hyperhiveRepo, + metaRepo, + }: + '' + import re + import secrets + + CA = "${caBundle}" + API = "${forgeUrl}/api/v1" + TOPLEVEL = "${toplevel}" + # What nix prints for a derivation it builds or plans to build. + BUILD_PATTERN = r"building '|will be built" + # The forge admin's password, made for this run. + password = secrets.token_hex(16) + + def forge_api(method, path, body, auth): + return machine.succeed( + f"curl --fail --silent --cacert {CA} -X {method} {API}{path} " + f"-H 'Content-Type: application/json' {auth} -d '{body}'" + ) + + machine.start() + machine.wait_for_unit("forgejo.service") + machine.wait_for_open_port(${toString forgePort}) + + with subtest("the forge holds both repos, private, at the fixtures' commits"): + machine.succeed( + "su -l forgejo -c 'GITEA_WORK_DIR=/var/lib/forgejo forgejo admin user create " + f"--admin --username ops --password {password} --email ops@${forgeHost} " + "--must-change-password=false'" + ) + token = machine.succeed( + f"curl --fail --silent --cacert {CA} -X POST {API}/users/ops/tokens " + f"-u ops:{password} -H 'Content-Type: application/json' " + "-d '{\"name\":\"fixtures\",\"scopes\":[\"all\"]}' | jq -r .sha1" + ).strip() + auth = f"-H 'Authorization: token {token}'" + for org, repo, fixture in [ + ("hyperhive", "hyperhive", "${hyperhiveRepo}"), + ("internal", "meta", "${metaRepo}"), + ]: + forge_api("POST", "/orgs", f'{{"username":"{org}","visibility":"private"}}', auth) + forge_api("POST", f"/orgs/{org}/repos", f'{{"name":"{repo}","private":true,"default_branch":"main"}}', auth) + remote = f"https://ops:{token}@${forgeAuthority}/{org}/{repo}.git" + machine.succeed(f"cp -r {fixture}/repo.git /tmp/{repo}.git && chmod -R u+w /tmp/{repo}.git") + machine.succeed(f"GIT_SSL_CAINFO={CA} git -C /tmp/{repo}.git push {remote} main") + pushed = machine.succeed(f"GIT_SSL_CAINFO={CA} git ls-remote {remote} refs/heads/main").split()[0] + assert pushed == machine.succeed(f"cat {fixture}/rev").strip(), (repo, pushed) + # The forge asks for a login: the builder's credential helper has to supply it. + machine.fail(f"GIT_SSL_CAINFO={CA} GIT_TERMINAL_PROMPT=0 git ls-remote ${forgeUrl}/internal/meta.git") + machine.succeed( + f"install -d -m 0755 /run/builder-offline && printf %s {token} > ${tokenFile} " + "&& chown builder ${tokenFile} && chmod 0400 ${tokenFile}" + ) + + with subtest("the toplevel is in the store before the build"): + machine.succeed(f"nix --extra-experimental-features nix-command path-info {TOPLEVEL}") + # The control for the lookup: a path that is not there is not found. + machine.fail("nix --extra-experimental-features nix-command path-info /nix/store/00000000000000000000000000000000-absent") + + with subtest("the builder's build finishes with nothing built"): + status, _ = machine.execute( + "runuser -u builder -- timeout 600 /run/current-system/sw/bin/meta-build >/tmp/out 2>/tmp/err" + ) + out = machine.succeed("cat /tmp/out").strip() + err = machine.succeed("cat /tmp/err") + assert status == 0, f"nix exited {status}:\n{err}" + assert out == TOPLEVEL, f"printed {out}, prebuilt {TOPLEVEL}" + assert not re.search(BUILD_PATTERN, err), err + + with subtest("control: a derivation that does build is found by the same pattern"): + machine.succeed("grep -x 'max-jobs = 0' /etc/nix/nix.conf") + machine.succeed( + "cp --remove-destination $(readlink -f /etc/nix/nix.conf) /etc/nix/nix.conf " + "&& sed -i 's/^max-jobs = 0$/max-jobs = 1/' /etc/nix/nix.conf " + "&& systemctl stop nix-daemon.socket nix-daemon.service " + "&& systemctl start nix-daemon.socket" + ) + machine.succeed( + "NIX_CONFIG='experimental-features = nix-command' " + "nix build --no-link --print-build-logs --impure -f ${control} 2>/tmp/control.err" + ) + assert re.search(BUILD_PATTERN, machine.succeed("cat /tmp/control.err")) + ''; in pkgs.testers.runNixOSTest { name = "swarm-builder-offline"; @@ -175,85 +265,12 @@ pkgs.testers.runNixOSTest { ]; }; - testScript = '' - import re - import secrets + testScript = scriptWith { inherit toplevel hyperhiveRepo metaRepo; }; - CA = "${caBundle}" - API = "${forgeUrl}/api/v1" - TOPLEVEL = "${toplevel}" - # What nix prints for a derivation it builds or plans to build. - BUILD_PATTERN = r"building '|will be built" - # The forge admin's password, made for this run. - password = secrets.token_hex(16) - - def forge_api(method, path, body, auth): - return machine.succeed( - f"curl --fail --silent --cacert {CA} -X {method} {API}{path} " - f"-H 'Content-Type: application/json' {auth} -d '{body}'" - ) - - machine.start() - machine.wait_for_unit("forgejo.service") - machine.wait_for_open_port(${toString forgePort}) - - with subtest("the forge holds both repos, private, at the fixtures' commits"): - machine.succeed( - "su -l forgejo -c 'GITEA_WORK_DIR=/var/lib/forgejo forgejo admin user create " - f"--admin --username ops --password {password} --email ops@${forgeHost} " - "--must-change-password=false'" - ) - token = machine.succeed( - f"curl --fail --silent --cacert {CA} -X POST {API}/users/ops/tokens " - f"-u ops:{password} -H 'Content-Type: application/json' " - "-d '{\"name\":\"fixtures\",\"scopes\":[\"all\"]}' | jq -r .sha1" - ).strip() - auth = f"-H 'Authorization: token {token}'" - for org, repo, fixture in [ - ("hyperhive", "hyperhive", "${hyperhiveRepo}"), - ("internal", "meta", "${metaRepo}"), - ]: - forge_api("POST", "/orgs", f'{{"username":"{org}","visibility":"private"}}', auth) - forge_api("POST", f"/orgs/{org}/repos", f'{{"name":"{repo}","private":true,"default_branch":"main"}}', auth) - remote = f"https://ops:{token}@${forgeAuthority}/{org}/{repo}.git" - machine.succeed(f"cp -r {fixture}/repo.git /tmp/{repo}.git && chmod -R u+w /tmp/{repo}.git") - machine.succeed(f"GIT_SSL_CAINFO={CA} git -C /tmp/{repo}.git push {remote} main") - pushed = machine.succeed(f"GIT_SSL_CAINFO={CA} git ls-remote {remote} refs/heads/main").split()[0] - assert pushed == machine.succeed(f"cat {fixture}/rev").strip(), (repo, pushed) - # The forge asks for a login: the builder's credential helper has to supply it. - machine.fail(f"GIT_SSL_CAINFO={CA} GIT_TERMINAL_PROMPT=0 git ls-remote ${forgeUrl}/internal/meta.git") - machine.succeed( - f"install -d -m 0755 /run/builder-offline && printf %s {token} > ${tokenFile} " - "&& chown builder ${tokenFile} && chmod 0400 ${tokenFile}" - ) - - with subtest("the toplevel is in the store before the build"): - machine.succeed(f"nix --extra-experimental-features nix-command path-info {TOPLEVEL}") - # The control for the lookup: a path that is not there is not found. - machine.fail("nix --extra-experimental-features nix-command path-info /nix/store/00000000000000000000000000000000-absent") - - with subtest("the builder's build finishes with nothing built"): - status, _ = machine.execute( - "runuser -u builder -- timeout 600 /run/current-system/sw/bin/meta-build >/tmp/out 2>/tmp/err" - ) - out = machine.succeed("cat /tmp/out").strip() - err = machine.succeed("cat /tmp/err") - assert status == 0, f"nix exited {status}:\n{err}" - assert out == TOPLEVEL, f"printed {out}, prebuilt {TOPLEVEL}" - assert not re.search(BUILD_PATTERN, err), err - - with subtest("control: a derivation that does build is found by the same pattern"): - machine.succeed("grep -x 'max-jobs = 0' /etc/nix/nix.conf") - machine.succeed( - "cp --remove-destination $(readlink -f /etc/nix/nix.conf) /etc/nix/nix.conf " - "&& sed -i 's/^max-jobs = 0$/max-jobs = 1/' /etc/nix/nix.conf " - "&& systemctl stop nix-daemon.socket nix-daemon.service " - "&& systemctl start nix-daemon.socket" - ) - machine.succeed( - "NIX_CONFIG='experimental-features = nix-command' " - "nix build --no-link --print-build-logs --impure -f ${control} 2>/tmp/control.err" - ) - assert re.search(BUILD_PATTERN, machine.succeed("cat /tmp/control.err")) - ''; + # What `nix/vm-test-typecheck.nix` checks: the same text with stand-in paths. + passthru.typecheckScript = scriptWith { + toplevel = "/nix/store/stub"; + hyperhiveRepo = "/nix/store/stub"; + metaRepo = "/nix/store/stub"; + }; }