nightly vm test: the builder's meta build finds a prebuilt agent in an offline store
This commit is contained in:
atlas
parent
90272ec3c0
commit
4d85ec4c21
1 changed files with 508 additions and 2 deletions
258
nix/vm-tests/swarm-builder-offline.nix
Normal file
258
nix/vm-tests/swarm-builder-offline.nix
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
# `vmTests.<system>.swarm-builder-offline`: the build the swarm builder runs for
|
||||
# a deploy gate, on a VM with no network and no substituter, finds the agent's
|
||||
# toplevel in its store and builds nothing.
|
||||
#
|
||||
# One node runs a forge over https (test CA) holding `hyperhive/hyperhive` and
|
||||
# `internal/meta` (./swarm-builder-offline-fixtures.nix). The agent's toplevel
|
||||
# and the source trees its meta flake pins are in the VM's store through
|
||||
# `virtualisation.additionalPaths`, built from `self` before the VM boots. The
|
||||
# script runs the builder's `nix build` of the toplevel from `internal/meta` with
|
||||
# the builder's flags, credential helper and environment, and checks that:
|
||||
# - it exits 0 and prints the prebuilt toplevel's path;
|
||||
# - the path was valid in the store before the command ran;
|
||||
# - nix's output names no build. The daemon runs with `max-jobs = 0`, so a
|
||||
# derivation that needs building fails at once instead of compiling the
|
||||
# workspace. A control build with `max-jobs` raised proves the pattern
|
||||
# finds a build where there is one.
|
||||
#
|
||||
# What it does not cover: the builder daemon (its queue, store and OIDC
|
||||
# logins, request handling) and the controller. The command is the one in
|
||||
# swarm-builder/src/nix_build.rs; a change there has to be copied here, and a
|
||||
# unit test there (`the_vm_test_copy_matches`) fails until it is.
|
||||
{ pkgs, self }:
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
agent = "rev1";
|
||||
forgeHost = "forge.test";
|
||||
forgePort = 3000;
|
||||
forgeAuthority = "${forgeHost}:${toString forgePort}";
|
||||
forgeUrl = "https://${forgeAuthority}";
|
||||
|
||||
fixtures = import ./swarm-builder-offline-fixtures.nix {
|
||||
inherit
|
||||
pkgs
|
||||
self
|
||||
forgeUrl
|
||||
agent
|
||||
;
|
||||
};
|
||||
inherit (fixtures) toplevel hyperhiveRepo metaRepo;
|
||||
|
||||
# A test CA and the forge's leaf for `forge.test`, made with the `openssl req`
|
||||
# calls of nix/host-modules/glue-bao-tls.nix.
|
||||
tls = pkgs.runCommand "swarm-builder-offline-tls" { nativeBuildInputs = [ pkgs.openssl ]; } ''
|
||||
mkdir $out
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 3650 \
|
||||
-keyout $TMPDIR/ca.key -out $out/ca.pem -subj /CN=builder-offline-test-ca
|
||||
openssl req -newkey rsa:2048 -nodes -sha256 \
|
||||
-keyout $out/forge.key -out $TMPDIR/forge.csr -subj /CN=${forgeHost}
|
||||
printf 'subjectAltName=DNS:${forgeHost},DNS:localhost,IP:127.0.0.1\nbasicConstraints=CA:FALSE\n' > $TMPDIR/ext
|
||||
openssl x509 -req -sha256 -days 3650 -in $TMPDIR/forge.csr \
|
||||
-CA $out/ca.pem -CAkey $TMPDIR/ca.key -CAcreateserial \
|
||||
-extfile $TMPDIR/ext -out $out/forge.pem
|
||||
'';
|
||||
|
||||
# The git credential helper of `credential_helper()` in swarm-builder/src/nix_build.rs, with
|
||||
# `TOKEN_VAR` and `HOST_VAR` as they are written there.
|
||||
credentialHelper = lib.concatStrings [
|
||||
"!f() { test \"$1\" = get || return 0; p=; h=; "
|
||||
"while IFS='=' read -r k v; do case \"$k\" in '') break ;; protocol) p=$v ;; host) h=$v ;; esac; done; "
|
||||
"test -n \"$SWARM_BUILDER_GIT_HOST\" && test \"$p\" = https && test \"$h\" = \"$SWARM_BUILDER_GIT_HOST\" "
|
||||
"&& printf 'username=builder\\npassword=%s\\n' \"$SWARM_BUILDER_GIT_TOKEN\"; return 0; }; f"
|
||||
];
|
||||
credentialHelperFile = pkgs.writeText "credential-helper" credentialHelper;
|
||||
|
||||
# A derivation no store holds, to check the pattern that finds a build.
|
||||
control = pkgs.writeText "control.nix" ''
|
||||
derivation {
|
||||
name = "build-control";
|
||||
system = "${pkgs.stdenv.hostPlatform.system}";
|
||||
builder = "/bin/sh";
|
||||
args = [ "-c" "echo built > $out" ];
|
||||
}
|
||||
'';
|
||||
|
||||
tokenFile = "/run/builder-offline/forge-token";
|
||||
caBundle = "${tls}/ca.pem";
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "swarm-builder-offline";
|
||||
|
||||
# The build itself takes seconds. A fixture that does not match leaves nix
|
||||
# failing at once (`max-jobs = 0`), so the limit only catches a hang.
|
||||
globalTimeout = 1200;
|
||||
|
||||
nodes.machine =
|
||||
{ config, ... }:
|
||||
{
|
||||
# The derivations the evaluation writes go to a disk overlay, not memory.
|
||||
virtualisation.memorySize = 4096;
|
||||
virtualisation.cores = 2;
|
||||
virtualisation.diskSize = 4096;
|
||||
virtualisation.writableStoreUseTmpfs = false;
|
||||
# The toplevel, and every source tree the meta flake's lock names, keyed by
|
||||
# narHash: nix takes a locked input from the store when its path is valid.
|
||||
virtualisation.additionalPaths = [
|
||||
toplevel
|
||||
self
|
||||
self.inputs.nixpkgs
|
||||
self.inputs.crane
|
||||
self.inputs.treefmt-nix
|
||||
hyperhiveRepo
|
||||
metaRepo
|
||||
];
|
||||
|
||||
# No substituter, and no build: a derivation missing from the store fails
|
||||
# the build at once.
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ ];
|
||||
max-jobs = 0;
|
||||
};
|
||||
|
||||
networking.hosts."127.0.0.1" = [ forgeHost ];
|
||||
security.pki.certificateFiles = [ caBundle ];
|
||||
|
||||
services.forgejo = {
|
||||
enable = true;
|
||||
database.type = "sqlite3";
|
||||
settings = {
|
||||
server = {
|
||||
PROTOCOL = "https";
|
||||
DOMAIN = forgeHost;
|
||||
HTTP_PORT = forgePort;
|
||||
ROOT_URL = "${forgeUrl}/";
|
||||
CERT_FILE = "${tls}/forge.pem";
|
||||
KEY_FILE = "${tls}/forge.key";
|
||||
};
|
||||
service.DISABLE_REGISTRATION = true;
|
||||
};
|
||||
};
|
||||
|
||||
# The builder's service user, which nix and git run as.
|
||||
users.users.builder = {
|
||||
isSystemUser = true;
|
||||
group = "builder";
|
||||
home = "/var/lib/builder";
|
||||
createHome = true;
|
||||
};
|
||||
users.groups.builder = { };
|
||||
|
||||
environment.systemPackages = [
|
||||
config.services.forgejo.package
|
||||
pkgs.git
|
||||
pkgs.jq
|
||||
# nix/host-modules/swarm-builder.nix sets `NIX_CONFIG` and `GIT_SSL_CAINFO`
|
||||
# in the builder's environment and puts `git` on its
|
||||
# PATH; the daemon is reached as `NIX_REMOTE=daemon`, which
|
||||
# is a non-root client's default here.
|
||||
(pkgs.writeShellScriptBin "meta-build" ''
|
||||
set -eu
|
||||
export PATH=${
|
||||
lib.makeBinPath [
|
||||
config.nix.package
|
||||
pkgs.git
|
||||
pkgs.coreutils
|
||||
pkgs.bash
|
||||
]
|
||||
}
|
||||
export HOME=/var/lib/builder XDG_CACHE_HOME=/var/lib/builder/.cache
|
||||
export NIX_CONFIG='experimental-features = nix-command flakes'
|
||||
export GIT_SSL_CAINFO=${caBundle}
|
||||
# `nix_command` in swarm-builder/src/nix_build.rs.
|
||||
SWARM_BUILDER_GIT_TOKEN="$(cat ${tokenFile})"
|
||||
export SWARM_BUILDER_GIT_TOKEN
|
||||
export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=1
|
||||
export SWARM_BUILDER_GIT_HOST=${forgeAuthority}
|
||||
export GIT_CONFIG_KEY_0=credential.${forgeUrl}.helper
|
||||
GIT_CONFIG_VALUE_0="$(cat ${credentialHelperFile})"
|
||||
export GIT_CONFIG_VALUE_0
|
||||
# `BUILD_FLAGS` and `meta_attr` in nix_build.rs, for the commit in
|
||||
# `internal/meta`.
|
||||
exec nix build --no-link --print-out-paths --no-write-lock-file \
|
||||
--option accept-flake-config false --print-build-logs \
|
||||
"git+${forgeUrl}/internal/meta.git?rev=$(cat ${metaRepo}/rev)#nixosConfigurations.${agent}.config.system.build.toplevel"
|
||||
'')
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import re
|
||||
import secrets
|
||||
|
||||
CA = "${caBundle}"
|
||||
API = "${forgeUrl}/api/v1"
|
||||
TOPLEVEL = "${toplevel}"
|
||||
# What nix prints for a derivation it builds or plans to build.
|
||||
BUILD_PATTERN = r"building '|will be built"
|
||||
# The forge admin's password, made for this run.
|
||||
password = secrets.token_hex(16)
|
||||
|
||||
def forge_api(method, path, body, auth):
|
||||
return machine.succeed(
|
||||
f"curl --fail --silent --cacert {CA} -X {method} {API}{path} "
|
||||
f"-H 'Content-Type: application/json' {auth} -d '{body}'"
|
||||
)
|
||||
|
||||
machine.start()
|
||||
machine.wait_for_unit("forgejo.service")
|
||||
machine.wait_for_open_port(${toString forgePort})
|
||||
|
||||
with subtest("the forge holds both repos, private, at the fixtures' commits"):
|
||||
machine.succeed(
|
||||
"su -l forgejo -c 'GITEA_WORK_DIR=/var/lib/forgejo forgejo admin user create "
|
||||
f"--admin --username ops --password {password} --email ops@${forgeHost} "
|
||||
"--must-change-password=false'"
|
||||
)
|
||||
token = machine.succeed(
|
||||
f"curl --fail --silent --cacert {CA} -X POST {API}/users/ops/tokens "
|
||||
f"-u ops:{password} -H 'Content-Type: application/json' "
|
||||
"-d '{\"name\":\"fixtures\",\"scopes\":[\"all\"]}' | jq -r .sha1"
|
||||
).strip()
|
||||
auth = f"-H 'Authorization: token {token}'"
|
||||
for org, repo, fixture in [
|
||||
("hyperhive", "hyperhive", "${hyperhiveRepo}"),
|
||||
("internal", "meta", "${metaRepo}"),
|
||||
]:
|
||||
forge_api("POST", "/orgs", f'{{"username":"{org}","visibility":"private"}}', auth)
|
||||
forge_api("POST", f"/orgs/{org}/repos", f'{{"name":"{repo}","private":true,"default_branch":"main"}}', auth)
|
||||
remote = f"https://ops:{token}@${forgeAuthority}/{org}/{repo}.git"
|
||||
machine.succeed(f"cp -r {fixture}/repo.git /tmp/{repo}.git && chmod -R u+w /tmp/{repo}.git")
|
||||
machine.succeed(f"GIT_SSL_CAINFO={CA} git -C /tmp/{repo}.git push {remote} main")
|
||||
pushed = machine.succeed(f"GIT_SSL_CAINFO={CA} git ls-remote {remote} refs/heads/main").split()[0]
|
||||
assert pushed == machine.succeed(f"cat {fixture}/rev").strip(), (repo, pushed)
|
||||
# The forge asks for a login: the builder's credential helper has to supply it.
|
||||
machine.fail(f"GIT_SSL_CAINFO={CA} GIT_TERMINAL_PROMPT=0 git ls-remote ${forgeUrl}/internal/meta.git")
|
||||
machine.succeed(
|
||||
f"install -d -m 0755 /run/builder-offline && printf %s {token} > ${tokenFile} "
|
||||
"&& chown builder ${tokenFile} && chmod 0400 ${tokenFile}"
|
||||
)
|
||||
|
||||
with subtest("the toplevel is in the store before the build"):
|
||||
machine.succeed(f"nix path-info {TOPLEVEL}")
|
||||
# The control for the lookup: a path that is not there is not found.
|
||||
machine.fail("nix path-info /nix/store/00000000000000000000000000000000-absent")
|
||||
|
||||
with subtest("the builder's build finishes with nothing built"):
|
||||
status, _ = machine.execute(
|
||||
"runuser -u builder -- timeout 600 /run/current-system/sw/bin/meta-build >/tmp/out 2>/tmp/err"
|
||||
)
|
||||
out = machine.succeed("cat /tmp/out").strip()
|
||||
err = machine.succeed("cat /tmp/err")
|
||||
assert status == 0, f"nix exited {status}:\n{err}"
|
||||
assert out == TOPLEVEL, f"printed {out}, prebuilt {TOPLEVEL}"
|
||||
assert not re.search(BUILD_PATTERN, err), err
|
||||
|
||||
with subtest("control: a derivation that does build is found by the same pattern"):
|
||||
machine.succeed("grep -x 'max-jobs = 0' /etc/nix/nix.conf")
|
||||
machine.succeed(
|
||||
"cp --remove-destination $(readlink -f /etc/nix/nix.conf) /etc/nix/nix.conf "
|
||||
"&& sed -i 's/^max-jobs = 0$/max-jobs = 1/' /etc/nix/nix.conf "
|
||||
"&& systemctl restart nix-daemon.socket nix-daemon.service"
|
||||
)
|
||||
machine.succeed(
|
||||
"NIX_CONFIG='experimental-features = nix-command' "
|
||||
"nix build --no-link --print-build-logs --impure -f ${control} 2>/tmp/control.err"
|
||||
)
|
||||
assert re.search(BUILD_PATTERN, machine.succeed("cat /tmp/control.err"))
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue