nightly vm test: the builder's meta build finds a prebuilt agent in an offline store
This commit is contained in:
atlas
parent
90272ec3c0
commit
4d85ec4c21
2 changed files with 508 additions and 2 deletions
113
nix/vm-tests/swarm-builder-offline-fixtures.nix
Normal file
113
nix/vm-tests/swarm-builder-offline-fixtures.nix
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
# What `./swarm-builder-offline.nix` puts on its forge, and the agent it
|
||||
# prebuilds: two deterministic one-commit repos and one toplevel, all built from
|
||||
# `self`.
|
||||
#
|
||||
# hyperhive/hyperhive the source tree of `self`
|
||||
# internal/meta the meta flake of one archetype agent, pinned to the
|
||||
# first repo's commit
|
||||
#
|
||||
# Both commits carry a fixed author and date, so their ids depend on the tree
|
||||
# alone. The meta flake's lock is built from hyperhive's own `flake.lock`, in the
|
||||
# shape `nix flake lock` writes for the pinned flake: every input carries its
|
||||
# `narHash`, so a nix with those source trees in its store needs no fetch for
|
||||
# them.
|
||||
{
|
||||
pkgs,
|
||||
self,
|
||||
forgeUrl,
|
||||
agent,
|
||||
}:
|
||||
let
|
||||
# `hive-meta-flake`'s `render_flake` output for one archetype agent; a unit
|
||||
# test there keeps the file equal to it.
|
||||
metaTemplate = ../../hive-meta-flake/testdata/meta-flake.nix.in;
|
||||
|
||||
# The same flake, evaluated here with `self` as its `hyperhive` input.
|
||||
toplevel =
|
||||
((import metaTemplate).outputs {
|
||||
self = null;
|
||||
hyperhive = self;
|
||||
}).nixosConfigurations.${agent}.config.system.build.toplevel;
|
||||
|
||||
commitEnv = ''
|
||||
export HOME=$TMPDIR
|
||||
export GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@forge.test
|
||||
export GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@forge.test
|
||||
export GIT_AUTHOR_DATE="946684800 +0000" GIT_COMMITTER_DATE="946684800 +0000"
|
||||
'';
|
||||
|
||||
hyperhiveRepo =
|
||||
pkgs.runCommand "hyperhive-fixture-repo"
|
||||
{
|
||||
nativeBuildInputs = [ pkgs.git ];
|
||||
}
|
||||
''
|
||||
${commitEnv}
|
||||
cp -a ${self} work
|
||||
chmod -R u+w work
|
||||
git -C work init -q -b main
|
||||
git -C work add -f .
|
||||
git -C work commit -q -m "hyperhive source tree"
|
||||
mkdir $out
|
||||
git clone -q --bare work $out/repo.git
|
||||
git -C work rev-parse HEAD > $out/rev
|
||||
'';
|
||||
|
||||
# `nix flake lock`'s result for the template: hyperhive's own nodes next to a
|
||||
# `hyperhive` node pinned by URL and rev, whose follows paths start at the
|
||||
# meta flake's root.
|
||||
lockProgram = pkgs.writeText "meta-lock.jq" ''
|
||||
def from_meta_root: if type == "array" then ["hyperhive"] + . else . end;
|
||||
.nodes.root.inputs as $hyperhiveInputs
|
||||
| {
|
||||
version: 7,
|
||||
root: "root",
|
||||
nodes: (
|
||||
(.nodes | del(.root) | map_values(if .inputs then .inputs |= map_values(from_meta_root) else . end))
|
||||
+ {
|
||||
hyperhive: {
|
||||
inputs: ($hyperhiveInputs | map_values(from_meta_root)),
|
||||
locked: {
|
||||
lastModified: ($lastModified | tonumber),
|
||||
narHash: $narHash,
|
||||
ref: "refs/heads/main",
|
||||
rev: $rev,
|
||||
revCount: 1,
|
||||
type: "git",
|
||||
url: $url
|
||||
},
|
||||
original: { rev: $rev, type: "git", url: $url }
|
||||
},
|
||||
root: { inputs: { hyperhive: "hyperhive", nixpkgs: ["hyperhive", "nixpkgs"] } }
|
||||
}
|
||||
)
|
||||
}
|
||||
'';
|
||||
|
||||
metaRepo =
|
||||
pkgs.runCommand "meta-fixture-repo"
|
||||
{
|
||||
nativeBuildInputs = [
|
||||
pkgs.git
|
||||
pkgs.jq
|
||||
];
|
||||
}
|
||||
''
|
||||
${commitEnv}
|
||||
rev=$(cat ${hyperhiveRepo}/rev)
|
||||
url=${forgeUrl}/hyperhive/hyperhive.git
|
||||
mkdir work
|
||||
substitute ${metaTemplate} work/flake.nix --replace-fail @HYPERHIVE_URL@ "git+$url?rev=$rev"
|
||||
jq --arg url "$url" --arg rev "$rev" --arg narHash ${self.narHash} --arg lastModified 946684800 \
|
||||
-f ${lockProgram} ${self}/flake.lock > work/flake.lock
|
||||
git -C work init -q -b main
|
||||
git -C work add .
|
||||
git -C work commit -q -m "meta flake"
|
||||
mkdir $out
|
||||
git clone -q --bare work $out/repo.git
|
||||
git -C work rev-parse HEAD > $out/rev
|
||||
'';
|
||||
in
|
||||
{
|
||||
inherit toplevel hyperhiveRepo metaRepo;
|
||||
}
|
||||
258
nix/vm-tests/swarm-builder-offline.nix
Normal file
258
nix/vm-tests/swarm-builder-offline.nix
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
# `vmTests.<system>.swarm-builder-offline`: the build the swarm builder runs for
|
||||
# a deploy gate, on a VM with no network and no substituter, finds the agent's
|
||||
# toplevel in its store and builds nothing.
|
||||
#
|
||||
# One node runs a forge over https (test CA) holding `hyperhive/hyperhive` and
|
||||
# `internal/meta` (./swarm-builder-offline-fixtures.nix). The agent's toplevel
|
||||
# and the source trees its meta flake pins are in the VM's store through
|
||||
# `virtualisation.additionalPaths`, built from `self` before the VM boots. The
|
||||
# script runs the builder's `nix build` of the toplevel from `internal/meta` with
|
||||
# the builder's flags, credential helper and environment, and checks that:
|
||||
# - it exits 0 and prints the prebuilt toplevel's path;
|
||||
# - the path was valid in the store before the command ran;
|
||||
# - nix's output names no build. The daemon runs with `max-jobs = 0`, so a
|
||||
# derivation that needs building fails at once instead of compiling the
|
||||
# workspace. A control build with `max-jobs` raised proves the pattern
|
||||
# finds a build where there is one.
|
||||
#
|
||||
# What it does not cover: the builder daemon (its queue, store and OIDC
|
||||
# logins, request handling) and the controller. The command is the one in
|
||||
# swarm-builder/src/nix_build.rs; a change there has to be copied here, and a
|
||||
# unit test there (`the_vm_test_copy_matches`) fails until it is.
|
||||
{ pkgs, self }:
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
agent = "rev1";
|
||||
forgeHost = "forge.test";
|
||||
forgePort = 3000;
|
||||
forgeAuthority = "${forgeHost}:${toString forgePort}";
|
||||
forgeUrl = "https://${forgeAuthority}";
|
||||
|
||||
fixtures = import ./swarm-builder-offline-fixtures.nix {
|
||||
inherit
|
||||
pkgs
|
||||
self
|
||||
forgeUrl
|
||||
agent
|
||||
;
|
||||
};
|
||||
inherit (fixtures) toplevel hyperhiveRepo metaRepo;
|
||||
|
||||
# A test CA and the forge's leaf for `forge.test`, made with the `openssl req`
|
||||
# calls of nix/host-modules/glue-bao-tls.nix.
|
||||
tls = pkgs.runCommand "swarm-builder-offline-tls" { nativeBuildInputs = [ pkgs.openssl ]; } ''
|
||||
mkdir $out
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 3650 \
|
||||
-keyout $TMPDIR/ca.key -out $out/ca.pem -subj /CN=builder-offline-test-ca
|
||||
openssl req -newkey rsa:2048 -nodes -sha256 \
|
||||
-keyout $out/forge.key -out $TMPDIR/forge.csr -subj /CN=${forgeHost}
|
||||
printf 'subjectAltName=DNS:${forgeHost},DNS:localhost,IP:127.0.0.1\nbasicConstraints=CA:FALSE\n' > $TMPDIR/ext
|
||||
openssl x509 -req -sha256 -days 3650 -in $TMPDIR/forge.csr \
|
||||
-CA $out/ca.pem -CAkey $TMPDIR/ca.key -CAcreateserial \
|
||||
-extfile $TMPDIR/ext -out $out/forge.pem
|
||||
'';
|
||||
|
||||
# The git credential helper of `credential_helper()` in swarm-builder/src/nix_build.rs, with
|
||||
# `TOKEN_VAR` and `HOST_VAR` as they are written there.
|
||||
credentialHelper = lib.concatStrings [
|
||||
"!f() { test \"$1\" = get || return 0; p=; h=; "
|
||||
"while IFS='=' read -r k v; do case \"$k\" in '') break ;; protocol) p=$v ;; host) h=$v ;; esac; done; "
|
||||
"test -n \"$SWARM_BUILDER_GIT_HOST\" && test \"$p\" = https && test \"$h\" = \"$SWARM_BUILDER_GIT_HOST\" "
|
||||
"&& printf 'username=builder\\npassword=%s\\n' \"$SWARM_BUILDER_GIT_TOKEN\"; return 0; }; f"
|
||||
];
|
||||
credentialHelperFile = pkgs.writeText "credential-helper" credentialHelper;
|
||||
|
||||
# A derivation no store holds, to check the pattern that finds a build.
|
||||
control = pkgs.writeText "control.nix" ''
|
||||
derivation {
|
||||
name = "build-control";
|
||||
system = "${pkgs.stdenv.hostPlatform.system}";
|
||||
builder = "/bin/sh";
|
||||
args = [ "-c" "echo built > $out" ];
|
||||
}
|
||||
'';
|
||||
|
||||
tokenFile = "/run/builder-offline/forge-token";
|
||||
caBundle = "${tls}/ca.pem";
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "swarm-builder-offline";
|
||||
|
||||
# The build itself takes seconds. A fixture that does not match leaves nix
|
||||
# failing at once (`max-jobs = 0`), so the limit only catches a hang.
|
||||
globalTimeout = 1200;
|
||||
|
||||
nodes.machine =
|
||||
{ config, ... }:
|
||||
{
|
||||
# The derivations the evaluation writes go to a disk overlay, not memory.
|
||||
virtualisation.memorySize = 4096;
|
||||
virtualisation.cores = 2;
|
||||
virtualisation.diskSize = 4096;
|
||||
virtualisation.writableStoreUseTmpfs = false;
|
||||
# The toplevel, and every source tree the meta flake's lock names, keyed by
|
||||
# narHash: nix takes a locked input from the store when its path is valid.
|
||||
virtualisation.additionalPaths = [
|
||||
toplevel
|
||||
self
|
||||
self.inputs.nixpkgs
|
||||
self.inputs.crane
|
||||
self.inputs.treefmt-nix
|
||||
hyperhiveRepo
|
||||
metaRepo
|
||||
];
|
||||
|
||||
# No substituter, and no build: a derivation missing from the store fails
|
||||
# the build at once.
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ ];
|
||||
max-jobs = 0;
|
||||
};
|
||||
|
||||
networking.hosts."127.0.0.1" = [ forgeHost ];
|
||||
security.pki.certificateFiles = [ caBundle ];
|
||||
|
||||
services.forgejo = {
|
||||
enable = true;
|
||||
database.type = "sqlite3";
|
||||
settings = {
|
||||
server = {
|
||||
PROTOCOL = "https";
|
||||
DOMAIN = forgeHost;
|
||||
HTTP_PORT = forgePort;
|
||||
ROOT_URL = "${forgeUrl}/";
|
||||
CERT_FILE = "${tls}/forge.pem";
|
||||
KEY_FILE = "${tls}/forge.key";
|
||||
};
|
||||
service.DISABLE_REGISTRATION = true;
|
||||
};
|
||||
};
|
||||
|
||||
# The builder's service user, which nix and git run as.
|
||||
users.users.builder = {
|
||||
isSystemUser = true;
|
||||
group = "builder";
|
||||
home = "/var/lib/builder";
|
||||
createHome = true;
|
||||
};
|
||||
users.groups.builder = { };
|
||||
|
||||
environment.systemPackages = [
|
||||
config.services.forgejo.package
|
||||
pkgs.git
|
||||
pkgs.jq
|
||||
# nix/host-modules/swarm-builder.nix sets `NIX_CONFIG` and `GIT_SSL_CAINFO`
|
||||
# in the builder's environment and puts `git` on its
|
||||
# PATH; the daemon is reached as `NIX_REMOTE=daemon`, which
|
||||
# is a non-root client's default here.
|
||||
(pkgs.writeShellScriptBin "meta-build" ''
|
||||
set -eu
|
||||
export PATH=${
|
||||
lib.makeBinPath [
|
||||
config.nix.package
|
||||
pkgs.git
|
||||
pkgs.coreutils
|
||||
pkgs.bash
|
||||
]
|
||||
}
|
||||
export HOME=/var/lib/builder XDG_CACHE_HOME=/var/lib/builder/.cache
|
||||
export NIX_CONFIG='experimental-features = nix-command flakes'
|
||||
export GIT_SSL_CAINFO=${caBundle}
|
||||
# `nix_command` in swarm-builder/src/nix_build.rs.
|
||||
SWARM_BUILDER_GIT_TOKEN="$(cat ${tokenFile})"
|
||||
export SWARM_BUILDER_GIT_TOKEN
|
||||
export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=1
|
||||
export SWARM_BUILDER_GIT_HOST=${forgeAuthority}
|
||||
export GIT_CONFIG_KEY_0=credential.${forgeUrl}.helper
|
||||
GIT_CONFIG_VALUE_0="$(cat ${credentialHelperFile})"
|
||||
export GIT_CONFIG_VALUE_0
|
||||
# `BUILD_FLAGS` and `meta_attr` in nix_build.rs, for the commit in
|
||||
# `internal/meta`.
|
||||
exec nix build --no-link --print-out-paths --no-write-lock-file \
|
||||
--option accept-flake-config false --print-build-logs \
|
||||
"git+${forgeUrl}/internal/meta.git?rev=$(cat ${metaRepo}/rev)#nixosConfigurations.${agent}.config.system.build.toplevel"
|
||||
'')
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import re
|
||||
import secrets
|
||||
|
||||
CA = "${caBundle}"
|
||||
API = "${forgeUrl}/api/v1"
|
||||
TOPLEVEL = "${toplevel}"
|
||||
# What nix prints for a derivation it builds or plans to build.
|
||||
BUILD_PATTERN = r"building '|will be built"
|
||||
# The forge admin's password, made for this run.
|
||||
password = secrets.token_hex(16)
|
||||
|
||||
def forge_api(method, path, body, auth):
|
||||
return machine.succeed(
|
||||
f"curl --fail --silent --cacert {CA} -X {method} {API}{path} "
|
||||
f"-H 'Content-Type: application/json' {auth} -d '{body}'"
|
||||
)
|
||||
|
||||
machine.start()
|
||||
machine.wait_for_unit("forgejo.service")
|
||||
machine.wait_for_open_port(${toString forgePort})
|
||||
|
||||
with subtest("the forge holds both repos, private, at the fixtures' commits"):
|
||||
machine.succeed(
|
||||
"su -l forgejo -c 'GITEA_WORK_DIR=/var/lib/forgejo forgejo admin user create "
|
||||
f"--admin --username ops --password {password} --email ops@${forgeHost} "
|
||||
"--must-change-password=false'"
|
||||
)
|
||||
token = machine.succeed(
|
||||
f"curl --fail --silent --cacert {CA} -X POST {API}/users/ops/tokens "
|
||||
f"-u ops:{password} -H 'Content-Type: application/json' "
|
||||
"-d '{\"name\":\"fixtures\",\"scopes\":[\"all\"]}' | jq -r .sha1"
|
||||
).strip()
|
||||
auth = f"-H 'Authorization: token {token}'"
|
||||
for org, repo, fixture in [
|
||||
("hyperhive", "hyperhive", "${hyperhiveRepo}"),
|
||||
("internal", "meta", "${metaRepo}"),
|
||||
]:
|
||||
forge_api("POST", "/orgs", f'{{"username":"{org}","visibility":"private"}}', auth)
|
||||
forge_api("POST", f"/orgs/{org}/repos", f'{{"name":"{repo}","private":true,"default_branch":"main"}}', auth)
|
||||
remote = f"https://ops:{token}@${forgeAuthority}/{org}/{repo}.git"
|
||||
machine.succeed(f"cp -r {fixture}/repo.git /tmp/{repo}.git && chmod -R u+w /tmp/{repo}.git")
|
||||
machine.succeed(f"GIT_SSL_CAINFO={CA} git -C /tmp/{repo}.git push {remote} main")
|
||||
pushed = machine.succeed(f"GIT_SSL_CAINFO={CA} git ls-remote {remote} refs/heads/main").split()[0]
|
||||
assert pushed == machine.succeed(f"cat {fixture}/rev").strip(), (repo, pushed)
|
||||
# The forge asks for a login: the builder's credential helper has to supply it.
|
||||
machine.fail(f"GIT_SSL_CAINFO={CA} GIT_TERMINAL_PROMPT=0 git ls-remote ${forgeUrl}/internal/meta.git")
|
||||
machine.succeed(
|
||||
f"install -d -m 0755 /run/builder-offline && printf %s {token} > ${tokenFile} "
|
||||
"&& chown builder ${tokenFile} && chmod 0400 ${tokenFile}"
|
||||
)
|
||||
|
||||
with subtest("the toplevel is in the store before the build"):
|
||||
machine.succeed(f"nix path-info {TOPLEVEL}")
|
||||
# The control for the lookup: a path that is not there is not found.
|
||||
machine.fail("nix path-info /nix/store/00000000000000000000000000000000-absent")
|
||||
|
||||
with subtest("the builder's build finishes with nothing built"):
|
||||
status, _ = machine.execute(
|
||||
"runuser -u builder -- timeout 600 /run/current-system/sw/bin/meta-build >/tmp/out 2>/tmp/err"
|
||||
)
|
||||
out = machine.succeed("cat /tmp/out").strip()
|
||||
err = machine.succeed("cat /tmp/err")
|
||||
assert status == 0, f"nix exited {status}:\n{err}"
|
||||
assert out == TOPLEVEL, f"printed {out}, prebuilt {TOPLEVEL}"
|
||||
assert not re.search(BUILD_PATTERN, err), err
|
||||
|
||||
with subtest("control: a derivation that does build is found by the same pattern"):
|
||||
machine.succeed("grep -x 'max-jobs = 0' /etc/nix/nix.conf")
|
||||
machine.succeed(
|
||||
"cp --remove-destination $(readlink -f /etc/nix/nix.conf) /etc/nix/nix.conf "
|
||||
"&& sed -i 's/^max-jobs = 0$/max-jobs = 1/' /etc/nix/nix.conf "
|
||||
"&& systemctl restart nix-daemon.socket nix-daemon.service"
|
||||
)
|
||||
machine.succeed(
|
||||
"NIX_CONFIG='experimental-features = nix-command' "
|
||||
"nix build --no-link --print-build-logs --impure -f ${control} 2>/tmp/control.err"
|
||||
)
|
||||
assert re.search(BUILD_PATTERN, machine.succeed("cat /tmp/control.err"))
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue