Watch
0
0
Fork
You've already forked hyperhive
0

nightly vm test: the builder's meta build finds a prebuilt agent in an offline store

This commit is contained in:
atlas 2026-11-01 20:21:35 +01:00
commit 4d85ec4c21
8 changed files with 508 additions and 2 deletions

View file

@ -8,3 +8,7 @@ Pure string rendering, with no IO and no environment reads. The caller passes
the hyperhive pin, the agents with their pins, and the values every agent the hyperhive pin, the agents with their pins, and the values every agent
container gets. swarm-controller calls `render_flake`; hive-c0re reads the container gets. swarm-controller calls `render_flake`; hive-c0re reads the
pins of the flake it renders (`pinned_rev`, `builds_agent_at`). pins of the flake it renders (`pinned_rev`, `builds_agent_at`).
`testdata/meta-flake.nix.in` is the render for one archetype agent with a
placeholder for the hyperhive pin. A unit test keeps it equal to `render_flake`'s
output, and the `swarm-builder-offline` VM test (`nix/vm-tests/`) serves it.

View file

@ -543,6 +543,29 @@ mod tests {
.unwrap() .unwrap()
} }
/// `testdata/meta-flake.nix.in` is the flake `nix/vm-tests/swarm-builder-offline.nix`
/// evaluates and serves, with `@HYPERHIVE_URL@` where the pin goes. It must be
/// what [`render_flake`] renders for one archetype agent.
#[test]
fn the_vm_test_fixture_is_what_render_flake_renders() {
let archetype = ArchetypeRef {
source: "hyperhive".to_owned(),
name: "reviewer".to_owned(),
};
let flake = render_flake(
"@HYPERHIVE_URL@",
&[spec("rev1", Some(archetype), None)],
&[],
false,
)
.unwrap();
assert_eq!(
flake,
include_str!("../testdata/meta-flake.nix.in"),
"update the fixture to the rendered flake"
);
}
#[test] #[test]
fn a_rendered_flake_names_the_commit_each_input_is_pinned_to() { fn a_rendered_flake_names_the_commit_each_input_is_pinned_to() {
let flake = rendered(); let flake = rendered();

View file

@ -0,0 +1,85 @@
{
description = "hyperhive deployed agents";
inputs = {
hyperhive.url = "@HYPERHIVE_URL@";
nixpkgs.follows = "hyperhive/nixpkgs";
};
outputs =
{ self, hyperhive, ... }@inputs:
let
mkAgent = { name, port, archetype ? null }:
let
base = hyperhive.nixosConfigurations.agent-base;
# Every flake an archetype can come from, by its source name.
archetypeSources = { inherit hyperhive; };
agentModule =
if archetype == null then
inputs."agent-${name}".nixosModules.default
else
(archetypeSources.${archetype.source}
or (throw "agent ${name}: archetype source ${archetype.source} is not an input of this hive's meta flake")
).hyperhiveArchetypes.${archetype.name}.module
or (throw "agent ${name}: this hive's ${archetype.source} input has no archetype ${archetype.name}; update that input");
service = "hive-agent";
in
base.extendModules {
modules = [
agentModule
{
# The harness service inside the container runs as a
# non-root unix user named after the agent (`damocles`,
# `iris`, `root`, …). UID auto-assigned by NixOS; the
# per-agent override here is what makes
# `services.hyperhive.agent.user.name` match the agent's identity
# instead of the harness default of `"agent"`.
services.hyperhive.agent.user.name = name;
# Without this an agent container is called `nixos` and ships
# every log line under that: nixpkgs sets the hostname from the
# attr name in the `containers.<name>` submodule, and an
# agent's system is its own `nixosSystem` off this flake, which
# never evaluates that option. The machine name, not the
# logical one — it is what `container.name` on our metrics and
# nspawn itself already call this container, so logs join to
# both without a prefix transform.
networking.hostName = "h-${name}";
programs.git.config.user = {
name = name;
email = "${name}@hyperhive.local";
};
# Container-wide env: every service + co-process daemon can
# resolve the agent's durable state dir without hard-coding it.
# `environment.variables` only writes /etc/environment (login
# shells); `systemd.globalEnvironment` is the analogue for
# systemd units so forge-avatar-sync /
# hive-matrix-daemon etc. can read `$HYPERHIVE_STATE_DIR`
# without each service having to redeclare it.
environment.variables = {
HIVE_LABEL = name;
HYPERHIVE_STATE_DIR = "/agents/${name}/state";
HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
};
systemd.globalEnvironment = {
HIVE_LABEL = name;
HYPERHIVE_STATE_DIR = "/agents/${name}/state";
HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
};
systemd.services.${service}.environment = {
HIVE_PORT = toString port;
HIVE_LABEL = name;
HYPERHIVE_STATE_DIR = "/agents/${name}/state";
HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
};
}
# Tells the harness the agent has no config repo of its own.
(if archetype == null then { } else {
systemd.services.${service}.environment.HIVE_ARCHETYPE = archetype.name;
})
];
};
in
{
nixosConfigurations = {
rev1 = mkAgent { name = "rev1"; port = 8297; archetype = { source = "hyperhive"; name = "reviewer"; }; };
};
};
}

View file

@ -36,13 +36,16 @@ rec {
# runtime from `$HIVE_ASSETS_DIR`. Net effect: only `*.rs` / # runtime from `$HIVE_ASSETS_DIR`. Net effect: only `*.rs` /
# `*.toml` / `Cargo.lock` edits rebuild the rust derivations. # `*.toml` / `Cargo.lock` edits rebuild the rust derivations.
# #
# The one other file in it is the repo's own review-classifier rules: a # The two other files in it are the repo's own review-classifier rules and
# `swarm-controller` test parses them, so they must be in the sandbox. # the meta flake fixture of the swarm-builder-offline VM test. A
# `swarm-controller` test parses the rules and a `hive-meta-flake` test
# compares the fixture, so both must be in the sandbox.
cleanSrc = lib.fileset.toSource { cleanSrc = lib.fileset.toSource {
root = ../.; root = ../.;
fileset = lib.fileset.unions [ fileset = lib.fileset.unions [
(craneLib.fileset.commonCargoSources ../.) (craneLib.fileset.commonCargoSources ../.)
../.hyperhive/review-classifier.yml ../.hyperhive/review-classifier.yml
../hive-meta-flake/testdata/meta-flake.nix.in
]; ];
}; };

View file

@ -10,4 +10,5 @@
agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; }; agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; };
wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; }; wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; }; hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };
} }

View file

@ -0,0 +1,113 @@
# What `./swarm-builder-offline.nix` puts on its forge, and the agent it
# prebuilds: two deterministic one-commit repos and one toplevel, all built from
# `self`.
#
# hyperhive/hyperhive the source tree of `self`
# internal/meta the meta flake of one archetype agent, pinned to the
# first repo's commit
#
# Both commits carry a fixed author and date, so their ids depend on the tree
# alone. The meta flake's lock is built from hyperhive's own `flake.lock`, in the
# shape `nix flake lock` writes for the pinned flake: every input carries its
# `narHash`, so a nix with those source trees in its store needs no fetch for
# them.
{
pkgs,
self,
forgeUrl,
agent,
}:
let
# `hive-meta-flake`'s `render_flake` output for one archetype agent; a unit
# test there keeps the file equal to it.
metaTemplate = ../../hive-meta-flake/testdata/meta-flake.nix.in;
# The same flake, evaluated here with `self` as its `hyperhive` input.
toplevel =
((import metaTemplate).outputs {
self = null;
hyperhive = self;
}).nixosConfigurations.${agent}.config.system.build.toplevel;
commitEnv = ''
export HOME=$TMPDIR
export GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@forge.test
export GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@forge.test
export GIT_AUTHOR_DATE="946684800 +0000" GIT_COMMITTER_DATE="946684800 +0000"
'';
hyperhiveRepo =
pkgs.runCommand "hyperhive-fixture-repo"
{
nativeBuildInputs = [ pkgs.git ];
}
''
${commitEnv}
cp -a ${self} work
chmod -R u+w work
git -C work init -q -b main
git -C work add -f .
git -C work commit -q -m "hyperhive source tree"
mkdir $out
git clone -q --bare work $out/repo.git
git -C work rev-parse HEAD > $out/rev
'';
# `nix flake lock`'s result for the template: hyperhive's own nodes next to a
# `hyperhive` node pinned by URL and rev, whose follows paths start at the
# meta flake's root.
lockProgram = pkgs.writeText "meta-lock.jq" ''
def from_meta_root: if type == "array" then ["hyperhive"] + . else . end;
.nodes.root.inputs as $hyperhiveInputs
| {
version: 7,
root: "root",
nodes: (
(.nodes | del(.root) | map_values(if .inputs then .inputs |= map_values(from_meta_root) else . end))
+ {
hyperhive: {
inputs: ($hyperhiveInputs | map_values(from_meta_root)),
locked: {
lastModified: ($lastModified | tonumber),
narHash: $narHash,
ref: "refs/heads/main",
rev: $rev,
revCount: 1,
type: "git",
url: $url
},
original: { rev: $rev, type: "git", url: $url }
},
root: { inputs: { hyperhive: "hyperhive", nixpkgs: ["hyperhive", "nixpkgs"] } }
}
)
}
'';
metaRepo =
pkgs.runCommand "meta-fixture-repo"
{
nativeBuildInputs = [
pkgs.git
pkgs.jq
];
}
''
${commitEnv}
rev=$(cat ${hyperhiveRepo}/rev)
url=${forgeUrl}/hyperhive/hyperhive.git
mkdir work
substitute ${metaTemplate} work/flake.nix --replace-fail @HYPERHIVE_URL@ "git+$url?rev=$rev"
jq --arg url "$url" --arg rev "$rev" --arg narHash ${self.narHash} --arg lastModified 946684800 \
-f ${lockProgram} ${self}/flake.lock > work/flake.lock
git -C work init -q -b main
git -C work add .
git -C work commit -q -m "meta flake"
mkdir $out
git clone -q --bare work $out/repo.git
git -C work rev-parse HEAD > $out/rev
'';
in
{
inherit toplevel hyperhiveRepo metaRepo;
}

View file

@ -0,0 +1,258 @@
# `vmTests.<system>.swarm-builder-offline`: the build the swarm builder runs for
# a deploy gate, on a VM with no network and no substituter, finds the agent's
# toplevel in its store and builds nothing.
#
# One node runs a forge over https (test CA) holding `hyperhive/hyperhive` and
# `internal/meta` (./swarm-builder-offline-fixtures.nix). The agent's toplevel
# and the source trees its meta flake pins are in the VM's store through
# `virtualisation.additionalPaths`, built from `self` before the VM boots. The
# script runs the builder's `nix build` of the toplevel from `internal/meta` with
# the builder's flags, credential helper and environment, and checks that:
# - it exits 0 and prints the prebuilt toplevel's path;
# - the path was valid in the store before the command ran;
# - nix's output names no build. The daemon runs with `max-jobs = 0`, so a
# derivation that needs building fails at once instead of compiling the
# workspace. A control build with `max-jobs` raised proves the pattern
# finds a build where there is one.
#
# What it does not cover: the builder daemon (its queue, store and OIDC
# logins, request handling) and the controller. The command is the one in
# swarm-builder/src/nix_build.rs; a change there has to be copied here, and a
# unit test there (`the_vm_test_copy_matches`) fails until it is.
{ pkgs, self }:
let
inherit (pkgs) lib;
agent = "rev1";
forgeHost = "forge.test";
forgePort = 3000;
forgeAuthority = "${forgeHost}:${toString forgePort}";
forgeUrl = "https://${forgeAuthority}";
fixtures = import ./swarm-builder-offline-fixtures.nix {
inherit
pkgs
self
forgeUrl
agent
;
};
inherit (fixtures) toplevel hyperhiveRepo metaRepo;
# A test CA and the forge's leaf for `forge.test`, made with the `openssl req`
# calls of nix/host-modules/glue-bao-tls.nix.
tls = pkgs.runCommand "swarm-builder-offline-tls" { nativeBuildInputs = [ pkgs.openssl ]; } ''
mkdir $out
openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 3650 \
-keyout $TMPDIR/ca.key -out $out/ca.pem -subj /CN=builder-offline-test-ca
openssl req -newkey rsa:2048 -nodes -sha256 \
-keyout $out/forge.key -out $TMPDIR/forge.csr -subj /CN=${forgeHost}
printf 'subjectAltName=DNS:${forgeHost},DNS:localhost,IP:127.0.0.1\nbasicConstraints=CA:FALSE\n' > $TMPDIR/ext
openssl x509 -req -sha256 -days 3650 -in $TMPDIR/forge.csr \
-CA $out/ca.pem -CAkey $TMPDIR/ca.key -CAcreateserial \
-extfile $TMPDIR/ext -out $out/forge.pem
'';
# The git credential helper of `credential_helper()` in swarm-builder/src/nix_build.rs, with
# `TOKEN_VAR` and `HOST_VAR` as they are written there.
credentialHelper = lib.concatStrings [
"!f() { test \"$1\" = get || return 0; p=; h=; "
"while IFS='=' read -r k v; do case \"$k\" in '') break ;; protocol) p=$v ;; host) h=$v ;; esac; done; "
"test -n \"$SWARM_BUILDER_GIT_HOST\" && test \"$p\" = https && test \"$h\" = \"$SWARM_BUILDER_GIT_HOST\" "
"&& printf 'username=builder\\npassword=%s\\n' \"$SWARM_BUILDER_GIT_TOKEN\"; return 0; }; f"
];
credentialHelperFile = pkgs.writeText "credential-helper" credentialHelper;
# A derivation no store holds, to check the pattern that finds a build.
control = pkgs.writeText "control.nix" ''
derivation {
name = "build-control";
system = "${pkgs.stdenv.hostPlatform.system}";
builder = "/bin/sh";
args = [ "-c" "echo built > $out" ];
}
'';
tokenFile = "/run/builder-offline/forge-token";
caBundle = "${tls}/ca.pem";
in
pkgs.testers.runNixOSTest {
name = "swarm-builder-offline";
# The build itself takes seconds. A fixture that does not match leaves nix
# failing at once (`max-jobs = 0`), so the limit only catches a hang.
globalTimeout = 1200;
nodes.machine =
{ config, ... }:
{
# The derivations the evaluation writes go to a disk overlay, not memory.
virtualisation.memorySize = 4096;
virtualisation.cores = 2;
virtualisation.diskSize = 4096;
virtualisation.writableStoreUseTmpfs = false;
# The toplevel, and every source tree the meta flake's lock names, keyed by
# narHash: nix takes a locked input from the store when its path is valid.
virtualisation.additionalPaths = [
toplevel
self
self.inputs.nixpkgs
self.inputs.crane
self.inputs.treefmt-nix
hyperhiveRepo
metaRepo
];
# No substituter, and no build: a derivation missing from the store fails
# the build at once.
nix.settings = {
substituters = lib.mkForce [ ];
max-jobs = 0;
};
networking.hosts."127.0.0.1" = [ forgeHost ];
security.pki.certificateFiles = [ caBundle ];
services.forgejo = {
enable = true;
database.type = "sqlite3";
settings = {
server = {
PROTOCOL = "https";
DOMAIN = forgeHost;
HTTP_PORT = forgePort;
ROOT_URL = "${forgeUrl}/";
CERT_FILE = "${tls}/forge.pem";
KEY_FILE = "${tls}/forge.key";
};
service.DISABLE_REGISTRATION = true;
};
};
# The builder's service user, which nix and git run as.
users.users.builder = {
isSystemUser = true;
group = "builder";
home = "/var/lib/builder";
createHome = true;
};
users.groups.builder = { };
environment.systemPackages = [
config.services.forgejo.package
pkgs.git
pkgs.jq
# nix/host-modules/swarm-builder.nix sets `NIX_CONFIG` and `GIT_SSL_CAINFO`
# in the builder's environment and puts `git` on its
# PATH; the daemon is reached as `NIX_REMOTE=daemon`, which
# is a non-root client's default here.
(pkgs.writeShellScriptBin "meta-build" ''
set -eu
export PATH=${
lib.makeBinPath [
config.nix.package
pkgs.git
pkgs.coreutils
pkgs.bash
]
}
export HOME=/var/lib/builder XDG_CACHE_HOME=/var/lib/builder/.cache
export NIX_CONFIG='experimental-features = nix-command flakes'
export GIT_SSL_CAINFO=${caBundle}
# `nix_command` in swarm-builder/src/nix_build.rs.
SWARM_BUILDER_GIT_TOKEN="$(cat ${tokenFile})"
export SWARM_BUILDER_GIT_TOKEN
export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=1
export SWARM_BUILDER_GIT_HOST=${forgeAuthority}
export GIT_CONFIG_KEY_0=credential.${forgeUrl}.helper
GIT_CONFIG_VALUE_0="$(cat ${credentialHelperFile})"
export GIT_CONFIG_VALUE_0
# `BUILD_FLAGS` and `meta_attr` in nix_build.rs, for the commit in
# `internal/meta`.
exec nix build --no-link --print-out-paths --no-write-lock-file \
--option accept-flake-config false --print-build-logs \
"git+${forgeUrl}/internal/meta.git?rev=$(cat ${metaRepo}/rev)#nixosConfigurations.${agent}.config.system.build.toplevel"
'')
];
};
testScript = ''
import re
import secrets
CA = "${caBundle}"
API = "${forgeUrl}/api/v1"
TOPLEVEL = "${toplevel}"
# What nix prints for a derivation it builds or plans to build.
BUILD_PATTERN = r"building '|will be built"
# The forge admin's password, made for this run.
password = secrets.token_hex(16)
def forge_api(method, path, body, auth):
return machine.succeed(
f"curl --fail --silent --cacert {CA} -X {method} {API}{path} "
f"-H 'Content-Type: application/json' {auth} -d '{body}'"
)
machine.start()
machine.wait_for_unit("forgejo.service")
machine.wait_for_open_port(${toString forgePort})
with subtest("the forge holds both repos, private, at the fixtures' commits"):
machine.succeed(
"su -l forgejo -c 'GITEA_WORK_DIR=/var/lib/forgejo forgejo admin user create "
f"--admin --username ops --password {password} --email ops@${forgeHost} "
"--must-change-password=false'"
)
token = machine.succeed(
f"curl --fail --silent --cacert {CA} -X POST {API}/users/ops/tokens "
f"-u ops:{password} -H 'Content-Type: application/json' "
"-d '{\"name\":\"fixtures\",\"scopes\":[\"all\"]}' | jq -r .sha1"
).strip()
auth = f"-H 'Authorization: token {token}'"
for org, repo, fixture in [
("hyperhive", "hyperhive", "${hyperhiveRepo}"),
("internal", "meta", "${metaRepo}"),
]:
forge_api("POST", "/orgs", f'{{"username":"{org}","visibility":"private"}}', auth)
forge_api("POST", f"/orgs/{org}/repos", f'{{"name":"{repo}","private":true,"default_branch":"main"}}', auth)
remote = f"https://ops:{token}@${forgeAuthority}/{org}/{repo}.git"
machine.succeed(f"cp -r {fixture}/repo.git /tmp/{repo}.git && chmod -R u+w /tmp/{repo}.git")
machine.succeed(f"GIT_SSL_CAINFO={CA} git -C /tmp/{repo}.git push {remote} main")
pushed = machine.succeed(f"GIT_SSL_CAINFO={CA} git ls-remote {remote} refs/heads/main").split()[0]
assert pushed == machine.succeed(f"cat {fixture}/rev").strip(), (repo, pushed)
# The forge asks for a login: the builder's credential helper has to supply it.
machine.fail(f"GIT_SSL_CAINFO={CA} GIT_TERMINAL_PROMPT=0 git ls-remote ${forgeUrl}/internal/meta.git")
machine.succeed(
f"install -d -m 0755 /run/builder-offline && printf %s {token} > ${tokenFile} "
"&& chown builder ${tokenFile} && chmod 0400 ${tokenFile}"
)
with subtest("the toplevel is in the store before the build"):
machine.succeed(f"nix path-info {TOPLEVEL}")
# The control for the lookup: a path that is not there is not found.
machine.fail("nix path-info /nix/store/00000000000000000000000000000000-absent")
with subtest("the builder's build finishes with nothing built"):
status, _ = machine.execute(
"runuser -u builder -- timeout 600 /run/current-system/sw/bin/meta-build >/tmp/out 2>/tmp/err"
)
out = machine.succeed("cat /tmp/out").strip()
err = machine.succeed("cat /tmp/err")
assert status == 0, f"nix exited {status}:\n{err}"
assert out == TOPLEVEL, f"printed {out}, prebuilt {TOPLEVEL}"
assert not re.search(BUILD_PATTERN, err), err
with subtest("control: a derivation that does build is found by the same pattern"):
machine.succeed("grep -x 'max-jobs = 0' /etc/nix/nix.conf")
machine.succeed(
"cp --remove-destination $(readlink -f /etc/nix/nix.conf) /etc/nix/nix.conf "
"&& sed -i 's/^max-jobs = 0$/max-jobs = 1/' /etc/nix/nix.conf "
"&& systemctl restart nix-daemon.socket nix-daemon.service"
)
machine.succeed(
"NIX_CONFIG='experimental-features = nix-command' "
"nix build --no-link --print-build-logs --impure -f ${control} 2>/tmp/control.err"
)
assert re.search(BUILD_PATTERN, machine.succeed("cat /tmp/control.err"))
'';
}

View file

@ -705,6 +705,25 @@ mod tests {
assert!(credential_helper().contains(&format!("${TOKEN_VAR}"))); assert!(credential_helper().contains(&format!("${TOKEN_VAR}")));
} }
/// `nix/vm-tests/swarm-builder-offline.nix` runs a copy of the build command:
/// its credential helper and `nix build` flags. Change those there when this
/// fails.
#[test]
fn the_vm_test_copy_matches() {
assert_eq!(
credential_helper(),
"!f() { test \"$1\" = get || return 0; p=; h=; \
while IFS='=' read -r k v; do case \"$k\" in '') break ;; protocol) p=$v ;; host) h=$v ;; esac; done; \
test -n \"$SWARM_BUILDER_GIT_HOST\" && test \"$p\" = https && test \"$h\" = \"$SWARM_BUILDER_GIT_HOST\" \
&& printf 'username=builder\\npassword=%s\\n' \"$SWARM_BUILDER_GIT_TOKEN\"; return 0; }; f"
);
assert_eq!(
BUILD_FLAGS.join(" "),
"build --no-link --print-out-paths --no-write-lock-file \
--option accept-flake-config false --print-build-logs"
);
}
#[test] #[test]
fn a_job_that_would_bend_the_reference_is_refused() { fn a_job_that_would_bend_the_reference_is_refused() {
for bad in [ for bad in [