151 lines
6.2 KiB
Nix
151 lines
6.2 KiB
Nix
# `vmTests.<system>.full-setup`: one VM runs a whole single-host swarm
|
|
# (`deploy.singleHostSwarm`) with no network and no substituter, and a script walks
|
|
# docs/1-getting-started/setup.md until the agents are deployed and need a login.
|
|
#
|
|
# Steps of setup.md, as the script runs them:
|
|
# 1 secret store `bao operator init`, the granter bootstrap with the root
|
|
# token, then the policy units reset and restarted.
|
|
# 4 unseal `bao operator unseal` (`deploy.bao.seal` is "shamir"; pkcs11
|
|
# needs a TPM). A pkcs11 store unseals itself at init, so here
|
|
# the unseal comes right after init, before the granter.
|
|
# 2 SSO account `swarmctl user add mara --group admins`.
|
|
# 8 spawn `POST /api/agents` on the controller socket, as the swarm
|
|
# UI's create form does, with the reviewer archetype's defaults.
|
|
# 9 first agent The same endpoint, for `iris`: `swarmctl agent create` has no
|
|
# archetype flag, and a custom agent's config repo would need
|
|
# inputs the VM cannot resolve offline.
|
|
# `swarmctl setup status` is read before the agents deploy: steps 1-6 done; step 9
|
|
# (the product's hard-coded `ruth`) stays undone, as no root agent exists here. Not
|
|
# scripted yet: step 3, the first SSO login to the forge (an OIDC code flow against
|
|
# authelia), and `swarmctl forge make-admin`.
|
|
# Green when `GET /api/agents/status` on the controller socket lists `iris` and
|
|
# `spawn` with `freshness` fresh and `needs_login` true.
|
|
#
|
|
# Offline: the test seeds the swarm's forge with `hyperhive/hyperhive` (one
|
|
# commit of `self`, ./hyperhive-fixture-repo.nix). The controller renders
|
|
# `internal/meta` and the builder locks and builds it as in production.
|
|
# `hive-meta-flake/testdata/meta-flake-swarm.nix.in`, filled in with the
|
|
# controller's settings, is that flake: the agents' toplevels are built from it
|
|
# here, so the builder finds them in the store. The daemon runs with
|
|
# `max-jobs = 0`, so a missing toplevel fails the build instead of compiling.
|
|
{ pkgs, self }:
|
|
let
|
|
inherit (pkgs) lib;
|
|
|
|
hive = "pr1ma";
|
|
swarmDomain = "swarm.test";
|
|
agents = [
|
|
"iris"
|
|
"spawn"
|
|
];
|
|
|
|
inherit (import ./agent-toplevels.nix { inherit pkgs self agents; })
|
|
hyperhiveRepo
|
|
metaTemplate
|
|
sourcePaths
|
|
toplevels
|
|
;
|
|
|
|
# A derivation no store holds, to show the daemon refuses to build.
|
|
control = pkgs.writeText "control.nix" ''
|
|
derivation {
|
|
name = "build-control";
|
|
system = "${pkgs.stdenv.hostPlatform.system}";
|
|
builder = "/bin/sh";
|
|
args = [ "-c" "echo built > $out" ];
|
|
}
|
|
'';
|
|
|
|
test = pkgs.testers.runNixOSTest (
|
|
{ config, ... }:
|
|
{
|
|
name = "full-setup";
|
|
|
|
# Boot of fourteen containers and the setup steps, with the waits' own
|
|
# bounds well inside.
|
|
globalTimeout = 7200;
|
|
|
|
nodes.machine =
|
|
{ config, ... }:
|
|
{
|
|
imports = [ self.nixosModules.default ];
|
|
|
|
services.hyperhive = {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.hive-controller.enable = true;
|
|
deploy.hive-controller.hiveName = hive;
|
|
deploy.bao.seal = "shamir";
|
|
# The test pushes its own hyperhive/hyperhive; a mirror of the
|
|
# upstream could not be created offline and would race the push.
|
|
deploy.forgejo.hyperhiveUpstream = null;
|
|
swarm.domain = swarmDomain;
|
|
swarm.hives.${hive} = { };
|
|
};
|
|
system.stateVersion = "25.11";
|
|
|
|
# 12 GB: fourteen service containers, the hive and a Rust-heavy agent
|
|
# closure through the controller, the builder and the cache. 4 cores: the
|
|
# containers boot in parallel. The disk holds the cache's copy of the
|
|
# closure; the store overlay is on disk, not in memory.
|
|
virtualisation.memorySize = 12288;
|
|
virtualisation.cores = 4;
|
|
virtualisation.diskSize = 40960;
|
|
virtualisation.writableStoreUseTmpfs = false;
|
|
|
|
# The toplevels, and every source tree the meta flake's lock names.
|
|
virtualisation.additionalPaths =
|
|
sourcePaths
|
|
++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
|
|
|
# No substituter, and no build.
|
|
nix.settings = {
|
|
substituters = lib.mkForce [ ];
|
|
max-jobs = 0;
|
|
};
|
|
|
|
# The store's raft data is on a tmpfs. Raft syncs every write, so on the
|
|
# VM's qcow2 root (`cache=writeback`, which passes each guest flush on to
|
|
# the host's disk) a login's lease write can outlast the 15 s the store's
|
|
# clients wait, and its calls then fail with `context canceled`. The
|
|
# state is under `/var/lib/private`, where the store's DynamicUser keeps
|
|
# it; the mount sits on that directory in the container's root, and
|
|
# nothing the test checks needs it to survive the VM.
|
|
fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
|
|
{
|
|
device = "tmpfs";
|
|
fsType = "tmpfs";
|
|
options = [
|
|
"mode=0700"
|
|
"size=1G"
|
|
];
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.jq
|
|
];
|
|
};
|
|
|
|
testScript = builtins.readFile ./full-setup.py;
|
|
}
|
|
);
|
|
|
|
inherit (test.config.nodes.machine.services.hyperhive) swarm deploy;
|
|
settingsJson =
|
|
test.config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
|
in
|
|
# The script is plain Python with no Nix in it: the build-time values reach it
|
|
# as environment variables of the test run, and the commands it runs in the VM
|
|
# find their binaries on the node's PATH.
|
|
test.overrideTestDerivation (old: {
|
|
env = (old.env or { }) // {
|
|
FULL_SETUP_SOCKET = deploy.swarm-controller.socketPath;
|
|
FULL_SETUP_FORGE_DOMAIN = swarm.forge.domain;
|
|
FULL_SETUP_HIVE = hive;
|
|
FULL_SETUP_TOPLEVELS = builtins.toJSON (map toString (toplevels settingsJson));
|
|
FULL_SETUP_META_SETTINGS = settingsJson;
|
|
FULL_SETUP_META_TEMPLATE = "${metaTemplate}";
|
|
FULL_SETUP_HYPERHIVE_REPO = hyperhiveRepo;
|
|
FULL_SETUP_CONTROL_NIX = control;
|
|
};
|
|
})
|