nightly vm test: full single-host swarm setup up to the meta flake
This commit is contained in:
parent
c490c72413
commit
af434e1374
8 changed files with 559 additions and 26 deletions
|
|
@ -13,3 +13,8 @@ helpers.
|
||||||
`testdata/meta-flake.nix.in` is the render for one archetype agent with a
|
`testdata/meta-flake.nix.in` is the render for one archetype agent with a
|
||||||
placeholder for the hyperhive pin. A unit test keeps it equal to `render_flake`'s
|
placeholder for the hyperhive pin. A unit test keeps it equal to `render_flake`'s
|
||||||
output, and the `swarm-builder-offline` VM test (`nix/vm-tests/`) serves it.
|
output, and the `swarm-builder-offline` VM test (`nix/vm-tests/`) serves it.
|
||||||
|
|
||||||
|
`testdata/meta-flake-swarm.nix.in` is the render for two archetype agents with
|
||||||
|
the forwarded values of a single-host swarm as `@<NAME>@` placeholders, kept equal
|
||||||
|
by a unit test the same way. The `full-setup` VM test fills it in with the
|
||||||
|
controller's own settings and builds the agents' toplevels from it.
|
||||||
|
|
|
||||||
|
|
@ -583,6 +583,41 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `testdata/meta-flake-swarm.nix.in` is the flake `nix/vm-tests/full-setup.nix`
|
||||||
|
/// evaluates and serves: two archetype agents, the seven forwarded values a
|
||||||
|
/// single-host swarm without a display name sets, each as `@<NAME>@`, and
|
||||||
|
/// `@HYPERHIVE_URL@` where the pin goes. It must be what [`render_flake`]
|
||||||
|
/// renders for them.
|
||||||
|
#[test]
|
||||||
|
fn the_full_setup_fixture_is_what_render_flake_renders() {
|
||||||
|
let archetype = || {
|
||||||
|
Some(ArchetypeRef {
|
||||||
|
source: "hyperhive".to_owned(),
|
||||||
|
name: "reviewer".to_owned(),
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let vars: Vec<(&'static str, String)> = FORWARDED_VARS
|
||||||
|
.iter()
|
||||||
|
.filter(|name| **name != "HYPERHIVE_SWARM_NAME")
|
||||||
|
.map(|&name| (name, format!("@{name}@")))
|
||||||
|
.collect();
|
||||||
|
let flake = render_flake(
|
||||||
|
"@HYPERHIVE_URL@",
|
||||||
|
&[
|
||||||
|
spec("iris", archetype(), None),
|
||||||
|
spec("ruth", archetype(), None),
|
||||||
|
],
|
||||||
|
&vars,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
flake,
|
||||||
|
include_str!("../testdata/meta-flake-swarm.nix.in"),
|
||||||
|
"update the fixture to the rendered flake"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_rendered_flake_names_the_commit_each_input_is_pinned_to() {
|
fn a_rendered_flake_names_the_commit_each_input_is_pinned_to() {
|
||||||
let flake = rendered();
|
let flake = rendered();
|
||||||
|
|
|
||||||
94
hive-meta-flake/testdata/meta-flake-swarm.nix.in
vendored
Normal file
94
hive-meta-flake/testdata/meta-flake-swarm.nix.in
vendored
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
{
|
||||||
|
description = "hyperhive deployed agents";
|
||||||
|
inputs = {
|
||||||
|
hyperhive.url = "@HYPERHIVE_URL@";
|
||||||
|
nixpkgs.follows = "hyperhive/nixpkgs";
|
||||||
|
};
|
||||||
|
outputs =
|
||||||
|
{ self, hyperhive, ... }@inputs:
|
||||||
|
let
|
||||||
|
mkAgent = { name, port, archetype ? null }:
|
||||||
|
let
|
||||||
|
base = hyperhive.nixosConfigurations.agent-base;
|
||||||
|
# Every flake an archetype can come from, by its source name.
|
||||||
|
archetypeSources = { inherit hyperhive; };
|
||||||
|
agentModule =
|
||||||
|
if archetype == null then
|
||||||
|
inputs."agent-${name}".nixosModules.default
|
||||||
|
else
|
||||||
|
(archetypeSources.${archetype.source}
|
||||||
|
or (throw "agent ${name}: archetype source ${archetype.source} is not an input of this hive's meta flake")
|
||||||
|
).hyperhiveArchetypes.${archetype.name}.module
|
||||||
|
or (throw "agent ${name}: this hive's ${archetype.source} input has no archetype ${archetype.name}; update that input");
|
||||||
|
service = "hive-agent";
|
||||||
|
# One attrset for `environment.variables` (login shells),
|
||||||
|
# `systemd.globalEnvironment` (every unit) and the harness service,
|
||||||
|
# so the three cannot disagree. The forwarded vars come from the host.
|
||||||
|
agentEnv = {
|
||||||
|
HIVE_LABEL = name;
|
||||||
|
HYPERHIVE_STATE_DIR = "/agents/${name}/state";
|
||||||
|
HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
|
||||||
|
HIVE_AGENT_BAO_ADDR = "@HIVE_AGENT_BAO_ADDR@";
|
||||||
|
HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = "@HIVE_AGENT_BAO_SERVICES_PKI_MOUNT@";
|
||||||
|
HIVE_AGENT_LOGS_QUERY_URL = "@HIVE_AGENT_LOGS_QUERY_URL@";
|
||||||
|
HIVE_AGENT_METRICS_URL = "@HIVE_AGENT_METRICS_URL@";
|
||||||
|
HIVE_FORGE_URL = "@HIVE_FORGE_URL@";
|
||||||
|
HIVE_FORGE_PUBLIC_URL = "@HIVE_FORGE_PUBLIC_URL@";
|
||||||
|
HIVE_MATRIX_URL = "@HIVE_MATRIX_URL@";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
base.extendModules {
|
||||||
|
modules = [
|
||||||
|
agentModule
|
||||||
|
{
|
||||||
|
services.hyperhive.agent.bao.addr = "@HIVE_AGENT_BAO_ADDR@";
|
||||||
|
services.hyperhive.agent.bao.servicesPkiMount = "@HIVE_AGENT_BAO_SERVICES_PKI_MOUNT@";
|
||||||
|
services.hyperhive.agent.logs.queryUrl = "@HIVE_AGENT_LOGS_QUERY_URL@";
|
||||||
|
services.hyperhive.agent.metrics.url = "@HIVE_AGENT_METRICS_URL@";
|
||||||
|
services.hyperhive.agent.forge.url = "@HIVE_FORGE_URL@";
|
||||||
|
services.hyperhive.agent.matrix.url = "@HIVE_MATRIX_URL@";
|
||||||
|
# The harness service inside the container runs as a
|
||||||
|
# non-root unix user named after the agent (`damocles`,
|
||||||
|
# `iris`, `root`, …). UID auto-assigned by NixOS; the
|
||||||
|
# per-agent override here is what makes
|
||||||
|
# `services.hyperhive.agent.user.name` match the agent's identity
|
||||||
|
# instead of the harness default of `"agent"`.
|
||||||
|
services.hyperhive.agent.user.name = name;
|
||||||
|
# Without this an agent container is called `nixos` and ships
|
||||||
|
# every log line under that: nixpkgs sets the hostname from the
|
||||||
|
# attr name in the `containers.<name>` submodule, and an
|
||||||
|
# agent's system is its own `nixosSystem` off this flake, which
|
||||||
|
# never evaluates that option. The machine name, not the
|
||||||
|
# logical one — it is what `container.name` on our metrics and
|
||||||
|
# nspawn itself already call this container, so logs join to
|
||||||
|
# both without a prefix transform.
|
||||||
|
networking.hostName = "h-${name}";
|
||||||
|
programs.git.config.user = {
|
||||||
|
name = name;
|
||||||
|
email = "${name}@hyperhive.local";
|
||||||
|
};
|
||||||
|
# Container-wide env, all from `agentEnv`: every unit and daemon
|
||||||
|
# (the bash-task runner where `hive-forge` and `git` run included)
|
||||||
|
# resolves the state dir and the forwarded service URLs without
|
||||||
|
# redeclaring them. `environment.variables` is what login shells
|
||||||
|
# read; `systemd.globalEnvironment` is the analogue for units.
|
||||||
|
environment.variables = agentEnv;
|
||||||
|
systemd.globalEnvironment = agentEnv;
|
||||||
|
systemd.services.${service}.environment = agentEnv // {
|
||||||
|
HIVE_PORT = toString port;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
# Tells the harness the agent has no config repo of its own.
|
||||||
|
(if archetype == null then { } else {
|
||||||
|
systemd.services.${service}.environment.HIVE_ARCHETYPE = archetype.name;
|
||||||
|
})
|
||||||
|
];
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
nixosConfigurations = {
|
||||||
|
iris = mkAgent { name = "iris"; port = 8178; archetype = { source = "hyperhive"; name = "reviewer"; }; };
|
||||||
|
ruth = mkAgent { name = "ruth"; port = 8232; archetype = { source = "hyperhive"; name = "reviewer"; }; };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -37,15 +37,16 @@ rec {
|
||||||
# `*.toml` / `Cargo.lock` edits rebuild the rust derivations.
|
# `*.toml` / `Cargo.lock` edits rebuild the rust derivations.
|
||||||
#
|
#
|
||||||
# The two other files in it are the repo's own review-classifier rules and
|
# The two other files in it are the repo's own review-classifier rules and
|
||||||
# the meta flake fixture of the swarm-builder-offline VM test. A
|
# the meta flake fixtures of the swarm-builder-offline and full-setup VM tests. A
|
||||||
# `swarm-controller` test parses the rules and a `hive-meta-flake` test
|
# `swarm-controller` test parses the rules and a `hive-meta-flake` test
|
||||||
# compares the fixture, so both must be in the sandbox.
|
# compares the fixtures, so all must be in the sandbox.
|
||||||
cleanSrc = lib.fileset.toSource {
|
cleanSrc = lib.fileset.toSource {
|
||||||
root = ../.;
|
root = ../.;
|
||||||
fileset = lib.fileset.unions [
|
fileset = lib.fileset.unions [
|
||||||
(craneLib.fileset.commonCargoSources ../.)
|
(craneLib.fileset.commonCargoSources ../.)
|
||||||
../.hyperhive/review-classifier.yml
|
../.hyperhive/review-classifier.yml
|
||||||
../hive-meta-flake/testdata/meta-flake.nix.in
|
../hive-meta-flake/testdata/meta-flake.nix.in
|
||||||
|
../hive-meta-flake/testdata/meta-flake-swarm.nix.in
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,4 +11,5 @@
|
||||||
wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
|
wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
|
||||||
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
|
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
|
||||||
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };
|
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };
|
||||||
|
full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
386
nix/vm-tests/full-setup.nix
Normal file
386
nix/vm-tests/full-setup.nix
Normal file
|
|
@ -0,0 +1,386 @@
|
||||||
|
# `vmTests.<system>.full-setup`: one VM runs a whole single-host swarm
|
||||||
|
# (`deploy.singleHostSwarm`) with no network and no substituter, and a script
|
||||||
|
# walks docs/1-getting-started/setup.md until the controller has committed the
|
||||||
|
# agents' meta flake to the swarm's forge.
|
||||||
|
#
|
||||||
|
# Steps of setup.md, as the script runs them:
|
||||||
|
# 1 secret store `bao operator init`, the granter bootstrap with the root
|
||||||
|
# token, then the policy units reset and restarted.
|
||||||
|
# 4 unseal `bao operator unseal` (`deploy.bao.seal` is "shamir"; pkcs11
|
||||||
|
# needs a TPM). A pkcs11 store unseals itself at init, so here
|
||||||
|
# the unseal comes right after init, before the granter.
|
||||||
|
# 2 SSO account `swarmctl user add mara --group admins`.
|
||||||
|
# 8 ruth `POST /api/agents` on the controller socket, as the swarm
|
||||||
|
# UI's create form does, plus `manage_root_agent`.
|
||||||
|
# 9 first agent The same endpoint, for `iris`: `swarmctl agent create` has no
|
||||||
|
# archetype flag, and a custom agent's config repo would need
|
||||||
|
# inputs the VM cannot resolve offline.
|
||||||
|
# Not scripted yet; a later slice adds them: step 3, the first SSO login to the
|
||||||
|
# forge (an OIDC code flow against authelia), `swarmctl forge make-admin`, and
|
||||||
|
# the agents' deploy up to the point where each says it needs a login. Green
|
||||||
|
# when `internal/meta` on the forge holds the flake the toplevels were built from.
|
||||||
|
#
|
||||||
|
# Offline: the test seeds the swarm's forge with `hyperhive/hyperhive` (one
|
||||||
|
# commit of `self`, ./hyperhive-fixture-repo.nix). The controller renders
|
||||||
|
# `internal/meta` and the builder locks and builds it as in production.
|
||||||
|
# `hive-meta-flake/testdata/meta-flake-swarm.nix.in`, filled in with the
|
||||||
|
# controller's settings, is that flake: the agents' toplevels are built from it
|
||||||
|
# here, so the builder finds them in the store. The daemon runs with
|
||||||
|
# `max-jobs = 0`, so a missing toplevel fails the build instead of compiling.
|
||||||
|
{ pkgs, self }:
|
||||||
|
let
|
||||||
|
inherit (pkgs) lib;
|
||||||
|
|
||||||
|
hive = "pr1ma";
|
||||||
|
swarmDomain = "swarm.test";
|
||||||
|
agents = [
|
||||||
|
"iris"
|
||||||
|
"ruth"
|
||||||
|
];
|
||||||
|
|
||||||
|
inherit (import ./hyperhive-fixture-repo.nix { inherit pkgs self; }) hyperhiveRepo;
|
||||||
|
|
||||||
|
metaTemplate = ../../hive-meta-flake/testdata/meta-flake-swarm.nix.in;
|
||||||
|
|
||||||
|
# The values the fixture has a placeholder for. A swarm whose controller sends
|
||||||
|
# another set renders another flake, so evaluation stops there.
|
||||||
|
forwardedVars = [
|
||||||
|
"HIVE_AGENT_BAO_ADDR"
|
||||||
|
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT"
|
||||||
|
"HIVE_AGENT_LOGS_QUERY_URL"
|
||||||
|
"HIVE_AGENT_METRICS_URL"
|
||||||
|
"HIVE_FORGE_PUBLIC_URL"
|
||||||
|
"HIVE_FORGE_URL"
|
||||||
|
"HIVE_MATRIX_URL"
|
||||||
|
];
|
||||||
|
|
||||||
|
# The meta flake the controller renders for `agents`, from its settings JSON
|
||||||
|
# (`SWARM_CONTROLLER_META_FLAKE`). `hyperhive` is `self`: the agents' toplevels
|
||||||
|
# do not depend on how the input is pinned.
|
||||||
|
filled =
|
||||||
|
settingsJson: pin:
|
||||||
|
let
|
||||||
|
settings = builtins.fromJSON settingsJson;
|
||||||
|
names = [ "HYPERHIVE_URL" ] ++ forwardedVars;
|
||||||
|
values = [ pin ] ++ map (name: settings.vars.${name}) forwardedVars;
|
||||||
|
in
|
||||||
|
assert lib.assertMsg
|
||||||
|
(builtins.attrNames settings.vars == forwardedVars && !settings.github_disabled)
|
||||||
|
"the controller's meta flake settings changed: update hive-meta-flake/testdata/meta-flake-swarm.nix.in";
|
||||||
|
builtins.replaceStrings (map (name: "@${name}@") names) values (builtins.readFile metaTemplate);
|
||||||
|
|
||||||
|
toplevels =
|
||||||
|
settingsJson:
|
||||||
|
let
|
||||||
|
flake = import (builtins.toFile "meta-flake.nix" (filled settingsJson "unused"));
|
||||||
|
evaluated = flake.outputs {
|
||||||
|
self = null;
|
||||||
|
hyperhive = self;
|
||||||
|
};
|
||||||
|
in
|
||||||
|
map (agent: evaluated.nixosConfigurations.${agent}.config.system.build.toplevel) agents;
|
||||||
|
in
|
||||||
|
pkgs.testers.runNixOSTest (
|
||||||
|
{ config, ... }:
|
||||||
|
let
|
||||||
|
inherit (config.nodes.machine.services.hyperhive) swarm deploy;
|
||||||
|
settingsJson =
|
||||||
|
config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
||||||
|
controllerSocket = deploy.swarm-controller.socketPath;
|
||||||
|
|
||||||
|
# A derivation no store holds, to show the daemon refuses to build.
|
||||||
|
control = pkgs.writeText "control.nix" ''
|
||||||
|
derivation {
|
||||||
|
name = "build-control";
|
||||||
|
system = "${pkgs.stdenv.hostPlatform.system}";
|
||||||
|
builder = "/bin/sh";
|
||||||
|
args = [ "-c" "echo built > $out" ];
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
{
|
||||||
|
name = "full-setup";
|
||||||
|
|
||||||
|
# Boot of fourteen containers and the setup steps, with the waits' own
|
||||||
|
# bounds well inside.
|
||||||
|
globalTimeout = 7200;
|
||||||
|
|
||||||
|
nodes.machine =
|
||||||
|
{ config, ... }:
|
||||||
|
{
|
||||||
|
imports = [ self.nixosModules.default ];
|
||||||
|
|
||||||
|
services.hyperhive = {
|
||||||
|
deploy.singleHostSwarm = true;
|
||||||
|
deploy.hive-controller.enable = true;
|
||||||
|
deploy.hive-controller.hiveName = hive;
|
||||||
|
deploy.bao.seal = "shamir";
|
||||||
|
swarm.domain = swarmDomain;
|
||||||
|
swarm.hives.${hive} = { };
|
||||||
|
};
|
||||||
|
system.stateVersion = "25.11";
|
||||||
|
|
||||||
|
# 12 GB: fourteen service containers, the hive and a Rust-heavy agent
|
||||||
|
# closure through the controller, the builder and the cache. 4 cores: the
|
||||||
|
# containers boot in parallel. The disk holds the cache's copy of the
|
||||||
|
# closure; the store overlay is on disk, not in memory.
|
||||||
|
virtualisation.memorySize = 12288;
|
||||||
|
virtualisation.cores = 4;
|
||||||
|
virtualisation.diskSize = 40960;
|
||||||
|
virtualisation.writableStoreUseTmpfs = false;
|
||||||
|
|
||||||
|
# The toplevels, and every source tree the meta flake's lock names, keyed
|
||||||
|
# by narHash: nix takes a locked input from the store when its path is
|
||||||
|
# valid.
|
||||||
|
virtualisation.additionalPaths = [
|
||||||
|
self
|
||||||
|
self.inputs.nixpkgs
|
||||||
|
self.inputs.crane
|
||||||
|
self.inputs.treefmt-nix
|
||||||
|
hyperhiveRepo
|
||||||
|
]
|
||||||
|
++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
||||||
|
|
||||||
|
# No substituter, and no build.
|
||||||
|
nix.settings = {
|
||||||
|
substituters = lib.mkForce [ ];
|
||||||
|
max-jobs = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
# The store's raft data is on a tmpfs. Raft syncs every write, so on the
|
||||||
|
# VM's qcow2 root (`cache=writeback`, which passes each guest flush on to
|
||||||
|
# the host's disk) a login's lease write can outlast the 15 s the store's
|
||||||
|
# clients wait, and its calls then fail with `context canceled`. The
|
||||||
|
# state is under `/var/lib/private`, where the store's DynamicUser keeps
|
||||||
|
# it; the mount sits on that directory in the container's root, and
|
||||||
|
# nothing the test checks needs it to survive the VM.
|
||||||
|
fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
|
||||||
|
{
|
||||||
|
device = "tmpfs";
|
||||||
|
fsType = "tmpfs";
|
||||||
|
options = [
|
||||||
|
"mode=0700"
|
||||||
|
"size=1G"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
environment.systemPackages = [
|
||||||
|
pkgs.git
|
||||||
|
pkgs.jq
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
testScript = ''
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
|
||||||
|
API = "http://localhost"
|
||||||
|
SOCKET = "${controllerSocket}"
|
||||||
|
FORGE = "https://${swarm.forge.domain}"
|
||||||
|
HIVE = "${hive}"
|
||||||
|
TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))}
|
||||||
|
INIT = "/root/bao-init.json"
|
||||||
|
FORGE_AUTH = "/root/forge-auth.header"
|
||||||
|
NIX = "nix --extra-experimental-features nix-command"
|
||||||
|
ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
|
||||||
|
POLICY_UNITS_ACTIVE = (
|
||||||
|
"for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
|
||||||
|
"swarm-bao-agent-pki.service; do "
|
||||||
|
"[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done"
|
||||||
|
)
|
||||||
|
|
||||||
|
def controller(method, path, body=None, check="--fail-with-body"):
|
||||||
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
||||||
|
return machine.succeed(
|
||||||
|
f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def forge(method, path, body=None):
|
||||||
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
||||||
|
return machine.succeed(
|
||||||
|
f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} "
|
||||||
|
f"-X {method} {FORGE}/api/v1{path} {data}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def diagnose(unit=None, container=None):
|
||||||
|
# What a timed-out wait leaves behind: the failed units, and the status
|
||||||
|
# and the end of the journal of the unit that was waited on.
|
||||||
|
where = f"--machine={container} " if container else ""
|
||||||
|
units = [unit] if isinstance(unit, str) else list(unit or [])
|
||||||
|
flags = " ".join(f"-u '{u}'" for u in units)
|
||||||
|
for command in [
|
||||||
|
f"systemctl {where}--failed --no-legend --no-pager",
|
||||||
|
*([f"systemctl {where}status --no-pager -n 0 -l {' '.join(units)} | head -c 4000"] if units else []),
|
||||||
|
*([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []),
|
||||||
|
]:
|
||||||
|
print(f"--- {command}")
|
||||||
|
print(machine.execute(command)[1])
|
||||||
|
|
||||||
|
def settle(command, timeout, unit=None, container=None):
|
||||||
|
try:
|
||||||
|
machine.wait_until_succeeds(command, timeout=timeout)
|
||||||
|
except Exception:
|
||||||
|
diagnose(unit, container)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def create_agent(body):
|
||||||
|
# The controller refuses a create until its dependencies answer (the
|
||||||
|
# archetype list comes from the builder's evaluation), so ask again.
|
||||||
|
for _ in range(60):
|
||||||
|
status, out = machine.execute(
|
||||||
|
f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' "
|
||||||
|
f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}"
|
||||||
|
)
|
||||||
|
if out.strip().startswith("2"):
|
||||||
|
return
|
||||||
|
machine.sleep(10)
|
||||||
|
diagnose("swarm-controller")
|
||||||
|
raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {machine.succeed('cat /root/create.out')}")
|
||||||
|
|
||||||
|
machine.start()
|
||||||
|
machine.wait_for_unit("multi-user.target")
|
||||||
|
|
||||||
|
with subtest("the agents' toplevels are in the store, and the daemon builds nothing"):
|
||||||
|
for toplevel in TOPLEVELS:
|
||||||
|
machine.succeed(f"{NIX} path-info {toplevel}")
|
||||||
|
# The control for the lookup: a path that is not there is not found.
|
||||||
|
machine.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent")
|
||||||
|
# The control for the missing build: a derivation no store holds is refused.
|
||||||
|
machine.fail("NIX_CONFIG='experimental-features = nix-command' nix build --no-link --impure -f ${control}")
|
||||||
|
|
||||||
|
# setup.md step 1: the secret store.
|
||||||
|
with subtest("step 1: bao operator init"):
|
||||||
|
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
||||||
|
machine.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")
|
||||||
|
|
||||||
|
with subtest("step 4: bao operator unseal, with the shamir seal"):
|
||||||
|
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
||||||
|
machine.succeed("bao status")
|
||||||
|
|
||||||
|
with subtest("step 1: the granter bootstrap"):
|
||||||
|
machine.succeed(
|
||||||
|
"set -o pipefail; "
|
||||||
|
f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
|
||||||
|
"bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
|
||||||
|
"bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
|
||||||
|
"| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
|
||||||
|
"unset BAO_TOKEN; "
|
||||||
|
"systemctl restart swarm-bao-granter-role"
|
||||||
|
)
|
||||||
|
machine.wait_until_succeeds(
|
||||||
|
"journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'",
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
|
||||||
|
with subtest("step 1: the policy units that waited for the granter are restarted"):
|
||||||
|
# Some of these units retry every 30 seconds until something outside
|
||||||
|
# them exists: the bao UI's viewer policy waits for authelia's secrets
|
||||||
|
# unit, in its container, to publish the UI's OIDC client secret, and
|
||||||
|
# that unit waits for the secret publisher's policy written here. So
|
||||||
|
# the units are started without waiting, and polled until all are active.
|
||||||
|
machine.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true")
|
||||||
|
machine.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service")
|
||||||
|
machine.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=600)
|
||||||
|
# Under the shamir seal the PKI unit leaves a restart of the store to the
|
||||||
|
# operator when its listener's client CA changed, and says which command.
|
||||||
|
if machine.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
|
||||||
|
machine.succeed("systemctl --machine=swarm-bao restart openbao.service")
|
||||||
|
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
||||||
|
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
||||||
|
settle(POLICY_UNITS_ACTIVE, 1200, "swarm-bao-*-policy")
|
||||||
|
machine.succeed(
|
||||||
|
"journalctl -u swarm-bao-controller-policy --no-pager "
|
||||||
|
"| grep -F 'Data written to: auth/cert/certs/swarm-controller'"
|
||||||
|
)
|
||||||
|
|
||||||
|
with subtest("authelia's mint has published the clients' secrets"):
|
||||||
|
# The operator viewer policy is left out of the poll above: it waits for
|
||||||
|
# the UI client's secret, which authelia's secrets unit publishes from
|
||||||
|
# its container once the store and the trust bundle it needs exist. The
|
||||||
|
# controller's client secret is published by the same unit, and the
|
||||||
|
# controller, the hive's store logins and the UI policy all wait for it.
|
||||||
|
settle(
|
||||||
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
||||||
|
"secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
|
||||||
|
1200,
|
||||||
|
"authelia-swarm-secrets",
|
||||||
|
"swarm-authelia",
|
||||||
|
)
|
||||||
|
settle("[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 300, "swarm-bao-operator-viewer-policy")
|
||||||
|
|
||||||
|
with subtest("step 2: swarmctl user add"):
|
||||||
|
machine.wait_until_succeeds(
|
||||||
|
"swarmctl user add mara --display-name Mara --email mara@example.com --group admins "
|
||||||
|
">/root/user-add.out 2>&1",
|
||||||
|
timeout=900,
|
||||||
|
)
|
||||||
|
machine.succeed("[ -s /root/user-add.out ]")
|
||||||
|
machine.succeed("swarmctl user list | grep -w mara")
|
||||||
|
|
||||||
|
with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
|
||||||
|
# The controller's forge token, from the store as the controller reads it.
|
||||||
|
# Minted and stored by `forgejo-swarm-controller-account` in the forge
|
||||||
|
# container, which waits for forgejo and for a login to the store.
|
||||||
|
settle(
|
||||||
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
||||||
|
"secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
|
||||||
|
900,
|
||||||
|
["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],
|
||||||
|
"hive-forge",
|
||||||
|
)
|
||||||
|
machine.succeed(
|
||||||
|
f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}"
|
||||||
|
)
|
||||||
|
# The token being published says nothing about the gateway's vhost for
|
||||||
|
# the forge, which the calls below go through.
|
||||||
|
machine.wait_until_succeeds(
|
||||||
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600
|
||||||
|
)
|
||||||
|
status = machine.succeed(
|
||||||
|
f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive"
|
||||||
|
).strip()
|
||||||
|
if status != "200":
|
||||||
|
forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"})
|
||||||
|
forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"})
|
||||||
|
machine.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git")
|
||||||
|
machine.succeed(
|
||||||
|
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
||||||
|
f"push {FORGE}/hyperhive/hyperhive.git main"
|
||||||
|
)
|
||||||
|
pushed = machine.succeed(
|
||||||
|
f"GIT_SSL_NO_VERIFY=1 git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
||||||
|
f"ls-remote {FORGE}/hyperhive/hyperhive.git refs/heads/main"
|
||||||
|
).split()[0]
|
||||||
|
assert pushed == machine.succeed("cat ${hyperhiveRepo}/rev").strip(), pushed
|
||||||
|
|
||||||
|
with subtest("step 8: ruth, through the endpoint the swarm UI calls"):
|
||||||
|
machine.wait_until_succeeds(
|
||||||
|
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=900
|
||||||
|
)
|
||||||
|
default = json.loads(controller("GET", "/api/permissions"))["default"]
|
||||||
|
create_agent({"name": "ruth", "hive": HIVE, "permissions": default + ["manage_root_agent"], "archetype": ARCHETYPE})
|
||||||
|
|
||||||
|
with subtest("step 9: the first agent"):
|
||||||
|
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
|
||||||
|
|
||||||
|
with subtest("internal/meta holds the flake the toplevels were built from"):
|
||||||
|
settle(
|
||||||
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/internal/meta/raw/flake.nix "
|
||||||
|
">/root/meta-flake.nix",
|
||||||
|
1800,
|
||||||
|
"swarm-controller",
|
||||||
|
)
|
||||||
|
rev = machine.succeed("cat ${hyperhiveRepo}/rev").strip()
|
||||||
|
settings = json.loads(${builtins.toJSON settingsJson})
|
||||||
|
expected = open("${metaTemplate}").read().replace(
|
||||||
|
"@HYPERHIVE_URL@", f"git+{settings['git_base']}/hyperhive/hyperhive.git?rev={rev}"
|
||||||
|
)
|
||||||
|
for name, value in settings["vars"].items():
|
||||||
|
expected = expected.replace(f"@{name}@", value)
|
||||||
|
committed = machine.succeed("cat /root/meta-flake.nix")
|
||||||
|
assert committed == expected, "the controller's flake differs from the one the toplevels came from:\n" + committed
|
||||||
|
|
||||||
|
# The agents' deploy and the check that they need a login are not scripted yet.
|
||||||
|
'';
|
||||||
|
}
|
||||||
|
)
|
||||||
33
nix/vm-tests/hyperhive-fixture-repo.nix
Normal file
33
nix/vm-tests/hyperhive-fixture-repo.nix
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
# The `hyperhive/hyperhive` repo the offline VM tests put on their forge: one
|
||||||
|
# commit holding the source tree of `self`, with a fixed author and date, so its
|
||||||
|
# id depends on the tree alone.
|
||||||
|
{ pkgs, self }:
|
||||||
|
let
|
||||||
|
commitEnv = ''
|
||||||
|
export HOME=$TMPDIR
|
||||||
|
export GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@forge.test
|
||||||
|
export GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@forge.test
|
||||||
|
export GIT_AUTHOR_DATE="946684800 +0000" GIT_COMMITTER_DATE="946684800 +0000"
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
{
|
||||||
|
inherit commitEnv;
|
||||||
|
|
||||||
|
# `repo.git` (bare, branch `main`) and `rev` (the commit id).
|
||||||
|
hyperhiveRepo =
|
||||||
|
pkgs.runCommand "hyperhive-fixture-repo"
|
||||||
|
{
|
||||||
|
nativeBuildInputs = [ pkgs.git ];
|
||||||
|
}
|
||||||
|
''
|
||||||
|
${commitEnv}
|
||||||
|
cp -a ${self} work
|
||||||
|
chmod -R u+w work
|
||||||
|
git -C work init -q -b main
|
||||||
|
git -C work add -f .
|
||||||
|
git -C work commit -q -m "hyperhive source tree"
|
||||||
|
mkdir $out
|
||||||
|
git clone -q --bare work $out/repo.git
|
||||||
|
git -C work rev-parse HEAD > $out/rev
|
||||||
|
'';
|
||||||
|
}
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
# prebuilds: two deterministic one-commit repos and one toplevel, all built from
|
# prebuilds: two deterministic one-commit repos and one toplevel, all built from
|
||||||
# `self`.
|
# `self`.
|
||||||
#
|
#
|
||||||
# hyperhive/hyperhive the source tree of `self`
|
# hyperhive/hyperhive the source tree of `self` (./hyperhive-fixture-repo.nix)
|
||||||
# internal/meta the meta flake of one archetype agent, pinned to the
|
# internal/meta the meta flake of one archetype agent, pinned to the
|
||||||
# first repo's commit
|
# first repo's commit
|
||||||
#
|
#
|
||||||
|
|
@ -29,29 +29,7 @@ let
|
||||||
hyperhive = self;
|
hyperhive = self;
|
||||||
}).nixosConfigurations.${agent}.config.system.build.toplevel;
|
}).nixosConfigurations.${agent}.config.system.build.toplevel;
|
||||||
|
|
||||||
commitEnv = ''
|
inherit (import ./hyperhive-fixture-repo.nix { inherit pkgs self; }) commitEnv hyperhiveRepo;
|
||||||
export HOME=$TMPDIR
|
|
||||||
export GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@forge.test
|
|
||||||
export GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@forge.test
|
|
||||||
export GIT_AUTHOR_DATE="946684800 +0000" GIT_COMMITTER_DATE="946684800 +0000"
|
|
||||||
'';
|
|
||||||
|
|
||||||
hyperhiveRepo =
|
|
||||||
pkgs.runCommand "hyperhive-fixture-repo"
|
|
||||||
{
|
|
||||||
nativeBuildInputs = [ pkgs.git ];
|
|
||||||
}
|
|
||||||
''
|
|
||||||
${commitEnv}
|
|
||||||
cp -a ${self} work
|
|
||||||
chmod -R u+w work
|
|
||||||
git -C work init -q -b main
|
|
||||||
git -C work add -f .
|
|
||||||
git -C work commit -q -m "hyperhive source tree"
|
|
||||||
mkdir $out
|
|
||||||
git clone -q --bare work $out/repo.git
|
|
||||||
git -C work rev-parse HEAD > $out/rev
|
|
||||||
'';
|
|
||||||
|
|
||||||
# `nix flake lock`'s result for the template: hyperhive's own nodes next to a
|
# `nix flake lock`'s result for the template: hyperhive's own nodes next to a
|
||||||
# `hyperhive` node pinned by URL and rev, whose follows paths start at the
|
# `hyperhive` node pinned by URL and rev, whose follows paths start at the
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue