nightly vm test: full single-host swarm setup up to the meta flake
This commit is contained in:
atlas
parent
c490c72413
commit
af434e1374
8 changed files with 559 additions and 26 deletions
|
|
@ -13,3 +13,8 @@ helpers.
|
|||
`testdata/meta-flake.nix.in` is the render for one archetype agent with a
|
||||
placeholder for the hyperhive pin. A unit test keeps it equal to `render_flake`'s
|
||||
output, and the `swarm-builder-offline` VM test (`nix/vm-tests/`) serves it.
|
||||
|
||||
`testdata/meta-flake-swarm.nix.in` is the render for two archetype agents with
|
||||
the forwarded values of a single-host swarm as `@<NAME>@` placeholders, kept equal
|
||||
by a unit test the same way. The `full-setup` VM test fills it in with the
|
||||
controller's own settings and builds the agents' toplevels from it.
|
||||
|
|
|
|||
|
|
@ -583,6 +583,41 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// `testdata/meta-flake-swarm.nix.in` is the flake `nix/vm-tests/full-setup.nix`
|
||||
/// evaluates and serves: two archetype agents, the seven forwarded values a
|
||||
/// single-host swarm without a display name sets, each as `@<NAME>@`, and
|
||||
/// `@HYPERHIVE_URL@` where the pin goes. It must be what [`render_flake`]
|
||||
/// renders for them.
|
||||
#[test]
|
||||
fn the_full_setup_fixture_is_what_render_flake_renders() {
|
||||
let archetype = || {
|
||||
Some(ArchetypeRef {
|
||||
source: "hyperhive".to_owned(),
|
||||
name: "reviewer".to_owned(),
|
||||
})
|
||||
};
|
||||
let vars: Vec<(&'static str, String)> = FORWARDED_VARS
|
||||
.iter()
|
||||
.filter(|name| **name != "HYPERHIVE_SWARM_NAME")
|
||||
.map(|&name| (name, format!("@{name}@")))
|
||||
.collect();
|
||||
let flake = render_flake(
|
||||
"@HYPERHIVE_URL@",
|
||||
&[
|
||||
spec("iris", archetype(), None),
|
||||
spec("ruth", archetype(), None),
|
||||
],
|
||||
&vars,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
flake,
|
||||
include_str!("../testdata/meta-flake-swarm.nix.in"),
|
||||
"update the fixture to the rendered flake"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rendered_flake_names_the_commit_each_input_is_pinned_to() {
|
||||
let flake = rendered();
|
||||
|
|
|
|||
94
hive-meta-flake/testdata/meta-flake-swarm.nix.in
vendored
Normal file
94
hive-meta-flake/testdata/meta-flake-swarm.nix.in
vendored
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
{
|
||||
description = "hyperhive deployed agents";
|
||||
inputs = {
|
||||
hyperhive.url = "@HYPERHIVE_URL@";
|
||||
nixpkgs.follows = "hyperhive/nixpkgs";
|
||||
};
|
||||
outputs =
|
||||
{ self, hyperhive, ... }@inputs:
|
||||
let
|
||||
mkAgent = { name, port, archetype ? null }:
|
||||
let
|
||||
base = hyperhive.nixosConfigurations.agent-base;
|
||||
# Every flake an archetype can come from, by its source name.
|
||||
archetypeSources = { inherit hyperhive; };
|
||||
agentModule =
|
||||
if archetype == null then
|
||||
inputs."agent-${name}".nixosModules.default
|
||||
else
|
||||
(archetypeSources.${archetype.source}
|
||||
or (throw "agent ${name}: archetype source ${archetype.source} is not an input of this hive's meta flake")
|
||||
).hyperhiveArchetypes.${archetype.name}.module
|
||||
or (throw "agent ${name}: this hive's ${archetype.source} input has no archetype ${archetype.name}; update that input");
|
||||
service = "hive-agent";
|
||||
# One attrset for `environment.variables` (login shells),
|
||||
# `systemd.globalEnvironment` (every unit) and the harness service,
|
||||
# so the three cannot disagree. The forwarded vars come from the host.
|
||||
agentEnv = {
|
||||
HIVE_LABEL = name;
|
||||
HYPERHIVE_STATE_DIR = "/agents/${name}/state";
|
||||
HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
|
||||
HIVE_AGENT_BAO_ADDR = "@HIVE_AGENT_BAO_ADDR@";
|
||||
HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = "@HIVE_AGENT_BAO_SERVICES_PKI_MOUNT@";
|
||||
HIVE_AGENT_LOGS_QUERY_URL = "@HIVE_AGENT_LOGS_QUERY_URL@";
|
||||
HIVE_AGENT_METRICS_URL = "@HIVE_AGENT_METRICS_URL@";
|
||||
HIVE_FORGE_URL = "@HIVE_FORGE_URL@";
|
||||
HIVE_FORGE_PUBLIC_URL = "@HIVE_FORGE_PUBLIC_URL@";
|
||||
HIVE_MATRIX_URL = "@HIVE_MATRIX_URL@";
|
||||
};
|
||||
in
|
||||
base.extendModules {
|
||||
modules = [
|
||||
agentModule
|
||||
{
|
||||
services.hyperhive.agent.bao.addr = "@HIVE_AGENT_BAO_ADDR@";
|
||||
services.hyperhive.agent.bao.servicesPkiMount = "@HIVE_AGENT_BAO_SERVICES_PKI_MOUNT@";
|
||||
services.hyperhive.agent.logs.queryUrl = "@HIVE_AGENT_LOGS_QUERY_URL@";
|
||||
services.hyperhive.agent.metrics.url = "@HIVE_AGENT_METRICS_URL@";
|
||||
services.hyperhive.agent.forge.url = "@HIVE_FORGE_URL@";
|
||||
services.hyperhive.agent.matrix.url = "@HIVE_MATRIX_URL@";
|
||||
# The harness service inside the container runs as a
|
||||
# non-root unix user named after the agent (`damocles`,
|
||||
# `iris`, `root`, …). UID auto-assigned by NixOS; the
|
||||
# per-agent override here is what makes
|
||||
# `services.hyperhive.agent.user.name` match the agent's identity
|
||||
# instead of the harness default of `"agent"`.
|
||||
services.hyperhive.agent.user.name = name;
|
||||
# Without this an agent container is called `nixos` and ships
|
||||
# every log line under that: nixpkgs sets the hostname from the
|
||||
# attr name in the `containers.<name>` submodule, and an
|
||||
# agent's system is its own `nixosSystem` off this flake, which
|
||||
# never evaluates that option. The machine name, not the
|
||||
# logical one — it is what `container.name` on our metrics and
|
||||
# nspawn itself already call this container, so logs join to
|
||||
# both without a prefix transform.
|
||||
networking.hostName = "h-${name}";
|
||||
programs.git.config.user = {
|
||||
name = name;
|
||||
email = "${name}@hyperhive.local";
|
||||
};
|
||||
# Container-wide env, all from `agentEnv`: every unit and daemon
|
||||
# (the bash-task runner where `hive-forge` and `git` run included)
|
||||
# resolves the state dir and the forwarded service URLs without
|
||||
# redeclaring them. `environment.variables` is what login shells
|
||||
# read; `systemd.globalEnvironment` is the analogue for units.
|
||||
environment.variables = agentEnv;
|
||||
systemd.globalEnvironment = agentEnv;
|
||||
systemd.services.${service}.environment = agentEnv // {
|
||||
HIVE_PORT = toString port;
|
||||
};
|
||||
}
|
||||
# Tells the harness the agent has no config repo of its own.
|
||||
(if archetype == null then { } else {
|
||||
systemd.services.${service}.environment.HIVE_ARCHETYPE = archetype.name;
|
||||
})
|
||||
];
|
||||
};
|
||||
in
|
||||
{
|
||||
nixosConfigurations = {
|
||||
iris = mkAgent { name = "iris"; port = 8178; archetype = { source = "hyperhive"; name = "reviewer"; }; };
|
||||
ruth = mkAgent { name = "ruth"; port = 8232; archetype = { source = "hyperhive"; name = "reviewer"; }; };
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -37,15 +37,16 @@ rec {
|
|||
# `*.toml` / `Cargo.lock` edits rebuild the rust derivations.
|
||||
#
|
||||
# The two other files in it are the repo's own review-classifier rules and
|
||||
# the meta flake fixture of the swarm-builder-offline VM test. A
|
||||
# the meta flake fixtures of the swarm-builder-offline and full-setup VM tests. A
|
||||
# `swarm-controller` test parses the rules and a `hive-meta-flake` test
|
||||
# compares the fixture, so both must be in the sandbox.
|
||||
# compares the fixtures, so all must be in the sandbox.
|
||||
cleanSrc = lib.fileset.toSource {
|
||||
root = ../.;
|
||||
fileset = lib.fileset.unions [
|
||||
(craneLib.fileset.commonCargoSources ../.)
|
||||
../.hyperhive/review-classifier.yml
|
||||
../hive-meta-flake/testdata/meta-flake.nix.in
|
||||
../hive-meta-flake/testdata/meta-flake-swarm.nix.in
|
||||
];
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -11,4 +11,5 @@
|
|||
wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
|
||||
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
|
||||
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };
|
||||
full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; };
|
||||
}
|
||||
|
|
|
|||
386
nix/vm-tests/full-setup.nix
Normal file
386
nix/vm-tests/full-setup.nix
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
# `vmTests.<system>.full-setup`: one VM runs a whole single-host swarm
|
||||
# (`deploy.singleHostSwarm`) with no network and no substituter, and a script
|
||||
# walks docs/1-getting-started/setup.md until the controller has committed the
|
||||
# agents' meta flake to the swarm's forge.
|
||||
#
|
||||
# Steps of setup.md, as the script runs them:
|
||||
# 1 secret store `bao operator init`, the granter bootstrap with the root
|
||||
# token, then the policy units reset and restarted.
|
||||
# 4 unseal `bao operator unseal` (`deploy.bao.seal` is "shamir"; pkcs11
|
||||
# needs a TPM). A pkcs11 store unseals itself at init, so here
|
||||
# the unseal comes right after init, before the granter.
|
||||
# 2 SSO account `swarmctl user add mara --group admins`.
|
||||
# 8 ruth `POST /api/agents` on the controller socket, as the swarm
|
||||
# UI's create form does, plus `manage_root_agent`.
|
||||
# 9 first agent The same endpoint, for `iris`: `swarmctl agent create` has no
|
||||
# archetype flag, and a custom agent's config repo would need
|
||||
# inputs the VM cannot resolve offline.
|
||||
# Not scripted yet; a later slice adds them: step 3, the first SSO login to the
|
||||
# forge (an OIDC code flow against authelia), `swarmctl forge make-admin`, and
|
||||
# the agents' deploy up to the point where each says it needs a login. Green
|
||||
# when `internal/meta` on the forge holds the flake the toplevels were built from.
|
||||
#
|
||||
# Offline: the test seeds the swarm's forge with `hyperhive/hyperhive` (one
|
||||
# commit of `self`, ./hyperhive-fixture-repo.nix). The controller renders
|
||||
# `internal/meta` and the builder locks and builds it as in production.
|
||||
# `hive-meta-flake/testdata/meta-flake-swarm.nix.in`, filled in with the
|
||||
# controller's settings, is that flake: the agents' toplevels are built from it
|
||||
# here, so the builder finds them in the store. The daemon runs with
|
||||
# `max-jobs = 0`, so a missing toplevel fails the build instead of compiling.
|
||||
{ pkgs, self }:
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
|
||||
hive = "pr1ma";
|
||||
swarmDomain = "swarm.test";
|
||||
agents = [
|
||||
"iris"
|
||||
"ruth"
|
||||
];
|
||||
|
||||
inherit (import ./hyperhive-fixture-repo.nix { inherit pkgs self; }) hyperhiveRepo;
|
||||
|
||||
metaTemplate = ../../hive-meta-flake/testdata/meta-flake-swarm.nix.in;
|
||||
|
||||
# The values the fixture has a placeholder for. A swarm whose controller sends
|
||||
# another set renders another flake, so evaluation stops there.
|
||||
forwardedVars = [
|
||||
"HIVE_AGENT_BAO_ADDR"
|
||||
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT"
|
||||
"HIVE_AGENT_LOGS_QUERY_URL"
|
||||
"HIVE_AGENT_METRICS_URL"
|
||||
"HIVE_FORGE_PUBLIC_URL"
|
||||
"HIVE_FORGE_URL"
|
||||
"HIVE_MATRIX_URL"
|
||||
];
|
||||
|
||||
# The meta flake the controller renders for `agents`, from its settings JSON
|
||||
# (`SWARM_CONTROLLER_META_FLAKE`). `hyperhive` is `self`: the agents' toplevels
|
||||
# do not depend on how the input is pinned.
|
||||
filled =
|
||||
settingsJson: pin:
|
||||
let
|
||||
settings = builtins.fromJSON settingsJson;
|
||||
names = [ "HYPERHIVE_URL" ] ++ forwardedVars;
|
||||
values = [ pin ] ++ map (name: settings.vars.${name}) forwardedVars;
|
||||
in
|
||||
assert lib.assertMsg
|
||||
(builtins.attrNames settings.vars == forwardedVars && !settings.github_disabled)
|
||||
"the controller's meta flake settings changed: update hive-meta-flake/testdata/meta-flake-swarm.nix.in";
|
||||
builtins.replaceStrings (map (name: "@${name}@") names) values (builtins.readFile metaTemplate);
|
||||
|
||||
toplevels =
|
||||
settingsJson:
|
||||
let
|
||||
flake = import (builtins.toFile "meta-flake.nix" (filled settingsJson "unused"));
|
||||
evaluated = flake.outputs {
|
||||
self = null;
|
||||
hyperhive = self;
|
||||
};
|
||||
in
|
||||
map (agent: evaluated.nixosConfigurations.${agent}.config.system.build.toplevel) agents;
|
||||
in
|
||||
pkgs.testers.runNixOSTest (
|
||||
{ config, ... }:
|
||||
let
|
||||
inherit (config.nodes.machine.services.hyperhive) swarm deploy;
|
||||
settingsJson =
|
||||
config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
||||
controllerSocket = deploy.swarm-controller.socketPath;
|
||||
|
||||
# A derivation no store holds, to show the daemon refuses to build.
|
||||
control = pkgs.writeText "control.nix" ''
|
||||
derivation {
|
||||
name = "build-control";
|
||||
system = "${pkgs.stdenv.hostPlatform.system}";
|
||||
builder = "/bin/sh";
|
||||
args = [ "-c" "echo built > $out" ];
|
||||
}
|
||||
'';
|
||||
in
|
||||
{
|
||||
name = "full-setup";
|
||||
|
||||
# Boot of fourteen containers and the setup steps, with the waits' own
|
||||
# bounds well inside.
|
||||
globalTimeout = 7200;
|
||||
|
||||
nodes.machine =
|
||||
{ config, ... }:
|
||||
{
|
||||
imports = [ self.nixosModules.default ];
|
||||
|
||||
services.hyperhive = {
|
||||
deploy.singleHostSwarm = true;
|
||||
deploy.hive-controller.enable = true;
|
||||
deploy.hive-controller.hiveName = hive;
|
||||
deploy.bao.seal = "shamir";
|
||||
swarm.domain = swarmDomain;
|
||||
swarm.hives.${hive} = { };
|
||||
};
|
||||
system.stateVersion = "25.11";
|
||||
|
||||
# 12 GB: fourteen service containers, the hive and a Rust-heavy agent
|
||||
# closure through the controller, the builder and the cache. 4 cores: the
|
||||
# containers boot in parallel. The disk holds the cache's copy of the
|
||||
# closure; the store overlay is on disk, not in memory.
|
||||
virtualisation.memorySize = 12288;
|
||||
virtualisation.cores = 4;
|
||||
virtualisation.diskSize = 40960;
|
||||
virtualisation.writableStoreUseTmpfs = false;
|
||||
|
||||
# The toplevels, and every source tree the meta flake's lock names, keyed
|
||||
# by narHash: nix takes a locked input from the store when its path is
|
||||
# valid.
|
||||
virtualisation.additionalPaths = [
|
||||
self
|
||||
self.inputs.nixpkgs
|
||||
self.inputs.crane
|
||||
self.inputs.treefmt-nix
|
||||
hyperhiveRepo
|
||||
]
|
||||
++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
||||
|
||||
# No substituter, and no build.
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ ];
|
||||
max-jobs = 0;
|
||||
};
|
||||
|
||||
# The store's raft data is on a tmpfs. Raft syncs every write, so on the
|
||||
# VM's qcow2 root (`cache=writeback`, which passes each guest flush on to
|
||||
# the host's disk) a login's lease write can outlast the 15 s the store's
|
||||
# clients wait, and its calls then fail with `context canceled`. The
|
||||
# state is under `/var/lib/private`, where the store's DynamicUser keeps
|
||||
# it; the mount sits on that directory in the container's root, and
|
||||
# nothing the test checks needs it to survive the VM.
|
||||
fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
|
||||
{
|
||||
device = "tmpfs";
|
||||
fsType = "tmpfs";
|
||||
options = [
|
||||
"mode=0700"
|
||||
"size=1G"
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.git
|
||||
pkgs.jq
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
import shlex
|
||||
|
||||
API = "http://localhost"
|
||||
SOCKET = "${controllerSocket}"
|
||||
FORGE = "https://${swarm.forge.domain}"
|
||||
HIVE = "${hive}"
|
||||
TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))}
|
||||
INIT = "/root/bao-init.json"
|
||||
FORGE_AUTH = "/root/forge-auth.header"
|
||||
NIX = "nix --extra-experimental-features nix-command"
|
||||
ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
|
||||
POLICY_UNITS_ACTIVE = (
|
||||
"for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
|
||||
"swarm-bao-agent-pki.service; do "
|
||||
"[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done"
|
||||
)
|
||||
|
||||
def controller(method, path, body=None, check="--fail-with-body"):
|
||||
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
||||
return machine.succeed(
|
||||
f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}"
|
||||
)
|
||||
|
||||
def forge(method, path, body=None):
|
||||
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
||||
return machine.succeed(
|
||||
f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} "
|
||||
f"-X {method} {FORGE}/api/v1{path} {data}"
|
||||
)
|
||||
|
||||
def diagnose(unit=None, container=None):
|
||||
# What a timed-out wait leaves behind: the failed units, and the status
|
||||
# and the end of the journal of the unit that was waited on.
|
||||
where = f"--machine={container} " if container else ""
|
||||
units = [unit] if isinstance(unit, str) else list(unit or [])
|
||||
flags = " ".join(f"-u '{u}'" for u in units)
|
||||
for command in [
|
||||
f"systemctl {where}--failed --no-legend --no-pager",
|
||||
*([f"systemctl {where}status --no-pager -n 0 -l {' '.join(units)} | head -c 4000"] if units else []),
|
||||
*([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []),
|
||||
]:
|
||||
print(f"--- {command}")
|
||||
print(machine.execute(command)[1])
|
||||
|
||||
def settle(command, timeout, unit=None, container=None):
|
||||
try:
|
||||
machine.wait_until_succeeds(command, timeout=timeout)
|
||||
except Exception:
|
||||
diagnose(unit, container)
|
||||
raise
|
||||
|
||||
def create_agent(body):
|
||||
# The controller refuses a create until its dependencies answer (the
|
||||
# archetype list comes from the builder's evaluation), so ask again.
|
||||
for _ in range(60):
|
||||
status, out = machine.execute(
|
||||
f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' "
|
||||
f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}"
|
||||
)
|
||||
if out.strip().startswith("2"):
|
||||
return
|
||||
machine.sleep(10)
|
||||
diagnose("swarm-controller")
|
||||
raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {machine.succeed('cat /root/create.out')}")
|
||||
|
||||
machine.start()
|
||||
machine.wait_for_unit("multi-user.target")
|
||||
|
||||
with subtest("the agents' toplevels are in the store, and the daemon builds nothing"):
|
||||
for toplevel in TOPLEVELS:
|
||||
machine.succeed(f"{NIX} path-info {toplevel}")
|
||||
# The control for the lookup: a path that is not there is not found.
|
||||
machine.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent")
|
||||
# The control for the missing build: a derivation no store holds is refused.
|
||||
machine.fail("NIX_CONFIG='experimental-features = nix-command' nix build --no-link --impure -f ${control}")
|
||||
|
||||
# setup.md step 1: the secret store.
|
||||
with subtest("step 1: bao operator init"):
|
||||
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
||||
machine.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")
|
||||
|
||||
with subtest("step 4: bao operator unseal, with the shamir seal"):
|
||||
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
||||
machine.succeed("bao status")
|
||||
|
||||
with subtest("step 1: the granter bootstrap"):
|
||||
machine.succeed(
|
||||
"set -o pipefail; "
|
||||
f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
|
||||
"bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
|
||||
"bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
|
||||
"| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
|
||||
"unset BAO_TOKEN; "
|
||||
"systemctl restart swarm-bao-granter-role"
|
||||
)
|
||||
machine.wait_until_succeeds(
|
||||
"journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'",
|
||||
timeout=300,
|
||||
)
|
||||
|
||||
with subtest("step 1: the policy units that waited for the granter are restarted"):
|
||||
# Some of these units retry every 30 seconds until something outside
|
||||
# them exists: the bao UI's viewer policy waits for authelia's secrets
|
||||
# unit, in its container, to publish the UI's OIDC client secret, and
|
||||
# that unit waits for the secret publisher's policy written here. So
|
||||
# the units are started without waiting, and polled until all are active.
|
||||
machine.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true")
|
||||
machine.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service")
|
||||
machine.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=600)
|
||||
# Under the shamir seal the PKI unit leaves a restart of the store to the
|
||||
# operator when its listener's client CA changed, and says which command.
|
||||
if machine.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
|
||||
machine.succeed("systemctl --machine=swarm-bao restart openbao.service")
|
||||
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
||||
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
||||
settle(POLICY_UNITS_ACTIVE, 1200, "swarm-bao-*-policy")
|
||||
machine.succeed(
|
||||
"journalctl -u swarm-bao-controller-policy --no-pager "
|
||||
"| grep -F 'Data written to: auth/cert/certs/swarm-controller'"
|
||||
)
|
||||
|
||||
with subtest("authelia's mint has published the clients' secrets"):
|
||||
# The operator viewer policy is left out of the poll above: it waits for
|
||||
# the UI client's secret, which authelia's secrets unit publishes from
|
||||
# its container once the store and the trust bundle it needs exist. The
|
||||
# controller's client secret is published by the same unit, and the
|
||||
# controller, the hive's store logins and the UI policy all wait for it.
|
||||
settle(
|
||||
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
||||
"secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
|
||||
1200,
|
||||
"authelia-swarm-secrets",
|
||||
"swarm-authelia",
|
||||
)
|
||||
settle("[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 300, "swarm-bao-operator-viewer-policy")
|
||||
|
||||
with subtest("step 2: swarmctl user add"):
|
||||
machine.wait_until_succeeds(
|
||||
"swarmctl user add mara --display-name Mara --email mara@example.com --group admins "
|
||||
">/root/user-add.out 2>&1",
|
||||
timeout=900,
|
||||
)
|
||||
machine.succeed("[ -s /root/user-add.out ]")
|
||||
machine.succeed("swarmctl user list | grep -w mara")
|
||||
|
||||
with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
|
||||
# The controller's forge token, from the store as the controller reads it.
|
||||
# Minted and stored by `forgejo-swarm-controller-account` in the forge
|
||||
# container, which waits for forgejo and for a login to the store.
|
||||
settle(
|
||||
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
||||
"secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
|
||||
900,
|
||||
["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],
|
||||
"hive-forge",
|
||||
)
|
||||
machine.succeed(
|
||||
f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}"
|
||||
)
|
||||
# The token being published says nothing about the gateway's vhost for
|
||||
# the forge, which the calls below go through.
|
||||
machine.wait_until_succeeds(
|
||||
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600
|
||||
)
|
||||
status = machine.succeed(
|
||||
f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive"
|
||||
).strip()
|
||||
if status != "200":
|
||||
forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"})
|
||||
forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"})
|
||||
machine.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git")
|
||||
machine.succeed(
|
||||
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
||||
f"push {FORGE}/hyperhive/hyperhive.git main"
|
||||
)
|
||||
pushed = machine.succeed(
|
||||
f"GIT_SSL_NO_VERIFY=1 git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
||||
f"ls-remote {FORGE}/hyperhive/hyperhive.git refs/heads/main"
|
||||
).split()[0]
|
||||
assert pushed == machine.succeed("cat ${hyperhiveRepo}/rev").strip(), pushed
|
||||
|
||||
with subtest("step 8: ruth, through the endpoint the swarm UI calls"):
|
||||
machine.wait_until_succeeds(
|
||||
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=900
|
||||
)
|
||||
default = json.loads(controller("GET", "/api/permissions"))["default"]
|
||||
create_agent({"name": "ruth", "hive": HIVE, "permissions": default + ["manage_root_agent"], "archetype": ARCHETYPE})
|
||||
|
||||
with subtest("step 9: the first agent"):
|
||||
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
|
||||
|
||||
with subtest("internal/meta holds the flake the toplevels were built from"):
|
||||
settle(
|
||||
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/internal/meta/raw/flake.nix "
|
||||
">/root/meta-flake.nix",
|
||||
1800,
|
||||
"swarm-controller",
|
||||
)
|
||||
rev = machine.succeed("cat ${hyperhiveRepo}/rev").strip()
|
||||
settings = json.loads(${builtins.toJSON settingsJson})
|
||||
expected = open("${metaTemplate}").read().replace(
|
||||
"@HYPERHIVE_URL@", f"git+{settings['git_base']}/hyperhive/hyperhive.git?rev={rev}"
|
||||
)
|
||||
for name, value in settings["vars"].items():
|
||||
expected = expected.replace(f"@{name}@", value)
|
||||
committed = machine.succeed("cat /root/meta-flake.nix")
|
||||
assert committed == expected, "the controller's flake differs from the one the toplevels came from:\n" + committed
|
||||
|
||||
# The agents' deploy and the check that they need a login are not scripted yet.
|
||||
'';
|
||||
}
|
||||
)
|
||||
33
nix/vm-tests/hyperhive-fixture-repo.nix
Normal file
33
nix/vm-tests/hyperhive-fixture-repo.nix
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# The `hyperhive/hyperhive` repo the offline VM tests put on their forge: one
|
||||
# commit holding the source tree of `self`, with a fixed author and date, so its
|
||||
# id depends on the tree alone.
|
||||
{ pkgs, self }:
|
||||
let
|
||||
commitEnv = ''
|
||||
export HOME=$TMPDIR
|
||||
export GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@forge.test
|
||||
export GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@forge.test
|
||||
export GIT_AUTHOR_DATE="946684800 +0000" GIT_COMMITTER_DATE="946684800 +0000"
|
||||
'';
|
||||
in
|
||||
{
|
||||
inherit commitEnv;
|
||||
|
||||
# `repo.git` (bare, branch `main`) and `rev` (the commit id).
|
||||
hyperhiveRepo =
|
||||
pkgs.runCommand "hyperhive-fixture-repo"
|
||||
{
|
||||
nativeBuildInputs = [ pkgs.git ];
|
||||
}
|
||||
''
|
||||
${commitEnv}
|
||||
cp -a ${self} work
|
||||
chmod -R u+w work
|
||||
git -C work init -q -b main
|
||||
git -C work add -f .
|
||||
git -C work commit -q -m "hyperhive source tree"
|
||||
mkdir $out
|
||||
git clone -q --bare work $out/repo.git
|
||||
git -C work rev-parse HEAD > $out/rev
|
||||
'';
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
# prebuilds: two deterministic one-commit repos and one toplevel, all built from
|
||||
# `self`.
|
||||
#
|
||||
# hyperhive/hyperhive the source tree of `self`
|
||||
# hyperhive/hyperhive the source tree of `self` (./hyperhive-fixture-repo.nix)
|
||||
# internal/meta the meta flake of one archetype agent, pinned to the
|
||||
# first repo's commit
|
||||
#
|
||||
|
|
@ -29,29 +29,7 @@ let
|
|||
hyperhive = self;
|
||||
}).nixosConfigurations.${agent}.config.system.build.toplevel;
|
||||
|
||||
commitEnv = ''
|
||||
export HOME=$TMPDIR
|
||||
export GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@forge.test
|
||||
export GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@forge.test
|
||||
export GIT_AUTHOR_DATE="946684800 +0000" GIT_COMMITTER_DATE="946684800 +0000"
|
||||
'';
|
||||
|
||||
hyperhiveRepo =
|
||||
pkgs.runCommand "hyperhive-fixture-repo"
|
||||
{
|
||||
nativeBuildInputs = [ pkgs.git ];
|
||||
}
|
||||
''
|
||||
${commitEnv}
|
||||
cp -a ${self} work
|
||||
chmod -R u+w work
|
||||
git -C work init -q -b main
|
||||
git -C work add -f .
|
||||
git -C work commit -q -m "hyperhive source tree"
|
||||
mkdir $out
|
||||
git clone -q --bare work $out/repo.git
|
||||
git -C work rev-parse HEAD > $out/rev
|
||||
'';
|
||||
inherit (import ./hyperhive-fixture-repo.nix { inherit pkgs self; }) commitEnv hyperhiveRepo;
|
||||
|
||||
# `nix flake lock`'s result for the template: hyperhive's own nodes next to a
|
||||
# `hyperhive` node pinned by URL and rev, whose follows paths start at the
|
||||
|
|
|
|||
Loading…
Reference in a new issue