d516bb890c
Each agent row on the hive dashboard, and the agent page's link menu, linked to the agent's forge profile and its config repo. Both are gone, with what only they used: - hive-agent no longer builds the two links, the `Forge` link kind, or the `forge_public_url` field of its state, and no longer reads `HIVE_FORGE_PUBLIC_URL`. The agent page's link menu and the dashboard row's icon strip (which skips a forge link an agent not yet rebuilt still sends) drop their forge handling. - hive-c0re's dashboard state no longer carries `forge_public_url`. - `HIVE_FORGE_PUBLIC_URL` is no longer forwarded to agents through the meta flake (the forwarded-variable list, its test fixture, the controller's settings and the VM test), so the next meta flake render differs by that one line. hive-c0re still reads it for the forge URL a hive publishes, so `swarm.forge.publicUrl` stays, with its description corrected. - The docs stop describing the links.
385 lines
18 KiB
Nix
385 lines
18 KiB
Nix
# `vmTests.<system>.full-setup`: one VM runs a whole single-host swarm
|
|
# (`deploy.singleHostSwarm`) with no network and no substituter, and a script
|
|
# walks docs/1-getting-started/setup.md until the controller has committed the
|
|
# agents' meta flake to the swarm's forge.
|
|
#
|
|
# Steps of setup.md, as the script runs them:
|
|
# 1 secret store `bao operator init`, the granter bootstrap with the root
|
|
# token, then the policy units reset and restarted.
|
|
# 4 unseal `bao operator unseal` (`deploy.bao.seal` is "shamir"; pkcs11
|
|
# needs a TPM). A pkcs11 store unseals itself at init, so here
|
|
# the unseal comes right after init, before the granter.
|
|
# 2 SSO account `swarmctl user add mara --group admins`.
|
|
# 8 ruth `POST /api/agents` on the controller socket, as the swarm
|
|
# UI's create form does, plus `manage_root_agent`.
|
|
# 9 first agent The same endpoint, for `iris`: `swarmctl agent create` has no
|
|
# archetype flag, and a custom agent's config repo would need
|
|
# inputs the VM cannot resolve offline.
|
|
# Not scripted yet; a later slice adds them: step 3, the first SSO login to the
|
|
# forge (an OIDC code flow against authelia), `swarmctl forge make-admin`, and
|
|
# the agents' deploy up to the point where each says it needs a login. Green
|
|
# when `internal/meta` on the forge holds the flake the toplevels were built from.
|
|
#
|
|
# Offline: the test seeds the swarm's forge with `hyperhive/hyperhive` (one
|
|
# commit of `self`, ./hyperhive-fixture-repo.nix). The controller renders
|
|
# `internal/meta` and the builder locks and builds it as in production.
|
|
# `hive-meta-flake/testdata/meta-flake-swarm.nix.in`, filled in with the
|
|
# controller's settings, is that flake: the agents' toplevels are built from it
|
|
# here, so the builder finds them in the store. The daemon runs with
|
|
# `max-jobs = 0`, so a missing toplevel fails the build instead of compiling.
|
|
{ pkgs, self }:
|
|
let
|
|
inherit (pkgs) lib;
|
|
|
|
hive = "pr1ma";
|
|
swarmDomain = "swarm.test";
|
|
agents = [
|
|
"iris"
|
|
"ruth"
|
|
];
|
|
|
|
inherit (import ./hyperhive-fixture-repo.nix { inherit pkgs self; }) hyperhiveRepo;
|
|
|
|
metaTemplate = ../../hive-meta-flake/testdata/meta-flake-swarm.nix.in;
|
|
|
|
# The values the fixture has a placeholder for. A swarm whose controller sends
|
|
# another set renders another flake, so evaluation stops there.
|
|
forwardedVars = [
|
|
"HIVE_AGENT_BAO_ADDR"
|
|
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT"
|
|
"HIVE_AGENT_LOGS_QUERY_URL"
|
|
"HIVE_AGENT_METRICS_URL"
|
|
"HIVE_FORGE_URL"
|
|
"HIVE_MATRIX_URL"
|
|
];
|
|
|
|
# The meta flake the controller renders for `agents`, from its settings JSON
|
|
# (`SWARM_CONTROLLER_META_FLAKE`). `hyperhive` is `self`: the agents' toplevels
|
|
# do not depend on how the input is pinned.
|
|
filled =
|
|
settingsJson: pin:
|
|
let
|
|
settings = builtins.fromJSON settingsJson;
|
|
names = [ "HYPERHIVE_URL" ] ++ forwardedVars;
|
|
values = [ pin ] ++ map (name: settings.vars.${name}) forwardedVars;
|
|
in
|
|
assert lib.assertMsg
|
|
(builtins.attrNames settings.vars == forwardedVars && !settings.github_disabled)
|
|
"the controller's meta flake settings changed: update hive-meta-flake/testdata/meta-flake-swarm.nix.in";
|
|
builtins.replaceStrings (map (name: "@${name}@") names) values (builtins.readFile metaTemplate);
|
|
|
|
toplevels =
|
|
settingsJson:
|
|
let
|
|
flake = import (builtins.toFile "meta-flake.nix" (filled settingsJson "unused"));
|
|
evaluated = flake.outputs {
|
|
self = null;
|
|
hyperhive = self;
|
|
};
|
|
in
|
|
map (agent: evaluated.nixosConfigurations.${agent}.config.system.build.toplevel) agents;
|
|
in
|
|
pkgs.testers.runNixOSTest (
|
|
{ config, ... }:
|
|
let
|
|
inherit (config.nodes.machine.services.hyperhive) swarm deploy;
|
|
settingsJson =
|
|
config.nodes.machine.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
|
controllerSocket = deploy.swarm-controller.socketPath;
|
|
|
|
# A derivation no store holds, to show the daemon refuses to build.
|
|
control = pkgs.writeText "control.nix" ''
|
|
derivation {
|
|
name = "build-control";
|
|
system = "${pkgs.stdenv.hostPlatform.system}";
|
|
builder = "/bin/sh";
|
|
args = [ "-c" "echo built > $out" ];
|
|
}
|
|
'';
|
|
in
|
|
{
|
|
name = "full-setup";
|
|
|
|
# Boot of fourteen containers and the setup steps, with the waits' own
|
|
# bounds well inside.
|
|
globalTimeout = 7200;
|
|
|
|
nodes.machine =
|
|
{ config, ... }:
|
|
{
|
|
imports = [ self.nixosModules.default ];
|
|
|
|
services.hyperhive = {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.hive-controller.enable = true;
|
|
deploy.hive-controller.hiveName = hive;
|
|
deploy.bao.seal = "shamir";
|
|
swarm.domain = swarmDomain;
|
|
swarm.hives.${hive} = { };
|
|
};
|
|
system.stateVersion = "25.11";
|
|
|
|
# 12 GB: fourteen service containers, the hive and a Rust-heavy agent
|
|
# closure through the controller, the builder and the cache. 4 cores: the
|
|
# containers boot in parallel. The disk holds the cache's copy of the
|
|
# closure; the store overlay is on disk, not in memory.
|
|
virtualisation.memorySize = 12288;
|
|
virtualisation.cores = 4;
|
|
virtualisation.diskSize = 40960;
|
|
virtualisation.writableStoreUseTmpfs = false;
|
|
|
|
# The toplevels, and every source tree the meta flake's lock names, keyed
|
|
# by narHash: nix takes a locked input from the store when its path is
|
|
# valid.
|
|
virtualisation.additionalPaths = [
|
|
self
|
|
self.inputs.nixpkgs
|
|
self.inputs.crane
|
|
self.inputs.treefmt-nix
|
|
hyperhiveRepo
|
|
]
|
|
++ toplevels config.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
|
|
|
# No substituter, and no build.
|
|
nix.settings = {
|
|
substituters = lib.mkForce [ ];
|
|
max-jobs = 0;
|
|
};
|
|
|
|
# The store's raft data is on a tmpfs. Raft syncs every write, so on the
|
|
# VM's qcow2 root (`cache=writeback`, which passes each guest flush on to
|
|
# the host's disk) a login's lease write can outlast the 15 s the store's
|
|
# clients wait, and its calls then fail with `context canceled`. The
|
|
# state is under `/var/lib/private`, where the store's DynamicUser keeps
|
|
# it; the mount sits on that directory in the container's root, and
|
|
# nothing the test checks needs it to survive the VM.
|
|
fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
|
|
{
|
|
device = "tmpfs";
|
|
fsType = "tmpfs";
|
|
options = [
|
|
"mode=0700"
|
|
"size=1G"
|
|
];
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.jq
|
|
];
|
|
};
|
|
|
|
testScript = ''
|
|
import json
|
|
import shlex
|
|
|
|
API = "http://localhost"
|
|
SOCKET = "${controllerSocket}"
|
|
FORGE = "https://${swarm.forge.domain}"
|
|
HIVE = "${hive}"
|
|
TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))}
|
|
INIT = "/root/bao-init.json"
|
|
FORGE_AUTH = "/root/forge-auth.header"
|
|
NIX = "nix --extra-experimental-features nix-command"
|
|
ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
|
|
POLICY_UNITS_ACTIVE = (
|
|
"for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
|
|
"swarm-bao-agent-pki.service; do "
|
|
"[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done"
|
|
)
|
|
|
|
def controller(method, path, body=None, check="--fail-with-body"):
|
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
|
return machine.succeed(
|
|
f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}"
|
|
)
|
|
|
|
def forge(method, path, body=None):
|
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
|
return machine.succeed(
|
|
f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} "
|
|
f"-X {method} {FORGE}/api/v1{path} {data}"
|
|
)
|
|
|
|
def diagnose(unit=None, container=None):
|
|
# What a timed-out wait leaves behind: the failed units, and the status
|
|
# and the end of the journal of the unit that was waited on.
|
|
where = f"--machine={container} " if container else ""
|
|
units = [unit] if isinstance(unit, str) else list(unit or [])
|
|
flags = " ".join(f"-u '{u}'" for u in units)
|
|
for command in [
|
|
f"systemctl {where}--failed --no-legend --no-pager",
|
|
*([f"systemctl {where}status --no-pager -n 0 -l {' '.join(units)} | head -c 4000"] if units else []),
|
|
*([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []),
|
|
]:
|
|
print(f"--- {command}")
|
|
print(machine.execute(command)[1])
|
|
|
|
def settle(command, timeout, unit=None, container=None):
|
|
try:
|
|
machine.wait_until_succeeds(command, timeout=timeout)
|
|
except Exception:
|
|
diagnose(unit, container)
|
|
raise
|
|
|
|
def create_agent(body):
|
|
# The controller refuses a create until its dependencies answer (the
|
|
# archetype list comes from the builder's evaluation), so ask again.
|
|
for _ in range(60):
|
|
status, out = machine.execute(
|
|
f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' "
|
|
f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}"
|
|
)
|
|
if out.strip().startswith("2"):
|
|
return
|
|
machine.sleep(10)
|
|
diagnose("swarm-controller")
|
|
raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {machine.succeed('cat /root/create.out')}")
|
|
|
|
machine.start()
|
|
machine.wait_for_unit("multi-user.target")
|
|
|
|
with subtest("the agents' toplevels are in the store, and the daemon builds nothing"):
|
|
for toplevel in TOPLEVELS:
|
|
machine.succeed(f"{NIX} path-info {toplevel}")
|
|
# The control for the lookup: a path that is not there is not found.
|
|
machine.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent")
|
|
# The control for the missing build: a derivation no store holds is refused.
|
|
machine.fail("NIX_CONFIG='experimental-features = nix-command' nix build --no-link --impure -f ${control}")
|
|
|
|
# setup.md step 1: the secret store.
|
|
with subtest("step 1: bao operator init"):
|
|
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
|
machine.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")
|
|
|
|
with subtest("step 4: bao operator unseal, with the shamir seal"):
|
|
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
|
machine.succeed("bao status")
|
|
|
|
with subtest("step 1: the granter bootstrap"):
|
|
machine.succeed(
|
|
"set -o pipefail; "
|
|
f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
|
|
"bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
|
|
"bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
|
|
"| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
|
|
"unset BAO_TOKEN; "
|
|
"systemctl restart swarm-bao-granter-role"
|
|
)
|
|
machine.wait_until_succeeds(
|
|
"journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'",
|
|
timeout=300,
|
|
)
|
|
|
|
with subtest("step 1: the policy units that waited for the granter are restarted"):
|
|
# Some of these units retry every 30 seconds until something outside
|
|
# them exists: the bao UI's viewer policy waits for authelia's secrets
|
|
# unit, in its container, to publish the UI's OIDC client secret, and
|
|
# that unit waits for the secret publisher's policy written here. So
|
|
# the units are started without waiting, and polled until all are active.
|
|
machine.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true")
|
|
machine.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service")
|
|
machine.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=600)
|
|
# Under the shamir seal the PKI unit leaves a restart of the store to the
|
|
# operator when its listener's client CA changed, and says which command.
|
|
if machine.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
|
|
machine.succeed("systemctl --machine=swarm-bao restart openbao.service")
|
|
machine.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
|
machine.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
|
settle(POLICY_UNITS_ACTIVE, 1200, "swarm-bao-*-policy")
|
|
machine.succeed(
|
|
"journalctl -u swarm-bao-controller-policy --no-pager "
|
|
"| grep -F 'Data written to: auth/cert/certs/swarm-controller'"
|
|
)
|
|
|
|
with subtest("authelia's mint has published the clients' secrets"):
|
|
# The operator viewer policy is left out of the poll above: it waits for
|
|
# the UI client's secret, which authelia's secrets unit publishes from
|
|
# its container once the store and the trust bundle it needs exist. The
|
|
# controller's client secret is published by the same unit, and the
|
|
# controller, the hive's store logins and the UI policy all wait for it.
|
|
settle(
|
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
|
"secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
|
|
1200,
|
|
"authelia-swarm-secrets",
|
|
"swarm-authelia",
|
|
)
|
|
settle("[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 300, "swarm-bao-operator-viewer-policy")
|
|
|
|
with subtest("step 2: swarmctl user add"):
|
|
machine.wait_until_succeeds(
|
|
"swarmctl user add mara --display-name Mara --email mara@example.com --group admins "
|
|
">/root/user-add.out 2>&1",
|
|
timeout=900,
|
|
)
|
|
machine.succeed("[ -s /root/user-add.out ]")
|
|
machine.succeed("swarmctl user list | grep -w mara")
|
|
|
|
with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
|
|
# The controller's forge token, from the store as the controller reads it.
|
|
# Minted and stored by `forgejo-swarm-controller-account` in the forge
|
|
# container, which waits for forgejo and for a login to the store.
|
|
settle(
|
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
|
"secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
|
|
900,
|
|
["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],
|
|
"hive-forge",
|
|
)
|
|
machine.succeed(
|
|
f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}"
|
|
)
|
|
# The token being published says nothing about the gateway's vhost for
|
|
# the forge, which the calls below go through.
|
|
machine.wait_until_succeeds(
|
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600
|
|
)
|
|
status = machine.succeed(
|
|
f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive"
|
|
).strip()
|
|
if status != "200":
|
|
forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"})
|
|
forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"})
|
|
machine.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git")
|
|
machine.succeed(
|
|
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
|
f"push {FORGE}/hyperhive/hyperhive.git main"
|
|
)
|
|
pushed = machine.succeed(
|
|
f"GIT_SSL_NO_VERIFY=1 git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
|
f"ls-remote {FORGE}/hyperhive/hyperhive.git refs/heads/main"
|
|
).split()[0]
|
|
assert pushed == machine.succeed("cat ${hyperhiveRepo}/rev").strip(), pushed
|
|
|
|
with subtest("step 8: ruth, through the endpoint the swarm UI calls"):
|
|
machine.wait_until_succeeds(
|
|
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=900
|
|
)
|
|
default = json.loads(controller("GET", "/api/permissions"))["default"]
|
|
create_agent({"name": "ruth", "hive": HIVE, "permissions": default + ["manage_root_agent"], "archetype": ARCHETYPE})
|
|
|
|
with subtest("step 9: the first agent"):
|
|
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
|
|
|
|
with subtest("internal/meta holds the flake the toplevels were built from"):
|
|
settle(
|
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/internal/meta/raw/flake.nix "
|
|
">/root/meta-flake.nix",
|
|
1800,
|
|
"swarm-controller",
|
|
)
|
|
rev = machine.succeed("cat ${hyperhiveRepo}/rev").strip()
|
|
settings = json.loads(${builtins.toJSON settingsJson})
|
|
expected = open("${metaTemplate}").read().replace(
|
|
"@HYPERHIVE_URL@", f"git+{settings['git_base']}/hyperhive/hyperhive.git?rev={rev}"
|
|
)
|
|
for name, value in settings["vars"].items():
|
|
expected = expected.replace(f"@{name}@", value)
|
|
committed = machine.succeed("cat /root/meta-flake.nix")
|
|
assert committed == expected, "the controller's flake differs from the one the toplevels came from:\n" + committed
|
|
|
|
# The agents' deploy and the check that they need a login are not scripted yet.
|
|
'';
|
|
}
|
|
)
|