vm-tests: semi-spread, the example on three machines and an outside client
This commit is contained in:
atlas
mara
parent
250118d558
commit
bcd698eb43
2 changed files with 630 additions and 0 deletions
|
|
@ -13,4 +13,5 @@
|
||||||
swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; };
|
swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; };
|
||||||
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };
|
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };
|
||||||
full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; };
|
full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; };
|
||||||
|
semi-spread = import ./vm-tests/semi-spread.nix { inherit pkgs self; };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
629
nix/vm-tests/semi-spread.nix
Normal file
629
nix/vm-tests/semi-spread.nix
Normal file
|
|
@ -0,0 +1,629 @@
|
||||||
|
# `vmTests.<system>.semi-spread`: the semi-spread example
|
||||||
|
# (docs/examples/semi-spread/, docs/2-handbook/swarm/semi-spread-deploy.md) on
|
||||||
|
# three VMs and an outside client.
|
||||||
|
#
|
||||||
|
# swarm swarm-central.nix and swarm-services.nix on one machine
|
||||||
|
# heavyduty swarm-heavyduty.nix: the builder, the CI runner, the cache
|
||||||
|
# hive hive.nix, as `hive-one`
|
||||||
|
# outside a plain client on the public network, not on wg0
|
||||||
|
#
|
||||||
|
# The example's files are imported from a copy of the directory with two
|
||||||
|
# changes: `mesh.nix` holds the VMs' endpoints and seed-derived keys and has no
|
||||||
|
# swarm-services entry (its machine is swarm-central here; `hive-two` stays as
|
||||||
|
# an entry nobody answers for), and `dns.nix` lists swarm-services' names under
|
||||||
|
# swarm-central, because the file leaves out the names its own machine serves.
|
||||||
|
#
|
||||||
|
# The script, in order:
|
||||||
|
# assert 1 wg0: every machine pings the other two over IPv4 and IPv6.
|
||||||
|
# assert 3 split DNS: `hive` resolves the internal names to wg0 addresses,
|
||||||
|
# A and AAAA; `outside` resolves the public names only.
|
||||||
|
# ceremony ./full-setup.nix's store steps on `swarm`, then the operator's
|
||||||
|
# copy of the store's CA and leaves to the other machines.
|
||||||
|
# assert 2 `outside` reaches forge, chat, auth and the swarm UI on the public
|
||||||
|
# address and nothing else; `hive` and `swarm` reach the private
|
||||||
|
# names over wg0.
|
||||||
|
# assert 4 two agents are created on the controller, built on `heavyduty`,
|
||||||
|
# pushed to its cache and pulled by `hive`, whose store holds
|
||||||
|
# neither toplevel before and which has no other substituter.
|
||||||
|
# Green when both are fresh and need a login.
|
||||||
|
{ pkgs, self }:
|
||||||
|
let
|
||||||
|
inherit (pkgs) lib;
|
||||||
|
|
||||||
|
hiveName = "one";
|
||||||
|
agents = [
|
||||||
|
"iris"
|
||||||
|
"spawn"
|
||||||
|
];
|
||||||
|
|
||||||
|
inherit (import ./agent-toplevels.nix { inherit pkgs self agents; })
|
||||||
|
hyperhiveRepo
|
||||||
|
sourcePaths
|
||||||
|
toplevels
|
||||||
|
;
|
||||||
|
|
||||||
|
example = ../../docs/examples/semi-spread;
|
||||||
|
exampleMesh = import (example + "/mesh.nix");
|
||||||
|
|
||||||
|
# Test-only keys: base64 of sha256(seed); wg clamps the scalar itself. The
|
||||||
|
# same seeds on every store build the same bytes.
|
||||||
|
mkKey =
|
||||||
|
name:
|
||||||
|
pkgs.runCommand "semi-spread-test-key-${name}"
|
||||||
|
{
|
||||||
|
nativeBuildInputs = [
|
||||||
|
pkgs.wireguard-tools
|
||||||
|
pkgs.openssl
|
||||||
|
];
|
||||||
|
}
|
||||||
|
''
|
||||||
|
mkdir $out
|
||||||
|
printf '%s' 'hyperhive-semi-spread-test-${name}' | openssl dgst -sha256 -binary | base64 > $out/private
|
||||||
|
wg pubkey < $out/private > $out/public
|
||||||
|
'';
|
||||||
|
keys = lib.genAttrs [
|
||||||
|
"swarm"
|
||||||
|
"heavyduty"
|
||||||
|
"hive"
|
||||||
|
# Never installed anywhere: the key of the hive-two entry.
|
||||||
|
"absent"
|
||||||
|
] mkKey;
|
||||||
|
pub = name: lib.removeSuffix "\n" (builtins.readFile "${keys.${name}}/public");
|
||||||
|
|
||||||
|
# The test driver numbers the nodes on vlan 1 in alphabetical order.
|
||||||
|
lanAddr = {
|
||||||
|
heavyduty = "192.168.1.1";
|
||||||
|
hive = "192.168.1.2";
|
||||||
|
outside = "192.168.1.3";
|
||||||
|
swarm = "192.168.1.4";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The example's addresses, with the endpoints and keys of the VMs.
|
||||||
|
entry =
|
||||||
|
machine: node:
|
||||||
|
exampleMesh.${machine}
|
||||||
|
// {
|
||||||
|
endpoint = "${lanAddr.${node}}:51820";
|
||||||
|
publicKey = pub node;
|
||||||
|
};
|
||||||
|
testMesh = {
|
||||||
|
swarm-central = entry "swarm-central" "swarm";
|
||||||
|
swarm-heavyduty = entry "swarm-heavyduty" "heavyduty";
|
||||||
|
hive-one = entry "hive-one" "hive";
|
||||||
|
hive-two = exampleMesh.hive-two // {
|
||||||
|
endpoint = "192.168.1.250:51820";
|
||||||
|
publicKey = pub "absent";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
meshFile = pkgs.writeText "mesh.nix" (
|
||||||
|
"builtins.fromJSON " + builtins.toJSON (builtins.toJSON testMesh)
|
||||||
|
);
|
||||||
|
|
||||||
|
exampleCopy = pkgs.runCommand "semi-spread-example" { nativeBuildInputs = [ pkgs.perl ]; } ''
|
||||||
|
cp -r ${example} $out
|
||||||
|
chmod -R u+w $out
|
||||||
|
cp ${meshFile} $out/mesh.nix
|
||||||
|
perl -0pi -e 's/(swarm\.bao\.ui\.domain\n)\s*\];\n\s*swarm-services = \[\n/$1/ or die "dns.nix: swarm-services entry not found"' $out/dns.nix
|
||||||
|
if grep -q 'swarm-services = \[' $out/dns.nix; then exit 1; fi
|
||||||
|
'';
|
||||||
|
|
||||||
|
wgKey = node: {
|
||||||
|
environment.etc."wireguard/wg0.key" = {
|
||||||
|
source = "${keys.${node}}/private";
|
||||||
|
mode = "0400";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# No substituter and no build: a closure comes from the swarm's cache or not
|
||||||
|
# at all.
|
||||||
|
offline = {
|
||||||
|
nix.settings = {
|
||||||
|
substituters = lib.mkForce [ ];
|
||||||
|
max-jobs = 0;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
tools = {
|
||||||
|
environment.systemPackages = [
|
||||||
|
pkgs.curl
|
||||||
|
pkgs.jq
|
||||||
|
pkgs.wireguard-tools
|
||||||
|
];
|
||||||
|
};
|
||||||
|
in
|
||||||
|
pkgs.testers.runNixOSTest (
|
||||||
|
{ config, ... }:
|
||||||
|
let
|
||||||
|
swarmCfg = config.nodes.swarm.services.hyperhive;
|
||||||
|
inherit (swarmCfg) swarm;
|
||||||
|
controllerSocket = swarmCfg.deploy.swarm-controller.socketPath;
|
||||||
|
settingsJson =
|
||||||
|
config.nodes.swarm.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_META_FLAKE;
|
||||||
|
|
||||||
|
# What the public DNS holds (semi-spread-deploy.md, "DNS"), and the rest of
|
||||||
|
# the swarm's names, which only the VPN resolves.
|
||||||
|
publicNames = {
|
||||||
|
ui = swarm.domain;
|
||||||
|
forge = swarm.forge.domain;
|
||||||
|
chat = swarm.matrix.gatewayHost;
|
||||||
|
auth = swarm.authelia.domain;
|
||||||
|
};
|
||||||
|
privateNames = {
|
||||||
|
grafana = swarm.grafana.domain;
|
||||||
|
metrics = swarm.victoriametrics.domain;
|
||||||
|
logs = swarm.victorialogs.domain;
|
||||||
|
otel = swarm.otel.domain;
|
||||||
|
bao-ui = swarm.bao.ui.domain;
|
||||||
|
attic = swarm.attic.ownDomain;
|
||||||
|
};
|
||||||
|
# Every internal name, with the machine whose VPN address it resolves to.
|
||||||
|
internalNames =
|
||||||
|
lib.genAttrs [
|
||||||
|
"ui"
|
||||||
|
"forge"
|
||||||
|
"chat"
|
||||||
|
"auth"
|
||||||
|
"grafana"
|
||||||
|
"metrics"
|
||||||
|
"logs"
|
||||||
|
"otel"
|
||||||
|
"bao-ui"
|
||||||
|
"bao"
|
||||||
|
"nats"
|
||||||
|
] (_: "swarm-central")
|
||||||
|
// {
|
||||||
|
attic = "swarm-heavyduty";
|
||||||
|
};
|
||||||
|
nameOf = {
|
||||||
|
inherit (publicNames)
|
||||||
|
ui
|
||||||
|
forge
|
||||||
|
chat
|
||||||
|
auth
|
||||||
|
;
|
||||||
|
inherit (privateNames)
|
||||||
|
grafana
|
||||||
|
metrics
|
||||||
|
logs
|
||||||
|
otel
|
||||||
|
bao-ui
|
||||||
|
attic
|
||||||
|
;
|
||||||
|
bao = swarm.bao.domain;
|
||||||
|
nats = swarm.nats.domain;
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
name = "semi-spread";
|
||||||
|
|
||||||
|
# Three hosts' boots, the setup ceremony and the agents' build and pull.
|
||||||
|
globalTimeout = 10800;
|
||||||
|
|
||||||
|
nodes.swarm =
|
||||||
|
{ config, ... }:
|
||||||
|
{
|
||||||
|
imports = [
|
||||||
|
self.nixosModules.default
|
||||||
|
tools
|
||||||
|
(wgKey "swarm")
|
||||||
|
"${exampleCopy}/common.nix"
|
||||||
|
"${exampleCopy}/swarm-central.nix"
|
||||||
|
"${exampleCopy}/swarm-services.nix"
|
||||||
|
];
|
||||||
|
networking.hostName = "swarm-central";
|
||||||
|
system.stateVersion = "25.11";
|
||||||
|
|
||||||
|
services.hyperhive = {
|
||||||
|
# swarm-services.nix says the store is elsewhere; here it is on this machine.
|
||||||
|
deploy.bao.enable = lib.mkForce true;
|
||||||
|
deploy.bao.seal = "shamir";
|
||||||
|
deploy.bao.bootstrapTokenFile = "/var/lib/swarm-bao-bootstrap/grant.token";
|
||||||
|
# Names this machine serves are not in the example's table, and a
|
||||||
|
# process on the machine reaches them on the machine.
|
||||||
|
gateway.localHostsEntry = true;
|
||||||
|
# The test pushes its own hyperhive/hyperhive; a mirror of the
|
||||||
|
# upstream could not be created offline and would race the push.
|
||||||
|
deploy.forgejo.hyperhiveUpstream = null;
|
||||||
|
};
|
||||||
|
|
||||||
|
# 8 GB: the store and the swarm's services in containers, the
|
||||||
|
# controller, and the forge's repositories.
|
||||||
|
virtualisation.memorySize = 8192;
|
||||||
|
virtualisation.cores = 4;
|
||||||
|
virtualisation.diskSize = 20480;
|
||||||
|
virtualisation.writableStoreUseTmpfs = false;
|
||||||
|
virtualisation.additionalPaths = [ hyperhiveRepo ];
|
||||||
|
|
||||||
|
# The store's raft data is on a tmpfs, for the reason in ./full-setup.nix.
|
||||||
|
fileSystems."/var/lib/nixos-containers/${config.services.hyperhive.swarm.bao.machine}/var/lib/private" =
|
||||||
|
{
|
||||||
|
device = "tmpfs";
|
||||||
|
fsType = "tmpfs";
|
||||||
|
options = [
|
||||||
|
"mode=0700"
|
||||||
|
"size=1G"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
environment.systemPackages = [ pkgs.git ];
|
||||||
|
};
|
||||||
|
|
||||||
|
nodes.heavyduty = {
|
||||||
|
imports = [
|
||||||
|
self.nixosModules.default
|
||||||
|
tools
|
||||||
|
offline
|
||||||
|
(wgKey "heavyduty")
|
||||||
|
"${exampleCopy}/common.nix"
|
||||||
|
"${exampleCopy}/swarm-heavyduty.nix"
|
||||||
|
];
|
||||||
|
networking.hostName = "swarm-heavyduty";
|
||||||
|
system.stateVersion = "25.11";
|
||||||
|
|
||||||
|
# 5 GB: the builder evaluates and builds the agents' flake, beside the
|
||||||
|
# cache and the CI runner. The disk holds the toplevels and the cache's
|
||||||
|
# copy of them.
|
||||||
|
virtualisation.memorySize = 5120;
|
||||||
|
virtualisation.cores = 4;
|
||||||
|
virtualisation.diskSize = 40960;
|
||||||
|
virtualisation.writableStoreUseTmpfs = false;
|
||||||
|
# The toplevels, and the source trees the meta flake's lock names.
|
||||||
|
virtualisation.additionalPaths = sourcePaths ++ toplevels settingsJson;
|
||||||
|
};
|
||||||
|
|
||||||
|
nodes.hive = {
|
||||||
|
imports = [
|
||||||
|
self.nixosModules.default
|
||||||
|
tools
|
||||||
|
offline
|
||||||
|
(wgKey "hive")
|
||||||
|
"${exampleCopy}/common.nix"
|
||||||
|
"${exampleCopy}/hive.nix"
|
||||||
|
];
|
||||||
|
networking.hostName = "hive-one";
|
||||||
|
system.stateVersion = "25.11";
|
||||||
|
|
||||||
|
# 4 GB: the hive and two agent containers. The disk holds the pulled
|
||||||
|
# closures.
|
||||||
|
virtualisation.memorySize = 4096;
|
||||||
|
virtualisation.cores = 2;
|
||||||
|
virtualisation.diskSize = 30720;
|
||||||
|
virtualisation.writableStoreUseTmpfs = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
nodes.outside = {
|
||||||
|
environment.systemPackages = [ pkgs.curl ];
|
||||||
|
# The public DNS: the four names, at the swarm machine's public address.
|
||||||
|
networking.hosts.${lanAddr.swarm} = lib.attrValues publicNames;
|
||||||
|
virtualisation.memorySize = 512;
|
||||||
|
};
|
||||||
|
|
||||||
|
testScript = ''
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
|
||||||
|
API = "http://localhost"
|
||||||
|
SOCKET = "${controllerSocket}"
|
||||||
|
FORGE = "https://${publicNames.forge}"
|
||||||
|
HIVE = "${hiveName}"
|
||||||
|
TOPLEVELS = ${builtins.toJSON (map toString (toplevels settingsJson))}
|
||||||
|
INIT = "/root/bao-init.json"
|
||||||
|
FORGE_AUTH = "/root/forge-auth.header"
|
||||||
|
NIX = "nix --extra-experimental-features nix-command"
|
||||||
|
ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
|
||||||
|
POOL = "/var/lib/swarm-pki"
|
||||||
|
MINTED = "/var/lib/swarm-bao-pki"
|
||||||
|
POLICY_UNITS_ACTIVE = (
|
||||||
|
"for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
|
||||||
|
"swarm-bao-agent-pki.service; do "
|
||||||
|
"[ \"$(systemctl is-active $u)\" = active ] || { echo \"$u is not active\"; exit 1; }; done"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Each machine's VPN addresses, from the example's mesh.nix.
|
||||||
|
WG = {
|
||||||
|
swarm: ("${exampleMesh.swarm-central.address}", "${exampleMesh.swarm-central.address6}"),
|
||||||
|
heavyduty: ("${exampleMesh.swarm-heavyduty.address}", "${exampleMesh.swarm-heavyduty.address6}"),
|
||||||
|
hive: ("${exampleMesh.hive-one.address}", "${exampleMesh.hive-one.address6}"),
|
||||||
|
}
|
||||||
|
LAN = {"swarm": "${lanAddr.swarm}", "heavyduty": "${lanAddr.heavyduty}", "hive": "${lanAddr.hive}", "outside": "${lanAddr.outside}"}
|
||||||
|
PUBLIC = {"ui": ("/", "23"), "forge": ("/api/healthz", "2"), "chat": ("/_matrix/client/versions", "2"), "auth": ("/api/health", "2")}
|
||||||
|
NAMES = ${builtins.toJSON nameOf}
|
||||||
|
INTERNAL = ${builtins.toJSON internalNames}
|
||||||
|
PRIVATE = ["grafana", "metrics", "logs", "otel", "bao-ui", "attic"]
|
||||||
|
|
||||||
|
def diagnose(m, unit=None, container=None):
|
||||||
|
# What a timed-out wait leaves behind: the failed units and the end of the
|
||||||
|
# journal of the units that were waited on.
|
||||||
|
where = f"--machine={container} " if container else ""
|
||||||
|
units = [unit] if isinstance(unit, str) else list(unit or [])
|
||||||
|
flags = " ".join(f"-u '{u}'" for u in units)
|
||||||
|
for command in [
|
||||||
|
f"systemctl {where}--failed --no-legend --no-pager",
|
||||||
|
*([f"journalctl {where}{flags} --no-pager -n 50 | tail -c 6000"] if units else []),
|
||||||
|
]:
|
||||||
|
print(f"--- {m.name}: {command}")
|
||||||
|
print(m.execute(command)[1])
|
||||||
|
|
||||||
|
def settle(m, command, timeout, unit=None, container=None):
|
||||||
|
try:
|
||||||
|
m.wait_until_succeeds(command, timeout=timeout)
|
||||||
|
except Exception:
|
||||||
|
diagnose(m, unit, container)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def controller(method, path, body=None, check="--fail-with-body"):
|
||||||
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
||||||
|
return swarm.succeed(
|
||||||
|
f"curl {check} --silent --unix-socket {SOCKET} -X {method} {API}{path} {data}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def forge(method, path, body=None):
|
||||||
|
data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
|
||||||
|
return swarm.succeed(
|
||||||
|
f"curl --fail-with-body --silent --insecure -H @{FORGE_AUTH} "
|
||||||
|
f"-X {method} {FORGE}/api/v1{path} {data}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_agent(body):
|
||||||
|
# The controller refuses a create until its dependencies answer (the
|
||||||
|
# archetype list comes from the builder's evaluation), so ask again.
|
||||||
|
for _ in range(60):
|
||||||
|
status, out = swarm.execute(
|
||||||
|
f"curl --silent --unix-socket {SOCKET} -o /root/create.out -w '%{{http_code}}' "
|
||||||
|
f"-X POST {API}/api/agents -H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}"
|
||||||
|
)
|
||||||
|
if out.strip().startswith("2"):
|
||||||
|
return
|
||||||
|
swarm.sleep(10)
|
||||||
|
diagnose(swarm, "swarm-controller")
|
||||||
|
raise Exception(f"the controller refused to create {body['name']}: HTTP {out}: {swarm.succeed('cat /root/create.out')}")
|
||||||
|
|
||||||
|
def http(m, name, path="/", resolve=None):
|
||||||
|
# The status code of an HTTPS request, 000 when nothing answers: a
|
||||||
|
# refused connection, and the gateway's catch-all, which closes the
|
||||||
|
# connection without a response.
|
||||||
|
pin = f"--resolve {name}:443:{resolve} " if resolve else ""
|
||||||
|
return m.execute(
|
||||||
|
f"curl --silent --insecure --max-time 10 {pin}-o /dev/null -w '%{{http_code}}' https://{name}{path}"
|
||||||
|
)[1].strip()
|
||||||
|
|
||||||
|
def answers(code):
|
||||||
|
return code not in ("000", "502", "503", "504")
|
||||||
|
|
||||||
|
def wait_http(m, name, accept, path="/", resolve=None, tries=90):
|
||||||
|
for _ in range(tries):
|
||||||
|
code = http(m, name, path, resolve)
|
||||||
|
if accept(code):
|
||||||
|
return code
|
||||||
|
m.sleep(10)
|
||||||
|
diagnose(m)
|
||||||
|
raise Exception(f"{m.name}: https://{name}{path} answered {code}")
|
||||||
|
|
||||||
|
def redeploy(m):
|
||||||
|
# What a second deploy does once the leaves are in place: start what
|
||||||
|
# was not started and restart what failed for want of them.
|
||||||
|
m.execute(
|
||||||
|
"failed=\"$(systemctl list-units --failed --plain --no-legend | cut -d' ' -f1)\"; "
|
||||||
|
"systemctl reset-failed; "
|
||||||
|
"[ -z \"$failed\" ] || systemctl restart --no-block $failed; "
|
||||||
|
"systemctl start --no-block multi-user.target"
|
||||||
|
)
|
||||||
|
|
||||||
|
def push_file(src_path, m, dest):
|
||||||
|
# One file from the store's host to another machine, through the
|
||||||
|
# driver: the operator's copy over the VPN.
|
||||||
|
payload = swarm.succeed(f"base64 -w0 {src_path}").strip()
|
||||||
|
m.succeed(f"install -d -m 0700 {POOL} && echo {payload} | base64 -d | install -m 0600 /dev/stdin {dest}")
|
||||||
|
|
||||||
|
start_all()
|
||||||
|
for m in (swarm, heavyduty, hive, outside):
|
||||||
|
m.wait_for_unit("multi-user.target")
|
||||||
|
|
||||||
|
with subtest("the VMs sit on the public network where the mesh expects them"):
|
||||||
|
for m, name in ((swarm, "swarm"), (heavyduty, "heavyduty"), (hive, "hive"), (outside, "outside")):
|
||||||
|
assert f"inet {LAN[name]}/" in m.succeed("ip -4 -o addr show"), name
|
||||||
|
|
||||||
|
with subtest("the toplevels are in heavyduty's store only, and nobody builds"):
|
||||||
|
for toplevel in TOPLEVELS:
|
||||||
|
heavyduty.succeed(f"{NIX} path-info {toplevel}")
|
||||||
|
hive.fail(f"{NIX} path-info {toplevel}")
|
||||||
|
# The control for the lookup: a path that is not there is not found.
|
||||||
|
heavyduty.fail(f"{NIX} path-info /nix/store/00000000000000000000000000000000-absent")
|
||||||
|
|
||||||
|
with subtest("assert 1: wg0 is up between all three machines, over IPv4 and IPv6"):
|
||||||
|
for m in (swarm, heavyduty, hive):
|
||||||
|
m.wait_for_unit("wireguard-wg0.service")
|
||||||
|
for src in (swarm, heavyduty, hive):
|
||||||
|
own = WG[src]
|
||||||
|
assert own[0] in src.succeed("ip -4 -o addr show dev wg0"), own
|
||||||
|
assert own[1] in src.succeed("ip -6 -o addr show dev wg0"), own
|
||||||
|
for dst in (swarm, heavyduty, hive):
|
||||||
|
if dst is src:
|
||||||
|
continue
|
||||||
|
v4, v6 = WG[dst]
|
||||||
|
src.wait_until_succeeds(f"ping -c 1 -W 2 {v4}", timeout=180)
|
||||||
|
src.wait_until_succeeds(f"ping -6 -c 1 -W 2 {v6}", timeout=180)
|
||||||
|
src.succeed(f"ping -c 3 -W 5 {v4}")
|
||||||
|
src.succeed(f"ping -6 -c 3 -W 5 {v6}")
|
||||||
|
# The address is routed through the tunnel, the public one is not.
|
||||||
|
assert "dev wg0" in hive.succeed(f"ip route get {WG[swarm][0]}")
|
||||||
|
assert "dev wg0" not in hive.succeed(f"ip route get {LAN['swarm']}")
|
||||||
|
# Negative control: the hive-two entry has no machine behind it.
|
||||||
|
hive.fail("ping -c 2 -W 2 ${exampleMesh.hive-two.address}")
|
||||||
|
|
||||||
|
with subtest("assert 3: split DNS"):
|
||||||
|
for key, machine in INTERNAL.items():
|
||||||
|
v4, v6 = WG[{"swarm-central": swarm, "swarm-heavyduty": heavyduty}[machine]]
|
||||||
|
got4 = hive.succeed(f"getent ahostsv4 {NAMES[key]}").split()[0]
|
||||||
|
got6 = hive.succeed(f"getent ahostsv6 {NAMES[key]}").split()[0]
|
||||||
|
assert got4 == v4, f"{NAMES[key]}: A {got4}, want {v4}"
|
||||||
|
assert got6 == v6, f"{NAMES[key]}: AAAA {got6}, want {v6}"
|
||||||
|
# The public names, from outside, are the swarm machine's public address.
|
||||||
|
for key in PUBLIC:
|
||||||
|
got = outside.succeed(f"getent ahostsv4 {NAMES[key]}").split()[0]
|
||||||
|
assert got == LAN["swarm"], f"{NAMES[key]}: {got}"
|
||||||
|
# The rest of the names do not resolve there, and none resolves to the VPN.
|
||||||
|
for key in PRIVATE + ["bao", "nats"]:
|
||||||
|
outside.fail(f"getent ahosts {NAMES[key]}")
|
||||||
|
for key in PUBLIC:
|
||||||
|
assert "10.77." not in outside.succeed(f"getent ahosts {NAMES[key]}")
|
||||||
|
|
||||||
|
with subtest("the example's leaves are copied to the machine that runs them"):
|
||||||
|
swarm.wait_until_succeeds(f"[ -s {MINTED}/hives/{HIVE}/hive-leaf.pem ] && [ -s {MINTED}/forge-publisher.pem ]", timeout=900)
|
||||||
|
# The store's host keeps the CA's and the granter's keys, and a hive's
|
||||||
|
# leaves are not another machine's.
|
||||||
|
swarm.succeed(
|
||||||
|
f"install -d -m 0700 {POOL} && cd {MINTED} && "
|
||||||
|
f"for f in *.pem; do case $f in ca-key.pem|granter*|server*) ;; *) install -m 0600 $f {POOL}/$f ;; esac; done"
|
||||||
|
)
|
||||||
|
swarm.succeed(f"[ -s {POOL}/ca.pem ] && [ -s {POOL}/services-issuer.pem ] && [ ! -e {POOL}/ca-key.pem ]")
|
||||||
|
redeploy(swarm)
|
||||||
|
|
||||||
|
with subtest("step 1: bao operator init"):
|
||||||
|
swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=900)
|
||||||
|
swarm.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")
|
||||||
|
|
||||||
|
with subtest("step 4: bao operator unseal, with the shamir seal"):
|
||||||
|
swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
||||||
|
swarm.succeed("bao status")
|
||||||
|
|
||||||
|
with subtest("step 1: the granter bootstrap"):
|
||||||
|
swarm.succeed(
|
||||||
|
"set -o pipefail; "
|
||||||
|
f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
|
||||||
|
"bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
|
||||||
|
"bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
|
||||||
|
"| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
|
||||||
|
"unset BAO_TOKEN; "
|
||||||
|
"systemctl restart swarm-bao-granter-role"
|
||||||
|
)
|
||||||
|
swarm.wait_until_succeeds(
|
||||||
|
"journalctl -u swarm-bao-granter-role --no-pager | grep -F 'Uploaded policy: bao-granter'",
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
|
||||||
|
with subtest("step 1: the policy units that waited for the granter are restarted"):
|
||||||
|
# See ./full-setup.nix for why these are started without waiting and polled.
|
||||||
|
swarm.succeed("systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service || true")
|
||||||
|
swarm.succeed("systemctl restart --no-block 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service")
|
||||||
|
swarm.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=900)
|
||||||
|
if swarm.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
|
||||||
|
swarm.succeed("systemctl --machine=swarm-bao restart openbao.service")
|
||||||
|
swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
|
||||||
|
swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
|
||||||
|
settle(swarm, POLICY_UNITS_ACTIVE, 1800, "swarm-bao-*-policy")
|
||||||
|
swarm.succeed(
|
||||||
|
"journalctl -u swarm-bao-controller-policy --no-pager "
|
||||||
|
"| grep -F 'Data written to: auth/cert/certs/swarm-controller'"
|
||||||
|
)
|
||||||
|
|
||||||
|
with subtest("the leaves for heavyduty and the hive are copied over, and their units start"):
|
||||||
|
# The builder, the CI runner and the cache each read the store with a
|
||||||
|
# leaf of their own; the hive with its reader and gateway leaves. The
|
||||||
|
# CA is what lets either check the store's certificate.
|
||||||
|
files = swarm.succeed(
|
||||||
|
f"cd {MINTED} && for f in *.pem; do case $f in ca-key.pem|granter*|server*) ;; *) echo $f ;; esac; done"
|
||||||
|
).split()
|
||||||
|
for name in files:
|
||||||
|
push_file(f"{MINTED}/{name}", heavyduty, f"{POOL}/{name}")
|
||||||
|
for name in ("client.pem", "client-key.pem", "hive-leaf.pem", "hive-leaf-key.pem"):
|
||||||
|
push_file(f"{MINTED}/hives/{HIVE}/{name}", hive, f"{POOL}/{name}")
|
||||||
|
push_file(f"{MINTED}/ca.pem", hive, f"{POOL}/ca.pem")
|
||||||
|
# Control: the lookup finds what was copied, and nothing of the CA's key.
|
||||||
|
heavyduty.succeed(f"[ -s {POOL}/builder.pem ] && [ -s {POOL}/ca.pem ] && [ ! -e {POOL}/ca-key.pem ]")
|
||||||
|
hive.succeed(f"[ -s {POOL}/hive-leaf.pem ] && [ ! -e {POOL}/builder.pem ]")
|
||||||
|
redeploy(heavyduty)
|
||||||
|
redeploy(hive)
|
||||||
|
|
||||||
|
with subtest("authelia's mint has published the clients' secrets"):
|
||||||
|
settle(
|
||||||
|
swarm,
|
||||||
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
||||||
|
"secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
|
||||||
|
1800,
|
||||||
|
"authelia-swarm-secrets",
|
||||||
|
"swarm-authelia",
|
||||||
|
)
|
||||||
|
settle(swarm, "[ \"$(systemctl is-active swarm-bao-operator-viewer-policy.service)\" = active ]", 600, "swarm-bao-operator-viewer-policy")
|
||||||
|
|
||||||
|
with subtest("assert 2: from outside, the forge, chat, SSO and the swarm UI answer on the public address"):
|
||||||
|
for key, (path, accept) in PUBLIC.items():
|
||||||
|
code = wait_http(outside, NAMES[key], lambda c, a=accept: c[:1] in a, path, tries=180)
|
||||||
|
print(f"{NAMES[key]}{path}: {code}")
|
||||||
|
|
||||||
|
with subtest("assert 2: from hive, over wg0, the private names answer"):
|
||||||
|
for key in PRIVATE:
|
||||||
|
code = wait_http(hive, NAMES[key], answers, tries=180)
|
||||||
|
print(f"{NAMES[key]}: {code}")
|
||||||
|
# The hive's dashboard, from the store's machine, over wg0.
|
||||||
|
code = wait_http(swarm, "${swarm.dashboardDomains.${hiveName}}", answers, resolve=WG[hive][0])
|
||||||
|
print(f"dashboard: {code}")
|
||||||
|
|
||||||
|
with subtest("assert 2: from outside, the private names get nothing, on any machine's public address"):
|
||||||
|
# The positive controls are the two subtests above: the same
|
||||||
|
# requests from inside answered. Here nginx is up on the swarm
|
||||||
|
# machine, so 000 is the catch-all closing the connection.
|
||||||
|
for key in PRIVATE:
|
||||||
|
code = http(outside, NAMES[key], resolve=LAN["swarm"])
|
||||||
|
assert code == "000", f"{NAMES[key]} at the swarm machine's public address: {code}"
|
||||||
|
code = http(outside, NAMES["attic"], resolve=LAN["heavyduty"])
|
||||||
|
assert code == "000", f"the cache at heavyduty's public address: {code}"
|
||||||
|
code = http(outside, "${swarm.dashboardDomains.${hiveName}}", resolve=LAN["hive"])
|
||||||
|
assert code == "000", f"the hive's dashboard at its public address: {code}"
|
||||||
|
# The public names are not on heavyduty or the hive either.
|
||||||
|
assert http(outside, NAMES["forge"], "/api/healthz", resolve=LAN["heavyduty"]) == "000"
|
||||||
|
|
||||||
|
with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
|
||||||
|
settle(
|
||||||
|
swarm,
|
||||||
|
f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
|
||||||
|
"secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
|
||||||
|
1800,
|
||||||
|
["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],
|
||||||
|
"hive-forge",
|
||||||
|
)
|
||||||
|
swarm.succeed(
|
||||||
|
f"umask 077 && printf 'Authorization: token %s\\n' \"$(cat /root/forge-token)\" > {FORGE_AUTH}"
|
||||||
|
)
|
||||||
|
swarm.wait_until_succeeds(
|
||||||
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/user >/dev/null", timeout=600
|
||||||
|
)
|
||||||
|
status = swarm.succeed(
|
||||||
|
f"curl --silent --insecure -o /dev/null -w '%{{http_code}}' -H @{FORGE_AUTH} {FORGE}/api/v1/orgs/hyperhive"
|
||||||
|
).strip()
|
||||||
|
if status != "200":
|
||||||
|
forge("POST", "/orgs", {"username": "hyperhive", "visibility": "public"})
|
||||||
|
forge("POST", "/orgs/hyperhive/repos", {"name": "hyperhive", "private": False, "default_branch": "main"})
|
||||||
|
swarm.succeed("cp -r ${hyperhiveRepo}/repo.git /root/hyperhive.git && chmod -R u+w /root/hyperhive.git")
|
||||||
|
swarm.succeed(
|
||||||
|
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
||||||
|
f"push {FORGE}/hyperhive/hyperhive.git main"
|
||||||
|
)
|
||||||
|
|
||||||
|
with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"):
|
||||||
|
swarm.wait_until_succeeds(
|
||||||
|
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800
|
||||||
|
)
|
||||||
|
create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE})
|
||||||
|
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
|
||||||
|
|
||||||
|
with subtest("assert 4: the agents are deployed on the hive, from the cache on heavyduty, and need a login"):
|
||||||
|
settle(
|
||||||
|
swarm,
|
||||||
|
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status "
|
||||||
|
"| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)"
|
||||||
|
" | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'",
|
||||||
|
3600,
|
||||||
|
["swarm-controller"],
|
||||||
|
)
|
||||||
|
# The closures came over the VPN from the cache: the hive's nix
|
||||||
|
# names it as its only extra substituter and holds the toplevels now.
|
||||||
|
fragment = hive.succeed("cat /run/hyperhive-nix/cache.conf")
|
||||||
|
assert "extra-substituters" in fragment and "${swarm.attic.domain}" in fragment, fragment
|
||||||
|
for toplevel in TOPLEVELS:
|
||||||
|
hive.succeed(f"{NIX} path-info {toplevel}")
|
||||||
|
# Not built on the swarm machine, which has no copy.
|
||||||
|
swarm.fail(f"{NIX} path-info {toplevel}")
|
||||||
|
'';
|
||||||
|
}
|
||||||
|
)
|
||||||
Loading…
Reference in a new issue