diff --git a/nix/vm-tests/semi-spread.nix b/nix/vm-tests/semi-spread.nix index c02f1109d..e11959071 100644 --- a/nix/vm-tests/semi-spread.nix +++ b/nix/vm-tests/semi-spread.nix @@ -313,6 +313,11 @@ pkgs.testers.runNixOSTest ( ARCHETYPE = {"source": "hyperhive", "name": "reviewer"} POOL = "/var/lib/swarm-pki" MINTED = "/var/lib/swarm-bao-pki" + # The store's listeners refuse a connection that presents no certificate + # from its CA, and this machine runs no hive, so its `bao` wrapper has + # none. Any leaf does for the CLI; the services-issuer's is in POOL + # before the first call. + BAO = f"env BAO_CLIENT_CERT={POOL}/services-issuer.pem BAO_CLIENT_KEY={POOL}/services-issuer-key.pem bao" POLICY_UNITS_ACTIVE = ( "for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) " "swarm-bao-agent-pki.service; do " @@ -351,6 +356,17 @@ pkgs.testers.runNixOSTest ( diagnose(m, unit, container) raise + def store_sealed(timeout): + # `bao status` exits 2 when it reaches a sealed store and 1 for any + # address, CA or client-certificate problem. + try: + swarm.wait_until_succeeds(f"[ \"$({BAO} status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=timeout) + except Exception: + out = swarm.execute(f"{BAO} status 2>&1; echo \"exit=$?\"; grep -h '^export BAO_' \"$(command -v bao)\" | tr '\\n' ' '")[1] + print(f"bao status on swarm: {' '.join(out.split())}") + diagnose(swarm, "openbao", "${swarm.bao.machine}") + raise + def controller(method, path, body=None, check="--fail-with-body"): data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else "" return swarm.succeed( @@ -480,19 +496,19 @@ pkgs.testers.runNixOSTest ( redeploy(swarm) with subtest("step 1: bao operator init"): - swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=900) - swarm.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}") + store_sealed(900) + swarm.succeed(f"umask 077 && {BAO} operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}") with subtest("step 4: bao operator unseal, with the shamir seal"): - swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") - swarm.succeed("bao status") + swarm.succeed(f"{BAO} operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") + swarm.succeed(f"{BAO} status") with subtest("step 1: the granter bootstrap"): swarm.succeed( "set -o pipefail; " f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; " - "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; " - "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token " + f"{BAO} policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; " + f"{BAO} token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token " "| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; " "unset BAO_TOKEN; " "systemctl restart swarm-bao-granter-role" @@ -509,8 +525,8 @@ pkgs.testers.runNixOSTest ( swarm.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=900) if swarm.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0: swarm.succeed("systemctl --machine=swarm-bao restart openbao.service") - swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600) - swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") + store_sealed(600) + swarm.succeed(f"{BAO} operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"") settle(swarm, POLICY_UNITS_ACTIVE, 1800, "swarm-bao-*-policy") swarm.succeed( "journalctl -u swarm-bao-controller-policy --no-pager " @@ -538,7 +554,7 @@ pkgs.testers.runNixOSTest ( with subtest("authelia's mint has published the clients' secrets"): settle( swarm, - f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value " + f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" {BAO} kv get -field=value " "secret/swarm/controller/swarm-controller/oidc/client >/dev/null", 1800, "authelia-swarm-secrets", @@ -576,7 +592,7 @@ pkgs.testers.runNixOSTest ( with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"): settle( swarm, - f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value " + f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" {BAO} kv get -field=value " "secret/swarm/controller/swarm-controller/forge/token > /root/forge-token", 1800, ["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],