semi-spread test: bao on the store host presents a client certificate

nix/vm-tests/semi-spread.nix

@@ -313,6 +313,11 @@ pkgs.testers.runNixOSTest (
       ARCHETYPE = {"source": "hyperhive", "name": "reviewer"}
       POOL = "/var/lib/swarm-pki"
       MINTED = "/var/lib/swarm-bao-pki"
+      # The store's listeners refuse a connection that presents no certificate
+      # from its CA, and this machine runs no hive, so its `bao` wrapper has
+      # none. Any leaf does for the CLI; the services-issuer's is in POOL
+      # before the first call.
+      BAO = f"env BAO_CLIENT_CERT={POOL}/services-issuer.pem BAO_CLIENT_KEY={POOL}/services-issuer-key.pem bao"
       POLICY_UNITS_ACTIVE = (
           "for u in $(systemctl list-unit-files 'swarm-bao-*-policy.service' --no-legend | cut -d' ' -f1 | grep -v operator-viewer) "
           "swarm-bao-agent-pki.service; do "
@@ -351,6 +356,17 @@ pkgs.testers.runNixOSTest (
               diagnose(m, unit, container)
               raise
 
+      def store_sealed(timeout):
+          # `bao status` exits 2 when it reaches a sealed store and 1 for any
+          # address, CA or client-certificate problem.
+          try:
+              swarm.wait_until_succeeds(f"[ \"$({BAO} status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=timeout)
+          except Exception:
+              out = swarm.execute(f"{BAO} status 2>&1; echo \"exit=$?\"; grep -h '^export BAO_' \"$(command -v bao)\" | tr '\\n' ' '")[1]
+              print(f"bao status on swarm: {' '.join(out.split())}")
+              diagnose(swarm, "openbao", "${swarm.bao.machine}")
+              raise
+
       def controller(method, path, body=None, check="--fail-with-body"):
           data = f"-H 'Content-Type: application/json' -d {shlex.quote(json.dumps(body))}" if body else ""
           return swarm.succeed(
@@ -480,19 +496,19 @@ pkgs.testers.runNixOSTest (
           redeploy(swarm)
 
       with subtest("step 1: bao operator init"):
-          swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=900)
+          store_sealed(900)
-          swarm.succeed(f"umask 077 && bao operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")
+          swarm.succeed(f"umask 077 && {BAO} operator init -key-shares=1 -key-threshold=1 -format=json > {INIT}")
 
       with subtest("step 4: bao operator unseal, with the shamir seal"):
-          swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
+          swarm.succeed(f"{BAO} operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
-          swarm.succeed("bao status")
+          swarm.succeed(f"{BAO} status")
 
       with subtest("step 1: the granter bootstrap"):
           swarm.succeed(
               "set -o pipefail; "
               f"export BAO_TOKEN=\"$(jq -r .root_token {INIT})\"; "
-              "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
+              f"{BAO} policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl; "
-              "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
+              f"{BAO} token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token "
               "| install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token; "
               "unset BAO_TOKEN; "
               "systemctl restart swarm-bao-granter-role"
@@ -509,8 +525,8 @@ pkgs.testers.runNixOSTest (
           swarm.wait_until_succeeds("[ \"$(systemctl is-active swarm-bao-agent-pki.service)\" = active ]", timeout=900)
           if swarm.execute("journalctl -u swarm-bao-agent-pki --no-pager | grep -F 'restart openbao.service'")[0] == 0:
               swarm.succeed("systemctl --machine=swarm-bao restart openbao.service")
-              swarm.wait_until_succeeds("[ \"$(bao status >/dev/null 2>&1; echo $?)\" = 2 ]", timeout=600)
+              store_sealed(600)
-              swarm.succeed(f"bao operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
+              swarm.succeed(f"{BAO} operator unseal \"$(jq -r '.unseal_keys_b64[0]' {INIT})\"")
           settle(swarm, POLICY_UNITS_ACTIVE, 1800, "swarm-bao-*-policy")
           swarm.succeed(
               "journalctl -u swarm-bao-controller-policy --no-pager "
@@ -538,7 +554,7 @@ pkgs.testers.runNixOSTest (
       with subtest("authelia's mint has published the clients' secrets"):
           settle(
               swarm,
-              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
+              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" {BAO} kv get -field=value "
               "secret/swarm/controller/swarm-controller/oidc/client >/dev/null",
               1800,
               "authelia-swarm-secrets",
@@ -576,7 +592,7 @@ pkgs.testers.runNixOSTest (
       with subtest("the forge holds hyperhive/hyperhive at the fixture's commit"):
           settle(
               swarm,
-              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" bao kv get -field=value "
+              f"BAO_TOKEN=\"$(jq -r .root_token {INIT})\" {BAO} kv get -field=value "
               "secret/swarm/controller/swarm-controller/forge/token > /root/forge-token",
               1800,
               ["forgejo", "forgejo-swarm-controller-account", "forgejo-sso-source"],