diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index 5522b5ec..e96f2c3f 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -123,11 +123,13 @@ the side that has one. **Confirm with `systemctl status swarm-bao-granter-role`**, which should log `Uploaded policy: bao-granter` and `Data written to: auth/cert/certs/bao-granter`. -Then restart the granting units that failed while they waited: +Then restart the granting units that failed while they waited. These two +names cover every unit that logs in as the granter, and CI fails when one +doesn't: ```bash -systemctl reset-failed 'swarm-bao-*-policy.service' -systemctl restart 'swarm-bao-*-policy.service' +systemctl reset-failed 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service +systemctl restart 'swarm-bao-*-policy.service' swarm-bao-agent-pki.service systemctl status swarm-bao-controller-policy # Uploaded policy, Data written to: auth/cert/certs/swarm-controller rm /var/lib/swarm-bao-bootstrap/grant.token ``` diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index bd2ee06e..065fc780 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -285,17 +285,24 @@ let ] ); + # Exit 0 only on a reachable, unsealed store; its error, if any, is appended + # to `$err`. `status` needs no token, but the CLI resolves a token helper + # under `$HOME` before it asks, and systemd sets no `HOME` for a unit without + # `User=`: there it exits 1 on a healthy store. An empty home holds no helper + # config and no token. + baoStoreUp = ''HOME=/var/empty bao status >/dev/null 2>>"$err"''; + # The login every granting unit starts with. It FAILS rather than skips: a # grant that was not written is otherwise invisible until whatever needs it - # fails somewhere else. `bao status` exits 0 only on a reachable, unsealed - # store, which separates "the granter is not set up" from "retry later"; - # either way bao's own message follows. + # fails somewhere else. `baoStoreUp` separates "the granter is not set up" + # from "retry later"; either way bao's own message follows. granterLogin = '' err="$(mktemp)" trap 'rm -f "$err"' EXIT if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - if bao status >/dev/null 2>&1; then + if ${baoStoreUp}; then echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2 + ${lib.optionalString haveBootstrapToken ''echo "swarm-bao-granter-role writes that role; journalctl -u swarm-bao-granter-role says why it has not." >&2''} echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps} else @@ -2379,13 +2386,31 @@ in BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" export BAO_TOKEN + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + # Every cert-auth role in this file lives under `auth/cert/`, and # nothing else creates that mount. # # Asked rather than attempted: `auth enable` errors on a mount # that already exists, and recognising that would tie a rebuild # to an error string we have never seen this store emit. - mounted="$(bao auth list -format=json)" + # + # The token's first use, and the bootstrap policy grants this read, + # so a refusal from a store that is up is the token's: expired, + # revoked, or minted without that policy. + if ! mounted="$(bao auth list -format=json 2>"$err")"; then + if ${baoStoreUp}; then + echo ${lib.escapeShellArg "the store is unsealed but refused the bootstrap token in ${baoDeploy.bootstrapTokenFile}: it has expired, been revoked, or does not carry the bao-bootstrap policy."} >&2 + echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 + ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps} + cat "$err" >&2 + exit 4 + fi + echo "the store is sealed or unreachable; retrying." >&2 + cat "$err" >&2 + exit 1 + fi case "$mounted" in *'"cert/"'*) ;; *) bao auth enable cert ;; diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index a32e0348..df191daa 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -821,6 +821,23 @@ let && lib.hasInfix "systemctl restart swarm-bao-granter-role" sc ) grantingUnitNames; } + { + # A token the store refuses gets the same step, under an exit status of + # its own. Both store checks run with a HOME: without one `bao status` + # exits 1 on a healthy store and every refusal reads as "sealed". + name = "a refused bootstrap token prints the one-time step and exits 4, and every store check has a HOME"; + ok = + let + s = baoGrantHere.systemd.services; + g = s.swarm-bao-granter-role.script; + in + lib.hasInfix "refused the bootstrap token in /run/secrets/bao-bootstrap.token" g + && lib.hasInfix "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl" g + && lib.hasInfix "exit 4" g + && lib.all (unit: lib.hasInfix "HOME=/var/empty bao status" s.${unit}.script) ( + [ "swarm-bao-granter-role" ] ++ grantingUnitNames + ); + } { # Every granting unit retries a sealed or late store for a day, in the # `[Unit]` section systemd reads it from, and waits for the unit that @@ -1164,6 +1181,34 @@ let && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues bootstrapUnits); } + { + # setup.md's restart after the one-time step: a granting unit it misses + # stays failed once its start limit is hit. + name = "setup.md's reset-failed and restart after the one-time step reach every granting unit"; + ok = + let + lines = lib.splitString "\n" (builtins.readFile ../../docs/getting-started/setup.md); + # The unit patterns on the one `systemctl 'swarm-bao-…` line. + argsOf = + verb: + map (l: map (lib.replaceStrings [ "'" ] [ "" ]) (lib.drop 2 (lib.splitString " " l))) ( + lib.filter (lib.hasPrefix "systemctl ${verb} 'swarm-bao-") lines + ); + covers = + pats: + lib.all ( + unit: + lib.any ( + p: builtins.match (lib.replaceStrings [ "." "*" ] [ "[.]" ".*" ] p) "${unit}.service" != null + ) pats + ) (lib.attrNames granterUnits); + in + lib.length (argsOf "restart") == 1 + && argsOf "reset-failed" == argsOf "restart" + && covers (lib.head (argsOf "restart")) + # The control: the policy glob alone misses one. + && !(covers [ "swarm-bao-*-policy.service" ]); + } { # And the grants side: a stanza the parser skipped would read as a # grant that is not there.