diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 784d087c..3b40a403 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -77,6 +77,20 @@ in deployCfg.bao.package pkgs.coreutils ]; + # Sized for the race this loses, not for an unseal. `swarm-bao` comes up + # seconds before this unit asks, and the cert-auth role it logs in + # against is written seconds after — so a few short attempts cover it. + # ⚠️ `swarm-bao-controller-policy`'s 2880 × 30s is NOT the model to copy: + # that unit blocks nothing, while this one is `Before=` the homeserver's + # container, so every retry is time the homeserver may spend waiting. A + # store still sealed after this window keeps the degrade below, which is + # the same place it ends up today. + # + # `StartLimit*` are `[Unit]` settings, so they go here and not in + # `serviceConfig` — systemd ignores them under `[Service]`. The window + # has to exceed `RestartSec × burst`. + startLimitBurst = 4; + startLimitIntervalSec = 300; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -84,6 +98,8 @@ in # left to systemd's default, so the number a boot waits on is in # the file that waits. TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -117,6 +133,11 @@ in # presents; without a token `bao` asks its token helper instead, and # that is a `sh` this unit's `path` does not carry. `-token-only` # answers on stdout and skips the helper on both sides. + # + # Fails LOUDLY, unlike the read below: the three states a login failure + # covers — store not up, sealed, role not written yet — are all things + # a retry fixes, and `Restart=on-failure` above is what retries. Exiting + # 0 here spends the whole boot on a condition that was seconds old. if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "could not log in to swarm-bao with this host's certificate; keeping the token hive-matrix already has." >&2 if [ -s "$err" ]; then @@ -124,7 +145,7 @@ in else echo "bao failed without writing a diagnostic." >&2 fi - exit 0 + exit 1 fi export BAO_TOKEN diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7111151b..acaf5b70 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1127,6 +1127,27 @@ let && !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x") && holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x"; } + { + # A login failure is the store being unreachable, sealed, or not yet + # holding this host's role — all of which a retry fixes. A read that + # answers "nothing there" is not, so only the first is allowed to fail + # the unit. + name = "the matrix token reader retries a failed login and still degrades on an empty read"; + ok = + let + u = baoWithMatrix.systemd.services.swarm-bao-matrix-token; + # Everything between the login's failure branch and the read's, which + # is where the exit that decides "retry or give up" lives. + afterLogin = lib.last (lib.splitString "bao login" u.script); + loginBranch = lib.head (lib.splitString "bao kv get" afterLogin); + in + u.serviceConfig.Restart or null == "on-failure" + && u.startLimitBurst or 0 > 0 + # The window has to outlast every attempt, or the burst is unreachable. + && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0) + && lib.hasInfix "exit 1" loginBranch + && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script)); + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts,