From 35c3f724f1a5c326f1808b0937daa9e699f0960f Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 12 Sep 2026 15:19:17 +0200 Subject: [PATCH] glue-matrix-bao-token: retry a failed login, keep degrading on an empty read The unit treats every failure as permanent: it prints why and `exit 0`s, with no `Restart=`, so one bad moment costs the whole boot. Two of its failure modes deserve that and one does not. `bao login` fails when the store is unreachable, sealed, or has not been given this host's cert-auth role yet. All three are transient. Measured on this morning's rebuild: 11:24:05 Started Container 'swarm-bao' 11:24:06 could not log in to swarm-bao with this host's certificate 11:24:07 Success! Data written to: auth/cert/certs/swarm-secret-publisher It lost by one second, and stayed degraded for the boot. The publisher next to it hit the same race and recovered on its first retry, because it has `Restart=on-failure`. `bao kv get` returning nothing is the opposite: the store answered, and holds no token at that path. A retry cannot improve it, so that branch keeps `exit 0` and the local token. The bound is sized for this race, not for an unseal. `swarm-bao-controller-policy` waits 2880 x 30s because a shamir unseal is a human action and that unit blocks nothing. This one is `Before=` the homeserver's container, so every retry is time the homeserver may spend waiting -- 4 x 15s covers a container-start race with margin, and a store still sealed after it degrades exactly as it does today. `StartLimit*` are `[Unit]` settings and are ignored under `[Service]`, so they are top-level attrs here. The module-eval case asserts the window outlasts `RestartSec x burst`, since a burst that cannot be reached is a unit that looks like it retries and does not. Refs #4303 --- nix/host-modules/glue-matrix-bao-token.nix | 23 +++++++++++++++++++++- nix/module-eval.nix | 21 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 784d087c..3b40a403 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -77,6 +77,20 @@ in deployCfg.bao.package pkgs.coreutils ]; + # Sized for the race this loses, not for an unseal. `swarm-bao` comes up + # seconds before this unit asks, and the cert-auth role it logs in + # against is written seconds after — so a few short attempts cover it. + # ⚠️ `swarm-bao-controller-policy`'s 2880 × 30s is NOT the model to copy: + # that unit blocks nothing, while this one is `Before=` the homeserver's + # container, so every retry is time the homeserver may spend waiting. A + # store still sealed after this window keeps the degrade below, which is + # the same place it ends up today. + # + # `StartLimit*` are `[Unit]` settings, so they go here and not in + # `serviceConfig` — systemd ignores them under `[Service]`. The window + # has to exceed `RestartSec × burst`. + startLimitBurst = 4; + startLimitIntervalSec = 300; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -84,6 +98,8 @@ in # left to systemd's default, so the number a boot waits on is in # the file that waits. TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -117,6 +133,11 @@ in # presents; without a token `bao` asks its token helper instead, and # that is a `sh` this unit's `path` does not carry. `-token-only` # answers on stdout and skips the helper on both sides. + # + # Fails LOUDLY, unlike the read below: the three states a login failure + # covers — store not up, sealed, role not written yet — are all things + # a retry fixes, and `Restart=on-failure` above is what retries. Exiting + # 0 here spends the whole boot on a condition that was seconds old. if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "could not log in to swarm-bao with this host's certificate; keeping the token hive-matrix already has." >&2 if [ -s "$err" ]; then @@ -124,7 +145,7 @@ in else echo "bao failed without writing a diagnostic." >&2 fi - exit 0 + exit 1 fi export BAO_TOKEN diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7111151b..acaf5b70 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1127,6 +1127,27 @@ let && !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x") && holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x"; } + { + # A login failure is the store being unreachable, sealed, or not yet + # holding this host's role — all of which a retry fixes. A read that + # answers "nothing there" is not, so only the first is allowed to fail + # the unit. + name = "the matrix token reader retries a failed login and still degrades on an empty read"; + ok = + let + u = baoWithMatrix.systemd.services.swarm-bao-matrix-token; + # Everything between the login's failure branch and the read's, which + # is where the exit that decides "retry or give up" lives. + afterLogin = lib.last (lib.splitString "bao login" u.script); + loginBranch = lib.head (lib.splitString "bao kv get" afterLogin); + in + u.serviceConfig.Restart or null == "on-failure" + && u.startLimitBurst or 0 > 0 + # The window has to outlast every attempt, or the burst is unreachable. + && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0) + && lib.hasInfix "exit 1" loginBranch + && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script)); + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts,