From 35c3f724f1a5c326f1808b0937daa9e699f0960f Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 12 Sep 2026 15:19:17 +0200 Subject: [PATCH 1/2] glue-matrix-bao-token: retry a failed login, keep degrading on an empty read The unit treats every failure as permanent: it prints why and `exit 0`s, with no `Restart=`, so one bad moment costs the whole boot. Two of its failure modes deserve that and one does not. `bao login` fails when the store is unreachable, sealed, or has not been given this host's cert-auth role yet. All three are transient. Measured on this morning's rebuild: 11:24:05 Started Container 'swarm-bao' 11:24:06 could not log in to swarm-bao with this host's certificate 11:24:07 Success! Data written to: auth/cert/certs/swarm-secret-publisher It lost by one second, and stayed degraded for the boot. The publisher next to it hit the same race and recovered on its first retry, because it has `Restart=on-failure`. `bao kv get` returning nothing is the opposite: the store answered, and holds no token at that path. A retry cannot improve it, so that branch keeps `exit 0` and the local token. The bound is sized for this race, not for an unseal. `swarm-bao-controller-policy` waits 2880 x 30s because a shamir unseal is a human action and that unit blocks nothing. This one is `Before=` the homeserver's container, so every retry is time the homeserver may spend waiting -- 4 x 15s covers a container-start race with margin, and a store still sealed after it degrades exactly as it does today. `StartLimit*` are `[Unit]` settings and are ignored under `[Service]`, so they are top-level attrs here. The module-eval case asserts the window outlasts `RestartSec x burst`, since a burst that cannot be reached is a unit that looks like it retries and does not. Refs #4303 --- nix/host-modules/glue-matrix-bao-token.nix | 23 +++++++++++++++++++++- nix/module-eval.nix | 21 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 784d087c..3b40a403 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -77,6 +77,20 @@ in deployCfg.bao.package pkgs.coreutils ]; + # Sized for the race this loses, not for an unseal. `swarm-bao` comes up + # seconds before this unit asks, and the cert-auth role it logs in + # against is written seconds after — so a few short attempts cover it. + # ⚠️ `swarm-bao-controller-policy`'s 2880 × 30s is NOT the model to copy: + # that unit blocks nothing, while this one is `Before=` the homeserver's + # container, so every retry is time the homeserver may spend waiting. A + # store still sealed after this window keeps the degrade below, which is + # the same place it ends up today. + # + # `StartLimit*` are `[Unit]` settings, so they go here and not in + # `serviceConfig` — systemd ignores them under `[Service]`. The window + # has to exceed `RestartSec × burst`. + startLimitBurst = 4; + startLimitIntervalSec = 300; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -84,6 +98,8 @@ in # left to systemd's default, so the number a boot waits on is in # the file that waits. TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -117,6 +133,11 @@ in # presents; without a token `bao` asks its token helper instead, and # that is a `sh` this unit's `path` does not carry. `-token-only` # answers on stdout and skips the helper on both sides. + # + # Fails LOUDLY, unlike the read below: the three states a login failure + # covers — store not up, sealed, role not written yet — are all things + # a retry fixes, and `Restart=on-failure` above is what retries. Exiting + # 0 here spends the whole boot on a condition that was seconds old. if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "could not log in to swarm-bao with this host's certificate; keeping the token hive-matrix already has." >&2 if [ -s "$err" ]; then @@ -124,7 +145,7 @@ in else echo "bao failed without writing a diagnostic." >&2 fi - exit 0 + exit 1 fi export BAO_TOKEN diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7111151b..acaf5b70 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1127,6 +1127,27 @@ let && !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x") && holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x"; } + { + # A login failure is the store being unreachable, sealed, or not yet + # holding this host's role — all of which a retry fixes. A read that + # answers "nothing there" is not, so only the first is allowed to fail + # the unit. + name = "the matrix token reader retries a failed login and still degrades on an empty read"; + ok = + let + u = baoWithMatrix.systemd.services.swarm-bao-matrix-token; + # Everything between the login's failure branch and the read's, which + # is where the exit that decides "retry or give up" lives. + afterLogin = lib.last (lib.splitString "bao login" u.script); + loginBranch = lib.head (lib.splitString "bao kv get" afterLogin); + in + u.serviceConfig.Restart or null == "on-failure" + && u.startLimitBurst or 0 > 0 + # The window has to outlast every attempt, or the burst is unreachable. + && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0) + && lib.hasInfix "exit 1" loginBranch + && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script)); + } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts, From 4db746a22c538a508382e37f084f5c5dfa51eebd Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 12 Sep 2026 15:31:07 +0200 Subject: [PATCH 2/2] glue-matrix-bao-token: do not assert what the Before= ordering does The retry comment claimed "every retry is time the homeserver may spend waiting". That is the conservative reading, not a measured one, and argus is right that it is probably wrong: systemd ordering typically resolves once a unit's first start job completes, success or failure, and an auto-restart after that is not a new ordering-relevant job. If that holds, the container proceeds after the first failed attempt -- same timing as today -- and what the retries buy is the token file being correct within ~60s for whatever starts next, rather than this boot's race getting a second chance. Both readings justify the same tight bound, for different reasons, so the comment now says the ordering behaviour is unverified instead of picking one. An agent container cannot reach a systemd manager to settle it. Refs #4303 --- nix/host-modules/glue-matrix-bao-token.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 3b40a403..4e84d8da 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -80,11 +80,12 @@ in # Sized for the race this loses, not for an unseal. `swarm-bao` comes up # seconds before this unit asks, and the cert-auth role it logs in # against is written seconds after — so a few short attempts cover it. - # ⚠️ `swarm-bao-controller-policy`'s 2880 × 30s is NOT the model to copy: - # that unit blocks nothing, while this one is `Before=` the homeserver's - # container, so every retry is time the homeserver may spend waiting. A - # store still sealed after this window keeps the degrade below, which is - # the same place it ends up today. + # ⚠️ `swarm-bao-controller-policy`'s 2880 × 30s is NOT the model to copy. + # That unit blocks nothing; this one is `Before=` the homeserver's + # container, and whether that ordering waits across an auto-restart is + # unverified — so an hours-long window would be a bet on an unknown, + # where a minute is not. A store still sealed after it keeps the degrade + # below, as today. # # `StartLimit*` are `[Unit]` settings, so they go here and not in # `serviceConfig` — systemd ignores them under `[Service]`. The window