retry the authelia mint and the appservice render until the store answers, and let tuwunel retry

docs/integrations/matrix.md

@@ -171,6 +171,7 @@ The swarm secret store holds the registration's tokens, at
 `swarm-matrix-ctl appservice render` fetches them and renders the
 registration tuwunel loads as a credential onto the `hive-matrix`
 container's tmpfs, before tuwunel starts.
+A render that can't reach the store fails and retries, and tuwunel, which fails to start without the registration, retries until one has landed.
 `appservice sync` runs beside tuwunel on a timer. The store's tokens win:
 when they differ from the rendered copy, it rewrites it. It
 restarts tuwunel whenever the registration is newer than tuwunel's last

docs/swarm/credentials.md

@@ -89,7 +89,7 @@ The same unit mints the client secrets, straight into the store. It reads
 only when the store answers and holds none does it run `authelia crypto hash generate`,
 create the leaf and write the digest. Authelia keeps only the digest on disk, in
 `oidc-clients/<id>.digest`, and starts from it while the store is down. A store that
-can't answer a read never mints, because a second secret would rotate the first. A client with a
+can't answer a read never mints, because a second secret would rotate the first. When it can't read the store, the unit still writes authelia's client file from the digests, then exits non-zero and retries every 15 seconds, so authelia starts meanwhile and `swarm-authelia-bridge` finds its secret once the store answers. A client with a
 secret in the store but no digest gets the digest of that secret and keeps the secret; `authelia crypto hash generate` computes the digest,
 with the password typed into its prompt through a pty, never in a command line.
 `swarm-authelia-bridge` fetches its own secret from

nix/host-modules/hive-matrix.nix

@@ -1044,23 +1044,29 @@ in
           # The swarm registration. The credential id carries a `.yaml`
           # suffix on purpose: `appservice_dir` above names this very
           # directory, and tuwunel takes only `.yaml`/`.yml` entries from
-          # it. A missing source fails this unit, which is why the render
-          # below is `requiredBy` it.
+          # it. A missing source fails this unit rather than starting it
+          # without the registration.
           ++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceRuntimeDir}/swarm.yaml";
 
+          # Upstream's `Restart=on-failure` retries a start that failed for
+          # want of the registration; a start limit would end the retries.
+          systemd.services.tuwunel.startLimitIntervalSec = lib.mkIf ctlActive 0;
+
           # The swarm registration, fetched from the store and rendered onto
           # the tmpfs before the homeserver loads it. A store that does not
-          # answer fails this unit, and with it the homeserver's start: the
-          # tokens are on no disk to start from. It retries like the other
-          # store fetches, and `swarm-matrix-appservice-sync` below starts the
-          # homeserver once the registration is there.
+          # answer fails this unit, and the homeserver's start fails on the
+          # missing credential: the tokens are on no disk to start from. Both
+          # retry, so the homeserver comes up on its own once a render has
+          # landed. `wantedBy`, not `requiredBy`: a start whose dependency
+          # failed is dropped without running, and the homeserver's own retry
+          # never begins.
           #
           # `RuntimeDirectoryPreserve` because the sync unit shares the
           # directory: neither unit's stop may remove it from under the other.
           systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive {
             description = "render the swarm's appservice registration";
             before = [ "tuwunel.service" ];
-            requiredBy = [ "tuwunel.service" ];
+            wantedBy = [ "tuwunel.service" ];
             inherit (storeRetry) startLimitBurst startLimitIntervalSec;
             serviceConfig = storeRetry.serviceConfig // {
               Type = "oneshot";

nix/host-modules/lib/oidc-client-secrets.nix

@@ -11,8 +11,10 @@
 # secret behind it. A store that cannot be read never mints, because a second
 # secret would rotate the first.
 #
-# Return statuses: 0 done, 1 retry later (the store could not answer, or a
-# digest could not be computed).
+# Return statuses: 0 done, 1 retry later (the store could not answer and the
+# client has no digest, or a digest could not be computed), 2 the client keeps
+# its digest but the store could not be read, so the caller finishes its work
+# from the digests and then retries.
 #
 # Pure function — NOT a NixOS module. Call it after `./store-seeded-secret.nix`
 # and `./atomic-write-secret.nix`, with `store_init` and `store_login` run and
@@ -92,7 +94,7 @@
       if [ -s "$dig" ]; then
         chmod 0600 "$dig"
         echo "oidc client $id: the store cannot be read; keeping its digest" >&2
-        return 0
+        return 2
       fi
       echo "oidc client $id: the store cannot be read and there is no digest; not minting" >&2
       return 1

nix/host-modules/swarm-authelia.nix

@@ -522,11 +522,15 @@ let
   # to disk. A digest on disk is enough for authelia to start while the store is
   # down; a client with no digest cannot be rendered, so that run fails and
   # retries (./lib/store-retry.nix). A client whose digest is missing gets it
-  # from the secret that exists, so nothing is minted again.
+  # from the secret that exists, so nothing is minted again. A store that
+  # cannot be read fails the run too, after the files are rendered from the
+  # digests (`STORE_UNREADABLE`, checked at the end of the unit's script), so
+  # the unit retries until the store answers and authelia has started from the
+  # digests meanwhile.
   storeMintScript = ''
     ${oidcClientSecrets}
     ${lib.concatMapStrings (c: ''
-      mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || exit $?
+      mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || { rc=$?; [ "$rc" = 2 ] || exit "$rc"; STORE_UNREADABLE=1; }
     '') secretClients}
   '';
 
@@ -1147,9 +1151,14 @@ in
           # itself, as its own user, under `PrivateUsers=true`.
           systemd.services."${unitName}-secrets" = {
             description = "Generate authelia's secrets on first boot";
-            wantedBy = [ "multi-user.target" ];
+            # With a store identity authelia `wants` this unit, not `requires`
+            # it: a run that fails because the store cannot be read must not
+            # stop authelia from starting on the digests and keys already on
+            # disk. Authelia still cannot start without a key file, which
+            # fails its credential load.
+            wantedBy = [ "multi-user.target" ] ++ lib.optional publishes "${unitName}.service";
             before = [ "${unitName}.service" ];
-            requiredBy = [ "${unitName}.service" ];
+            requiredBy = lib.optional (!publishes) "${unitName}.service";
             # `deployCfg.authelia.package` is here for its CLI, not its
             # daemon: the client secrets are minted with `authelia crypto
             # hash generate`, which is the only way to produce a digest in
@@ -1167,10 +1176,9 @@ in
               pkgs.systemd
               pkgs.util-linux
             ];
-            # A store that cannot answer, with a key missing on disk, is a
-            # failed run that retries (./lib/store-retry.nix).
-            startLimitBurst = lib.mkIf publishes storeRetry.startLimitBurst;
-            startLimitIntervalSec = lib.mkIf publishes storeRetry.startLimitIntervalSec;
+            # A store that cannot answer is a failed run that retries, with no
+            # start limit, until it does (./lib/store-retry.nix).
+            startLimitIntervalSec = lib.mkIf publishes 0;
             serviceConfig = {
               Type = "oneshot";
               RemainAfterExit = true;
@@ -1184,6 +1192,7 @@ in
             // lib.optionalAttrs publishes (
               storeRetry.serviceConfig
               // {
+                RestartSec = 15;
                 LoadCredential = lib.attrNames storeFiles;
                 TimeoutStartSec = 120;
               }
@@ -1217,6 +1226,12 @@ in
                 echo "seeded $users with the disabled placeholder ${placeholderUser}"
               fi
               chmod 0600 "$users"
+              ${lib.optionalString (publishes && oidcEnabled) ''
+                if [ -n "''${STORE_UNREADABLE-}" ]; then
+                  echo "the store could not be read; authelia runs on its digests and this unit retries" >&2
+                  exit 1
+                fi
+              ''}
             '';
           };
 

nix/module-eval/bao-matrix-reader.nix

@@ -264,15 +264,23 @@ let
     {
       # A missing `LoadCredential` source fails tuwunel, so the render has to
       # have run first. It fetches from the store, so it carries the store's
-      # address and matrix-ctl's identity, and retries on failure: a store
-      # that does not answer keeps the homeserver down until it does.
-      name = "the swarm registration is rendered before tuwunel and required by it, from the store, with retries";
+      # address and matrix-ctl's identity, and retries on failure. It is
+      # `wantedBy` tuwunel, not `requiredBy`: a start whose dependency failed
+      # never runs, so tuwunel's own retry never began and nothing brought it
+      # back. tuwunel retries without a start limit, so it starts once a
+      # render has landed.
+      name = "the swarm registration is rendered before tuwunel and wanted by it, from the store, and tuwunel retries without a limit";
       ok =
         let
           r = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-render;
         in
         lib.elem "tuwunel.service" r.before
-        && lib.elem "tuwunel.service" r.requiredBy
+        && lib.elem "tuwunel.service" r.wantedBy
+        && !(lib.elem "tuwunel.service" (r.requiredBy or [ ]))
+        &&
+          baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.serviceConfig.Restart
+          == "on-failure"
+        && baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.startLimitIntervalSec == 0
         && lib.hasSuffix "swarm-matrix-ctl appservice render" r.serviceConfig.ExecStart
         && r.environment.BAO_ADDR != ""
         && r.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/matrix-ctl.pem"
@@ -284,7 +292,7 @@ let
       # down, so it must run when the render has failed: it is after the
       # render, and neither requires nor binds to it. It stays off tuwunel's
       # own start path, and tuwunel names it nowhere. The presence arms are the
-      # render's own ordering (the same lookup finds `before`/`requiredBy`
+      # render's own ordering (the same lookup finds `before`/`wantedBy`
       # where they are) and the sync's order after the render.
       name = "the sync unit runs after the render without requiring it, and is off tuwunel's start path";
       ok =

nix/module-eval/secret-publisher.nix

@@ -424,6 +424,29 @@ let
         )
         && lib.any (lib.hasPrefix "--load-credential=") secretPublisherHere.containers.swarm-authelia.extraFlags;
     }
+    {
+      # A mint that cannot read the store fails after rendering from the
+      # digests, and retries without a limit; authelia `wants` it, so a failed
+      # run does not stop authelia starting from them. The control is the
+      # no-identity shape, where the generator still gates authelia.
+      name = "a mint that cannot read the store retries without a limit and does not hold authelia back";
+      ok =
+        let
+          here = inAuthelia secretPublisherHere;
+          u = here.systemd.services.authelia-swarm-secrets;
+          files = (inAuthelia secretPublisherNoIdentity).systemd.services.authelia-swarm-secrets;
+        in
+        lib.elem "authelia-swarm.service" u.wantedBy
+        && !(lib.elem "authelia-swarm.service" (u.requiredBy or [ ]))
+        && lib.elem "authelia-swarm.service" u.before
+        && u.serviceConfig.Restart == "on-failure"
+        && u.serviceConfig.RestartSec == 15
+        && u.startLimitIntervalSec == 0
+        && lib.hasInfix "STORE_UNREADABLE=1" u.script
+        && lib.hasInfix "chmod 0600 \"$users\"\nif [ -n \"\${STORE_UNREADABLE-}\" ]; then" u.script
+        && lib.elem "authelia-swarm.service" files.requiredBy
+        && !(lib.hasInfix "STORE_UNREADABLE" files.script);
+    }
     {
       # With a store identity no unit writes a plaintext client secret under
       # /var/lib: the mint keeps no `$sec` file, no redirect in the container

nix/script-tests/oidc-client-secrets.sh

@@ -127,20 +127,20 @@ held minted-secret
 digest_is '$pbkdf2-sha512$fake'
 if [ "$minted" != 1 ]; then fail "did not report the mint"; fi
 
-setup "mint: a store that refuses the login leaves the digest, and mints nothing"
+setup "mint: a store that refuses the login leaves the digest, mints nothing and asks to retry"
 printf 'old-digest\n' >"$digest"
 export FAKE_BAO_LOGIN_RC=1
 run mint_client_secret
-expect 0
+expect 2
 digest_is old-digest
 if ran_authelia; then fail "minted with the store down"; fi
 if store_called; then fail "called kv after a refused login"; fi
 
-setup "mint: a store that refuses the read leaves the digest, and mints nothing"
+setup "mint: a store that refuses the read leaves the digest, mints nothing and asks to retry"
 printf 'old-digest\n' >"$digest"
 export FAKE_BAO_DENY_READ=1
 run mint_client_secret
-expect 0
+expect 2
 digest_is old-digest
 if ran_authelia; then fail "minted although the read was refused"; fi
 if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote although the read was refused"; fi
@@ -205,12 +205,12 @@ digest_is old-digest
 if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote to a leaf that held a value"; fi
 if ran_authelia; then fail "minted"; fi
 
-setup "mint: a file with a store that refuses the login keeps the digest and mints nothing"
+setup "mint: a file with a store that refuses the login keeps the digest, mints nothing and asks to retry"
 printf 'old-secret\n' >"$legacy"
 printf 'old-digest\n' >"$digest"
 export FAKE_BAO_LOGIN_RC=1
 run mint_client_secret
-expect 0
+expect 2
 digest_is old-digest
 if ran_authelia; then fail "minted with the store down"; fi