retry the authelia mint and the appservice render until the store answers, and let tuwunel retry

nix/host-modules/hive-matrix.nix

@@ -1044,23 +1044,29 @@ in
           # The swarm registration. The credential id carries a `.yaml`
           # suffix on purpose: `appservice_dir` above names this very
           # directory, and tuwunel takes only `.yaml`/`.yml` entries from
-          # it. A missing source fails this unit, which is why the render
-          # below is `requiredBy` it.
+          # it. A missing source fails this unit rather than starting it
+          # without the registration.
           ++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceRuntimeDir}/swarm.yaml";
 
+          # Upstream's `Restart=on-failure` retries a start that failed for
+          # want of the registration; a start limit would end the retries.
+          systemd.services.tuwunel.startLimitIntervalSec = lib.mkIf ctlActive 0;
+
           # The swarm registration, fetched from the store and rendered onto
           # the tmpfs before the homeserver loads it. A store that does not
-          # answer fails this unit, and with it the homeserver's start: the
-          # tokens are on no disk to start from. It retries like the other
-          # store fetches, and `swarm-matrix-appservice-sync` below starts the
-          # homeserver once the registration is there.
+          # answer fails this unit, and the homeserver's start fails on the
+          # missing credential: the tokens are on no disk to start from. Both
+          # retry, so the homeserver comes up on its own once a render has
+          # landed. `wantedBy`, not `requiredBy`: a start whose dependency
+          # failed is dropped without running, and the homeserver's own retry
+          # never begins.
           #
           # `RuntimeDirectoryPreserve` because the sync unit shares the
           # directory: neither unit's stop may remove it from under the other.
           systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive {
             description = "render the swarm's appservice registration";
             before = [ "tuwunel.service" ];
-            requiredBy = [ "tuwunel.service" ];
+            wantedBy = [ "tuwunel.service" ];
             inherit (storeRetry) startLimitBurst startLimitIntervalSec;
             serviceConfig = storeRetry.serviceConfig // {
               Type = "oneshot";

nix/host-modules/lib/oidc-client-secrets.nix

@@ -11,8 +11,10 @@
 # secret behind it. A store that cannot be read never mints, because a second
 # secret would rotate the first.
 #
-# Return statuses: 0 done, 1 retry later (the store could not answer, or a
-# digest could not be computed).
+# Return statuses: 0 done, 1 retry later (the store could not answer and the
+# client has no digest, or a digest could not be computed), 2 the client keeps
+# its digest but the store could not be read, so the caller finishes its work
+# from the digests and then retries.
 #
 # Pure function — NOT a NixOS module. Call it after `./store-seeded-secret.nix`
 # and `./atomic-write-secret.nix`, with `store_init` and `store_login` run and
@@ -92,7 +94,7 @@
       if [ -s "$dig" ]; then
         chmod 0600 "$dig"
         echo "oidc client $id: the store cannot be read; keeping its digest" >&2
-        return 0
+        return 2
       fi
       echo "oidc client $id: the store cannot be read and there is no digest; not minting" >&2
       return 1

nix/host-modules/swarm-authelia.nix

@@ -522,11 +522,15 @@ let
   # to disk. A digest on disk is enough for authelia to start while the store is
   # down; a client with no digest cannot be rendered, so that run fails and
   # retries (./lib/store-retry.nix). A client whose digest is missing gets it
-  # from the secret that exists, so nothing is minted again.
+  # from the secret that exists, so nothing is minted again. A store that
+  # cannot be read fails the run too, after the files are rendered from the
+  # digests (`STORE_UNREADABLE`, checked at the end of the unit's script), so
+  # the unit retries until the store answers and authelia has started from the
+  # digests meanwhile.
   storeMintScript = ''
     ${oidcClientSecrets}
     ${lib.concatMapStrings (c: ''
-      mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || exit $?
+      mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || { rc=$?; [ "$rc" = 2 ] || exit "$rc"; STORE_UNREADABLE=1; }
     '') secretClients}
   '';
 
@@ -1147,9 +1151,14 @@ in
           # itself, as its own user, under `PrivateUsers=true`.
           systemd.services."${unitName}-secrets" = {
             description = "Generate authelia's secrets on first boot";
-            wantedBy = [ "multi-user.target" ];
+            # With a store identity authelia `wants` this unit, not `requires`
+            # it: a run that fails because the store cannot be read must not
+            # stop authelia from starting on the digests and keys already on
+            # disk. Authelia still cannot start without a key file, which
+            # fails its credential load.
+            wantedBy = [ "multi-user.target" ] ++ lib.optional publishes "${unitName}.service";
             before = [ "${unitName}.service" ];
-            requiredBy = [ "${unitName}.service" ];
+            requiredBy = lib.optional (!publishes) "${unitName}.service";
             # `deployCfg.authelia.package` is here for its CLI, not its
             # daemon: the client secrets are minted with `authelia crypto
             # hash generate`, which is the only way to produce a digest in
@@ -1167,10 +1176,9 @@ in
               pkgs.systemd
               pkgs.util-linux
             ];
-            # A store that cannot answer, with a key missing on disk, is a
-            # failed run that retries (./lib/store-retry.nix).
-            startLimitBurst = lib.mkIf publishes storeRetry.startLimitBurst;
-            startLimitIntervalSec = lib.mkIf publishes storeRetry.startLimitIntervalSec;
+            # A store that cannot answer is a failed run that retries, with no
+            # start limit, until it does (./lib/store-retry.nix).
+            startLimitIntervalSec = lib.mkIf publishes 0;
             serviceConfig = {
               Type = "oneshot";
               RemainAfterExit = true;
@@ -1184,6 +1192,7 @@ in
             // lib.optionalAttrs publishes (
               storeRetry.serviceConfig
               // {
+                RestartSec = 15;
                 LoadCredential = lib.attrNames storeFiles;
                 TimeoutStartSec = 120;
               }
@@ -1217,6 +1226,12 @@ in
                 echo "seeded $users with the disabled placeholder ${placeholderUser}"
               fi
               chmod 0600 "$users"
+              ${lib.optionalString (publishes && oidcEnabled) ''
+                if [ -n "''${STORE_UNREADABLE-}" ]; then
+                  echo "the store could not be read; authelia runs on its digests and this unit retries" >&2
+                  exit 1
+                fi
+              ''}
             '';
           };