diff --git a/docs/integrations/matrix.md b/docs/integrations/matrix.md index 80a45369..1649b621 100644 --- a/docs/integrations/matrix.md +++ b/docs/integrations/matrix.md @@ -171,6 +171,7 @@ The swarm secret store holds the registration's tokens, at `swarm-matrix-ctl appservice render` fetches them and renders the registration tuwunel loads as a credential onto the `hive-matrix` container's tmpfs, before tuwunel starts. +A render that can't reach the store fails and retries, and tuwunel, which fails to start without the registration, retries until one has landed. `appservice sync` runs beside tuwunel on a timer. The store's tokens win: when they differ from the rendered copy, it rewrites it. It restarts tuwunel whenever the registration is newer than tuwunel's last diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 12a57d91..a2204efd 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -89,7 +89,7 @@ The same unit mints the client secrets, straight into the store. It reads only when the store answers and holds none does it run `authelia crypto hash generate`, create the leaf and write the digest. Authelia keeps only the digest on disk, in `oidc-clients/.digest`, and starts from it while the store is down. A store that -can't answer a read never mints, because a second secret would rotate the first. A client with a +can't answer a read never mints, because a second secret would rotate the first. When it can't read the store, the unit still writes authelia's client file from the digests, then exits non-zero and retries every 15 seconds, so authelia starts meanwhile and `swarm-authelia-bridge` finds its secret once the store answers. A client with a secret in the store but no digest gets the digest of that secret and keeps the secret; `authelia crypto hash generate` computes the digest, with the password typed into its prompt through a pty, never in a command line. `swarm-authelia-bridge` fetches its own secret from diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index 8f80dfd6..6cb59955 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -1044,23 +1044,29 @@ in # The swarm registration. The credential id carries a `.yaml` # suffix on purpose: `appservice_dir` above names this very # directory, and tuwunel takes only `.yaml`/`.yml` entries from - # it. A missing source fails this unit, which is why the render - # below is `requiredBy` it. + # it. A missing source fails this unit rather than starting it + # without the registration. ++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceRuntimeDir}/swarm.yaml"; + # Upstream's `Restart=on-failure` retries a start that failed for + # want of the registration; a start limit would end the retries. + systemd.services.tuwunel.startLimitIntervalSec = lib.mkIf ctlActive 0; + # The swarm registration, fetched from the store and rendered onto # the tmpfs before the homeserver loads it. A store that does not - # answer fails this unit, and with it the homeserver's start: the - # tokens are on no disk to start from. It retries like the other - # store fetches, and `swarm-matrix-appservice-sync` below starts the - # homeserver once the registration is there. + # answer fails this unit, and the homeserver's start fails on the + # missing credential: the tokens are on no disk to start from. Both + # retry, so the homeserver comes up on its own once a render has + # landed. `wantedBy`, not `requiredBy`: a start whose dependency + # failed is dropped without running, and the homeserver's own retry + # never begins. # # `RuntimeDirectoryPreserve` because the sync unit shares the # directory: neither unit's stop may remove it from under the other. systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive { description = "render the swarm's appservice registration"; before = [ "tuwunel.service" ]; - requiredBy = [ "tuwunel.service" ]; + wantedBy = [ "tuwunel.service" ]; inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { Type = "oneshot"; diff --git a/nix/host-modules/lib/oidc-client-secrets.nix b/nix/host-modules/lib/oidc-client-secrets.nix index 57e8bdfe..4014aad9 100644 --- a/nix/host-modules/lib/oidc-client-secrets.nix +++ b/nix/host-modules/lib/oidc-client-secrets.nix @@ -11,8 +11,10 @@ # secret behind it. A store that cannot be read never mints, because a second # secret would rotate the first. # -# Return statuses: 0 done, 1 retry later (the store could not answer, or a -# digest could not be computed). +# Return statuses: 0 done, 1 retry later (the store could not answer and the +# client has no digest, or a digest could not be computed), 2 the client keeps +# its digest but the store could not be read, so the caller finishes its work +# from the digests and then retries. # # Pure function — NOT a NixOS module. Call it after `./store-seeded-secret.nix` # and `./atomic-write-secret.nix`, with `store_init` and `store_login` run and @@ -92,7 +94,7 @@ if [ -s "$dig" ]; then chmod 0600 "$dig" echo "oidc client $id: the store cannot be read; keeping its digest" >&2 - return 0 + return 2 fi echo "oidc client $id: the store cannot be read and there is no digest; not minting" >&2 return 1 diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index b52e878a..4a8d7a3f 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -522,11 +522,15 @@ let # to disk. A digest on disk is enough for authelia to start while the store is # down; a client with no digest cannot be rendered, so that run fails and # retries (./lib/store-retry.nix). A client whose digest is missing gets it - # from the secret that exists, so nothing is minted again. + # from the secret that exists, so nothing is minted again. A store that + # cannot be read fails the run too, after the files are rendered from the + # digests (`STORE_UNREADABLE`, checked at the end of the unit's script), so + # the unit retries until the store answers and authelia has started from the + # digests meanwhile. storeMintScript = '' ${oidcClientSecrets} ${lib.concatMapStrings (c: '' - mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || exit $? + mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || { rc=$?; [ "$rc" = 2 ] || exit "$rc"; STORE_UNREADABLE=1; } '') secretClients} ''; @@ -1147,9 +1151,14 @@ in # itself, as its own user, under `PrivateUsers=true`. systemd.services."${unitName}-secrets" = { description = "Generate authelia's secrets on first boot"; - wantedBy = [ "multi-user.target" ]; + # With a store identity authelia `wants` this unit, not `requires` + # it: a run that fails because the store cannot be read must not + # stop authelia from starting on the digests and keys already on + # disk. Authelia still cannot start without a key file, which + # fails its credential load. + wantedBy = [ "multi-user.target" ] ++ lib.optional publishes "${unitName}.service"; before = [ "${unitName}.service" ]; - requiredBy = [ "${unitName}.service" ]; + requiredBy = lib.optional (!publishes) "${unitName}.service"; # `deployCfg.authelia.package` is here for its CLI, not its # daemon: the client secrets are minted with `authelia crypto # hash generate`, which is the only way to produce a digest in @@ -1167,10 +1176,9 @@ in pkgs.systemd pkgs.util-linux ]; - # A store that cannot answer, with a key missing on disk, is a - # failed run that retries (./lib/store-retry.nix). - startLimitBurst = lib.mkIf publishes storeRetry.startLimitBurst; - startLimitIntervalSec = lib.mkIf publishes storeRetry.startLimitIntervalSec; + # A store that cannot answer is a failed run that retries, with no + # start limit, until it does (./lib/store-retry.nix). + startLimitIntervalSec = lib.mkIf publishes 0; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -1184,6 +1192,7 @@ in // lib.optionalAttrs publishes ( storeRetry.serviceConfig // { + RestartSec = 15; LoadCredential = lib.attrNames storeFiles; TimeoutStartSec = 120; } @@ -1217,6 +1226,12 @@ in echo "seeded $users with the disabled placeholder ${placeholderUser}" fi chmod 0600 "$users" + ${lib.optionalString (publishes && oidcEnabled) '' + if [ -n "''${STORE_UNREADABLE-}" ]; then + echo "the store could not be read; authelia runs on its digests and this unit retries" >&2 + exit 1 + fi + ''} ''; }; diff --git a/nix/module-eval/bao-matrix-reader.nix b/nix/module-eval/bao-matrix-reader.nix index dd9afe56..553b1384 100644 --- a/nix/module-eval/bao-matrix-reader.nix +++ b/nix/module-eval/bao-matrix-reader.nix @@ -264,15 +264,23 @@ let { # A missing `LoadCredential` source fails tuwunel, so the render has to # have run first. It fetches from the store, so it carries the store's - # address and matrix-ctl's identity, and retries on failure: a store - # that does not answer keeps the homeserver down until it does. - name = "the swarm registration is rendered before tuwunel and required by it, from the store, with retries"; + # address and matrix-ctl's identity, and retries on failure. It is + # `wantedBy` tuwunel, not `requiredBy`: a start whose dependency failed + # never runs, so tuwunel's own retry never began and nothing brought it + # back. tuwunel retries without a start limit, so it starts once a + # render has landed. + name = "the swarm registration is rendered before tuwunel and wanted by it, from the store, and tuwunel retries without a limit"; ok = let r = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-render; in lib.elem "tuwunel.service" r.before - && lib.elem "tuwunel.service" r.requiredBy + && lib.elem "tuwunel.service" r.wantedBy + && !(lib.elem "tuwunel.service" (r.requiredBy or [ ])) + && + baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.serviceConfig.Restart + == "on-failure" + && baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.startLimitIntervalSec == 0 && lib.hasSuffix "swarm-matrix-ctl appservice render" r.serviceConfig.ExecStart && r.environment.BAO_ADDR != "" && r.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/matrix-ctl.pem" @@ -284,7 +292,7 @@ let # down, so it must run when the render has failed: it is after the # render, and neither requires nor binds to it. It stays off tuwunel's # own start path, and tuwunel names it nowhere. The presence arms are the - # render's own ordering (the same lookup finds `before`/`requiredBy` + # render's own ordering (the same lookup finds `before`/`wantedBy` # where they are) and the sync's order after the render. name = "the sync unit runs after the render without requiring it, and is off tuwunel's start path"; ok = diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index 115095c5..ed5ffe92 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -424,6 +424,29 @@ let ) && lib.any (lib.hasPrefix "--load-credential=") secretPublisherHere.containers.swarm-authelia.extraFlags; } + { + # A mint that cannot read the store fails after rendering from the + # digests, and retries without a limit; authelia `wants` it, so a failed + # run does not stop authelia starting from them. The control is the + # no-identity shape, where the generator still gates authelia. + name = "a mint that cannot read the store retries without a limit and does not hold authelia back"; + ok = + let + here = inAuthelia secretPublisherHere; + u = here.systemd.services.authelia-swarm-secrets; + files = (inAuthelia secretPublisherNoIdentity).systemd.services.authelia-swarm-secrets; + in + lib.elem "authelia-swarm.service" u.wantedBy + && !(lib.elem "authelia-swarm.service" (u.requiredBy or [ ])) + && lib.elem "authelia-swarm.service" u.before + && u.serviceConfig.Restart == "on-failure" + && u.serviceConfig.RestartSec == 15 + && u.startLimitIntervalSec == 0 + && lib.hasInfix "STORE_UNREADABLE=1" u.script + && lib.hasInfix "chmod 0600 \"$users\"\nif [ -n \"\${STORE_UNREADABLE-}\" ]; then" u.script + && lib.elem "authelia-swarm.service" files.requiredBy + && !(lib.hasInfix "STORE_UNREADABLE" files.script); + } { # With a store identity no unit writes a plaintext client secret under # /var/lib: the mint keeps no `$sec` file, no redirect in the container diff --git a/nix/script-tests/oidc-client-secrets.sh b/nix/script-tests/oidc-client-secrets.sh index e6a5f97f..ae344665 100644 --- a/nix/script-tests/oidc-client-secrets.sh +++ b/nix/script-tests/oidc-client-secrets.sh @@ -127,20 +127,20 @@ held minted-secret digest_is '$pbkdf2-sha512$fake' if [ "$minted" != 1 ]; then fail "did not report the mint"; fi -setup "mint: a store that refuses the login leaves the digest, and mints nothing" +setup "mint: a store that refuses the login leaves the digest, mints nothing and asks to retry" printf 'old-digest\n' >"$digest" export FAKE_BAO_LOGIN_RC=1 run mint_client_secret -expect 0 +expect 2 digest_is old-digest if ran_authelia; then fail "minted with the store down"; fi if store_called; then fail "called kv after a refused login"; fi -setup "mint: a store that refuses the read leaves the digest, and mints nothing" +setup "mint: a store that refuses the read leaves the digest, mints nothing and asks to retry" printf 'old-digest\n' >"$digest" export FAKE_BAO_DENY_READ=1 run mint_client_secret -expect 0 +expect 2 digest_is old-digest if ran_authelia; then fail "minted although the read was refused"; fi if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote although the read was refused"; fi @@ -205,12 +205,12 @@ digest_is old-digest if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote to a leaf that held a value"; fi if ran_authelia; then fail "minted"; fi -setup "mint: a file with a store that refuses the login keeps the digest and mints nothing" +setup "mint: a file with a store that refuses the login keeps the digest, mints nothing and asks to retry" printf 'old-secret\n' >"$legacy" printf 'old-digest\n' >"$digest" export FAKE_BAO_LOGIN_RC=1 run mint_client_secret -expect 0 +expect 2 digest_is old-digest if ran_authelia; then fail "minted with the store down"; fi