vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

retry the authelia mint and the appservice render until the store answers, and let tuwunel retry

Browse source
This commit is contained in:
atlas 2026-11-01 10:47:50 +01:00
commit c4924d227b
6 changed files with 85 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

20
nix/host-modules/hive-matrix.nix
View file

@ -1044,23 +1044,29 @@ in
# The swarm registration. The credential id carries a `.yaml`
# suffix on purpose: `appservice_dir` above names this very
# directory, and tuwunel takes only `.yaml`/`.yml` entries from
# it. A missing source fails this unit, which is why the render
# below is `requiredBy` it.
# it. A missing source fails this unit rather than starting it
# without the registration.
++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceRuntimeDir}/swarm.yaml";
# Upstream's `Restart=on-failure` retries a start that failed for
# want of the registration; a start limit would end the retries.
systemd.services.tuwunel.startLimitIntervalSec = lib.mkIf ctlActive 0;
# The swarm registration, fetched from the store and rendered onto
# the tmpfs before the homeserver loads it. A store that does not
# answer fails this unit, and with it the homeserver's start: the
# tokens are on no disk to start from. It retries like the other
# store fetches, and `swarm-matrix-appservice-sync` below starts the
# homeserver once the registration is there.
# answer fails this unit, and the homeserver's start fails on the
# missing credential: the tokens are on no disk to start from. Both
# retry, so the homeserver comes up on its own once a render has
# landed. `wantedBy`, not `requiredBy`: a start whose dependency
# failed is dropped without running, and the homeserver's own retry
# never begins.
#
# `RuntimeDirectoryPreserve` because the sync unit shares the
# directory: neither unit's stop may remove it from under the other.
systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive {
description = "render the swarm's appservice registration";
before = [ "tuwunel.service" ];
requiredBy = [ "tuwunel.service" ];
wantedBy = [ "tuwunel.service" ];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";

8
nix/host-modules/lib/oidc-client-secrets.nix
View file

@ -11,8 +11,10 @@
# secret behind it. A store that cannot be read never mints, because a second
# secret would rotate the first.
#
# Return statuses: 0 done, 1 retry later (the store could not answer, or a
# digest could not be computed).
# Return statuses: 0 done, 1 retry later (the store could not answer and the
# client has no digest, or a digest could not be computed), 2 the client keeps
# its digest but the store could not be read, so the caller finishes its work
# from the digests and then retries.
#
# Pure function — NOT a NixOS module. Call it after `./store-seeded-secret.nix`
# and `./atomic-write-secret.nix`, with `store_init` and `store_login` run and
@ -92,7 +94,7 @@
if [ -s "$dig" ]; then
chmod 0600 "$dig"
echo "oidc client $id: the store cannot be read; keeping its digest" >&2
return 0
return 2
fi
echo "oidc client $id: the store cannot be read and there is no digest; not minting" >&2
return 1

31
nix/host-modules/swarm-authelia.nix
View file

@ -522,11 +522,15 @@ let
# to disk. A digest on disk is enough for authelia to start while the store is
# down; a client with no digest cannot be rendered, so that run fails and
# retries (./lib/store-retry.nix). A client whose digest is missing gets it
# from the secret that exists, so nothing is minted again.
# from the secret that exists, so nothing is minted again. A store that
# cannot be read fails the run too, after the files are rendered from the
# digests (`STORE_UNREADABLE`, checked at the end of the unit's script), so
# the unit retries until the store answers and authelia has started from the
# digests meanwhile.
storeMintScript = ''
${oidcClientSecrets}
${lib.concatMapStrings (c: ''
mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || exit $?
mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || { rc=$?; [ "$rc" = 2 ] || exit "$rc"; STORE_UNREADABLE=1; }
'') secretClients}
'';
@ -1147,9 +1151,14 @@ in
# itself, as its own user, under `PrivateUsers=true`.
systemd.services."${unitName}-secrets" = {
description = "Generate authelia's secrets on first boot";
wantedBy = [ "multi-user.target" ];
# With a store identity authelia `wants` this unit, not `requires`
# it: a run that fails because the store cannot be read must not
# stop authelia from starting on the digests and keys already on
# disk. Authelia still cannot start without a key file, which
# fails its credential load.
wantedBy = [ "multi-user.target" ] ++ lib.optional publishes "${unitName}.service";
before = [ "${unitName}.service" ];
requiredBy = [ "${unitName}.service" ];
requiredBy = lib.optional (!publishes) "${unitName}.service";
# `deployCfg.authelia.package` is here for its CLI, not its
# daemon: the client secrets are minted with `authelia crypto
# hash generate`, which is the only way to produce a digest in
@ -1167,10 +1176,9 @@ in
pkgs.systemd
pkgs.util-linux
];
# A store that cannot answer, with a key missing on disk, is a
# failed run that retries (./lib/store-retry.nix).
startLimitBurst = lib.mkIf publishes storeRetry.startLimitBurst;
startLimitIntervalSec = lib.mkIf publishes storeRetry.startLimitIntervalSec;
# A store that cannot answer is a failed run that retries, with no
# start limit, until it does (./lib/store-retry.nix).
startLimitIntervalSec = lib.mkIf publishes 0;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
@ -1184,6 +1192,7 @@ in
// lib.optionalAttrs publishes (
storeRetry.serviceConfig
// {
RestartSec = 15;
LoadCredential = lib.attrNames storeFiles;
TimeoutStartSec = 120;
}
@ -1217,6 +1226,12 @@ in
echo "seeded $users with the disabled placeholder ${placeholderUser}"
fi
chmod 0600 "$users"
${lib.optionalString (publishes && oidcEnabled) ''
if [ -n "''${STORE_UNREADABLE-}" ]; then
echo "the store could not be read; authelia runs on its digests and this unit retries" >&2
exit 1
fi
''}
'';
};

18
nix/module-eval/bao-matrix-reader.nix
View file

@ -264,15 +264,23 @@ let
{
# A missing `LoadCredential` source fails tuwunel, so the render has to
# have run first. It fetches from the store, so it carries the store's
# address and matrix-ctl's identity, and retries on failure: a store
# that does not answer keeps the homeserver down until it does.
name = "the swarm registration is rendered before tuwunel and required by it, from the store, with retries";
# address and matrix-ctl's identity, and retries on failure. It is
# `wantedBy` tuwunel, not `requiredBy`: a start whose dependency failed
# never runs, so tuwunel's own retry never began and nothing brought it
# back. tuwunel retries without a start limit, so it starts once a
# render has landed.
name = "the swarm registration is rendered before tuwunel and wanted by it, from the store, and tuwunel retries without a limit";
ok =
let
r = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-render;
in
lib.elem "tuwunel.service" r.before
&& lib.elem "tuwunel.service" r.requiredBy
&& lib.elem "tuwunel.service" r.wantedBy
&& !(lib.elem "tuwunel.service" (r.requiredBy or [ ]))
&&
baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.serviceConfig.Restart
== "on-failure"
&& baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.startLimitIntervalSec == 0
&& lib.hasSuffix "swarm-matrix-ctl appservice render" r.serviceConfig.ExecStart
&& r.environment.BAO_ADDR != ""
&& r.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/matrix-ctl.pem"
@ -284,7 +292,7 @@ let
# down, so it must run when the render has failed: it is after the
# render, and neither requires nor binds to it. It stays off tuwunel's
# own start path, and tuwunel names it nowhere. The presence arms are the
# render's own ordering (the same lookup finds `before`/`requiredBy`
# render's own ordering (the same lookup finds `before`/`wantedBy`
# where they are) and the sync's order after the render.
name = "the sync unit runs after the render without requiring it, and is off tuwunel's start path";
ok =

23
nix/module-eval/secret-publisher.nix
View file

@ -424,6 +424,29 @@ let
)
&& lib.any (lib.hasPrefix "--load-credential=") secretPublisherHere.containers.swarm-authelia.extraFlags;
}
{
# A mint that cannot read the store fails after rendering from the
# digests, and retries without a limit; authelia `wants` it, so a failed
# run does not stop authelia starting from them. The control is the
# no-identity shape, where the generator still gates authelia.
name = "a mint that cannot read the store retries without a limit and does not hold authelia back";
ok =
let
here = inAuthelia secretPublisherHere;
u = here.systemd.services.authelia-swarm-secrets;
files = (inAuthelia secretPublisherNoIdentity).systemd.services.authelia-swarm-secrets;
in
lib.elem "authelia-swarm.service" u.wantedBy
&& !(lib.elem "authelia-swarm.service" (u.requiredBy or [ ]))
&& lib.elem "authelia-swarm.service" u.before
&& u.serviceConfig.Restart == "on-failure"
&& u.serviceConfig.RestartSec == 15
&& u.startLimitIntervalSec == 0
&& lib.hasInfix "STORE_UNREADABLE=1" u.script
&& lib.hasInfix "chmod 0600 \"$users\"\nif [ -n \"\${STORE_UNREADABLE-}\" ]; then" u.script
&& lib.elem "authelia-swarm.service" files.requiredBy
&& !(lib.hasInfix "STORE_UNREADABLE" files.script);
}
{
# With a store identity no unit writes a plaintext client secret under
# /var/lib: the mint keeps no `$sec` file, no redirect in the container

12
nix/script-tests/oidc-client-secrets.sh
View file

@ -127,20 +127,20 @@ held minted-secret
digest_is '$pbkdf2-sha512$fake'
if [ "$minted" != 1 ]; then fail "did not report the mint"; fi
setup "mint: a store that refuses the login leaves the digest, and mints nothing"
setup "mint: a store that refuses the login leaves the digest, mints nothing and asks to retry"
printf 'old-digest\n' >"$digest"
export FAKE_BAO_LOGIN_RC=1
run mint_client_secret
expect 0
expect 2
digest_is old-digest
if ran_authelia; then fail "minted with the store down"; fi
if store_called; then fail "called kv after a refused login"; fi
setup "mint: a store that refuses the read leaves the digest, and mints nothing"
setup "mint: a store that refuses the read leaves the digest, mints nothing and asks to retry"
printf 'old-digest\n' >"$digest"
export FAKE_BAO_DENY_READ=1
run mint_client_secret
expect 0
expect 2
digest_is old-digest
if ran_authelia; then fail "minted although the read was refused"; fi
if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote although the read was refused"; fi
@ -205,12 +205,12 @@ digest_is old-digest
if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote to a leaf that held a value"; fi
if ran_authelia; then fail "minted"; fi
setup "mint: a file with a store that refuses the login keeps the digest and mints nothing"
setup "mint: a file with a store that refuses the login keeps the digest, mints nothing and asks to retry"
printf 'old-secret\n' >"$legacy"
printf 'old-digest\n' >"$digest"
export FAKE_BAO_LOGIN_RC=1
run mint_client_secret
expect 0
expect 2
digest_is old-digest
if ran_authelia; then fail "minted with the store down"; fi