vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-matrix: keep tuwunel's OIDC secret on the host's /run, not in the container's root

Browse source 
swarm-bao-matrix-oidc still reads the secret from bao under matrix-ctl's
leaf, but now writes it to /run/hive-matrix-oidc on the host (tmpfs), which
the container bind-mounts read-only for tuwunel's LoadCredential. Before, it
wrote it to
/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/<id>.secret on
persistent disk, where a superseded secret stayed after rotation and went
into backups of the container root. Host tmpfiles now removes that old file.

clientSecretFile must now lie under /run/hive-matrix-oidc/, the one directory
the container shares; any other value fails at evaluation.

Rotation is unchanged: the ten-minute timer rewrites the file when the value
differs and refresh_consumer restarts tuwunel. After a reboot the secret is
fetched again; until the store answers, tuwunel does not start.

Part of #4998 (slice 4: tuwunel).
This commit is contained in:
atlas 2026-10-15 21:33:19 +02:00 • committed by mara
commit 25d282427f
4 changed files with 74 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

65
nix/host-modules/hive-matrix.nix
View file

@ -33,16 +33,20 @@ let
baoCfg = config.services.hyperhive.swarm.bao;
autheliaUrl = autheliaCfg.url;
# Where the plaintext lands inside the matrix container. Under /var/lib
# rather than /run: the homeserver may start before the delivery unit on
# a later boot, and a secret that evaporates on reboot turns a working
# login into an intermittent one.
matrixSecretPath = "/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# On the host's /run (tmpfs), so the secret never reaches persistent disk:
# after a reboot `swarm-bao-matrix-oidc` fetches it again. The container
# sees the directory read-only at the same path, and the file is tuwunel's
# `LoadCredential` source. The file starts out empty (tmpfiles, below) so
# the container starts before the first fetch; tuwunel does not start on an
# empty secret, and the fetched one starts it.
matrixSecretDir = "/run/hive-matrix-oidc";
matrixSecretPath = "${matrixSecretDir}/${cfg.sso.clientId}.secret";
# The same file seen from the host, where `swarm-bao-matrix-oidc` writes it.
# Follows `clientSecretFile`, so an operator-chosen path is where the
# fetched secret lands.
hostSecretPath = "/var/lib/nixos-containers/hive-matrix${toString deployCfg.matrix.sso.clientSecretFile}";
hostSecretPath = toString deployCfg.matrix.sso.clientSecretFile;
# Where the secret sat on persistent disk before it moved to /run.
legacyHostSecretPath = "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# matrix-ctl's store role grants read on this one path.
@ -358,7 +362,7 @@ in
sso.clientSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/tuwunel-oidc/tuwunel.secret";
example = "/run/hive-matrix-oidc/tuwunel.secret";
description = ''
Path **inside the matrix container** holding the client
secret's plaintext.
@ -368,11 +372,14 @@ in
needs the plaintext), and a literal written here would be
rendered into the world-readable nix store.
Set to `/var/lib/tuwunel-oidc/<client id>.secret` wherever the
homeserver runs. `swarm-bao-matrix-oidc` writes the secret here
from the swarm secret store, whichever path this names. The id
this pairs with is `swarm.matrix.sso.clientId`, which is
swarm-wide because authelia's host registers it.
Set to `/run/hive-matrix-oidc/<client id>.secret` wherever the
homeserver runs. `swarm-bao-matrix-oidc` writes the secret to
that path on the host's `/run` (tmpfs) from the swarm secret
store, and the container sees the directory read-only. A path
outside `/run/hive-matrix-oidc/` fails at evaluation: only that
directory is shared with the container. The id this pairs with
is `swarm.matrix.sso.clientId`, which is swarm-wide because
authelia's host registers it.
'';
};
@ -550,6 +557,20 @@ in
secret store. Remove the definition that set it to null.
'';
}
{
# Only `matrixSecretDir` is shared with the container; a path elsewhere
# would be read by tuwunel where the delivery unit does not write.
assertion =
deployCfg.matrix.sso.clientSecretFile == null
|| lib.hasPrefix "${matrixSecretDir}/" deployCfg.matrix.sso.clientSecretFile;
message = ''
services.hyperhive.deploy.matrix.sso.clientSecretFile must lie under
${matrixSecretDir}/, the one directory the homeserver's container
shares with the host's /run. swarm-bao-matrix-oidc writes the secret
there and nowhere else. Remove the definition, or move it under that
directory.
'';
}
{
# The callback URL must name the homeserver itself, and with no
# gateway vhost there is no public name for it to be built from.
@ -628,6 +649,11 @@ in
${atomicWriteSecret}
${refreshConsumer}
# The container's bind source, so it starts even when the store does
# not answer; tmpfiles creates it too, but unordered with the
# container on a switch.
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostSecretPath)}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
@ -662,7 +688,6 @@ in
# Root-owned 0400: tuwunel runs under `DynamicUser`, so there is no
# stable uid to hand the file to, and `LoadCredential` reads it as
# root before the sandbox exists.
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostSecretPath)}
if secret_differs ${lib.escapeShellArg hostSecretPath} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg hostSecretPath} "$secret"
fi
@ -739,6 +764,12 @@ in
# Everything else is kept out.
systemd.tmpfiles.rules = [
"d ${socketDir} 0770 root ${toString nginxGid} - -"
# The bind source must exist when the container starts.
"d ${matrixSecretDir} 0700 root root -"
# `LoadCredential=` fails the unit when the file is missing; an empty
# one lets the container start and tuwunel report the missing secret.
"f ${hostSecretPath} 0400 root root -"
"r ${legacyHostSecretPath}"
];
# The container hangs off the bridge, takes its address from the
@ -760,6 +791,10 @@ in
hostPath = socketDir;
isReadOnly = false;
};
${matrixSecretDir} = {
hostPath = matrixSecretDir;
isReadOnly = true;
};
}
// caTrust.bindMount
# matrix-ctl's bao client identity, and nothing else of the hive's. See

23
nix/module-eval/matrix-core.nix
View file

@ -74,22 +74,35 @@ let
}
{
# The homeserver's OIDC secret has one route, the store, under
# matrix-ctl's leaf, landing where `clientSecretFile` points inside the
# container. No unit copies it out of authelia's tree, even on a host
# that runs authelia.
# matrix-ctl's leaf, landing where `clientSecretFile` points: the host's
# /run (tmpfs), which the container sees read-only. No unit copies it out
# of authelia's tree, even on a host that runs authelia, and nothing
# writes it to persistent disk.
name = "the homeserver reads its OIDC secret from the store under matrix-ctl's leaf";
ok =
let
s = matrixBodyCap.systemd.services;
u = s.swarm-bao-matrix-oidc;
m = matrixBodyCap.services.hyperhive.deploy.matrix;
bind = matrixBodyCap.containers.hive-matrix.bindMounts."/run/hive-matrix-oidc";
tmpfiles = matrixBodyCap.systemd.tmpfiles.rules;
in
matrixBodyCap.services.hyperhive.deploy.authelia.enable
&& u.environment.BAO_CLIENT_CERT == m.ctlBaoClientCertFile
&& u.environment.BAO_CLIENT_KEY == m.ctlBaoClientKeyFile
&& m.sso.clientSecretFile == "/var/lib/tuwunel-oidc/tuwunel.secret"
&& m.sso.clientSecretFile == "/run/hive-matrix-oidc/tuwunel.secret"
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/tuwunel/oidc/client" u.script
&& lib.hasInfix "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/tuwunel.secret" u.script
&& lib.hasInfix "atomic_write_secret 0400 \"\" /run/hive-matrix-oidc/tuwunel.secret" u.script
&& !(lib.hasInfix "/var/lib" u.script)
&& !(lib.hasInfix "nixos-containers" u.script)
# The container sees the directory the unit writes, and cannot write it.
&& bind.hostPath == "/run/hive-matrix-oidc"
&& bind.isReadOnly
# tmpfiles keeps the bind source and the credential source in place and
# removes the copy an earlier version left on persistent disk.
&& lib.elem "d /run/hive-matrix-oidc 0700 root root -" tmpfiles
&& lib.elem "f /run/hive-matrix-oidc/tuwunel.secret 0400 root root -" tmpfiles
&& lib.elem "r /var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/tuwunel.secret" tmpfiles
&& lib.hasInfix "refresh_consumer hive-matrix tuwunel.service" u.script
# An unpublished or empty secret fails the unit, so `storeRetry` retries it.
&& !(lib.hasInfix "exit 0" u.script)