diff --git a/docs/integrations/matrix.md b/docs/integrations/matrix.md index 6ee3876f..b6a8e2aa 100644 --- a/docs/integrations/matrix.md +++ b/docs/integrations/matrix.md @@ -400,10 +400,12 @@ SSO is unconditional, so the three below are requirements of running a homeserver at all rather than of a setting: - **Keep `deploy.matrix.sso.clientSecretFile` set** — it defaults to - `/var/lib/tuwunel-oidc/.secret` wherever the homeserver runs, + `/run/hive-matrix-oidc/.secret` wherever the homeserver runs, and `swarm-bao-matrix-oidc` writes the secret there from the swarm secret - store under matrix-ctl's leaf. A null fails at eval, not at boot: tuwunel - reads its identity providers from the config file, so a half-configured + store under matrix-ctl's leaf. The path is on the host's tmpfs, so every + reboot fetches the secret again and it never reaches persistent disk. A path + outside `/run/hive-matrix-oidc/`, or a null, fails at eval, not at boot: + tuwunel reads its identity providers from the config file, so a half-configured one can stop the homeserver from starting outright rather than merely hiding a login button. - **Set `swarm.authelia.url`** — without a provider URL there diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index f33738d5..32ac8276 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -253,7 +253,7 @@ such as the root token. | hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | | the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | | the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | -| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/var/lib/tuwunel-oidc/.secret`, handed to tuwunel through `LoadCredential` | +| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | | the agent containers' queue credential | authelia, published to the store by `swarm-secret-publish` | `/secret` (`0600`) and `/client_id` (`0644`) | The forge and the homeserver read their copies out of the store, the way diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index 3fb1b1a3..8dc158ce 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -33,16 +33,20 @@ let baoCfg = config.services.hyperhive.swarm.bao; autheliaUrl = autheliaCfg.url; - # Where the plaintext lands inside the matrix container. Under /var/lib - # rather than /run: the homeserver may start before the delivery unit on - # a later boot, and a secret that evaporates on reboot turns a working - # login into an intermittent one. - matrixSecretPath = "/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret"; + # On the host's /run (tmpfs), so the secret never reaches persistent disk: + # after a reboot `swarm-bao-matrix-oidc` fetches it again. The container + # sees the directory read-only at the same path, and the file is tuwunel's + # `LoadCredential` source. The file starts out empty (tmpfiles, below) so + # the container starts before the first fetch; tuwunel does not start on an + # empty secret, and the fetched one starts it. + matrixSecretDir = "/run/hive-matrix-oidc"; + matrixSecretPath = "${matrixSecretDir}/${cfg.sso.clientId}.secret"; # The same file seen from the host, where `swarm-bao-matrix-oidc` writes it. - # Follows `clientSecretFile`, so an operator-chosen path is where the - # fetched secret lands. - hostSecretPath = "/var/lib/nixos-containers/hive-matrix${toString deployCfg.matrix.sso.clientSecretFile}"; + hostSecretPath = toString deployCfg.matrix.sso.clientSecretFile; + + # Where the secret sat on persistent disk before it moved to /run. + legacyHostSecretPath = "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret"; # Where swarm-secret-publish on authelia's host leaves this client's secret. # matrix-ctl's store role grants read on this one path. @@ -358,7 +362,7 @@ in sso.clientSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; - example = "/var/lib/tuwunel-oidc/tuwunel.secret"; + example = "/run/hive-matrix-oidc/tuwunel.secret"; description = '' Path **inside the matrix container** holding the client secret's plaintext. @@ -368,11 +372,14 @@ in needs the plaintext), and a literal written here would be rendered into the world-readable nix store. - Set to `/var/lib/tuwunel-oidc/.secret` wherever the - homeserver runs. `swarm-bao-matrix-oidc` writes the secret here - from the swarm secret store, whichever path this names. The id - this pairs with is `swarm.matrix.sso.clientId`, which is - swarm-wide because authelia's host registers it. + Set to `/run/hive-matrix-oidc/.secret` wherever the + homeserver runs. `swarm-bao-matrix-oidc` writes the secret to + that path on the host's `/run` (tmpfs) from the swarm secret + store, and the container sees the directory read-only. A path + outside `/run/hive-matrix-oidc/` fails at evaluation: only that + directory is shared with the container. The id this pairs with + is `swarm.matrix.sso.clientId`, which is swarm-wide because + authelia's host registers it. ''; }; @@ -550,6 +557,20 @@ in secret store. Remove the definition that set it to null. ''; } + { + # Only `matrixSecretDir` is shared with the container; a path elsewhere + # would be read by tuwunel where the delivery unit does not write. + assertion = + deployCfg.matrix.sso.clientSecretFile == null + || lib.hasPrefix "${matrixSecretDir}/" deployCfg.matrix.sso.clientSecretFile; + message = '' + services.hyperhive.deploy.matrix.sso.clientSecretFile must lie under + ${matrixSecretDir}/, the one directory the homeserver's container + shares with the host's /run. swarm-bao-matrix-oidc writes the secret + there and nowhere else. Remove the definition, or move it under that + directory. + ''; + } { # The callback URL must name the homeserver itself, and with no # gateway vhost there is no public name for it to be built from. @@ -628,6 +649,11 @@ in ${atomicWriteSecret} ${refreshConsumer} + # The container's bind source, so it starts even when the store does + # not answer; tmpfiles creates it too, but unordered with the + # container on a switch. + install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostSecretPath)} + err="$(mktemp)" trap 'rm -f "$err"' EXIT @@ -662,7 +688,6 @@ in # Root-owned 0400: tuwunel runs under `DynamicUser`, so there is no # stable uid to hand the file to, and `LoadCredential` reads it as # root before the sandbox exists. - install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostSecretPath)} if secret_differs ${lib.escapeShellArg hostSecretPath} "$secret"; then atomic_write_secret 0400 "" ${lib.escapeShellArg hostSecretPath} "$secret" fi @@ -739,6 +764,12 @@ in # Everything else is kept out. systemd.tmpfiles.rules = [ "d ${socketDir} 0770 root ${toString nginxGid} - -" + # The bind source must exist when the container starts. + "d ${matrixSecretDir} 0700 root root -" + # `LoadCredential=` fails the unit when the file is missing; an empty + # one lets the container start and tuwunel report the missing secret. + "f ${hostSecretPath} 0400 root root -" + "r ${legacyHostSecretPath}" ]; # The container hangs off the bridge, takes its address from the @@ -760,6 +791,10 @@ in hostPath = socketDir; isReadOnly = false; }; + ${matrixSecretDir} = { + hostPath = matrixSecretDir; + isReadOnly = true; + }; } // caTrust.bindMount # matrix-ctl's bao client identity, and nothing else of the hive's. See diff --git a/nix/module-eval/matrix-core.nix b/nix/module-eval/matrix-core.nix index ee79531f..9be157d9 100644 --- a/nix/module-eval/matrix-core.nix +++ b/nix/module-eval/matrix-core.nix @@ -74,22 +74,35 @@ let } { # The homeserver's OIDC secret has one route, the store, under - # matrix-ctl's leaf, landing where `clientSecretFile` points inside the - # container. No unit copies it out of authelia's tree, even on a host - # that runs authelia. + # matrix-ctl's leaf, landing where `clientSecretFile` points: the host's + # /run (tmpfs), which the container sees read-only. No unit copies it out + # of authelia's tree, even on a host that runs authelia, and nothing + # writes it to persistent disk. name = "the homeserver reads its OIDC secret from the store under matrix-ctl's leaf"; ok = let s = matrixBodyCap.systemd.services; u = s.swarm-bao-matrix-oidc; m = matrixBodyCap.services.hyperhive.deploy.matrix; + bind = matrixBodyCap.containers.hive-matrix.bindMounts."/run/hive-matrix-oidc"; + tmpfiles = matrixBodyCap.systemd.tmpfiles.rules; in matrixBodyCap.services.hyperhive.deploy.authelia.enable && u.environment.BAO_CLIENT_CERT == m.ctlBaoClientCertFile && u.environment.BAO_CLIENT_KEY == m.ctlBaoClientKeyFile - && m.sso.clientSecretFile == "/var/lib/tuwunel-oidc/tuwunel.secret" + && m.sso.clientSecretFile == "/run/hive-matrix-oidc/tuwunel.secret" && lib.hasInfix "bao kv get -field=value secret/swarm/services/tuwunel/oidc/client" u.script - && lib.hasInfix "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/tuwunel.secret" u.script + && lib.hasInfix "atomic_write_secret 0400 \"\" /run/hive-matrix-oidc/tuwunel.secret" u.script + && !(lib.hasInfix "/var/lib" u.script) + && !(lib.hasInfix "nixos-containers" u.script) + # The container sees the directory the unit writes, and cannot write it. + && bind.hostPath == "/run/hive-matrix-oidc" + && bind.isReadOnly + # tmpfiles keeps the bind source and the credential source in place and + # removes the copy an earlier version left on persistent disk. + && lib.elem "d /run/hive-matrix-oidc 0700 root root -" tmpfiles + && lib.elem "f /run/hive-matrix-oidc/tuwunel.secret 0400 root root -" tmpfiles + && lib.elem "r /var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/tuwunel.secret" tmpfiles && lib.hasInfix "refresh_consumer hive-matrix tuwunel.service" u.script # An unpublished or empty secret fails the unit, so `storeRetry` retries it. && !(lib.hasInfix "exit 0" u.script)