Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/host-modules/hive-matrix.nix
atlas db6648dfe6 swarm: nats and matrix OIDC fetch units fail until the secret is published
An unpublished or empty secret in swarm-bao-nats-auth-oidc and
swarm-bao-matrix-oidc now exits 1 instead of 0, the shape
swarm-bao-forwarder-oidc already uses. storeRetry then re-fetches every 30s
until swarm-secret-publish has written the secret, and refresh_consumer
starts the responder / tuwunel once the file lands. Before, a fresh swarm's
first fetch found nothing, exited 0, and neither consumer started until the
unit was restarted by hand.
2026-10-05 09:58:02 +02:00

1041 lines
49 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.matrix;
networkCfg = config.services.hyperhive.network;
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
gatewayCfg = config.services.hyperhive.gateway;
hyperhiveDomain = config.services.hyperhive.domain;
# Same runtime→build-time bridge hive-ci and hive-forge already cross:
# binds the hive trust bundle (which folds in the swarm root) into the
# container and orders the container after `hive-tls-ca.service`. The
# assembled bundle itself comes from `caTrust.trustBundle`, imported in
# the container config below.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
swarmDomain = config.services.hyperhive.swarm.domain;
# `url` is the half of the authelia module that exists on EVERY hive —
# null when no SSO provider is configured anywhere, which the assertion
# below turns into an eval failure rather than a discovery request to
# `null/.well-known/…`.
autheliaCfg = config.services.hyperhive.swarm.authelia;
deployCfg = config.services.hyperhive.deploy;
# Where the swarm's secret store answers. Swarm-tier, identical on every
# host, which is what lets the container address it without knowing whether
# it stands here.
baoCfg = config.services.hyperhive.swarm.bao;
autheliaUrl = autheliaCfg.url;
# Where the plaintext lands inside the matrix container. Under /var/lib
# rather than /run: the homeserver may start before the delivery unit on
# a later boot, and a secret that evaporates on reboot turns a working
# login into an intermittent one.
matrixSecretPath = "/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# The same file seen from the host, where `swarm-bao-matrix-oidc` writes it.
# Follows `clientSecretFile`, so an operator-chosen path is where the
# fetched secret lands.
hostSecretPath = "/var/lib/nixos-containers/hive-matrix${toString deployCfg.matrix.sso.clientSecretFile}";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# matrix-ctl's store role grants read on this one path.
storeSecretPath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeRetry = import ./lib/store-retry.nix { };
# ⚠️ tuwunel does NOT read the host path directly, and this indirection
# is not ceremony. Upstream's own words: "under systemd the path must be
# visible to the service after sandboxing (ReadWritePaths / ProtectHome),
# typically by placing the file under /etc/tuwunel/" — which this
# container has no writable etc for. `LoadCredential` is the answer
# already in use for the swarm appservice registration below, and for the
# same reason: it keeps `DynamicUser=true` + `PrivateUsers=true` intact
# with no host-side chown or GID pinning.
matrixSecretCredential = "/run/credentials/tuwunel.service/oidc_client_secret";
# Where tuwunel reads appservice registrations from: its own credentials
# directory. A registration is a 0600 root-owned file and the homeserver
# runs under `DynamicUser=true`, so `LoadCredential` copies it to a 0400
# dynamic-user-owned path as root, before the sandbox and that user exist.
#
# Safe rather than clever: tuwunel's loader takes only `.yaml`/`.yml`
# entries from the directory and skips every other file, so the sibling
# credentials (the OIDC secret) are invisible to it. A credential id is a
# free-form filename, which is what lets a registration's end in `.yaml`.
appserviceCredentialDir = "/run/credentials/tuwunel.service";
# ── swarm-matrix-ctl ────────────────────────────────────────────────
#
# The container's own store identity, which the swarm appservice units below
# run under. Gated on the identity, not on `deploy.bao.enable`: a swarm's ONE
# homeserver is the host least likely to also be the host running the store,
# so "co-located with bao" would leave the intended deployment silently
# publishing nothing. Same rule ./swarm-secret-publisher.nix's
# `haveClientIdentity` states, for a sharper reason.
ctlActive =
deployCfg.matrix.ctlBaoClientCertFile != null && deployCfg.matrix.ctlBaoClientKeyFile != null;
# The role on the store's `cert` auth mount, and so the single source of the
# string both ends must agree on: ./swarm-bao.nix writes the role under
# `matrixCtlPolicyName` and this hands it to the binary in the
# environment, so the binary itself spells no role at all.
ctlCertRole = "swarm-matrix-ctl";
# Every host directory matrix-ctl's bao identity is spread across. Normally
# one — ./glue-bao-tls.nix puts all three files in ./glue's PKI dir — but
# derived rather than assumed, because an operator naming paths by hand is
# exactly the deployment the gate above is written for.
#
# Directories rather than the files: a re-issued leaf is a new inode, and
# binding the file would pin the one the container saw when it started.
ctlPkiDirs = lib.optionals ctlActive (
lib.unique (
map builtins.dirOf (
[
deployCfg.matrix.ctlBaoClientCertFile
deployCfg.matrix.ctlBaoClientKeyFile
]
++ lib.optional (deployCfg.bao.serverCaFile != null) deployCfg.bao.serverCaFile
)
)
);
ctlBindMounts = lib.genAttrs ctlPkiDirs (dir: {
hostPath = dir;
isReadOnly = true;
});
# ── the swarm's own appservice ──────────────────────────────────────
#
# The homeserver's one appservice registration, and the swarm's identity on
# it: `swarm-controller` creates every agent's account and every hive's
# `@hive-<hive>` sender with its token. Minted INSIDE this container by
# `swarm-matrix-ctl appservice render` and published to the store by
# `appservice publish`, which is why it is gated on matrix-ctl's store
# identity: with nobody to publish it, a registration here would be an
# admin credential nobody reads.
#
# Its sender is promoted to homeserver admin at boot (`admin_execute`
# below), so its token goes only to a store path no hive's policy reaches.
# `swarm` is in ../reserved-names.nix, so no agent can be created as this
# account.
swarmSenderLocalpart = "swarm";
# Inside the container: the render unit's `StateDirectory`. `ephemeral =
# false` keeps it across restarts, so "mint when absent" is mint once.
swarmAppserviceDir = "/var/lib/swarm-matrix-appservice";
swarmAppserviceCredentialId = "swarm-appservice.yaml";
# Every local user the swarm appservice may create — agents and every
# hive's `@hive-<hive>:` sender — which is the whole matrix localpart charset.
#
# ⚠️ Anchored deliberately: tuwunel compiles a namespace into a `RegexSet`
# and asks it for a MATCH, not a full match, so an unanchored
# `@[a-z0-9]+:this.example` also matches `@x:this.example.evil.test`.
#
# Non-exclusive deliberately: an exclusive namespace does not widen what
# the appservice may do, it narrows what everything ELSE may do — an SSO
# login adopting an existing account, or `!admin users create-user`, would
# start failing with `M_EXCLUSIVE`. The appservice needs the right to
# create these names, not a monopoly on them.
appserviceUserRegex = "^@[a-z0-9._=/-]+:${lib.escapeRegex effectiveServerName}$";
# Falls back to the SWARM domain: a swarm runs one homeserver, so its
# identifier belongs to the swarm rather than to whichever hive happens
# to host it — otherwise moving the container between hives would look
# like a different homeserver.
#
# ⚠️ Changing this default is a BREAKING change in a way that moving
# `gatewayHost` was not: `serverName` is baked irrevocably into every
# user and room id, so a deployment that rebuilds onto a new one is a
# *different homeserver*, not a renamed one. Existing hives pin the old
# value explicitly (see the option's description); the default is what
# a fresh swarm gets.
#
# Total on a null swarm domain, deliberately: the required-domain
# assertion in hive-network.nix is what should fire, not a coercion
# error from an unrelated option interpolating null.
effectiveServerName =
if cfg.serverName != null then
cfg.serverName
else if swarmDomain != null then
swarmDomain
else
"invalid";
# fluffychat-web build fixes: nixpkgs's `flutter341.buildFlutterApplication`
# skips the dart web-worker compile + the emscripten native_imaging
# build. Two derivations below cover both. Full rationale (why
# passthru.pubspecLock.dependencySources, why `dontConfigure`, why
# `make -C js`, why build-CWD-relative dart path): docs/integrations/matrix.md::
# fluffychat-web build fixes.
fluffychat-web-imaging = pkgs.stdenv.mkDerivation {
pname = "fluffychat-web-imaging";
version = pkgs.fluffychat-web.passthru.pubspecLock.dependencyVersions.native_imaging;
src = pkgs.fluffychat-web.passthru.pubspecLock.dependencySources.native_imaging;
nativeBuildInputs = with pkgs; [
emscripten
cmake
gnumake
jq
];
# cmake runs inside js/Makefile via `emcmake cmake`; the default
# configurePhase would invoke cmake at the package root (no
# CMakeLists) and fail.
dontConfigure = true;
buildPhase = ''
runHook preBuild
# emscripten on-demand sysroot build needs writable HOME + cache.
export HOME=$TMPDIR
export EM_CACHE=$TMPDIR/.emscriptencache
mkdir -p $EM_CACHE
# `make -C js` keeps pwd at source root for the installPhase.
make -C js Imaging.js Imaging.wasm
runHook postBuild
'';
installPhase = ''
runHook preInstall
mkdir -p $out
install -m 644 js/Imaging.js $out/Imaging.js
install -m 644 js/Imaging.wasm $out/Imaging.wasm
runHook postInstall
'';
meta = with pkgs.lib; {
description = "Imaging.js + Imaging.wasm built from the native_imaging dart package for fluffychat-web";
homepage = "https://pub.dev/packages/native_imaging";
license = licenses.agpl3Plus;
};
};
fluffychat-web-fixed = pkgs.fluffychat-web.overrideAttrs (old: {
# dart from the flutter341 closure (already pulled, no incremental
# cost) to compile the web-worker entry point.
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.flutter341.dart ];
postInstall = (old.postInstall or "") + ''
# `web/...` is BUILD-CWD-relative (not `$src/...`) so dart's
# package_config walk-up hits buildFlutterApplication's
# pub-get output `.dart_tool/`.
${pkgs.flutter341.dart}/bin/dart compile js \
-o $out/native_executor.js \
web/native_executor.dart
install -m 644 ${fluffychat-web-imaging}/Imaging.js $out/Imaging.js
install -m 644 ${fluffychat-web-imaging}/Imaging.wasm $out/Imaging.wasm
'';
});
# Shared host netns — agents reach tuwunel at localhost:<port>.
privateNetwork = false;
in
{
# Private matrix-tuwunel homeserver wrapped in a nixos-container,
# optional fluffychat-web client at chat.<swarm-domain>/ (gatewayHost's
# default). Container shape,
# serverName vs gatewayHost split, provisioning flow (registration
# token + LoadCredential), assertion rationale, initial rollout
# settings: docs/integrations/matrix.md. Vhost map + discovery flow + tuning
# knobs: docs/networking/gateway.md.
# Matrix moved under `swarm` when the swarm-global services were
# consolidated. One rename for the namespace: the subtree comes with it,
# so existing hives keep evaluating and get one warning naming both paths.
imports = [
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "matrix" ]
[ "services" "hyperhive" "swarm" "matrix" ]
)
# "does THIS host run it" is a per-host decision, and `swarm.*` is the
# namespace every host in the swarm agrees on. What stays under
# `swarm.matrix` is the homeserver's identity as every hive sees it —
# its server name, its addresses, its federation policy.
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "matrix" "enable" ]
[ "services" "hyperhive" "deploy" "matrix" "enable" ]
)
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "matrix" "sso" "enable" ] ''
SSO is no longer optional: a homeserver that runs at all delegates
login to the swarm's authelia.
Removed rather than defaulted to true so a config that turned it
OFF fails here, where the line is, instead of silently gaining a
login flow on the next rebuild. Drop the line; if it was false,
set services.hyperhive.deploy.matrix.sso.clientSecretFile and
services.hyperhive.swarm.authelia.url as the assertions describe.
'')
];
# `swarm.matrix`, in ./hive-matrix-service.nix, is what the homeserver IS from
# any hive's point of view: the name it answers to, the ports and URLs it is
# reached on, and the client id it is registered under. What lives here is what the host running it
# decides — which build it runs, whether it is exposed, which peers it trusts,
# how large a request it accepts, and where its host-local secrets sit. Same rule as
# ./swarm-victorialogs.nix; `enable` already lives in ./deploy.nix, which also
# carries the renames.
options.services.hyperhive.deploy.matrix = {
package = lib.mkOption {
type = lib.types.package;
default = pkgs.matrix-tuwunel;
defaultText = lib.literalExpression "pkgs.matrix-tuwunel";
description = ''
matrix-tuwunel package to run inside the container. Defaults
to nixpkgs's `pkgs.matrix-tuwunel`. Override to pin a
specific upstream if you need an unreleased feature.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Open `httpPort` in the host firewall. Off by default
(secure-by-default): hive-c0re and agent containers reach the
homeserver at `gatewayHost`'s vhost (`chat.<swarm-domain>` by
default) via the gateway, so the open port only matters for
direct access to `httpPort` from outside the host. Flip to `true` when announcing
the homeserver to other hives or when an external matrix
client needs to reach the client-server API directly.
**Breaking change**: this used to default to `true`. If you
relied on the old default for external reach, add
`services.hyperhive.deploy.matrix.openFirewall = true;` to your host
config before rebuilding.
Note: federation (the matrix-spec well-known port 8448) is
intentionally not opened here. tuwunel serves the federation
API on the same `httpPort` as the client-server API by
default; reaching it on 8448 requires either binding tuwunel
to that port explicitly OR a reverse-proxy + `.well-known/
matrix/server` delegation, neither of which lives in this
module. Add that proxy config alongside whatever serves your
dashboard or forge on 443.
'';
};
trustedServers = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "matrix.org" ];
description = ''
List of trusted matrix servers (homeservers whose signing
keys this server will fetch identity-server-style). Empty
by default — federation is enabled at the protocol level
but no peer is trusted until listed here, so the homeserver
is effectively closed until the operator declares hive
peers explicitly.
'';
};
maxRequestSize = lib.mkOption {
type = lib.types.ints.positive;
default = 20000000;
description = ''
Maximum size in bytes of a single matrix client request body.
Default 20 MB matches the matrix-spec recommendation for
media uploads + the upstream tuwunel default.
This is the only limit to set: the gateway's own body cap is
derived from it, with headroom, so the homeserver stays the one
that rejects an oversized upload.
'';
};
gui.package = lib.mkOption {
type = lib.types.package;
default = fluffychat-web-fixed;
defaultText = lib.literalMD ''
`pkgs.fluffychat-web` with a `postInstall` patch that adds
the three files `flutter341.buildFlutterApplication` skips.
'';
description = ''
Static web client dist served at `gatewayHost`'s vhost
(`chat.<swarm-domain>` by default). Override
to swap fluffychat for hydrogen-web, cinny, element-web, or
an out-of-tree dist — any replacement is mounted at the
sub-domain root with the upstream-default `<base href "/">`,
no sub-path gymnastics needed.
'';
};
sso.clientSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/tuwunel-oidc/tuwunel.secret";
description = ''
Path **inside the matrix container** holding the client
secret's plaintext.
A path, never a value: an OIDC client secret has two holders
in two containers (authelia keeps a hash, this homeserver
needs the plaintext), and a literal written here would be
rendered into the world-readable nix store.
Set to `/var/lib/tuwunel-oidc/<client id>.secret` wherever the
homeserver runs. `swarm-bao-matrix-oidc` writes the secret here
from the swarm secret store, whichever path this names. The id
this pairs with is `swarm.matrix.sso.clientId`, which is
swarm-wide because authelia's host registers it.
'';
};
ctlPackage = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-matrix-ctl";
description = ''
The `swarm-matrix-ctl` build run inside the matrix container.
⚠️ Named `ctlPackage`, not folded into `package` above: that one is
the homeserver, and this is a hyperhive binary that happens to run
beside it. Same split, and same reason, as
{option}`services.hyperhive.deploy.nats.authPackage`.
'';
};
ctlBaoClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
Client certificate the matrix container's `swarm-matrix-ctl` presents to the
swarm's secret store. Its subject must be
{option}`services.hyperhive.deploy.bao.matrixCtlCommonName` — cert
auth matches on the CN, and the role accepts nothing else.
⚠️ **Not the hive's own leaf**, and that is the whole deliverable of
giving this container an identity: the hive's certificate reads every
secret in the store, while this one may write a single path. Pointing
this at `deploy.bao.clientCertFile` would evaluate, deploy and work —
and give away the separation in one line.
No default: a module that guessed would be holding the CA opinion
./swarm-bao.nix deliberately does not hold.
./glue-matrix-ctl-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints, where this host mints one.
The file and its key are bind-mounted into the container read-only.
Co-located in the hive's filesystem and therefore readable by hive
**root** — accepted: the boundary this buys is identity (no hive
*process* holds the appservice token), not physical isolation.
'';
};
ctlBaoClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
Private key for
{option}`services.hyperhive.deploy.matrix.ctlBaoClientCertFile`.
Both or neither — the unit does not exist unless each is set.
'';
};
};
config = lib.mkIf deployCfg.matrix.enable {
# Matrix's own gateway surface: the sub-domain vhost, the name the
# hive resolver answers for, and the Accept-header map that vhost's
# SPA fallback reads. All three are matrix knowledge and none of
# them is the gateway's business.
#
# `gatewayHost = null` means matrix is reachable directly rather
# than fronted, so there is no name to claim and no vhost to serve —
# every clause below carries that guard.
services.hyperhive.gateway.localNames = lib.optional (cfg.gatewayHost != null) cfg.gatewayHost;
services.hyperhive.gateway.enable = lib.mkIf (cfg.gatewayHost != null) (lib.mkDefault true);
# Unguarded, unlike the two above: the homeserver container resolves
# through the hive's dnsmasq whether or not the gateway fronts it.
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Accept-header SPA map, used only by the `/` location below (see
# docs/networking/gateway.md "SPA fallback"): text/html → index.html, else a
# sentinel so `try_files` falls through to 404. `appendHttpConfig`
# is a `lines` option, so this merges with anything else the host
# contributes instead of replacing it.
#
# The dashboard needs no equivalent — it routes by path.
services.nginx.appendHttpConfig = ''
map $http_accept $matrix_spa_target {
default "/__matrix_spa_no_html_fallback";
"~*text/html" "/index.html";
}
'';
# `server_name = gatewayHost`. `/_matrix/*` → tuwunel (CORS `*`, 50M
# body cap, 1h long-poll timeout). `/` serves fluffychat. nginx's
# longest-prefix rule puts `/_matrix/`
# ahead of `/` with no ordering needed.
#
# ⚠️ The `.well-known/matrix/*` delegation is deliberately NOT here.
# It stays on the hive's own vhost because the spec requires it to
# be served at the *server name*, which is the hive domain — it is
# the hive answering "where is my homeserver", not the homeserver
# answering for itself.
services.nginx.virtualHosts = lib.optionalAttrs (cfg.gatewayHost != null) {
"${cfg.gatewayHost}" = (gatewayCfg.lib.tlsFor cfg.gatewayHost) // {
listen = gatewayCfg.lib.listen;
extraConfig = gatewayCfg.lib.securityHeaders;
locations = {
"/_matrix/" = {
proxyPass = "http://127.0.0.1:${toString cfg.httpPort}";
proxyWebsockets = true;
extraConfig = ''
proxy_buffering off;
# Tracks `maxRequestSize` with headroom so the homeserver stays
# the tighter limit: its rejection is a matrix error a client can
# act on, where a 413 here names neither matrix nor the option the
# operator just raised. A second literal beside it was free to
# disagree, and did — raising the option past the old 50M changed
# nothing.
client_max_body_size ${toString (deployCfg.matrix.maxRequestSize + 1048576)};
proxy_read_timeout 1h;
proxy_send_timeout 1h;
${gatewayCfg.lib.securityHeaders}
add_header Access-Control-Allow-Origin *;
'';
};
# fluffychat at sub-domain root, SPA-fallback via the
# Accept-header `$matrix_spa_target` map above.
"/" = {
alias = "${deployCfg.matrix.gui.package}/";
extraConfig = ''
try_files $uri $uri/ $matrix_spa_target =404;
'';
};
# FluffyChat boot-config pre-fill so the client's
# `.well-known/matrix/client` lookup hits the right delegation
# endpoint. `domain` is required, so this is always present.
"= /config.json" = {
extraConfig = ''
default_type application/json;
return 200 '{"defaultHomeserver":"${hyperhiveDomain}"}';
'';
};
};
};
};
# `serverName` is irrevocably embedded in user/room IDs; it derives
# from `services.hyperhive.domain` (required, asserted in
# hive-network.nix) when not set explicitly, so no separate
# domain/serverName assertion is needed here. gatewayHost may not be
# "" (same footgun as forge.domain — nginx rejects an empty
# server_name). docs/integrations/matrix.md::Assertion rationale.
assertions = [
{
assertion = cfg.gatewayHost == null || cfg.gatewayHost != "";
message = ''
services.hyperhive.swarm.matrix.gatewayHost = "" is rejected. The
rendered URLs would be invalid (nginx wildcard catch-all for
an empty server_name, /etc/hosts rejects empty entries).
Use `null` to disable the gateway vhost entirely (tuwunel
stays direct on httpPort), or set a non-empty hostname like
"matrix.example.com" or "homeserver.internal".
'';
}
{
# Fail at EVAL, not at boot. tuwunel reads its identity providers
# from the config file, so a half-configured one does not hide a
# login button — it can stop the homeserver from starting at all.
assertion = deployCfg.matrix.sso.clientSecretFile != null;
message = ''
This homeserver's SSO login flow requires
services.hyperhive.deploy.matrix.sso.clientSecretFile — the path
(inside the matrix container) holding the OIDC client secret's
plaintext.
It is set by default wherever the homeserver runs, and
swarm-bao-matrix-oidc writes the secret there from the swarm
secret store. Remove the definition that set it to null.
'';
}
{
# The callback URL must name the homeserver itself, and with no
# gateway vhost there is no public name for it to be built from.
assertion = cfg.gatewayHost != null;
message = ''
This homeserver's SSO login flow requires
services.hyperhive.swarm.matrix.gatewayHost.
tuwunel's SSO callback URL is format-locked to
`<homeserver>/_matrix/client/unstable/login/sso/callback/<client_id>`,
and the identity provider redirects a browser to it — so it
has to be a name the browser can reach, which is exactly what
`gatewayHost` is. With it null the homeserver is direct on
httpPort and has no such name.
'';
}
];
# Where the homeserver finds its OIDC client secret, inside the container.
services.hyperhive.deploy.matrix.sso.clientSecretFile = lib.mkDefault matrixSecretPath;
# The homeserver's OIDC client secret, out of the swarm secret store under
# matrix-ctl's leaf — the one route, wherever authelia runs. Same shape as
# ./swarm-bao.nix's `swarm-bao-forwarder-oidc`: the login and the read both
# fail loudly, and `storeRetry` retries until the publisher on authelia's
# host has written the secret, because tuwunel does not start without it.
#
# ⚠️ tuwunel requires the file to exist and be non-empty, and its
# `LoadCredential` source must exist for the unit to start, so a host with
# no matrix-ctl leaf fails here, naming the options, rather than rendering
# no unit.
systemd.services.swarm-bao-matrix-oidc = {
description = "fetch the homeserver's OIDC client secret from the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
# absent unit fails the job, so off-host the timeout below bounds the
# attempt instead.
after = lib.optionals deployCfg.bao.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
before = [ "container@hive-matrix.service" ];
wantedBy = [
"multi-user.target"
"container@hive-matrix.service"
];
path = [
deployCfg.bao.package
pkgs.coreutils
pkgs.systemd
];
# ./lib/store-retry.nix. The container waits for one attempt; a secret a
# later attempt lands restarts tuwunel (below).
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-matrix-oidc";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs ctlActive {
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script =
if !ctlActive then
''
echo "the homeserver's host has no matrix-ctl store identity, so it cannot read the homeserver's OIDC client secret: set services.hyperhive.deploy.matrix.ctlBaoClientCertFile and ctlBaoClientKeyFile" >&2
exit 1
''
else
''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with matrix-ctl's certificate; leaving the homeserver's OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; the homeserver has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; the homeserver has no OIDC client secret yet." >&2
exit 1
fi
# Root-owned 0400: tuwunel runs under `DynamicUser`, so there is no
# stable uid to hand the file to, and `LoadCredential` reads it as
# root before the sandbox exists.
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostSecretPath)}
if secret_differs ${lib.escapeShellArg hostSecretPath} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg hostSecretPath} "$secret"
fi
# `LoadCredential` copies the file once, at tuwunel's start.
refresh_consumer hive-matrix tuwunel.service ${lib.escapeShellArg hostSecretPath}
'';
};
# ⚠️ Deliberately NO `networking.hosts` entry for authelia's name. This
# container resolves through the hive's dnsmasq at `bridgeIp` (see the
# static resolv.conf below), and every `gateway.localNames` entry —
# authelia's domain among them — is already mapped there. Adding a
# loopback override would only create a second answer that can
# disagree with the first.
# Activation-time token generation — without this the bind-mount
# would hand tuwunel an empty file on first boot and break every
# registration until restart. Idempotent;
# docs/integrations/matrix.md::Provisioning flow.
# Tell an operator whose homeserver already exists that `serverName` is
# unpinned, at the one moment they are looking: the rebuild.
#
# ⚠️ An activation script and NOT `warnings`, which is where this
# obviously belongs and does not work. The condition needs the host
# filesystem — does a homeserver already exist here? — and
# `nixos-rebuild switch --flake` evaluates PURELY, where
# `builtins.pathExists "/var/lib/…"` answers **false** rather than
# throwing. A `warnings` entry gated on it would evaluate, deploy, and
# print nothing, on every real deployment. Same shape as an option whose
# consumer is disabled: renders perfectly, does nothing.
#
# Only rendered when `serverName` is null, so a hive that pinned it
# cannot be nagged — the script does not exist there rather than
# existing and choosing to stay quiet. A guard that cries wolf at a
# correctly-configured deployment is worse than no guard, because the
# next real one is read as noise too.
#
# Never fails. This warns about a choice that cannot be undone; refusing
# the activation would break the rebuild of a hive that had already made
# that choice deliberately, which is the opposite of helping.
system.activationScripts.hive-matrix-servername-pin = lib.mkIf (cfg.serverName == null) (
lib.stringAfter [ "var" ] ''
# The homeserver's own database, asked of the container's evaluated
# config rather than hardcoded: a guessed path resolves cleanly and
# silently never matches, which is exactly the failure this guard
# exists to avoid one level up.
dbDir=${
lib.escapeShellArg (
# Same host-side container-root prefix this module already writes
# by hand for the SSO secret copy above — not a second convention.
"/var/lib/nixos-containers/hive-matrix"
+ config.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.database_path
)
}
if [ -d "$dbDir" ]; then
echo "hive-matrix: WARNING — services.hyperhive.swarm.matrix.serverName is unset, and this host already has a homeserver at $dbDir."
echo "hive-matrix: it is defaulting to ${effectiveServerName}, which is baked into every NEW user and room id."
echo "hive-matrix: if ids here were minted under a different name, existing accounts and rooms are stranded — reverting the config does NOT undo it."
echo "hive-matrix: pin whichever name this homeserver already uses, e.g.:"
echo "hive-matrix: services.hyperhive.swarm.matrix.serverName = \"''${HIVE_MATRIX_EXISTING_SERVER_NAME:-<the name already in use>}\";"
fi
''
);
containers.hive-matrix = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
inherit privateNetwork;
bindMounts =
caTrust.bindMount
# matrix-ctl's bao client identity, and nothing else of the hive's. See
# `ctlPkiDirs` above for why it is a derived set of directories
# rather than one literal.
// ctlBindMounts;
config =
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
# tuwunel's rustls verifier resolves through `rustls-native-certs`
# → `openssl-probe`, which reads `SSL_CERT_FILE` — so the
# openssl-shaped variable is the lever despite tuwunel linking no
# openssl.
#
# ⚠️ The helper CONCATENATES, and that is load-bearing here beyond
# the usual reason: `SSL_CERT_FILE` replaces the default location,
# so naming the hive anchor alone would drop every public CA and
# break federation with the wider matrix network — trading a small
# outage for a much larger one.
# A literal, not an option: this module names its container
# `containers.hive-matrix` directly and declares no `machine`
# option to derive it from.
(caTrust.trustBundle {
inherit pkgs;
name = "hive-matrix";
consumers = [ "tuwunel" ];
})
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Swarm-internal trust reaches tuwunel at RUNTIME, not via
# `security.pki.certificateFiles`. That option is read when the
# system is BUILT, and the swarm root is deliberately a runtime
# file (`swarm.ca.stateDir`) because its key must never enter the
# store — so there is nothing build-time to name. The bind-mount
# above plus the bundle service below are what replaced it.
# tuwunel hard-fails to boot if `/etc/resolv.conf` has no
# `nameserver` line (`Failed to configure DNS resolver ... no
# nameservers found in config` → exit 1). This declarative
# nixos-container comes up with an EMPTY resolv.conf even with
# `networking.nameservers` set: the nixos-container default
# `useHostResolvConf = true` puts in-container resolvconf in
# host-tracking mode (ignores `networking.nameservers`, and never
# gets the host file across the shared-netns boundary), so it
# regenerates an empty file and tuwunel dies at boot.
#
# Trusting resolvconf to honour `networking.nameservers` doesn't
# work either — that's a RUNTIME resolvconf behaviour, not
# verifiable at eval time, and it still comes up empty in
# practice. So take resolvconf out of the loop entirely and
# write a STATIC `/etc/resolv.conf` from `bridgeIp` that nothing
# regenerates. Eval-proven: the generated
# `environment.etc."resolv.conf".text` is `nameserver <bridgeIp>`.
# This container always shares the host netns
# (`privateNetwork = false`), so it reaches `bridgeIp` regardless
# of agent-container isolation. See `docs/networking/network.md`.
# resolvconf is off through `writesOwnResolvConf` (its default);
# the static `environment.etc."resolv.conf"` below is the sole
# source of the resolver file (no `nameservers` — nothing would
# read it).
networking.useHostResolvConf = lib.mkForce false;
# resolvconf is disabled above, so write the static resolver file
# explicitly — NixOS won't synthesise one from `nameservers` once
# resolvconf is off, and this is the file tuwunel parses at boot.
environment.etc."resolv.conf".text = ''
nameserver ${networkCfg.bridgeIp}
options edns0
'';
services.matrix-tuwunel = {
enable = true;
package = deployCfg.matrix.package;
settings.global = {
server_name = effectiveServerName;
# `address` + `port` are upstream `listOf` — wrap singles.
address = [ "0.0.0.0" ];
port = [ cfg.httpPort ];
max_request_size = deployCfg.matrix.maxRequestSize;
# Federation enabled at the protocol level; empty
# trustedServers keeps it effectively closed.
allow_federation = true;
trusted_servers = deployCfg.matrix.trustedServers;
# Nobody registers themselves here. Accounts are created by
# the swarm's appservice, which this flag does not gate —
# tuwunel checks it only for requests that arrive WITHOUT an
# appservice token, so swarm-controller provisions and
# everyone else is refused outright.
#
# ⚠️ Not a hardening afterthought: `allow_registration = true`
# with no registration token configured makes tuwunel REFUSE
# TO START (it demands
# `yes_i_am_very_very_sure_…_open_registration_…` instead). So
# dropping the token and leaving this true is not a lax
# homeserver, it is a homeserver that does not boot.
allow_registration = false;
# Where the swarm's appservice registration is read from — the
# credentials directory, for the reasons at
# `appserviceCredentialDir`'s own definition. Loaded inside
# `Services::start()`, before the listener accepts anything.
appservice_dir = appserviceCredentialDir;
}
# Exactly one account is promoted to homeserver admin at boot:
# the swarm appservice's sender. Its token is read by
# matrix-ctl and swarm-controller only. Each `@hive-<hive>`
# stays an ordinary user, because its hive reads its
# credential; what it does (the Space, the chat room, the
# invites) rides on being the rooms' creator, not on admin.
#
# The registration load creates the sender inside
# `Services::start()`, and `admin_execute` runs after that and
# before the listener accepts anything. `make-user-admin` is a
# no-op on an account that is already admin.
# `admin_execute_errors_ignore` is load-bearing: a failing
# command aborts startup when it is false.
// lib.optionalAttrs ctlActive {
admin_execute = [
"users make-user-admin @${swarmSenderLocalpart}:${effectiveServerName}"
];
admin_execute_errors_ignore = true;
}
// {
# Server-side E2EE is opt-in (default off); the agent matrix
# client always supports decryption regardless.
allow_encryption = cfg.allowEncryption;
# Tuwunel's default suffix is " 💕" — suppress it so agent
# display names are clean (just the agent name, no emoji).
new_user_displayname_suffix = "";
# tuwunel's OIDC server and this list are the two ends of one
# pipe: `oidc_native_auth` stays false (its default), which
# upstream defines as "the OIDC server runs only to broker
# for a configured identity_provider". So the client-facing
# half needs no configuration — only the upstream half does.
identity_provider = [
{
# A free, case-insensitive string, not an enum: a
# recognised brand gets defaults and provider-specific
# workarounds, an unrecognised one simply gets neither.
# Which is why `issuer_url` below is not optional for us
# — the pre-supplied issuers cover public providers only.
brand = "authelia";
client_id = cfg.sso.clientId;
client_secret_file = matrixSecretCredential;
issuer_url = toString autheliaUrl;
callback_url = cfg.sso.redirectUri;
# Explicit, though a lone provider is auto-defaulted:
# relying on that logs a warning every startup, and a
# recurring warning that is expected is one nobody reads.
default = true;
# Upstream's rule is "only ever set `trusted` for
# identity providers you self-host and fully control",
# and this module cannot point anywhere else: the issuer
# is `swarm.authelia.url`, whose client, secret and user
# database are all ours. It does mean whoever can make
# authelia emit a given name gets that account — for our
# own identity provider that IS the identity.
# Without it, an SSO login cannot adopt an account that
# already exists; it can only ever create a new one.
trusted = true;
# One claim instead of upstream's ladder
# (`preferred_username` → `username` → `nickname` →
# `login` → `email`). The tail is the hazard: an email
# local part is a different namespace, so a login can
# land on a name that means someone else here.
userid_claims = [ "preferred_username" ];
# The default (`true`) makes a name collision SILENT —
# tuwunel invents a random localpart and the login
# succeeds as the wrong user. `false` errors instead,
# which is the only form of this an operator can act on.
unique_id_fallbacks = false;
}
];
};
};
# Keeps DynamicUser=true + PrivateUsers=true intact — no
# host-side chown :tuwunel / GID-pin gymnastics needed.
# See `man systemd.exec` → LoadCredential.
systemd.services.tuwunel.serviceConfig.LoadCredential = [
# tuwunel re-reads this file on every OAuth exchange, not just at startup, so it has to
# outlive the unit's start — a credentials path does.
"oidc_client_secret:${toString deployCfg.matrix.sso.clientSecretFile}"
]
# The swarm registration. The credential id carries a `.yaml`
# suffix on purpose: `appservice_dir` above names this very
# directory, and tuwunel takes only `.yaml`/`.yml` entries from
# it. A missing source fails this unit, which is why the render
# below is `requiredBy` it and local only.
++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceDir}/swarm.yaml";
# The swarm registration, minted and rendered before the homeserver
# loads it. No network and no store: this is on tuwunel's start
# path, and a store outage must not keep the homeserver down.
systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive {
description = "render the swarm's appservice registration";
before = [ "tuwunel.service" ];
requiredBy = [ "tuwunel.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice render";
StateDirectory = baseNameOf swarmAppserviceDir;
StateDirectoryMode = "0700";
UMask = "0077";
SyslogIdentifier = "swarm-matrix-appservice";
};
environment = {
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
# Agents' localparts are bare `[a-z0-9-]`, so nothing narrower
# covers them without also covering people.
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
};
};
# Hand the swarm registration's token to swarm-controller, through
# the store, under matrix-ctl's identity; the write lands at a path
# only matrix-ctl and the controller are granted (./swarm-bao.nix).
# `Restart=on-failure` covers a sealed store or one still starting;
# the start timeout is bounded for the reason
# ./swarm-secret-publisher.nix states: a sealed store answers on the
# port and never answers the read.
systemd.services.swarm-matrix-appservice-publish = lib.mkIf ctlActive {
description = "publish the swarm's appservice token to the swarm secret store";
after = [ "swarm-matrix-appservice-render.service" ];
requires = [ "swarm-matrix-appservice-render.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice publish";
Restart = "on-failure";
RestartSec = 30;
TimeoutStartSec = 60;
SyslogIdentifier = "swarm-matrix-appservice";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
MATRIX_APPSERVICE_CERT_ROLE = ctlCertRole;
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
};
environment.systemPackages = [ deployCfg.matrix.package ];
};
};
networking.firewall = lib.mkIf deployCfg.matrix.openFirewall {
allowedTCPPorts = [
cfg.httpPort
];
};
# The matrix container's resolver is the hive's dnsmasq (bound at
# `bridgeIp`). Order the matrix container start after it so the
# resolver is up before tuwunel's first federation lookups. tuwunel
# boots fine without this — it configures the resolver from
# `/etc/resolv.conf` at startup and only queries on-demand (the boot
# failure this module guards against is an *empty* resolv.conf, a
# parse error, not a connectivity one) — so this is robustness, not a
# boot requirement. Soft `after` ordering (not `requires`) keeps the
# matrix container's lifecycle decoupled from the resolver's.
#
# `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also
# sets `after`/`requires` (so the bound trust bundle exists before
# nspawn wires the mount up), and two plain assignments to the same
# unit would conflict rather than combine.
systemd.services."container@hive-matrix" = lib.mkMerge [
{ after = [ "dnsmasq.service" ]; }
caTrust.containerOrdering
];
};
}