Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/host-modules/hive-matrix.nix
atlas ce44b6e77e hive-forge, hive-matrix: forward each container's journal to the swarm collector
Each container now runs the journal forwarder the CI runner's, Grafana's and
the log store's containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers hive-forge and hive-matrix). Each authenticates as
a new machine OIDC client of its own (swarm.forge.otel.clientId,
swarm.matrix.otel.clientId), registered by a glue file wherever authelia runs,
minted by authelia and published to the store by the secret publisher. The
forge's and the homeserver's browser login clients are unchanged. The client
ids are swarm-forge-collector and swarm-matrix-collector, not
hive-<x>-collector, which is the client id a hive named <x>-collector gets.

The fetch unit inside each container reads the client secret with the store
identity the host already holds for that service. The forge publisher's leaf
is handed into the forge container as an nspawn system credential, as the
grafana and log store containers do. matrix-ctl's leaf is already bound into
the homeserver's container, so its forwarder reads it at those paths, as
swarm-matrix-appservice-publish does; with no matrix-ctl identity the
homeserver's forwarder is off. Each of the two policies gains one read stanza
on its forwarder's client path. The host units that use those identities are
unchanged.

The two containers drop --link-journal=host, and the host collector stops
reading them.
2026-10-16 13:50:23 +02:00

1110 lines
52 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.matrix;
networkCfg = config.services.hyperhive.network;
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
gatewayCfg = config.services.hyperhive.gateway;
hyperhiveDomain = config.services.hyperhive.domain;
# Same runtime→build-time bridge hive-forge and swarm-ci-runner cross:
# binds the hive trust bundle (which folds in the store's root) into the
# container and orders the container after `hive-tls-ca.service`. The
# assembled bundle itself comes from `caTrust.trustBundle`, imported in
# the container config below.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
swarmDomain = config.services.hyperhive.swarm.domain;
# `url` is the half of the authelia module that exists on EVERY hive —
# null when no SSO provider is configured anywhere, which the assertion
# below turns into an eval failure rather than a discovery request to
# `null/.well-known/…`.
autheliaCfg = config.services.hyperhive.swarm.authelia;
deployCfg = config.services.hyperhive.deploy;
# Where the swarm's secret store answers. Swarm-tier, identical on every
# host, which is what lets the container address it without knowing whether
# it stands here.
baoCfg = config.services.hyperhive.swarm.bao;
autheliaUrl = autheliaCfg.url;
# On the host's /run (tmpfs), so the secret never reaches persistent disk:
# after a reboot `swarm-bao-matrix-oidc` fetches it again. The container
# sees the directory read-only at the same path, and the file is tuwunel's
# `LoadCredential` source. The file starts out empty (tmpfiles, below) so
# the container starts before the first fetch; tuwunel does not start on an
# empty secret, and the fetched one starts it.
matrixSecretDir = "/run/hive-matrix-oidc";
matrixSecretPath = "${matrixSecretDir}/${cfg.sso.clientId}.secret";
# The same file seen from the host, where `swarm-bao-matrix-oidc` writes it.
hostSecretPath = toString deployCfg.matrix.sso.clientSecretFile;
# Where the secret sat on persistent disk before it moved to /run.
legacyHostSecretPath = "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# matrix-ctl's store role grants read on this path.
storeSecretPath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
# ⚠️ tuwunel does NOT read the host path directly, and this indirection
# is not ceremony. Upstream's own words: "under systemd the path must be
# visible to the service after sandboxing (ReadWritePaths / ProtectHome),
# typically by placing the file under /etc/tuwunel/" — which this
# container has no writable etc for. `LoadCredential` is the answer
# already in use for the swarm appservice registration below, and for the
# same reason: it keeps `DynamicUser=true` + `PrivateUsers=true` intact
# with no host-side chown or GID pinning.
matrixSecretCredential = "/run/credentials/tuwunel.service/oidc_client_secret";
# Where tuwunel reads appservice registrations from: its own credentials
# directory. A registration is a 0600 root-owned file and the homeserver
# runs under `DynamicUser=true`, so `LoadCredential` copies it to a 0400
# dynamic-user-owned path as root, before the sandbox and that user exist.
#
# Safe rather than clever: tuwunel's loader takes only `.yaml`/`.yml`
# entries from the directory and skips every other file, so the sibling
# credentials (the OIDC secret) are invisible to it. A credential id is a
# free-form filename, which is what lets a registration's end in `.yaml`.
appserviceCredentialDir = "/run/credentials/tuwunel.service";
# ── swarm-matrix-ctl ────────────────────────────────────────────────
#
# The container's own store identity, which the swarm appservice units below
# run under. Gated on the identity, not on `deploy.bao.enable`: a swarm's ONE
# homeserver is the host least likely to also be the host running the store,
# so "co-located with bao" would leave the intended deployment silently
# publishing nothing. Same rule ./swarm-secret-publisher.nix's
# `haveClientIdentity` states, for a sharper reason.
ctlActive =
deployCfg.matrix.ctlBaoClientCertFile != null && deployCfg.matrix.ctlBaoClientKeyFile != null;
# The role on the store's `cert` auth mount, and so the single source of the
# string both ends must agree on: ./swarm-bao.nix writes the role under
# `matrixCtlPolicyName` and this hands it to the binary in the
# environment, so the binary itself spells no role at all.
ctlCertRole = "swarm-matrix-ctl";
# Every host directory matrix-ctl's bao identity is spread across. Normally
# one — ./glue-bao-tls.nix puts all three files in ./glue's PKI dir — but
# derived rather than assumed, because an operator naming paths by hand is
# exactly the deployment the gate above is written for.
#
# Directories rather than the files: a re-issued leaf is a new inode, and
# binding the file would pin the one the container saw when it started.
ctlPkiDirs = lib.optionals ctlActive (
lib.unique (
map builtins.dirOf (
[
deployCfg.matrix.ctlBaoClientCertFile
deployCfg.matrix.ctlBaoClientKeyFile
]
++ lib.optional (deployCfg.bao.serverCaFile != null) deployCfg.bao.serverCaFile
)
)
);
ctlBindMounts = lib.genAttrs ctlPkiDirs (dir: {
hostPath = dir;
isReadOnly = true;
});
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.hive-matrix` in
# ./swarm-otel-service.nix declares, as a client of its own rather than the
# homeserver login's. The port suffix is elided on 443, as in
# ./swarm-bao.nix. It reads its client secret with matrix-ctl's identity,
# bound in above: one identity for the whole container.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}${gatewayPortSuffix}/hive-matrix";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8899;
# ── the swarm's own appservice ──────────────────────────────────────
#
# The homeserver's one appservice registration, and the swarm's identity on
# it: `swarm-controller` creates every agent's account and every hive's
# `@hive-<hive>` sender with its token. Minted INSIDE this container by
# `swarm-matrix-ctl appservice render` and published to the store by
# `appservice publish`, which is why it is gated on matrix-ctl's store
# identity: with nobody to publish it, a registration here would be an
# admin credential nobody reads.
#
# Its sender is promoted to homeserver admin at boot (`admin_execute`
# below), so its token goes only to a store path no hive's policy reaches.
# `swarm` is in ../reserved-names.nix, so no agent can be created as this
# account.
swarmSenderLocalpart = "swarm";
# Inside the container: the render unit's `StateDirectory`. `ephemeral =
# false` keeps it across restarts, so "mint when absent" is mint once.
swarmAppserviceDir = "/var/lib/swarm-matrix-appservice";
swarmAppserviceCredentialId = "swarm-appservice.yaml";
# Every local user the swarm appservice may create — agents and every
# hive's `@hive-<hive>:` sender — which is the whole matrix localpart charset.
#
# ⚠️ Anchored deliberately: tuwunel compiles a namespace into a `RegexSet`
# and asks it for a MATCH, not a full match, so an unanchored
# `@[a-z0-9]+:this.example` also matches `@x:this.example.evil.test`.
#
# Non-exclusive deliberately: an exclusive namespace does not widen what
# the appservice may do, it narrows what everything ELSE may do — an SSO
# login adopting an existing account, or `!admin users create-user`, would
# start failing with `M_EXCLUSIVE`. The appservice needs the right to
# create these names, not a monopoly on them.
appserviceUserRegex = "^@[a-z0-9._=/-]+:${lib.escapeRegex effectiveServerName}$";
# Falls back to the SWARM domain: a swarm runs one homeserver, so its
# identifier belongs to the swarm rather than to whichever hive happens
# to host it — otherwise moving the container between hives would look
# like a different homeserver.
#
# ⚠️ Changing this default is breaking: `serverName` is baked
# irrevocably into every user and room id, so a deployment that
# rebuilds onto a new one is a *different homeserver*. Existing hives
# pin their value explicitly (see the option's description); the
# default is what a fresh swarm gets.
#
# Total on a null swarm domain, deliberately: the required-domain
# assertion in hive-network.nix is what should fire, not a coercion
# error from an unrelated option interpolating null.
effectiveServerName =
if cfg.serverName != null then
cfg.serverName
else if swarmDomain != null then
swarmDomain
else
"invalid";
# fluffychat-web build fixes: nixpkgs's `flutter341.buildFlutterApplication`
# skips the dart web-worker compile + the emscripten native_imaging
# build. Two derivations below cover both. Full rationale (why
# passthru.pubspecLock.dependencySources, why `dontConfigure`, why
# `make -C js`, why build-CWD-relative dart path): docs/integrations/matrix.md::
# fluffychat-web build fixes.
fluffychat-web-imaging = pkgs.stdenv.mkDerivation {
pname = "fluffychat-web-imaging";
version = pkgs.fluffychat-web.passthru.pubspecLock.dependencyVersions.native_imaging;
src = pkgs.fluffychat-web.passthru.pubspecLock.dependencySources.native_imaging;
nativeBuildInputs = with pkgs; [
emscripten
cmake
gnumake
jq
];
# cmake runs inside js/Makefile via `emcmake cmake`; the default
# configurePhase would invoke cmake at the package root (no
# CMakeLists) and fail.
dontConfigure = true;
buildPhase = ''
runHook preBuild
# emscripten on-demand sysroot build needs writable HOME + cache.
export HOME=$TMPDIR
export EM_CACHE=$TMPDIR/.emscriptencache
mkdir -p $EM_CACHE
# `make -C js` keeps pwd at source root for the installPhase.
make -C js Imaging.js Imaging.wasm
runHook postBuild
'';
installPhase = ''
runHook preInstall
mkdir -p $out
install -m 644 js/Imaging.js $out/Imaging.js
install -m 644 js/Imaging.wasm $out/Imaging.wasm
runHook postInstall
'';
meta = with pkgs.lib; {
description = "Imaging.js + Imaging.wasm built from the native_imaging dart package for fluffychat-web";
homepage = "https://pub.dev/packages/native_imaging";
license = licenses.agpl3Plus;
};
};
fluffychat-web-fixed = pkgs.fluffychat-web.overrideAttrs (old: {
# dart from the flutter341 closure (already pulled, no incremental
# cost) to compile the web-worker entry point.
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.flutter341.dart ];
postInstall = (old.postInstall or "") + ''
# `web/...` is BUILD-CWD-relative (not `$src/...`) so dart's
# package_config walk-up hits buildFlutterApplication's
# pub-get output `.dart_tool/`.
${pkgs.flutter341.dart}/bin/dart compile js \
-o $out/native_executor.js \
web/native_executor.dart
install -m 644 ${fluffychat-web-imaging}/Imaging.js $out/Imaging.js
install -m 644 ${fluffychat-web-imaging}/Imaging.wasm $out/Imaging.wasm
'';
});
# A netns of its own on the hive bridge: the homeserver dials the store,
# authelia and federation peers by name, and the gateway reaches it over
# `socketPath` below, so it binds no TCP port.
privateNetwork = true;
# Fixed, not an option: the directory's mode is what keeps the socket
# private, so it holds this one socket and nothing else.
socketDir = "/run/hive-matrix";
socketPath = "${socketDir}/tuwunel.sock";
# The gid the gateway's nginx runs as. Only the number crosses into the
# container, which knows it as `gateway-nginx`.
nginxGid = config.ids.gids.nginx;
in
{
# Private matrix-tuwunel homeserver wrapped in a nixos-container,
# optional fluffychat-web client at chat.<swarm-domain>/ (gatewayHost's
# default). Container shape,
# serverName vs gatewayHost split, provisioning flow (registration
# token + LoadCredential), assertion rationale, initial rollout
# settings: docs/integrations/matrix.md. Vhost map + discovery flow + tuning
# knobs: docs/networking/gateway.md.
# Matrix lives under `swarm`, beside the other swarm-global services. One
# rename for the top-level `services.hyperhive.matrix` namespace: the
# subtree comes with it, so existing hives keep evaluating and get one
# warning naming both paths.
imports = [
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "matrix" ]
[ "services" "hyperhive" "swarm" "matrix" ]
)
# "does THIS host run it" is a per-host decision, and `swarm.*` is the
# namespace every host in the swarm agrees on. What stays under
# `swarm.matrix` is the homeserver's identity as every hive sees it —
# its server name, its addresses, its federation policy.
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "matrix" "enable" ]
[ "services" "hyperhive" "deploy" "matrix" "enable" ]
)
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "matrix" "sso" "enable" ] ''
SSO is no longer optional: a homeserver that runs at all delegates
login to the swarm's authelia.
Removed rather than defaulted to true so a config that turned it
OFF fails here, where the line is, instead of silently gaining a
login flow on the next rebuild. Drop the line; if it was false,
set services.hyperhive.deploy.matrix.sso.clientSecretFile and
services.hyperhive.swarm.authelia.url as the assertions describe.
'')
];
# `swarm.matrix`, in ./hive-matrix-service.nix, is what the homeserver IS from
# any hive's point of view: the name it answers to, the URLs it is
# reached on, and the client id it is registered under. What lives here is what the host running it
# decides — which build it runs, whether it is exposed, which peers it trusts,
# how large a request it accepts, and where its host-local secrets sit. Same rule as
# ./swarm-victorialogs.nix; `enable` already lives in ./deploy.nix, which also
# carries the renames.
options.services.hyperhive.deploy.matrix = {
package = lib.mkOption {
type = lib.types.package;
default = pkgs.matrix-tuwunel;
defaultText = lib.literalExpression "pkgs.matrix-tuwunel";
description = ''
matrix-tuwunel package to run inside the container. Defaults
to nixpkgs's `pkgs.matrix-tuwunel`. Override to pin a
specific upstream if you need an unreleased feature.
'';
};
trustedServers = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "matrix.org" ];
description = ''
List of trusted matrix servers (homeservers whose signing
keys this server will fetch identity-server-style). Empty
by default — federation is enabled at the protocol level
but no peer is trusted until listed here, so the homeserver
is effectively closed until the operator declares hive
peers explicitly.
'';
};
maxRequestSize = lib.mkOption {
type = lib.types.ints.positive;
default = 20000000;
description = ''
Maximum size in bytes of a single matrix client request body.
Default 20 MB matches the matrix-spec recommendation for
media uploads + the upstream tuwunel default.
This is the only limit to set: the gateway's own body cap is
derived from it, with headroom, so the homeserver stays the one
that rejects an oversized upload.
'';
};
gui.package = lib.mkOption {
type = lib.types.package;
default = fluffychat-web-fixed;
defaultText = lib.literalMD ''
`pkgs.fluffychat-web` with a `postInstall` patch that adds
the three files `flutter341.buildFlutterApplication` skips.
'';
description = ''
Static web client dist served at `gatewayHost`'s vhost
(`chat.<swarm-domain>` by default). Override
to swap fluffychat for hydrogen-web, cinny, element-web, or
an out-of-tree dist — any replacement is mounted at the
sub-domain root with the upstream-default `<base href "/">`,
no sub-path gymnastics needed.
'';
};
sso.clientSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/run/hive-matrix-oidc/tuwunel.secret";
description = ''
Path **inside the matrix container** holding the client
secret's plaintext.
A path, never a value: an OIDC client secret has two holders
in two containers (authelia keeps a hash, this homeserver
needs the plaintext), and a literal written here would be
rendered into the world-readable nix store.
Set to `/run/hive-matrix-oidc/<client id>.secret` wherever the
homeserver runs. `swarm-bao-matrix-oidc` writes the secret to
that path on the host's `/run` (tmpfs) from the swarm secret
store, and the container sees the directory read-only. A path
outside `/run/hive-matrix-oidc/` fails at evaluation: only that
directory is shared with the container. The id this pairs with
is `swarm.matrix.sso.clientId`, which is swarm-wide because
authelia's host registers it.
'';
};
ctlPackage = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-matrix-ctl";
description = ''
The `swarm-matrix-ctl` build run inside the matrix container.
⚠️ Named `ctlPackage`, not folded into `package` above: that one is
the homeserver, and this is a hyperhive binary that happens to run
beside it. Same split, and same reason, as
{option}`services.hyperhive.deploy.nats.authPackage`.
'';
};
ctlBaoClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
Client certificate the matrix container's `swarm-matrix-ctl` presents to the
swarm's secret store. Its subject must be
{option}`services.hyperhive.deploy.bao.matrixCtlCommonName` — cert
auth matches on the CN, and the role accepts nothing else.
⚠️ **Not the hive's own leaf**, and that is the whole deliverable of
giving this container an identity: the hive's certificate reads every
agent credential in the swarm, while this one may write a single path. Pointing
this at `deploy.bao.clientCertFile` would evaluate, deploy and work —
and give away the separation in one line.
No default: a module that guessed would be holding the CA opinion
./swarm-bao.nix deliberately does not hold.
./glue-matrix-ctl-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints on a host that runs the store
({option}`services.hyperhive.deploy.bao.enable`); on any other
homeserver host it is a file the operator places. Never derived from
{option}`services.hyperhive.deploy.bao.clientCertFile`, which is the
hive's own reader.
The file and its key are bind-mounted into the container read-only.
Co-located in the hive's filesystem and therefore readable by hive
**root** — accepted: the boundary this buys is identity (no hive
*process* holds the appservice token), not physical isolation.
'';
};
ctlBaoClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
Private key for
{option}`services.hyperhive.deploy.matrix.ctlBaoClientCertFile`.
Both or neither — the unit does not exist unless each is set.
'';
};
};
config = lib.mkIf deployCfg.matrix.enable {
# Matrix's own gateway surface: the sub-domain vhost, the name the
# hive resolver answers for, and the Accept-header map that vhost's
# SPA fallback reads. All three are matrix knowledge and none of
# them is the gateway's business.
#
# `gatewayHost = null` means no vhost, so there is no name to claim
# and nothing to serve — every clause below carries that guard.
services.hyperhive.gateway.localNames = lib.optional (cfg.gatewayHost != null) cfg.gatewayHost;
services.hyperhive.gateway.enable = lib.mkIf (cfg.gatewayHost != null) (lib.mkDefault true);
# Unguarded, unlike the two above: the homeserver container resolves
# through the hive's dnsmasq whether or not the gateway fronts it.
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Accept-header SPA map, used only by the `/` location below (see
# docs/networking/gateway.md "SPA fallback"): text/html → index.html, else a
# sentinel so `try_files` falls through to 404. `appendHttpConfig`
# is a `lines` option, so this merges with anything else the host
# contributes instead of replacing it.
#
# The dashboard needs no equivalent — it routes by path.
services.nginx.appendHttpConfig = ''
map $http_accept $matrix_spa_target {
default "/__matrix_spa_no_html_fallback";
"~*text/html" "/index.html";
}
'';
# `server_name = gatewayHost`. `/_matrix/*` → tuwunel (CORS `*`, 50M
# body cap, 1h long-poll timeout). `/` serves fluffychat. nginx's
# longest-prefix rule puts `/_matrix/`
# ahead of `/` with no ordering needed.
#
# ⚠️ The `.well-known/matrix/*` delegation is deliberately NOT here.
# It stays on the hive's own vhost because the spec requires it to
# be served at the *server name*, which is the hive domain — it is
# the hive answering "where is my homeserver", not the homeserver
# answering for itself.
services.hyperhive.gateway.vhosts = lib.optionalAttrs (cfg.gatewayHost != null) {
"${cfg.gatewayHost}" = {
locations = {
"/_matrix/" = {
# No URI after the socket, so tuwunel gets the request URI as the
# client sent it.
proxyPass = "http://unix:${socketPath}:";
proxyWebsockets = true;
extraConfig = ''
proxy_buffering off;
# Tracks `maxRequestSize` with headroom so the homeserver stays
# the tighter limit: its rejection is a matrix error a client can
# act on, where a 413 here names neither matrix nor the option the
# operator just raised. A second literal beside it was free to
# disagree, and did — raising the option past the old 50M changed
# nothing.
client_max_body_size ${toString (deployCfg.matrix.maxRequestSize + 1048576)};
proxy_read_timeout 1h;
proxy_send_timeout 1h;
${gatewayCfg.lib.securityHeaders}
add_header Access-Control-Allow-Origin *;
'';
};
# fluffychat at sub-domain root, SPA-fallback via the
# Accept-header `$matrix_spa_target` map above.
"/" = {
alias = "${deployCfg.matrix.gui.package}/";
extraConfig = ''
try_files $uri $uri/ $matrix_spa_target =404;
'';
};
# FluffyChat boot-config pre-fill so the client's
# `.well-known/matrix/client` lookup hits the right delegation
# endpoint. `domain` is required, so this is always present.
"= /config.json" = {
extraConfig = ''
default_type application/json;
return 200 '{"defaultHomeserver":"${hyperhiveDomain}"}';
'';
};
};
};
};
# `serverName` is irrevocably embedded in user/room IDs; it derives
# from `services.hyperhive.domain` (required, asserted in
# hive-network.nix) when not set explicitly, so no separate
# domain/serverName assertion is needed here. gatewayHost may not be
# "" (same footgun as forge.domain — nginx rejects an empty
# server_name). docs/integrations/matrix.md::Assertion rationale.
assertions = [
{
assertion = cfg.gatewayHost == null || cfg.gatewayHost != "";
message = ''
services.hyperhive.swarm.matrix.gatewayHost = "" is rejected. The
rendered URLs would be invalid (nginx wildcard catch-all for
an empty server_name, /etc/hosts rejects empty entries).
Use `null` to disable the gateway vhost entirely, or set a
non-empty hostname like
"matrix.example.com" or "homeserver.internal".
'';
}
{
# Fail at EVAL, not at boot. tuwunel reads its identity providers
# from the config file, so a half-configured one does not hide a
# login button — it can stop the homeserver from starting at all.
assertion = deployCfg.matrix.sso.clientSecretFile != null;
message = ''
This homeserver's SSO login flow requires
services.hyperhive.deploy.matrix.sso.clientSecretFile — the path
(inside the matrix container) holding the OIDC client secret's
plaintext.
It is set by default wherever the homeserver runs, and
swarm-bao-matrix-oidc writes the secret there from the swarm
secret store. Remove the definition that set it to null.
'';
}
{
# Only `matrixSecretDir` is shared with the container; a path elsewhere
# would be read by tuwunel where the delivery unit does not write.
assertion =
deployCfg.matrix.sso.clientSecretFile == null
|| lib.hasPrefix "${matrixSecretDir}/" deployCfg.matrix.sso.clientSecretFile;
message = ''
services.hyperhive.deploy.matrix.sso.clientSecretFile must lie under
${matrixSecretDir}/, the one directory the homeserver's container
shares with the host's /run. swarm-bao-matrix-oidc writes the secret
there and nowhere else. Remove the definition, or move it under that
directory.
'';
}
{
# The callback URL must name the homeserver itself, and with no
# gateway vhost there is no public name for it to be built from.
assertion = cfg.gatewayHost != null;
message = ''
This homeserver's SSO login flow requires
services.hyperhive.swarm.matrix.gatewayHost.
tuwunel's SSO callback URL is format-locked to
`<homeserver>/_matrix/client/unstable/login/sso/callback/<client_id>`,
and the identity provider redirects a browser to it — so it
has to be a name the browser can reach, which is exactly what
`gatewayHost` is. With it null the homeserver has no such name.
'';
}
{
assertion =
(config.services.hyperhive.swarm.otel.serviceProducers.hive-matrix.clientId or null)
== forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"hive-matrix" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for the homeserver's journal forwarder.
Declare it in swarm-otel-service.nix.
'';
}
];
# Where the homeserver finds its OIDC client secret, inside the container.
services.hyperhive.deploy.matrix.sso.clientSecretFile = lib.mkDefault matrixSecretPath;
# The homeserver's OIDC client secret, out of the swarm secret store under
# matrix-ctl's leaf — the one route, wherever authelia runs. Same shape as
# ./swarm-bao.nix's `swarm-bao-forwarder-oidc`: the login and the read both
# fail loudly, and `storeRetry` retries until the publisher on authelia's
# host has written the secret, because tuwunel does not start without it.
#
# ⚠️ tuwunel requires the file to exist and be non-empty, and its
# `LoadCredential` source must exist for the unit to start, so a host with
# no matrix-ctl leaf fails here, naming the options, rather than rendering
# no unit.
systemd.services.swarm-bao-matrix-oidc = {
description = "fetch the homeserver's OIDC client secret from the swarm secret store";
# Ordered after the store where it runs here
# (./glue-bao-readers-store-order.nix); elsewhere the timeout below
# bounds the attempt.
before = [ "container@hive-matrix.service" ];
wantedBy = [
"multi-user.target"
"container@hive-matrix.service"
];
path = [
deployCfg.bao.package
pkgs.coreutils
pkgs.systemd
];
# ./lib/store-retry.nix. The container waits for one attempt; a secret a
# later attempt lands restarts tuwunel (below).
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
# No `RemainAfterExit`: the timer below starts this again, and
# starting an active unit is a no-op.
Type = "oneshot";
SyslogIdentifier = "swarm-bao-matrix-oidc";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs ctlActive {
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script =
if !ctlActive then
''
echo "the homeserver's host has no matrix-ctl store identity, so it cannot read the homeserver's OIDC client secret: set services.hyperhive.deploy.matrix.ctlBaoClientCertFile and ctlBaoClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2
exit 1
''
else
''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
# The container's bind source, so it starts even when the store does
# not answer; tmpfiles creates it too, but unordered with the
# container on a switch.
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostSecretPath)}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with matrix-ctl's certificate; leaving the homeserver's OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; the homeserver has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; the homeserver has no OIDC client secret yet." >&2
exit 1
fi
# Root-owned 0400: tuwunel runs under `DynamicUser`, so there is no
# stable uid to hand the file to, and `LoadCredential` reads it as
# root before the sandbox exists.
if secret_differs ${lib.escapeShellArg hostSecretPath} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg hostSecretPath} "$secret"
fi
# `LoadCredential` copies the file once, at tuwunel's start.
refresh_consumer hive-matrix tuwunel.service ${lib.escapeShellArg hostSecretPath}
'';
};
systemd.timers.swarm-bao-matrix-oidc = storeRefreshTimer "re-read the homeserver's OIDC client secret from the swarm secret store";
# ⚠️ Deliberately NO `networking.hosts` entry for authelia's name. This
# container resolves through the hive's dnsmasq at `bridgeIp` (see
# `networking.nameservers` below), and every `gateway.localNames` entry —
# authelia's domain among them — is already mapped there. Adding a
# loopback override would only create a second answer that can
# disagree with the first.
# Activation-time token generation — without this the bind-mount
# would hand tuwunel an empty file on first boot and break every
# registration until restart. Idempotent;
# docs/integrations/matrix.md::Provisioning flow.
# Tell an operator whose homeserver already exists that `serverName` is
# unpinned, at the one moment they are looking: the rebuild.
#
# ⚠️ An activation script and NOT `warnings`, which is where this
# obviously belongs and does not work. The condition needs the host
# filesystem — does a homeserver already exist here? — and
# `nixos-rebuild switch --flake` evaluates PURELY, where
# `builtins.pathExists "/var/lib/…"` answers **false** rather than
# throwing. A `warnings` entry gated on it would evaluate, deploy, and
# print nothing, on every real deployment. Same shape as an option whose
# consumer is disabled: renders perfectly, does nothing.
#
# Only rendered when `serverName` is null, so a hive that pinned it
# cannot be nagged — the script does not exist there rather than
# existing and choosing to stay quiet. A guard that cries wolf at a
# correctly-configured deployment is worse than no guard, because the
# next real one is read as noise too.
#
# Never fails. This warns about a choice that cannot be undone; refusing
# the activation would break the rebuild of a hive that had already made
# that choice deliberately, which is the opposite of helping.
system.activationScripts.hive-matrix-servername-pin = lib.mkIf (cfg.serverName == null) (
lib.stringAfter [ "var" ] ''
# The homeserver's own database, asked of the container's evaluated
# config rather than hardcoded: a guessed path resolves cleanly and
# silently never matches, which is exactly the failure this guard
# exists to avoid one level up.
dbDir=${
lib.escapeShellArg (
# Same host-side container-root prefix this module already writes
# by hand for the SSO secret copy above — not a second convention.
"/var/lib/nixos-containers/hive-matrix"
+ config.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.database_path
)
}
if [ -d "$dbDir" ]; then
echo "hive-matrix: WARNING — services.hyperhive.swarm.matrix.serverName is unset, and this host already has a homeserver at $dbDir."
echo "hive-matrix: it is defaulting to ${effectiveServerName}, which is baked into every NEW user and room id."
echo "hive-matrix: if ids here were minted under a different name, existing accounts and rooms are stranded — reverting the config does NOT undo it."
echo "hive-matrix: pin whichever name this homeserver already uses, e.g.:"
echo "hive-matrix: services.hyperhive.swarm.matrix.serverName = \"''${HIVE_MATRIX_EXISTING_SERVER_NAME:-<the name already in use>}\";"
fi
''
);
# The socket directory, created host-side before the container starts:
# nixos-container refuses to start when a bind source is missing.
#
# Root-owned and group nginx, `0770`: tuwunel creates its socket here
# through the group (its uid is allocated inside the container, so the
# host cannot name it), and the gateway traverses it the same way.
# Everything else is kept out.
systemd.tmpfiles.rules = [
"d ${socketDir} 0770 root ${toString nginxGid} - -"
# The bind source must exist when the container starts.
"d ${matrixSecretDir} 0700 root root -"
# `LoadCredential=` fails the unit when the file is missing; an empty
# one lets the container start and tuwunel report the missing secret.
"f ${hostSecretPath} 0400 root root -"
"r ${legacyHostSecretPath}"
];
# The container hangs off the bridge, takes its address from the
# gateway's DHCP and resolves through its dnsmasq.
services.hyperhive.network.enable = lib.mkDefault true;
containers.hive-matrix = {
autoStart = true;
ephemeral = false;
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = {
# A bind mount rather than a path both sides happen to know: a
# container's `/run` is an nspawn tmpfs the host cannot see into.
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
${matrixSecretDir} = {
hostPath = matrixSecretDir;
isReadOnly = true;
};
}
// caTrust.bindMount
# matrix-ctl's bao client identity, and nothing else of the hive's. See
# `ctlPkiDirs` above for why it is a derived set of directories
# rather than one literal.
// ctlBindMounts;
config =
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
# tuwunel's rustls verifier resolves through `rustls-native-certs`
# → `openssl-probe`, which reads `SSL_CERT_FILE` — so the
# openssl-shaped variable is the lever despite tuwunel linking no
# openssl.
#
# ⚠️ The helper CONCATENATES, and that is load-bearing here beyond
# the usual reason: `SSL_CERT_FILE` replaces the default location,
# so naming the hive anchor alone would drop every public CA and
# break federation with the wider matrix network — trading a small
# outage for a much larger one.
# A literal, not an option: this module names its container
# `containers.hive-matrix` directly and declares no `machine`
# option to derive it from.
(caTrust.trustBundle {
inherit pkgs;
name = "hive-matrix";
consumers = [
"tuwunel"
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
})
];
# Authenticated as the forwarder's own OIDC client, with matrix-ctl's
# store identity at the paths bound in from the host. With no such
# identity there is nothing to fetch the secret with, as for the
# appservice units below.
services.hyperhive.swarmJournalForwarder = lib.mkIf ctlActive {
enable = true;
serviceName = "tuwunel";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = deployCfg.bao.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = deployCfg.matrix.ctlBaoClientCertFile;
clientKeyFile = deployCfg.matrix.ctlBaoClientKeyFile;
caFile = deployCfg.bao.serverCaFile;
};
};
# Its own netns, so it keeps its own firewall; resolvconf stays on
# and serves `networking.nameservers` below.
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
# Swarm-internal trust reaches tuwunel at RUNTIME, not via
# `security.pki.certificateFiles`. That option is read when the
# system is BUILT, and the store's root is a runtime file fetched
# from the store at boot — so there is nothing build-time to name.
# The bind-mount above plus the bundle service below carry it.
# The gateway's dnsmasq on the bridge answers every name in
# `gateway.localNames` with the bridge IP and forwards the rest
# upstream. tuwunel exits at boot when `/etc/resolv.conf` has no
# `nameserver` line.
networking.nameservers = [ networkCfg.bridgeIp ];
networking.interfaces.eth0.useDHCP = true;
# The socket's group is tuwunel's primary group, so that group is
# nginx's gid. The user and group both exist statically, so
# systemd runs the unit as them despite upstream's `DynamicUser`.
users.groups.gateway-nginx.gid = nginxGid;
services.matrix-tuwunel = {
enable = true;
package = deployCfg.matrix.package;
group = "gateway-nginx";
settings.global = {
server_name = effectiveServerName;
# ⚠️ `address` stays unset: tuwunel binds TCP as well as the
# socket whenever it is set.
unix_socket_path = socketPath;
unix_socket_perms = 660;
max_request_size = deployCfg.matrix.maxRequestSize;
# Federation enabled at the protocol level; empty
# trustedServers keeps it effectively closed.
allow_federation = true;
trusted_servers = deployCfg.matrix.trustedServers;
# Nobody registers themselves here. Accounts are created by
# the swarm's appservice, which this flag does not gate —
# tuwunel checks it only for requests that arrive WITHOUT an
# appservice token, so swarm-controller provisions and
# everyone else is refused outright.
#
# ⚠️ Not a hardening afterthought: `allow_registration = true`
# with no registration token configured makes tuwunel REFUSE
# TO START (it demands
# `yes_i_am_very_very_sure_…_open_registration_…` instead). So
# dropping the token and leaving this true is not a lax
# homeserver, it is a homeserver that does not boot.
allow_registration = false;
# Where the swarm's appservice registration is read from — the
# credentials directory, for the reasons at
# `appserviceCredentialDir`'s own definition. Loaded inside
# `Services::start()`, before the listener accepts anything.
appservice_dir = appserviceCredentialDir;
}
# Exactly one account is promoted to homeserver admin at boot:
# the swarm appservice's sender. Its token is read by
# matrix-ctl and swarm-controller only. Each `@hive-<hive>`
# stays an ordinary user, because its hive reads its
# credential; what it does (the Space, the chat room, the
# invites) rides on being the rooms' creator, not on admin.
#
# The registration load creates the sender inside
# `Services::start()`, and `admin_execute` runs after that and
# before the listener accepts anything. `make-user-admin` is a
# no-op on an account that is already admin.
# `admin_execute_errors_ignore` is load-bearing: a failing
# command aborts startup when it is false.
// lib.optionalAttrs ctlActive {
admin_execute = [
"users make-user-admin @${swarmSenderLocalpart}:${effectiveServerName}"
];
admin_execute_errors_ignore = true;
}
// {
# Server-side E2EE is opt-in (default off); the agent matrix
# client always supports decryption regardless.
allow_encryption = cfg.allowEncryption;
# Tuwunel's default suffix is " 💕" — suppress it so agent
# display names are clean (just the agent name, no emoji).
new_user_displayname_suffix = "";
# tuwunel's OIDC server and this list are the two ends of one
# pipe: `oidc_native_auth` stays false (its default), which
# upstream defines as "the OIDC server runs only to broker
# for a configured identity_provider". So the client-facing
# half needs no configuration — only the upstream half does.
identity_provider = [
{
# A free, case-insensitive string, not an enum: a
# recognised brand gets defaults and provider-specific
# workarounds, an unrecognised one simply gets neither.
# Which is why `issuer_url` below is not optional for us
# — the pre-supplied issuers cover public providers only.
brand = "authelia";
client_id = cfg.sso.clientId;
client_secret_file = matrixSecretCredential;
issuer_url = toString autheliaUrl;
callback_url = cfg.sso.redirectUri;
# Explicit, though a lone provider is auto-defaulted:
# relying on that logs a warning every startup, and a
# recurring warning that is expected is one nobody reads.
default = true;
# Upstream's rule is "only ever set `trusted` for
# identity providers you self-host and fully control",
# and this module cannot point anywhere else: the issuer
# is `swarm.authelia.url`, whose client, secret and user
# database are all ours. It does mean whoever can make
# authelia emit a given name gets that account — for our
# own identity provider that IS the identity.
# Without it, an SSO login cannot adopt an account that
# already exists; it can only ever create a new one.
trusted = true;
# One claim instead of upstream's ladder
# (`preferred_username` → `username` → `nickname` →
# `login` → `email`). The tail is the hazard: an email
# local part is a different namespace, so a login can
# land on a name that means someone else here.
userid_claims = [ "preferred_username" ];
# The default (`true`) makes a name collision SILENT —
# tuwunel invents a random localpart and the login
# succeeds as the wrong user. `false` errors instead,
# which is the only form of this an operator can act on.
unique_id_fallbacks = false;
}
];
};
};
# Upstream's `ProtectSystem = "strict"` leaves `/run` read-only.
systemd.services.tuwunel.serviceConfig.ReadWritePaths = [ socketDir ];
# Keeps DynamicUser=true + PrivateUsers=true intact — no
# host-side chown :tuwunel / GID-pin gymnastics needed.
# See `man systemd.exec` → LoadCredential.
systemd.services.tuwunel.serviceConfig.LoadCredential = [
# tuwunel re-reads this file on every OAuth exchange, not just at startup, so it has to
# outlive the unit's start — a credentials path does.
"oidc_client_secret:${toString deployCfg.matrix.sso.clientSecretFile}"
]
# The swarm registration. The credential id carries a `.yaml`
# suffix on purpose: `appservice_dir` above names this very
# directory, and tuwunel takes only `.yaml`/`.yml` entries from
# it. A missing source fails this unit, which is why the render
# below is `requiredBy` it and local only.
++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceDir}/swarm.yaml";
# The swarm registration, minted and rendered before the homeserver
# loads it. No network and no store: this is on tuwunel's start
# path, and a store outage must not keep the homeserver down.
systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive {
description = "render the swarm's appservice registration";
before = [ "tuwunel.service" ];
requiredBy = [ "tuwunel.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice render";
StateDirectory = baseNameOf swarmAppserviceDir;
StateDirectoryMode = "0700";
UMask = "0077";
SyslogIdentifier = "swarm-matrix-appservice";
};
environment = {
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
# Agents' localparts are bare `[a-z0-9-]`, so nothing narrower
# covers them without also covering people.
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
};
};
# Hand the swarm registration's token to swarm-controller, through
# the store, under matrix-ctl's identity; the write lands at a path
# only matrix-ctl and the controller are granted (./swarm-bao.nix).
# `Restart=on-failure` covers a sealed store or one still starting;
# the start timeout is bounded for the reason
# ./swarm-secret-publisher.nix states: a sealed store answers on the
# port and never answers the read.
systemd.services.swarm-matrix-appservice-publish = lib.mkIf ctlActive {
description = "publish the swarm's appservice token to the swarm secret store";
after = [ "swarm-matrix-appservice-render.service" ];
requires = [ "swarm-matrix-appservice-render.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice publish";
Restart = "on-failure";
RestartSec = 30;
TimeoutStartSec = 60;
SyslogIdentifier = "swarm-matrix-appservice";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
MATRIX_APPSERVICE_CERT_ROLE = ctlCertRole;
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
};
environment.systemPackages = [ deployCfg.matrix.package ];
};
};
# The matrix container's lease and resolver are the hive's dnsmasq
# (bound at `bridgeIp`). Order the matrix container start after it so
# both are up before tuwunel's first federation lookups. Soft `after`
# ordering (not `requires`) keeps the matrix container's lifecycle
# decoupled from the resolver's.
#
# `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also
# sets `after`/`requires` (so the bound trust bundle exists before
# nspawn wires the mount up), and two plain assignments to the same
# unit would conflict rather than combine.
systemd.services."container@hive-matrix" = lib.mkMerge [
{ after = [ "dnsmasq.service" ]; }
caTrust.containerOrdering
];
};
}