vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-matrix: run the homeserver in its own netns, behind a unix socket

Browse source 
tuwunel bound `0.0.0.0:<httpPort>` in the host netns, so the homeserver
port was open on every host address, and the gateway proxied to it on
host loopback.

The container now sets `privateNetwork = true` and hangs off the hive
bridge with a DHCP lease, the same attach the CI runner, builder and
review classifier use (resolvconf on, `networking.nameservers` the bridge
IP). tuwunel listens only on `/run/hive-matrix/tuwunel.sock`, a host
directory (root:nginx 0770) bind-mounted into the container; its primary
group is nginx's gid and the socket is 0660. The gateway's `/_matrix/`
location proxies to the socket.

The static resolv.conf goes: it existed because resolvconf's host
tracking came up empty in the shared netns.

`swarm.matrix.httpPort` and `{swarm,deploy}.matrix.openFirewall` are
removed with a message: there is no host port to choose or open. The
`.well-known/matrix/*` fallback to `<hive>:<httpPort>` is dropped with
them; it only rendered with `gatewayHost = null`, which the homeserver's
SSO assertion already rejects.

Part of #4815.
This commit is contained in:
atlas 2026-10-14 23:58:54 +02:00 • committed by mara
commit 34e142bb5c
11 changed files with 191 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

28
nix/host-modules/deploy.nix
View file

@ -275,16 +275,32 @@ in
)
# The homeserver, split the same way. `swarm.matrix` keeps what it IS to
# every hive — the name it answers to, the ports and URLs it is reached
# on, the client id it is registered under. These six are what the host
# every hive — the name it answers to, the URLs it is reached on, the
# client id it is registered under. These six are what the host
# running it decides; its two packages are renamed further down with
# the other `*.package` entries. ⚠️ `enable` is NOT here: its shim lives in
# ./hive-matrix.nix, so that file is the one
# to check before assuming this table is the whole matrix story.
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "matrix" "openFirewall" ]
[ "services" "hyperhive" "deploy" "matrix" "openFirewall" ]
)
#
# The homeserver's port and its firewall switch, removed rather than
# renamed: tuwunel listens on a unix socket only the gateway reaches, so
# there is no host port to choose or open.
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "matrix" "httpPort" ] ''
The homeserver no longer listens on a TCP port: it runs in a network
namespace of its own and the gateway reaches it over a unix socket.
Remove this definition. Clients and federation peers reach it at
services.hyperhive.swarm.matrix.gatewayHost.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "matrix" "openFirewall" ] ''
The homeserver no longer listens on a host port, so there is no port
to open. Remove this definition. Reach from outside the host goes
through the gateway: services.hyperhive.gateway.openFirewall.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "matrix" "openFirewall" ] ''
The homeserver no longer listens on a host port, so there is no port
to open. Remove this definition. Reach from outside the host goes
through the gateway: services.hyperhive.gateway.openFirewall.
'')
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "matrix" "trustedServers" ]
[ "services" "hyperhive" "deploy" "matrix" "trustedServers" ]

22
nix/host-modules/hive-gateway/vhosts.nix
View file

@ -61,17 +61,13 @@ let
}
);
# `.well-known/matrix/{client,server}` discovery JSON. Points
# clients at `matrixCfg.gatewayHost` when set; falls back to direct
# `<hive>:<httpPort>`. CORS `*` per matrix spec. The `m.server`
# port-8448 carve-out is documented inline. See `docs/networking/gateway.md`.
wellKnownLocations = lib.optionalAttrs matrixDeployCfg.enable (
# `.well-known/matrix/{client,server}` discovery JSON, pointing clients
# and peers at `matrixCfg.gatewayHost`: the homeserver has no other
# address. CORS `*` per matrix spec. The `m.server` port-8448 carve-out
# is documented inline. See `docs/networking/gateway.md`.
wellKnownLocations = lib.optionalAttrs (matrixDeployCfg.enable && matrixCfg.gatewayHost != null) (
let
clientBaseUrl =
if matrixCfg.gatewayHost != null then
"${publicScheme}://${matrixCfg.gatewayHost}${publicPortSuffix}"
else
"${publicScheme}://${hyperhiveDomain}:${toString matrixCfg.httpPort}";
clientBaseUrl = "${publicScheme}://${matrixCfg.gatewayHost}${publicPortSuffix}";
# `m.server` is NOT a URL: per the matrix server-server spec
# (Resolving Server Names) a delegated host with NO port resolves
# to the federation default 8448 (after the SRV check) — the
@ -81,11 +77,7 @@ let
# wrong for federation delegation. Without this, peers federate to
# <gatewayHost>:8448 (closed) while the endpoint actually lives on
# the gateway's 443 vhost. See docs/networking/gateway.md discovery flow.
serverHostPort =
if matrixCfg.gatewayHost != null then
"${matrixCfg.gatewayHost}:${toString publicPort}"
else
"${hyperhiveDomain}:${toString matrixCfg.httpPort}";
serverHostPort = "${matrixCfg.gatewayHost}:${toString publicPort}";
in
{
"= /.well-known/matrix/client" = {

18
nix/host-modules/hive-matrix-service.nix
View file

@ -1,5 +1,5 @@
# The swarm's matrix homeserver as every hive sees it: the name it mints ids
# under, the ports and URLs it is reached on, its encryption policy and the
# under, the URLs it is reached on, its encryption policy and the
# OIDC client id it is registered under, identical on every host. What the
# host running it decides, and the container itself, are in ./hive-matrix.nix.
{
@ -49,18 +49,6 @@ in
'';
};
httpPort = lib.mkOption {
type = lib.types.port;
default = 8008;
description = ''
TCP port tuwunel serves the matrix client-server API on.
Default 8008 is the matrix-spec well-known port. Sits
outside hyperhive's claimed ranges (dashboard 7000, every
agent in 8100..8999 via FNV-1a hash). Federation listens on
`federationPort` separately.
'';
};
apiUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = if cfg.gatewayHost == null then null else "https://${cfg.gatewayHost}";
@ -108,8 +96,8 @@ in
Public hostname for the matrix homeserver behind the gateway.
Defaults to `chat.''${services.hyperhive.swarm.domain}` — the
swarm's domain, because a swarm runs **one** homeserver. Set to
`null` to skip the gateway vhost (tuwunel stays direct on
`httpPort`). See `docs/networking/gateway.md` for the vhost map + matrix
`null` to skip the gateway vhost, which leaves nothing able to
reach the homeserver. See `docs/networking/gateway.md` for the vhost map + matrix
discovery flow, and the federation port-8448 caveat at the
bottom of that doc.

176
nix/host-modules/hive-matrix.nix
View file

@ -239,8 +239,19 @@ let
'';
});
# Shared host netns — agents reach tuwunel at localhost:<port>.
privateNetwork = false;
# A netns of its own on the hive bridge: the homeserver dials the store,
# authelia and federation peers by name, and the gateway reaches it over
# `socketPath` below, so it binds no TCP port.
privateNetwork = true;
# Fixed, not an option: the directory's mode is what keeps the socket
# private, so it holds this one socket and nothing else.
socketDir = "/run/hive-matrix";
socketPath = "${socketDir}/tuwunel.sock";
# The gid the gateway's nginx runs as. Only the number crosses into the
# container, which knows it as `gateway-nginx`.
nginxGid = config.ids.gids.nginx;
in
{
# Private matrix-tuwunel homeserver wrapped in a nixos-container,
@ -281,7 +292,7 @@ in
];
# `swarm.matrix`, in ./hive-matrix-service.nix, is what the homeserver IS from
# any hive's point of view: the name it answers to, the ports and URLs it is
# any hive's point of view: the name it answers to, the URLs it is
# reached on, and the client id it is registered under. What lives here is what the host running it
# decides — which build it runs, whether it is exposed, which peers it trusts,
# how large a request it accepts, and where its host-local secrets sit. Same rule as
@ -299,30 +310,6 @@ in
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Open `httpPort` in the host firewall. Off by default
(secure-by-default): hive-c0re and agent containers reach the
homeserver at `gatewayHost`'s vhost (`chat.<swarm-domain>` by
default) via the gateway, so the open port only matters for
direct access to `httpPort` from outside the host. Flip to `true` when announcing
the homeserver to other hives or when an external matrix
client needs to reach the client-server API directly.
Note: federation (the matrix-spec well-known port 8448) is
intentionally not opened here. tuwunel serves the federation
API on the same `httpPort` as the client-server API by
default; reaching it on 8448 requires either binding tuwunel
to that port explicitly OR a reverse-proxy + `.well-known/
matrix/server` delegation, neither of which lives in this
module. Add that proxy config alongside whatever serves your
dashboard or forge on 443.
'';
};
trustedServers = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
@ -450,9 +437,8 @@ in
# SPA fallback reads. All three are matrix knowledge and none of
# them is the gateway's business.
#
# `gatewayHost = null` means matrix is reachable directly rather
# than fronted, so there is no name to claim and no vhost to serve —
# every clause below carries that guard.
# `gatewayHost = null` means no vhost, so there is no name to claim
# and nothing to serve — every clause below carries that guard.
services.hyperhive.gateway.localNames = lib.optional (cfg.gatewayHost != null) cfg.gatewayHost;
services.hyperhive.gateway.enable = lib.mkIf (cfg.gatewayHost != null) (lib.mkDefault true);
@ -490,7 +476,9 @@ in
extraConfig = gatewayCfg.lib.securityHeaders;
locations = {
"/_matrix/" = {
proxyPass = "http://127.0.0.1:${toString cfg.httpPort}";
# No URI after the socket, so tuwunel gets the request URI as the
# client sent it.
proxyPass = "http://unix:${socketPath}:";
proxyWebsockets = true;
extraConfig = ''
proxy_buffering off;
@ -541,8 +529,8 @@ in
services.hyperhive.swarm.matrix.gatewayHost = "" is rejected. The
rendered URLs would be invalid (nginx wildcard catch-all for
an empty server_name, /etc/hosts rejects empty entries).
Use `null` to disable the gateway vhost entirely (tuwunel
stays direct on httpPort), or set a non-empty hostname like
Use `null` to disable the gateway vhost entirely, or set a
non-empty hostname like
"matrix.example.com" or "homeserver.internal".
'';
}
@ -574,8 +562,7 @@ in
`<homeserver>/_matrix/client/unstable/login/sso/callback/<client_id>`,
and the identity provider redirects a browser to it — so it
has to be a name the browser can reach, which is exactly what
`gatewayHost` is. With it null the homeserver is direct on
httpPort and has no such name.
`gatewayHost` is. With it null the homeserver has no such name.
'';
}
];
@ -688,8 +675,8 @@ in
systemd.timers.swarm-bao-matrix-oidc = storeRefreshTimer "re-read the homeserver's OIDC client secret from the swarm secret store";
# ⚠️ Deliberately NO `networking.hosts` entry for authelia's name. This
# container resolves through the hive's dnsmasq at `bridgeIp` (see the
# static resolv.conf below), and every `gateway.localNames` entry —
# container resolves through the hive's dnsmasq at `bridgeIp` (see
# `networking.nameservers` below), and every `gateway.localNames` entry —
# authelia's domain among them — is already mapped there. Adding a
# loopback override would only create a second answer that can
# disagree with the first.
@ -743,6 +730,21 @@ in
''
);
# The socket directory, created host-side before the container starts:
# nixos-container refuses to start when a bind source is missing.
#
# Root-owned and group nginx, `0770`: tuwunel creates its socket here
# through the group (its uid is allocated inside the container, so the
# host cannot name it), and the gateway traverses it the same way.
# Everything else is kept out.
systemd.tmpfiles.rules = [
"d ${socketDir} 0770 root ${toString nginxGid} - -"
];
# The container hangs off the bridge, takes its address from the
# gateway's DHCP and resolves through its dnsmasq.
services.hyperhive.network.enable = lib.mkDefault true;
containers.hive-matrix = {
autoStart = true;
ephemeral = false;
@ -750,12 +752,20 @@ in
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
inherit privateNetwork;
bindMounts =
caTrust.bindMount
# matrix-ctl's bao client identity, and nothing else of the hive's. See
# `ctlPkiDirs` above for why it is a derived set of directories
# rather than one literal.
// ctlBindMounts;
hostBridge = networkCfg.bridgeName;
bindMounts = {
# A bind mount rather than a path both sides happen to know: a
# container's `/run` is an nspawn tmpfs the host cannot see into.
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
}
// caTrust.bindMount
# matrix-ctl's bao client identity, and nothing else of the hive's. See
# `ctlPkiDirs` above for why it is a derived set of directories
# rather than one literal.
// ctlBindMounts;
config =
{ ... }:
{
@ -781,7 +791,12 @@ in
})
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Its own netns, so it keeps its own firewall; resolvconf stays on
# and serves `networking.nameservers` below.
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
# Swarm-internal trust reaches tuwunel at RUNTIME, not via
# `security.pki.certificateFiles`. That option is read when the
@ -789,48 +804,28 @@ in
# from the store at boot — so there is nothing build-time to name.
# The bind-mount above plus the bundle service below carry it.
# tuwunel hard-fails to boot if `/etc/resolv.conf` has no
# `nameserver` line (`Failed to configure DNS resolver ... no
# nameservers found in config` → exit 1). This declarative
# nixos-container comes up with an EMPTY resolv.conf even with
# `networking.nameservers` set: the nixos-container default
# `useHostResolvConf = true` puts in-container resolvconf in
# host-tracking mode (ignores `networking.nameservers`, and never
# gets the host file across the shared-netns boundary), so it
# regenerates an empty file and tuwunel dies at boot.
#
# Trusting resolvconf to honour `networking.nameservers` doesn't
# work either — that's a RUNTIME resolvconf behaviour, not
# verifiable at eval time, and it still comes up empty in
# practice. So take resolvconf out of the loop entirely and
# write a STATIC `/etc/resolv.conf` from `bridgeIp` that nothing
# regenerates. Eval-proven: the generated
# `environment.etc."resolv.conf".text` is `nameserver <bridgeIp>`.
# This container always shares the host netns
# (`privateNetwork = false`), so it reaches `bridgeIp` regardless
# of agent-container isolation. See `docs/networking/network.md`.
# resolvconf is off through `writesOwnResolvConf` (its default);
# the static `environment.etc."resolv.conf"` below is the sole
# source of the resolver file (no `nameservers` — nothing would
# read it).
networking.useHostResolvConf = lib.mkForce false;
# The gateway's dnsmasq on the bridge answers every name in
# `gateway.localNames` with the bridge IP and forwards the rest
# upstream. tuwunel exits at boot when `/etc/resolv.conf` has no
# `nameserver` line.
networking.nameservers = [ networkCfg.bridgeIp ];
networking.interfaces.eth0.useDHCP = true;
# resolvconf is disabled above, so write the static resolver file
# explicitly — NixOS won't synthesise one from `nameservers` once
# resolvconf is off, and this is the file tuwunel parses at boot.
environment.etc."resolv.conf".text = ''
nameserver ${networkCfg.bridgeIp}
options edns0
'';
# The socket's group is tuwunel's primary group, so that group is
# nginx's gid. The user and group both exist statically, so
# systemd runs the unit as them despite upstream's `DynamicUser`.
users.groups.gateway-nginx.gid = nginxGid;
services.matrix-tuwunel = {
enable = true;
package = deployCfg.matrix.package;
group = "gateway-nginx";
settings.global = {
server_name = effectiveServerName;
# `address` + `port` are upstream `listOf` — wrap singles.
address = [ "0.0.0.0" ];
port = [ cfg.httpPort ];
# ⚠️ `address` stays unset: tuwunel binds TCP as well as the
# socket whenever it is set.
unix_socket_path = socketPath;
unix_socket_perms = 660;
max_request_size = deployCfg.matrix.maxRequestSize;
# Federation enabled at the protocol level; empty
# trustedServers keeps it effectively closed.
@ -933,6 +928,9 @@ in
];
};
};
# Upstream's `ProtectSystem = "strict"` leaves `/run` read-only.
systemd.services.tuwunel.serviceConfig.ReadWritePaths = [ socketDir ];
# Keeps DynamicUser=true + PrivateUsers=true intact — no
# host-side chown :tuwunel / GID-pin gymnastics needed.
# See `man systemd.exec` → LoadCredential.
@ -1009,21 +1007,11 @@ in
};
};
networking.firewall = lib.mkIf deployCfg.matrix.openFirewall {
allowedTCPPorts = [
cfg.httpPort
];
};
# The matrix container's resolver is the hive's dnsmasq (bound at
# `bridgeIp`). Order the matrix container start after it so the
# resolver is up before tuwunel's first federation lookups. tuwunel
# boots fine without this — it configures the resolver from
# `/etc/resolv.conf` at startup and only queries on-demand (the boot
# failure this module guards against is an *empty* resolv.conf, a
# parse error, not a connectivity one) — so this is robustness, not a
# boot requirement. Soft `after` ordering (not `requires`) keeps the
# matrix container's lifecycle decoupled from the resolver's.
# The matrix container's lease and resolver are the hive's dnsmasq
# (bound at `bridgeIp`). Order the matrix container start after it so
# both are up before tuwunel's first federation lookups. Soft `after`
# ordering (not `requires`) keeps the matrix container's lifecycle
# decoupled from the resolver's.
#
# `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also
# sets `after`/`requires` (so the bound trust bundle exists before

2
nix/host-modules/swarm-authelia-service.nix
View file

@ -30,7 +30,7 @@ in
description = ''
TCP port authelia listens on. 9091 is upstream's default and
sits outside hyperhive's claimed ranges (dashboard 7000, forge
3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash).
3000, every agent in 8100..8999 via FNV-1a hash).
'';
};

2
nix/host-modules/swarm-nats-service.nix
View file

@ -54,7 +54,7 @@ in
description = ''
TCP port the queue listens on. 4222 is upstream's default and
sits outside hyperhive's claimed ranges (dashboard 7000, forge
3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash).
3000, every agent in 8100..8999 via FNV-1a hash).
Opened on the bridge interface, for agent containers, and on
`wg-hive` when this host is on the mesh, for other hives. Never

4
nix/module-eval/bao-controller.nix
View file

@ -57,8 +57,8 @@ let
swarm.matrix.gatewayHost = "matrix.hive.t.local";
};
# No matrix vhost for this swarm at all: tuwunel stays direct on
# `httpPort`, so there is no homeserver URL to hand the controller.
# No matrix vhost for this swarm at all, so there is no homeserver URL
# to hand the controller.
controllerNoGatewayHost = hive {
deploy.swarm-controller.enable = true;
swarm.matrix.gatewayHost = null;

46
nix/module-eval/swarm-services-switch.nix
View file

@ -163,8 +163,8 @@ let
lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links;
# Every service container at once: the services-only host plus the CI
# runner. The CI runner, the review classifier, the builder and the metrics
# store are the four containers on a netns of their own.
# runner. The CI runner, the review classifier, the builder, the metrics
# store and the homeserver are the five containers on a netns of their own.
serviceContainersWithCi = hive {
deploy.hive-controller.enable = false;
deploy.allSwarmServices = true;
@ -522,7 +522,7 @@ let
let
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
in
lib.length shared == 8
lib.length shared == 7
&& lib.all (
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
) shared;
@ -575,6 +575,46 @@ let
&& unit.serviceConfig.UMask == "0007"
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
}
{
# On the bridge by DHCP, like the CI runner, and its socket the only
# listener the gateway dials. `address` unset is what keeps tuwunel off
# TCP. Every vhost location that forwards goes through the socket; the
# `/_matrix/` one is named so an empty filter can't pass.
name = "the homeserver runs on a netns of its own and the gateway reaches it over its socket";
ok =
let
host = serviceContainersWithCi;
net = host.services.hyperhive.network;
c = host.containers.hive-matrix;
dir = "/run/hive-matrix";
sock = "${dir}/tuwunel.sock";
locs =
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.matrix.gatewayHost}.locations;
forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs));
tuwunel = c.config.services.matrix-tuwunel;
unit = c.config.systemd.services.tuwunel;
nginxGid = host.ids.gids.nginx;
in
c.privateNetwork
&& c.hostBridge == net.bridgeName
&& c.hostAddress == null
&& c.localAddress == null
&& c.config.networking.interfaces.eth0.useDHCP
&& c.config.networking.nameservers == [ net.bridgeIp ]
&& c.config.networking.firewall.enable
&& c.config.networking.resolvconf.enable
&& (tuwunel.settings.global.address or null) == null
&& tuwunel.settings.global.unix_socket_path == sock
&& tuwunel.settings.global.unix_socket_perms == 660
&& locs."/_matrix/".proxyPass == "http://unix:${sock}:"
&& forwarded == [ "http://unix:${sock}:" ]
&& c.bindMounts.${dir}.hostPath == dir
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
&& c.config.users.groups.gateway-nginx.gid == nginxGid
&& tuwunel.group == "gateway-nginx"
&& unit.serviceConfig.Group == "gateway-nginx"
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
}
{
# The DynamicUser layout keeps the history under /var/lib/private; the
# static user's StateDirectory is the plain path. The move runs as its