diff --git a/docs/integrations/matrix.md b/docs/integrations/matrix.md index 6ee3876f..bd363f73 100644 --- a/docs/integrations/matrix.md +++ b/docs/integrations/matrix.md @@ -6,9 +6,9 @@ client at `chat./` (the `gatewayHost` vhost). A swarm runs one homeserver, on one host. Two namespaces configure it: - `services.hyperhive.swarm.matrix.*` — what the homeserver **is**, as - every hive sees it: `serverName`, `gatewayHost`, `apiUrl`, `allowEncryption`. + every hive sees it: `serverName`, `gatewayHost`, ports, `allowEncryption`. - `services.hyperhive.deploy.matrix.*` — what the host running it decides: - `enable`, `gui.package`, `trustedServers`, + `enable`, `gui.package`, `openFirewall`, `trustedServers`, `maxRequestSize`, `sso.clientSecretFile`. Vhost routing lives in [`gateway.md`](../networking/gateway.md). @@ -23,10 +23,10 @@ Same shape as [`gateway.md::hive-forge container shape`](../networking/gateway.m - Keeps hive-matrix from fighting any `services.matrix-*` the operator already runs on the host — separate systemd namespace, separate state dir. -- Container runs in a network namespace of its own - (`privateNetwork = true`) on the hive bridge, and binds no TCP port. Agents - reach the homeserver at `chat.` via the gateway, which - dials its unix socket (see [Reach](#reach)). +- Container shares the host network namespace + (`privateNetwork = false`) for state + systemd-unit isolation. Agents + reach the homeserver at `chat.` via the gateway (agents + run in private netns and can't access host loopback directly). - Persistent state at `/var/lib/nixos-containers/hive-matrix/var/lib/matrix-tuwunel/` survives container restart / host reboot. To wipe, destroy the @@ -44,7 +44,7 @@ Two distinct hostnames: - **`gatewayHost`** — the API listener hostname, where the gateway's matrix vhost proxies `/_matrix/*` to tuwunel. Defaults to `chat.`. Set to `null` to skip the - gateway vhost, which leaves nothing able to reach the homeserver. + gateway vhost (tuwunel stays direct on `httpPort`). Both default under the **swarm** domain, because a swarm runs one homeserver: tying its identity to a single hive's domain would make @@ -80,15 +80,14 @@ pin the value before the homeserver mints any ids under the default. -## Reach +## Default-closed firewall -tuwunel binds no TCP port. It listens on the unix socket -`/run/hive-matrix/tuwunel.sock`, in a host directory (`root:nginx`, -`0770`) bind-mounted into the container, and the gateway's -`/_matrix/` location proxies to that socket. Clients, agents, hive-c0re -and federation peers all reach the homeserver at `chat.` -through the gateway; from outside the host that takes -`gateway.openFirewall`. +`deploy.matrix.openFirewall` defaults to `false`: hive-c0re and agent +containers reach the homeserver at `chat.` via the gateway, +so the firewall hole only matters for direct access to `httpPort` from +*outside* the host. Flip to `true` when announcing the homeserver to +other hives or when an external matrix client needs to reach the +client-server API directly. hive-c0re provisions matrix (its own sender login, the hive Space and chat room, invites) through `swarm.matrix.apiUrl`, which defaults to @@ -97,15 +96,11 @@ the hive's matrix access token, and on a hive that doesn't run the homeserver it crosses the network, so the default is TLS; hive-c0re verifies the gateway's certificate against the hive trust bundle. -Federation peers find the homeserver through `.well-known/matrix/server` -on the hive domain, which names `:` -([`gateway.md::Discovery flow`](../networking/gateway.md#discovery-flow-matrix)). -Port 8448 isn't used. - -Outbound, the container sits on the hive bridge like the CI runner: it -leases an address from the gateway's DHCP, resolves through its dnsmasq, -and leaves through the bridge's NAT. It reaches the secret store, -authelia and federation peers by name. +Federation port 8448 is intentionally not opened here — tuwunel +serves the federation API on the same `httpPort` as client-server +by default. Reaching it on 8448 needs either an explicit tuwunel +bind to that port OR a reverse-proxy + `.well-known/matrix/server` +delegation (the latter lives in `gateway.md::Discovery flow`). ## Provisioning flow (appservice) diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index aaca83bb..a84c9ecb 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -16,7 +16,7 @@ You rarely switch it on yourself. `gateway.enable` defaults to off, and every mo | `auth./` | authelia (`9091`) | `swarm-authelia.nix`, `deploy.authelia.enable` | | `auth./bridge/*` | swarm-authelia-bridge (`9092`), prefix stripped; answers swarm-controller's client only | `swarm-authelia.nix`, `deploy.authelia.enable` | | `forge./` | forgejo (`3000`) | `hive-forge/`, `deploy.forgejo.enable` | -| `chat./_matrix/*` | tuwunel (unix socket `/run/hive-matrix/tuwunel.sock`) | `hive-matrix.nix`, `swarm.matrix.gatewayHost != null` | +| `chat./_matrix/*` | tuwunel (`8008`) | `hive-matrix.nix`, `swarm.matrix.gatewayHost != null` | | `chat./` | fluffychat-web static | `hive-matrix.nix`, `swarm.matrix.gatewayHost != null` | | `chat./config.json` | inline JSON (FluffyChat boot config) | `hive-matrix.nix`, `swarm.matrix.gatewayHost != null` | | `/metrics` | the queue exporter's `/metrics` on loopback, behind an authelia `auth_request`; every other path, `/` included, returns `403`, so `/varz` and `/connz` are never reachable | `swarm-nats.nix`, the queue's host | @@ -221,7 +221,7 @@ The operator points a client at ``: 1. The client fetches `https:///.well-known/matrix/client` → `{"m.homeserver":{"base_url":"https://chat."}}` (no port suffix when `httpsPort` is 443). 2. It connects to `chat./_matrix/client/...`. -3. The gateway proxies `/_matrix/*` to tuwunel over its unix socket, `/run/hive-matrix/tuwunel.sock`. +3. The gateway proxies `/_matrix/*` to tuwunel at `127.0.0.1:8008`. matrix-dart-sdk (FluffyChat and others) always fetches the well-known over `https`, which is why [self-signed TLS](#self-signed-tls-default) is on by default. diff --git a/docs/networking/network.md b/docs/networking/network.md index f2f828c3..c4678ca4 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -22,11 +22,12 @@ that touches it. │ uplink NIC — NAT MASQUERADE for the │ bridge subnet (10.42.0.0/24 default) ┌──────────────────────────┴─────────────────────────────────────────┐ -│ host netns — the host itself plus gateway / forge │ +│ host netns — the host itself plus gateway / forge / matrix │ │ │ │ nginx :80/:443 [hive-gateway] │ │ dnsmasq 10.42.0.1:53 (DNS) + :67 (DHCP) [hive-gateway] │ │ forgejo :3000 http, :2222 git-ssh [hive-forge] │ +│ tuwunel :8008 client API [hive-matrix] │ │ hive-c0re dashboard 127.0.0.1:7000 (host service) │ │ wg-hive :51820/udp — swarm mesh, when enabled (host iface) │ │ │ @@ -45,7 +46,7 @@ that touches it. | ------------------------- | ----------------------- | -------------- | ----------------------------------------------------------------------------------------------- | | gateway (host) | host | host addresses | nginx `:80`/`:443` (every vhost); dnsmasq `bridgeIp:53` + DHCP `:67` on the bridge | | `hive-forge` | host (shared) | host addresses | forgejo `:3000` http, `:2222` git-ssh; fronted by the `forge.` vhost | -| `hive-matrix` | private, veth on bridge | DHCP pool | tuwunel on UDS `/run/hive-matrix/tuwunel.sock` → the matrix vhost; out to store, SSO, peers | +| `hive-matrix` | host (shared) | host addresses | tuwunel `:8008` (+ optional federation port); fronted by the matrix vhost | | `swarm-ci-runner` | private, veth on bridge | DHCP pool | outbound only (runner → forge); no inbound surface | | `swarm-review-classifier` | private, veth on bridge | DHCP pool | outbound only (classifier → queue, secret store, hive collector); no inbound surface | | `h-` | private, veth on bridge | DHCP pool | web UI via UDS `/run/hive-agent/` → nginx sub-path; in-container UI port hashed 8100–8999 | @@ -63,7 +64,7 @@ The flows, end to end: - **HTTP** — `forge.` and `chat.` (under `swarm.domain`) plus the hive's own dashboard name resolve to the bridge IP, land on nginx `:80`/`:443`, and proxy to forgejo - `:3000`, tuwunel's unix socket, hive-c0re `127.0.0.1:7000`, or a per-agent + `:3000`, tuwunel `:8008`, hive-c0re `127.0.0.1:7000`, or a per-agent UI unix socket. - **Internet egress** — agent default route points at the bridge IP; the host forwards + masquerades out its uplink. @@ -164,7 +165,7 @@ agent containers. UI proxies, and any other HTTP services. The **host** firewall is the only firewall. The swarm service -containers that share the host netns (forge, authelia, bao, the +containers that share the host netns (forge, matrix, authelia, bao, the metrics and log stores) run with `networking.firewall.enable = false` (`nix/container-modules/swarm-container.nix`): a NixOS firewall inside a shared-netns container runs against the _host_ ruleset — at container @@ -172,7 +173,7 @@ boot its `firewall-start` flushes the `nixos-fw` chains, rebuilds them from the container's (empty) port list, and deletes the host's `nixos-nat-*` chains without recreating them, silently wiping the bridge holes above plus the agents' NAT. Private-netns containers -(every container with `privateNetwork = true`: agents and the service containers on the bridge) keep their own firewall — it's scoped to their +(agents, swarm-ci-runner) may keep their own firewall — it's scoped to their namespace. ### Reaching host services (`exposeHostPorts`) diff --git a/hive-c0re/src/matrix.rs b/hive-c0re/src/matrix.rs index 803a259a..143aaeda 100644 --- a/hive-c0re/src/matrix.rs +++ b/hive-c0re/src/matrix.rs @@ -18,7 +18,7 @@ use reqwest::StatusCode; /// /// `None` means **this hive has no homeserver to provision against** and /// every matrix path no-ops — see [`is_present`]. There is deliberately no -/// fallback: a loopback default is right only when the homeserver happens to +/// fallback: `localhost:8008` is right only when the homeserver happens to /// share this daemon's netns, and an address baked into the binary is one /// that builds fine and then talks to the wrong machine. The nix module /// defaults it to `https://` on the homeserver's gateway vhost, on every diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 7791e721..a8c72b17 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -275,32 +275,16 @@ in ) # The homeserver, split the same way. `swarm.matrix` keeps what it IS to - # every hive — the name it answers to, the URLs it is reached on, the - # client id it is registered under. These six are what the host + # every hive — the name it answers to, the ports and URLs it is reached + # on, the client id it is registered under. These six are what the host # running it decides; its two packages are renamed further down with # the other `*.package` entries. ⚠️ `enable` is NOT here: its shim lives in # ./hive-matrix.nix, so that file is the one # to check before assuming this table is the whole matrix story. - # - # The homeserver's port and its firewall switch, removed rather than - # renamed: tuwunel listens on a unix socket only the gateway reaches, so - # there is no host port to choose or open. - (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "matrix" "httpPort" ] '' - The homeserver no longer listens on a TCP port: it runs in a network - namespace of its own and the gateway reaches it over a unix socket. - Remove this definition. Clients and federation peers reach it at - services.hyperhive.swarm.matrix.gatewayHost. - '') - (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "matrix" "openFirewall" ] '' - The homeserver no longer listens on a host port, so there is no port - to open. Remove this definition. Reach from outside the host goes - through the gateway: services.hyperhive.gateway.openFirewall. - '') - (lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "matrix" "openFirewall" ] '' - The homeserver no longer listens on a host port, so there is no port - to open. Remove this definition. Reach from outside the host goes - through the gateway: services.hyperhive.gateway.openFirewall. - '') + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "matrix" "openFirewall" ] + [ "services" "hyperhive" "deploy" "matrix" "openFirewall" ] + ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "matrix" "trustedServers" ] [ "services" "hyperhive" "deploy" "matrix" "trustedServers" ] diff --git a/nix/host-modules/hive-gateway/vhosts.nix b/nix/host-modules/hive-gateway/vhosts.nix index 03661eb0..9d7fd640 100644 --- a/nix/host-modules/hive-gateway/vhosts.nix +++ b/nix/host-modules/hive-gateway/vhosts.nix @@ -61,13 +61,17 @@ let } ); - # `.well-known/matrix/{client,server}` discovery JSON, pointing clients - # and peers at `matrixCfg.gatewayHost`: the homeserver has no other - # address. CORS `*` per matrix spec. The `m.server` port-8448 carve-out - # is documented inline. See `docs/networking/gateway.md`. - wellKnownLocations = lib.optionalAttrs (matrixDeployCfg.enable && matrixCfg.gatewayHost != null) ( + # `.well-known/matrix/{client,server}` discovery JSON. Points + # clients at `matrixCfg.gatewayHost` when set; falls back to direct + # `:`. CORS `*` per matrix spec. The `m.server` + # port-8448 carve-out is documented inline. See `docs/networking/gateway.md`. + wellKnownLocations = lib.optionalAttrs matrixDeployCfg.enable ( let - clientBaseUrl = "${publicScheme}://${matrixCfg.gatewayHost}${publicPortSuffix}"; + clientBaseUrl = + if matrixCfg.gatewayHost != null then + "${publicScheme}://${matrixCfg.gatewayHost}${publicPortSuffix}" + else + "${publicScheme}://${hyperhiveDomain}:${toString matrixCfg.httpPort}"; # `m.server` is NOT a URL: per the matrix server-server spec # (Resolving Server Names) a delegated host with NO port resolves # to the federation default 8448 (after the SRV check) — the @@ -77,7 +81,11 @@ let # wrong for federation delegation. Without this, peers federate to # :8448 (closed) while the endpoint actually lives on # the gateway's 443 vhost. See docs/networking/gateway.md discovery flow. - serverHostPort = "${matrixCfg.gatewayHost}:${toString publicPort}"; + serverHostPort = + if matrixCfg.gatewayHost != null then + "${matrixCfg.gatewayHost}:${toString publicPort}" + else + "${hyperhiveDomain}:${toString matrixCfg.httpPort}"; in { "= /.well-known/matrix/client" = { diff --git a/nix/host-modules/hive-matrix-service.nix b/nix/host-modules/hive-matrix-service.nix index d5d3b54d..ad009222 100644 --- a/nix/host-modules/hive-matrix-service.nix +++ b/nix/host-modules/hive-matrix-service.nix @@ -1,5 +1,5 @@ # The swarm's matrix homeserver as every hive sees it: the name it mints ids -# under, the URLs it is reached on, its encryption policy and the +# under, the ports and URLs it is reached on, its encryption policy and the # OIDC client id it is registered under, identical on every host. What the # host running it decides, and the container itself, are in ./hive-matrix.nix. { @@ -49,6 +49,18 @@ in ''; }; + httpPort = lib.mkOption { + type = lib.types.port; + default = 8008; + description = '' + TCP port tuwunel serves the matrix client-server API on. + Default 8008 is the matrix-spec well-known port. Sits + outside hyperhive's claimed ranges (dashboard 7000, every + agent in 8100..8999 via FNV-1a hash). Federation listens on + `federationPort` separately. + ''; + }; + apiUrl = lib.mkOption { type = lib.types.nullOr lib.types.str; default = if cfg.gatewayHost == null then null else "https://${cfg.gatewayHost}"; @@ -96,8 +108,8 @@ in Public hostname for the matrix homeserver behind the gateway. Defaults to `chat.''${services.hyperhive.swarm.domain}` — the swarm's domain, because a swarm runs **one** homeserver. Set to - `null` to skip the gateway vhost, which leaves nothing able to - reach the homeserver. See `docs/networking/gateway.md` for the vhost map + matrix + `null` to skip the gateway vhost (tuwunel stays direct on + `httpPort`). See `docs/networking/gateway.md` for the vhost map + matrix discovery flow, and the federation port-8448 caveat at the bottom of that doc. diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index 3fb1b1a3..f5b16fdd 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -239,19 +239,8 @@ let ''; }); - # A netns of its own on the hive bridge: the homeserver dials the store, - # authelia and federation peers by name, and the gateway reaches it over - # `socketPath` below, so it binds no TCP port. - privateNetwork = true; - - # Fixed, not an option: the directory's mode is what keeps the socket - # private, so it holds this one socket and nothing else. - socketDir = "/run/hive-matrix"; - socketPath = "${socketDir}/tuwunel.sock"; - - # The gid the gateway's nginx runs as. Only the number crosses into the - # container, which knows it as `gateway-nginx`. - nginxGid = config.ids.gids.nginx; + # Shared host netns — agents reach tuwunel at localhost:. + privateNetwork = false; in { # Private matrix-tuwunel homeserver wrapped in a nixos-container, @@ -292,7 +281,7 @@ in ]; # `swarm.matrix`, in ./hive-matrix-service.nix, is what the homeserver IS from - # any hive's point of view: the name it answers to, the URLs it is + # any hive's point of view: the name it answers to, the ports and URLs it is # reached on, and the client id it is registered under. What lives here is what the host running it # decides — which build it runs, whether it is exposed, which peers it trusts, # how large a request it accepts, and where its host-local secrets sit. Same rule as @@ -310,6 +299,30 @@ in ''; }; + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Open `httpPort` in the host firewall. Off by default + (secure-by-default): hive-c0re and agent containers reach the + homeserver at `gatewayHost`'s vhost (`chat.` by + default) via the gateway, so the open port only matters for + direct access to `httpPort` from outside the host. Flip to `true` when announcing + the homeserver to other hives or when an external matrix + client needs to reach the client-server API directly. + + Note: federation (the matrix-spec well-known port 8448) is + intentionally not opened here. tuwunel serves the federation + API on the same `httpPort` as the client-server API by + default; reaching it on 8448 requires either binding tuwunel + to that port explicitly OR a reverse-proxy + `.well-known/ + matrix/server` delegation, neither of which lives in this + module. Add that proxy config alongside whatever serves your + dashboard or forge on 443. + ''; + }; + trustedServers = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; @@ -437,8 +450,9 @@ in # SPA fallback reads. All three are matrix knowledge and none of # them is the gateway's business. # - # `gatewayHost = null` means no vhost, so there is no name to claim - # and nothing to serve — every clause below carries that guard. + # `gatewayHost = null` means matrix is reachable directly rather + # than fronted, so there is no name to claim and no vhost to serve — + # every clause below carries that guard. services.hyperhive.gateway.localNames = lib.optional (cfg.gatewayHost != null) cfg.gatewayHost; services.hyperhive.gateway.enable = lib.mkIf (cfg.gatewayHost != null) (lib.mkDefault true); @@ -476,9 +490,7 @@ in extraConfig = gatewayCfg.lib.securityHeaders; locations = { "/_matrix/" = { - # No URI after the socket, so tuwunel gets the request URI as the - # client sent it. - proxyPass = "http://unix:${socketPath}:"; + proxyPass = "http://127.0.0.1:${toString cfg.httpPort}"; proxyWebsockets = true; extraConfig = '' proxy_buffering off; @@ -529,8 +541,8 @@ in services.hyperhive.swarm.matrix.gatewayHost = "" is rejected. The rendered URLs would be invalid (nginx wildcard catch-all for an empty server_name, /etc/hosts rejects empty entries). - Use `null` to disable the gateway vhost entirely, or set a - non-empty hostname like + Use `null` to disable the gateway vhost entirely (tuwunel + stays direct on httpPort), or set a non-empty hostname like "matrix.example.com" or "homeserver.internal". ''; } @@ -562,7 +574,8 @@ in `/_matrix/client/unstable/login/sso/callback/`, and the identity provider redirects a browser to it — so it has to be a name the browser can reach, which is exactly what - `gatewayHost` is. With it null the homeserver has no such name. + `gatewayHost` is. With it null the homeserver is direct on + httpPort and has no such name. ''; } ]; @@ -675,8 +688,8 @@ in systemd.timers.swarm-bao-matrix-oidc = storeRefreshTimer "re-read the homeserver's OIDC client secret from the swarm secret store"; # ⚠️ Deliberately NO `networking.hosts` entry for authelia's name. This - # container resolves through the hive's dnsmasq at `bridgeIp` (see - # `networking.nameservers` below), and every `gateway.localNames` entry — + # container resolves through the hive's dnsmasq at `bridgeIp` (see the + # static resolv.conf below), and every `gateway.localNames` entry — # authelia's domain among them — is already mapped there. Adding a # loopback override would only create a second answer that can # disagree with the first. @@ -730,21 +743,6 @@ in '' ); - # The socket directory, created host-side before the container starts: - # nixos-container refuses to start when a bind source is missing. - # - # Root-owned and group nginx, `0770`: tuwunel creates its socket here - # through the group (its uid is allocated inside the container, so the - # host cannot name it), and the gateway traverses it the same way. - # Everything else is kept out. - systemd.tmpfiles.rules = [ - "d ${socketDir} 0770 root ${toString nginxGid} - -" - ]; - - # The container hangs off the bridge, takes its address from the - # gateway's DHCP and resolves through its dnsmasq. - services.hyperhive.network.enable = lib.mkDefault true; - containers.hive-matrix = { autoStart = true; ephemeral = false; @@ -752,20 +750,12 @@ in # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. extraFlags = [ "--link-journal=host" ]; inherit privateNetwork; - hostBridge = networkCfg.bridgeName; - bindMounts = { - # A bind mount rather than a path both sides happen to know: a - # container's `/run` is an nspawn tmpfs the host cannot see into. - ${socketDir} = { - hostPath = socketDir; - isReadOnly = false; - }; - } - // caTrust.bindMount - # matrix-ctl's bao client identity, and nothing else of the hive's. See - # `ctlPkiDirs` above for why it is a derived set of directories - # rather than one literal. - // ctlBindMounts; + bindMounts = + caTrust.bindMount + # matrix-ctl's bao client identity, and nothing else of the hive's. See + # `ctlPkiDirs` above for why it is a derived set of directories + # rather than one literal. + // ctlBindMounts; config = { ... }: { @@ -791,12 +781,7 @@ in }) ]; - # Its own netns, so it keeps its own firewall; resolvconf stays on - # and serves `networking.nameservers` below. - services.hyperhive.swarmContainer = { - inherit privateNetwork; - writesOwnResolvConf = false; - }; + services.hyperhive.swarmContainer = { inherit privateNetwork; }; # Swarm-internal trust reaches tuwunel at RUNTIME, not via # `security.pki.certificateFiles`. That option is read when the @@ -804,28 +789,48 @@ in # from the store at boot — so there is nothing build-time to name. # The bind-mount above plus the bundle service below carry it. - # The gateway's dnsmasq on the bridge answers every name in - # `gateway.localNames` with the bridge IP and forwards the rest - # upstream. tuwunel exits at boot when `/etc/resolv.conf` has no - # `nameserver` line. - networking.nameservers = [ networkCfg.bridgeIp ]; - networking.interfaces.eth0.useDHCP = true; + # tuwunel hard-fails to boot if `/etc/resolv.conf` has no + # `nameserver` line (`Failed to configure DNS resolver ... no + # nameservers found in config` → exit 1). This declarative + # nixos-container comes up with an EMPTY resolv.conf even with + # `networking.nameservers` set: the nixos-container default + # `useHostResolvConf = true` puts in-container resolvconf in + # host-tracking mode (ignores `networking.nameservers`, and never + # gets the host file across the shared-netns boundary), so it + # regenerates an empty file and tuwunel dies at boot. + # + # Trusting resolvconf to honour `networking.nameservers` doesn't + # work either — that's a RUNTIME resolvconf behaviour, not + # verifiable at eval time, and it still comes up empty in + # practice. So take resolvconf out of the loop entirely and + # write a STATIC `/etc/resolv.conf` from `bridgeIp` that nothing + # regenerates. Eval-proven: the generated + # `environment.etc."resolv.conf".text` is `nameserver `. + # This container always shares the host netns + # (`privateNetwork = false`), so it reaches `bridgeIp` regardless + # of agent-container isolation. See `docs/networking/network.md`. + # resolvconf is off through `writesOwnResolvConf` (its default); + # the static `environment.etc."resolv.conf"` below is the sole + # source of the resolver file (no `nameservers` — nothing would + # read it). + networking.useHostResolvConf = lib.mkForce false; - # The socket's group is tuwunel's primary group, so that group is - # nginx's gid. The user and group both exist statically, so - # systemd runs the unit as them despite upstream's `DynamicUser`. - users.groups.gateway-nginx.gid = nginxGid; + # resolvconf is disabled above, so write the static resolver file + # explicitly — NixOS won't synthesise one from `nameservers` once + # resolvconf is off, and this is the file tuwunel parses at boot. + environment.etc."resolv.conf".text = '' + nameserver ${networkCfg.bridgeIp} + options edns0 + ''; services.matrix-tuwunel = { enable = true; package = deployCfg.matrix.package; - group = "gateway-nginx"; settings.global = { server_name = effectiveServerName; - # ⚠️ `address` stays unset: tuwunel binds TCP as well as the - # socket whenever it is set. - unix_socket_path = socketPath; - unix_socket_perms = 660; + # `address` + `port` are upstream `listOf` — wrap singles. + address = [ "0.0.0.0" ]; + port = [ cfg.httpPort ]; max_request_size = deployCfg.matrix.maxRequestSize; # Federation enabled at the protocol level; empty # trustedServers keeps it effectively closed. @@ -928,9 +933,6 @@ in ]; }; }; - # Upstream's `ProtectSystem = "strict"` leaves `/run` read-only. - systemd.services.tuwunel.serviceConfig.ReadWritePaths = [ socketDir ]; - # Keeps DynamicUser=true + PrivateUsers=true intact — no # host-side chown :tuwunel / GID-pin gymnastics needed. # See `man systemd.exec` → LoadCredential. @@ -1007,11 +1009,21 @@ in }; }; - # The matrix container's lease and resolver are the hive's dnsmasq - # (bound at `bridgeIp`). Order the matrix container start after it so - # both are up before tuwunel's first federation lookups. Soft `after` - # ordering (not `requires`) keeps the matrix container's lifecycle - # decoupled from the resolver's. + networking.firewall = lib.mkIf deployCfg.matrix.openFirewall { + allowedTCPPorts = [ + cfg.httpPort + ]; + }; + + # The matrix container's resolver is the hive's dnsmasq (bound at + # `bridgeIp`). Order the matrix container start after it so the + # resolver is up before tuwunel's first federation lookups. tuwunel + # boots fine without this — it configures the resolver from + # `/etc/resolv.conf` at startup and only queries on-demand (the boot + # failure this module guards against is an *empty* resolv.conf, a + # parse error, not a connectivity one) — so this is robustness, not a + # boot requirement. Soft `after` ordering (not `requires`) keeps the + # matrix container's lifecycle decoupled from the resolver's. # # `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also # sets `after`/`requires` (so the bound trust bundle exists before diff --git a/nix/host-modules/swarm-authelia-service.nix b/nix/host-modules/swarm-authelia-service.nix index 00a520b5..9b527bcd 100644 --- a/nix/host-modules/swarm-authelia-service.nix +++ b/nix/host-modules/swarm-authelia-service.nix @@ -30,7 +30,7 @@ in description = '' TCP port authelia listens on. 9091 is upstream's default and sits outside hyperhive's claimed ranges (dashboard 7000, forge - 3000, every agent in 8100..8999 via FNV-1a hash). + 3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash). ''; }; diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 082aa2fb..463dd855 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -492,7 +492,7 @@ let mv ${lib.escapeShellArg "${clientsFile}.tmp"} ${lib.escapeShellArg clientsFile} ''; - # Shared host netns, like the forge container: the + # Shared host netns, like the forge and matrix containers: the # gateway reaches authelia at 127.0.0.1:. privateNetwork = false; in diff --git a/nix/host-modules/swarm-nats-service.nix b/nix/host-modules/swarm-nats-service.nix index 6440f2bb..aeeea857 100644 --- a/nix/host-modules/swarm-nats-service.nix +++ b/nix/host-modules/swarm-nats-service.nix @@ -54,7 +54,7 @@ in description = '' TCP port the queue listens on. 4222 is upstream's default and sits outside hyperhive's claimed ranges (dashboard 7000, forge - 3000, every agent in 8100..8999 via FNV-1a hash). + 3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash). Opened on the bridge interface, for agent containers, and on `wg-hive` when this host is on the mesh, for other hives. Never diff --git a/nix/module-eval/bao-controller.nix b/nix/module-eval/bao-controller.nix index d5c83d9c..99fa88b5 100644 --- a/nix/module-eval/bao-controller.nix +++ b/nix/module-eval/bao-controller.nix @@ -57,8 +57,8 @@ let swarm.matrix.gatewayHost = "matrix.hive.t.local"; }; - # No matrix vhost for this swarm at all, so there is no homeserver URL - # to hand the controller. + # No matrix vhost for this swarm at all: tuwunel stays direct on + # `httpPort`, so there is no homeserver URL to hand the controller. controllerNoGatewayHost = hive { deploy.swarm-controller.enable = true; swarm.matrix.gatewayHost = null; diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 95ae8613..2d101347 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -163,8 +163,8 @@ let lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links; # Every service container at once: the services-only host plus the CI - # runner. The CI runner, the review classifier, the builder, the metrics - # store and the homeserver are the five containers on a netns of their own. + # runner. The CI runner, the review classifier, the builder and the metrics + # store are the four containers on a netns of their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; @@ -522,7 +522,7 @@ let let shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers); in - lib.length shared == 7 + lib.length shared == 8 && lib.all ( c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable ) shared; @@ -575,46 +575,6 @@ let && unit.serviceConfig.UMask == "0007" && lib.hasInfix "--unix-socket ${sock}" unit.postStart; } - { - # On the bridge by DHCP, like the CI runner, and its socket the only - # listener the gateway dials. `address` unset is what keeps tuwunel off - # TCP. Every vhost location that forwards goes through the socket; the - # `/_matrix/` one is named so an empty filter can't pass. - name = "the homeserver runs on a netns of its own and the gateway reaches it over its socket"; - ok = - let - host = serviceContainersWithCi; - net = host.services.hyperhive.network; - c = host.containers.hive-matrix; - dir = "/run/hive-matrix"; - sock = "${dir}/tuwunel.sock"; - locs = - host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.matrix.gatewayHost}.locations; - forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs)); - tuwunel = c.config.services.matrix-tuwunel; - unit = c.config.systemd.services.tuwunel; - nginxGid = host.ids.gids.nginx; - in - c.privateNetwork - && c.hostBridge == net.bridgeName - && c.hostAddress == null - && c.localAddress == null - && c.config.networking.interfaces.eth0.useDHCP - && c.config.networking.nameservers == [ net.bridgeIp ] - && c.config.networking.firewall.enable - && c.config.networking.resolvconf.enable - && (tuwunel.settings.global.address or null) == null - && tuwunel.settings.global.unix_socket_path == sock - && tuwunel.settings.global.unix_socket_perms == 660 - && locs."/_matrix/".proxyPass == "http://unix:${sock}:" - && forwarded == [ "http://unix:${sock}:" ] - && c.bindMounts.${dir}.hostPath == dir - && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules - && c.config.users.groups.gateway-nginx.gid == nginxGid - && tuwunel.group == "gateway-nginx" - && unit.serviceConfig.Group == "gateway-nginx" - && lib.elem dir unit.serviceConfig.ReadWritePaths; - } { # The DynamicUser layout keeps the history under /var/lib/private; the # static user's StateDirectory is the plain path. The move runs as its