0b1b08dfe6
Per mara: a general fix, not one name in one container. Every container inherits a COPY of the host's /etc/resolv.conf at start (nixos-containers.nix: cp --remove-destination, one shot, not a bind-mount), so the address written there is the address every container tries - in its own netns. That makes the value load-bearing: value host host-netns containers bridged containers 127.0.0.1 ok ok THEIR OWN loopback bridge IP ok ok ok dnsmasq binds lo and the bridge, so the bridge IP works for the host too. It is the only value correct on both sides of a netns boundary. resolveLocalQueries goes ON for its plumbing, not its address: it points dnsmasq's own upstreams at a separate resolv-file, without which dnsmasq reads /etc/resolv.conf and every non-hive query loops the moment the host is pointed at dnsmasq. Its two loopback-publishing effects (networking.nameservers and resolvconf.useLocalResolver) are overridden. Cost: the host's DNS now depends on dnsmasq being up. Every container already did. The forge container keeps its hosts entry from the previous commit - not redundancy, a fallback in a different failure domain: it works with no DNS at all, so SSO does not ride on a host-wide resolver change.
368 lines
17 KiB
Nix
368 lines
17 KiB
Nix
# Single nginx in front of every hyperhive web surface — dashboard,
|
||
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
||
# delegations — plus the hive-internal dnsmasq resolver. Both run on the
|
||
# HOST, next to hive-c0re: nginx binds the host's :80/:443 and dnsmasq
|
||
# answers on the hive bridge, so neither can be confined to a network
|
||
# namespace of its own.
|
||
# Full vhost map + discovery flow + design rationale in
|
||
# `docs/gateway.md`. Layout: ./options.nix (option declarations),
|
||
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
|
||
# (styled static pages), ./dnsmasq.nix (resolver + DHCP config).
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.gateway;
|
||
hyperhiveDomain = config.services.hyperhive.domain;
|
||
# Derived once in ../swarm.nix; the vhosts that get the swarm-services
|
||
# cert are exactly the names that cert is issued for, so both read the
|
||
# same list rather than each deciding what "a swarm service" means.
|
||
swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains;
|
||
matrixCfg = config.services.hyperhive.swarm.matrix;
|
||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||
uiCfg = config.services.hyperhive.swarm.ui;
|
||
forgeCfg = config.services.hyperhive.swarm.forge;
|
||
networkCfg = config.services.hyperhive.network;
|
||
|
||
# Dashboard SPA dist, static-served by nginx.
|
||
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
||
|
||
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
||
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
||
# (see vhosts.nix's `swaggerUiLocations` and
|
||
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
||
# module (not c0re's) — hive-c0re has no relationship to it.
|
||
swaggerUiTheme = cfg.swaggerUiTheme;
|
||
|
||
# Self-signed TLS is the implicit floor: when neither an operator cert
|
||
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
||
# generates + serves a hive-CA-signed leaf (see hive-tls.nix). There is no
|
||
# explicit toggle and no http-only mode — matrix discovery requires https,
|
||
# so the gateway always terminates TLS.
|
||
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
||
useSelfSigned = cfg.useSelfSigned;
|
||
|
||
# nginx's own state dir. Kept at the historical `/var/lib/hive-gateway`
|
||
# path rather than renamed with the move: it holds the imported leaf
|
||
# across reboots, and renaming it would strand every existing hive's
|
||
# certs for no gain.
|
||
tlsDir = "/var/lib/hive-gateway/tls";
|
||
# TLS cert + key paths.
|
||
# - self-signed (default): the hive-CA-signed leaf, imported into the
|
||
# state dir by `hive-gateway-self-signed-cert` below.
|
||
# - tls.certDir set: the operator's own cert dir, read directly.
|
||
tlsCert =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
||
tlsKey =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
||
# The swarm-services pair, used only by the vhosts whose names this
|
||
# hive's CA cannot sign. Self-signed mode only: with an operator cert
|
||
# or ACME the operator owns every name and there is no second issuer.
|
||
svcCert = "${tlsDir}/swarm-services.pem";
|
||
svcKey = "${tlsDir}/swarm-services-key.pem";
|
||
|
||
nginxTree = import ./vhosts.nix {
|
||
inherit
|
||
lib
|
||
cfg
|
||
forgeCfg
|
||
matrixCfg
|
||
autheliaCfg
|
||
uiCfg
|
||
hyperhiveDomain
|
||
dashboardDist
|
||
swaggerUiTheme
|
||
tlsCert
|
||
tlsKey
|
||
svcCert
|
||
svcKey
|
||
swarmServiceDomains
|
||
;
|
||
errorPages = import ./error-pages.nix { inherit pkgs; };
|
||
};
|
||
in
|
||
{
|
||
imports = [ ./options.nix ];
|
||
|
||
config = lib.mkIf config.services.hyperhive.enable {
|
||
assertions = [
|
||
{
|
||
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true and
|
||
tls.certDir are mutually exclusive. Pick one TLS mode.
|
||
'';
|
||
}
|
||
{
|
||
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true requires
|
||
services.hyperhive.gateway.tls.acme.email to be set —
|
||
Let's Encrypt needs a contact address for the ACME account.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Ensure the gateway state dirs exist at host boot, before anything
|
||
# reads or writes them: these rules cover the fresh-boot window
|
||
# before c0re has run, and pin owner + mode rather than leaving it
|
||
# to whoever creates the path first.
|
||
#
|
||
# /run/hive-agent — per-agent UDS socket dir, written by c0re's
|
||
# set_nspawn_flags when agents start. Owned by `hive-core` (the
|
||
# unprivileged coordinator user): c0re does the
|
||
# `create_dir_all(/run/hive-agent/<name>)` itself, so a root-owned
|
||
# parent would EACCES on the very first agent create on a fresh host
|
||
# (hive-priv only chowns the subdir afterwards, it doesn't make it).
|
||
#
|
||
# ⚠️ There is deliberately NO rule for /var/lib/hyperhive here. One
|
||
# used to declare it `0755 root root` and could never win:
|
||
# `hive-c0re.service` sets `StateDirectory = "hyperhive"` with
|
||
# `StateDirectoryMode = "0750"`, which systemd re-applies on every
|
||
# start. Two mechanisms owning one path, and the loser still read as
|
||
# a guarantee — it is why the resulting outage was first misdiagnosed
|
||
# as someone having changed the mode. c0re's own unit owns that dir;
|
||
# this module no longer has an opinion about it.
|
||
systemd.tmpfiles.rules = [
|
||
# Must stay in step with the identical rule hive-priv generates into
|
||
# /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare
|
||
# different owners for this path.
|
||
"d /run/hive-agent 0755 hive-core hive-core - -"
|
||
# The gateway's own config dir — NOT under /var/lib/hyperhive. c0re
|
||
# writes here, nginx reads here, and neither needs any access to the
|
||
# other's tree: no shared parent to traverse means no group
|
||
# membership handing nginx c0re's broker db and everything else
|
||
# beside it. Sibling of `tls/`, which already lived under this root.
|
||
#
|
||
# Owned by hive-core because c0re is the writer; 0755 so the
|
||
# unprivileged nginx user can traverse and read. This is now the
|
||
# ONLY declaration of these paths' modes — nothing re-applies a
|
||
# different owner on top the way `StateDirectory=` does for
|
||
# /var/lib/hyperhive.
|
||
"d /var/lib/hive-gateway 0755 root root - -"
|
||
"d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -"
|
||
"f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n"
|
||
# Pre-create both files so nginx's config test can open them before
|
||
# c0re has ever written: nginx names them, and an `include` of a
|
||
# missing file is a fatal config error, not an empty one. tmpfiles
|
||
# runs before services, which is the whole ordering guarantee —
|
||
# content arrives when c0re writes and reloads, which it does on
|
||
# every topology change. `f` = create-if-absent, never overwrite.
|
||
#
|
||
# An empty htpasswd causes all auth checks to return 401 (no valid
|
||
# credentials), which is the correct no-users behaviour.
|
||
"f /var/lib/hive-gateway/conf/gateway.htpasswd 0644 hive-core hive-core - -"
|
||
];
|
||
|
||
# The host asks the hive's own resolver, at the BRIDGE IP.
|
||
#
|
||
# Every container inherits a COPY of this host's `/etc/resolv.conf`
|
||
# at start (`nixos-containers.nix`: `cp --remove-destination`, one
|
||
# shot, not a bind-mount) — so whatever address is written here is
|
||
# the address every container will try, in its own netns.
|
||
#
|
||
# 🚨 That is why this is the bridge IP and not `127.0.0.1`, and the
|
||
# distinction is load-bearing rather than stylistic:
|
||
#
|
||
# value host host-netns containers bridged containers
|
||
# 127.0.0.1 ok ok THEIR OWN loopback
|
||
# bridge IP ok ok ok
|
||
#
|
||
# dnsmasq binds both `lo` and the bridge (./dnsmasq.nix), so the
|
||
# bridge IP is reachable from the host too — it is the only value
|
||
# correct on both sides of a netns boundary. `resolveLocalQueries`
|
||
# publishes loopback by default, hence both overrides here; the
|
||
# flag stays on for its `resolv-file` plumbing, which is what keeps
|
||
# dnsmasq's own upstreams out of the file we are pointing at it.
|
||
#
|
||
# Cost, stated because it is real: the host's DNS now depends on
|
||
# dnsmasq being up. Every container already did.
|
||
networking.nameservers = lib.mkForce [ networkCfg.bridgeIp ];
|
||
networking.resolvconf.useLocalResolver = lib.mkForce false;
|
||
|
||
# ACME (Let's Encrypt) integration. nginx vhosts set
|
||
# `enableACME = true` via the vhost builder; this provides the
|
||
# shared ACME config (acceptTerms + email).
|
||
security.acme = lib.mkIf cfg.tls.acme.enable {
|
||
acceptTerms = true;
|
||
defaults.email = cfg.tls.acme.email;
|
||
};
|
||
|
||
# Import the hive-CA leaf into nginx's state dir before nginx starts.
|
||
#
|
||
# 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It
|
||
# does TWO jobs:
|
||
#
|
||
# (1) It re-modes the leaf. `hive-tls-ca` writes the key 0600
|
||
# root:root; nginx's pre-start `nginx -t` runs as the nginx
|
||
# *user*, so a 0600 key fails the config test with
|
||
# `BIO_new_file() … Permission denied` and blocks the unit —
|
||
# hence the 0640 root:nginx copy below.
|
||
# (2) It guarantees that **every cert path the nginx config names
|
||
# exists** — which is what the swarm-services fallback at the
|
||
# bottom of the script is for. nginx refuses to load a config
|
||
# naming a missing cert file, so a leaf that never issues takes
|
||
# the whole gateway down rather than one vhost; that has already
|
||
# happened once and it took the forge, dashboard and matrix with
|
||
# it. Removing this unit re-creates it exactly.
|
||
#
|
||
# nginx
|
||
# `Requires=` this via `requiredBy`, so it refuses to start until
|
||
# the copy succeeds. ALWAYS runs (no ConditionPathExists) and is
|
||
# idempotent — necessary to reconcile broken state from prior
|
||
# failed boots (a 0700 dir from a stale UMask, a truncated copy
|
||
# from an interrupted oneshot, etc.). The leaf covers the bare
|
||
# hive domain plus `forge.`, `matrix.` and `*.${hyperhiveDomain}`
|
||
# so all sub-domains validate under the same cert + the hive CA.
|
||
# See `docs/gateway.md` ("Self-signed TLS").
|
||
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
||
description = "Import host-generated TLS leaf for hive-gateway";
|
||
wantedBy = [ "multi-user.target" ];
|
||
before = [ "nginx.service" ];
|
||
requiredBy = [ "nginx.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
# Pin the journal identity (else it's the `script` store-path wrapper).
|
||
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -eu
|
||
mkdir -p ${tlsDir}
|
||
# 0755 on BOTH the cert dir and its parent so the nginx
|
||
# user can traverse the full path. The parent
|
||
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
||
# StateDirectory / mkdir umask depending on which service
|
||
# created it first), which on its own blocks traversal.
|
||
# Re-applied every boot in case a prior run left a tighter
|
||
# mode behind.
|
||
chmod 0755 ${builtins.dirOf tlsDir}
|
||
chmod 0755 ${tlsDir}
|
||
# Copy the leaf in. `install` writes atomically with the
|
||
# target mode; runs as root so the 0600 root:root key written
|
||
# by hive-tls-ca is readable. Key ends up root:nginx 0640 so
|
||
# nginx-pre-start
|
||
# (which runs `nginx -t` as the nginx user, not root) can
|
||
# read it — a 0600 root:root key passes the master load but
|
||
# fails the pre-start config test with `BIO_new_file() …
|
||
# Permission denied`, blocking the unit. Cert is world-read.
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${tlsCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${tlsKey}
|
||
|
||
# The swarm-services leaf, when this host issues one. It is
|
||
# a separate pair rather than more SANs on the one above
|
||
# because no hive CA can sign these names — each is
|
||
# constrained to its own hive's domain and the service
|
||
# names are siblings of it.
|
||
#
|
||
# Absent is a normal state, not a failure: the leaf exists
|
||
# only where the swarm CA is autoconfigured, and issuance
|
||
# can also fail on a host that wants one.
|
||
#
|
||
# ⚠️ When it is absent the HIVE leaf goes to this path
|
||
# anyway, and that fallback is load-bearing rather than
|
||
# tidy. nginx refuses to load a config naming a cert file
|
||
# that does not exist — `cannot load certificate … no such
|
||
# file` fails the pre-start test, so the vhost does not
|
||
# degrade, the ENTIRE gateway dies and takes the forge, the
|
||
# dashboard and matrix with it. Serving the hive leaf on a
|
||
# swarm-service name is a name mismatch: browsers warn,
|
||
# strict clients refuse, everything else keeps working, and
|
||
# the operator gets a bad cert instead of no hive.
|
||
#
|
||
# Measured, not theorised: this exact path took pr1ma's
|
||
# gateway down when the services sub-CA failed to issue.
|
||
if [ -s ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ]; then
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ${svcCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/swarm-services-key.pem ${svcKey}
|
||
else
|
||
echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${svcCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${svcKey}
|
||
fi
|
||
'';
|
||
};
|
||
|
||
# nginx reload is triggered from the HOST side by hive-c0re
|
||
# after each agents.conf write, through hive-priv (c0re is
|
||
# unprivileged and cannot act on a system unit).
|
||
#
|
||
# It stays an explicit trigger rather than a systemd path unit
|
||
# watching the file: the write and the reload belong in one causal
|
||
# chain c0re can retry and report on (see RELOAD_PENDING), not two
|
||
# independent units racing on an inotify event.
|
||
|
||
services.nginx = {
|
||
enable = true;
|
||
recommendedProxySettings = true;
|
||
recommendedTlsSettings = true;
|
||
recommendedGzipSettings = true;
|
||
recommendedOptimisation = true;
|
||
inherit (nginxTree) appendHttpConfig virtualHosts;
|
||
};
|
||
|
||
# ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
|
||
# absence is load-bearing rather than an omission.
|
||
#
|
||
# It used to be here, to let nginx traverse `/var/lib/hyperhive` and
|
||
# reach the config fragments that lived inside: `hive-c0re.service`
|
||
# declares `StateDirectory = "hyperhive"` with `StateDirectoryMode =
|
||
# "0750"` owned by `hive-core`, and systemd re-applies that on every
|
||
# c0re start — beating this module's own tmpfiles rule for the same
|
||
# path. Without the group, nginx failed its config test and never
|
||
# started (`nginx: [emerg] open() ".../agents.conf" failed (13:
|
||
# Permission denied)`), taking the gateway and every hive domain
|
||
# behind it down.
|
||
#
|
||
# The group fixed the symptom and paid for it: it also gave nginx
|
||
# read access to everything ELSE group-readable under that dir,
|
||
# including the broker sqlite — i.e. every message between every
|
||
# agent, reachable by the process whose entire job is parsing
|
||
# untrusted network input. Moving the fragments to the gateway's own
|
||
# dir removes the need and the exposure together. Re-adding this line
|
||
# would restore both.
|
||
|
||
# dnsmasq is a host service alongside nginx, so it reads the host's
|
||
# /etc/resolv.conf directly and picks up network changes as they
|
||
# happen — no copy to keep in sync.
|
||
services.dnsmasq = import ./dnsmasq.nix {
|
||
inherit
|
||
lib
|
||
networkCfg
|
||
forgeCfg
|
||
matrixCfg
|
||
autheliaCfg
|
||
uiCfg
|
||
hyperhiveDomain
|
||
;
|
||
};
|
||
|
||
networking.firewall = lib.mkIf cfg.openFirewall {
|
||
allowedTCPPorts = [
|
||
cfg.port
|
||
# The gateway always terminates TLS (self-signed floor), so
|
||
# `httpsPort` is always opened alongside the plain-http `port`.
|
||
cfg.httpsPort
|
||
];
|
||
};
|
||
|
||
# `/etc/hosts` entries for local dev — bare hive domain + any
|
||
# sub-domain modules that are on. `lib.unique` dedupes if any
|
||
# sub-domain happens to equal another. See `docs/gateway.md`
|
||
# ("Local dev").
|
||
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
||
"127.0.0.1" = lib.unique (
|
||
[ hyperhiveDomain ]
|
||
++ lib.optional (config.services.hyperhive.swarm.forge.behindGateway or false
|
||
) config.services.hyperhive.swarm.forge.domain
|
||
++ lib.optional (matrixCfg.enable && matrixCfg.gatewayHost != null) matrixCfg.gatewayHost
|
||
++ lib.optional autheliaCfg.enable autheliaCfg.domain
|
||
++ lib.optional uiCfg.enable uiCfg.domain
|
||
);
|
||
};
|
||
};
|
||
}
|