98d895cf9e
Per review: docs represent current state. Every "used to" / "no longer" clause this branch introduced is gone — including the History section in network.md, which was a whole subsection about a sync mechanism that doesn't exist. Where the removed clause was carrying a real constraint, the constraint stays and is stated in the present tense instead of as a delta: nothing narrows what the gateway's nginx can reach except the directory permissions in front of a socket, and nothing bounds `ReloadGatewayNginx` except the hard-coded unit name. Those read as rules now rather than as the story of how they came to be rules.
321 lines
15 KiB
Nix
321 lines
15 KiB
Nix
# Single nginx in front of every hyperhive web surface — dashboard,
|
||
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
||
# delegations — plus the hive-internal dnsmasq resolver. Both run on the
|
||
# HOST, next to hive-c0re: nginx binds the host's :80/:443 and dnsmasq
|
||
# answers on the hive bridge, so neither can be confined to a network
|
||
# namespace of its own.
|
||
# Full vhost map + discovery flow + design rationale in
|
||
# `docs/gateway.md`. Layout: ./options.nix (option declarations),
|
||
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
|
||
# (styled static pages), ./dnsmasq.nix (resolver + DHCP config).
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.gateway;
|
||
hyperhiveDomain = config.services.hyperhive.domain;
|
||
# Derived once in ../swarm.nix; the vhosts that get the swarm-services
|
||
# cert are exactly the names that cert is issued for, so both read the
|
||
# same list rather than each deciding what "a swarm service" means.
|
||
swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains;
|
||
matrixCfg = config.services.hyperhive.swarm.matrix;
|
||
forgeCfg = config.services.hyperhive.swarm.forge;
|
||
networkCfg = config.services.hyperhive.network;
|
||
|
||
# Dashboard SPA dist, static-served by nginx. Read in OUTER scope so
|
||
# `config` is the host's (inside the container block it'd be the
|
||
# container's).
|
||
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
||
|
||
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
||
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
||
# (see vhosts.nix's `swaggerUiLocations` and
|
||
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
||
# module (not c0re's) — hive-c0re has no relationship to it.
|
||
swaggerUiTheme = cfg.swaggerUiTheme;
|
||
|
||
# Self-signed TLS is the implicit floor: when neither an operator cert
|
||
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
||
# generates + serves a hive-CA-signed leaf (see hive-tls.nix). There is no
|
||
# explicit toggle and no http-only mode — matrix discovery requires https,
|
||
# so the gateway always terminates TLS.
|
||
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
||
useSelfSigned = cfg.useSelfSigned;
|
||
|
||
# nginx's own state dir. Kept at the historical `/var/lib/hive-gateway`
|
||
# path rather than renamed with the move: it holds the imported leaf
|
||
# across reboots, and renaming it would strand every existing hive's
|
||
# certs for no gain.
|
||
tlsDir = "/var/lib/hive-gateway/tls";
|
||
# TLS cert + key paths.
|
||
# - self-signed (default): the hive-CA-signed leaf, imported into the
|
||
# state dir by `hive-gateway-self-signed-cert` below.
|
||
# - tls.certDir set: the operator's own cert dir, read directly.
|
||
tlsCert =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
||
tlsKey =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
||
# The swarm-services pair, used only by the vhosts whose names this
|
||
# hive's CA cannot sign. Self-signed mode only: with an operator cert
|
||
# or ACME the operator owns every name and there is no second issuer.
|
||
svcCert = "${tlsDir}/swarm-services.pem";
|
||
svcKey = "${tlsDir}/swarm-services-key.pem";
|
||
|
||
nginxTree = import ./vhosts.nix {
|
||
inherit
|
||
lib
|
||
cfg
|
||
forgeCfg
|
||
matrixCfg
|
||
hyperhiveDomain
|
||
dashboardDist
|
||
swaggerUiTheme
|
||
tlsCert
|
||
tlsKey
|
||
svcCert
|
||
svcKey
|
||
swarmServiceDomains
|
||
;
|
||
errorPages = import ./error-pages.nix { inherit pkgs; };
|
||
};
|
||
in
|
||
{
|
||
imports = [ ./options.nix ];
|
||
|
||
config = lib.mkIf config.services.hyperhive.enable {
|
||
assertions = [
|
||
{
|
||
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true and
|
||
tls.certDir are mutually exclusive. Pick one TLS mode.
|
||
'';
|
||
}
|
||
{
|
||
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true requires
|
||
services.hyperhive.gateway.tls.acme.email to be set —
|
||
Let's Encrypt needs a contact address for the ACME account.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Ensure the gateway state dirs exist at host boot, before anything
|
||
# reads or writes them: these rules cover the fresh-boot window
|
||
# before c0re has run, and pin owner + mode rather than leaving it
|
||
# to whoever creates the path first.
|
||
#
|
||
# /run/hive-agent — per-agent UDS socket dir, written by c0re's
|
||
# set_nspawn_flags when agents start. Owned by `hive-core` (the
|
||
# unprivileged coordinator user): c0re does the
|
||
# `create_dir_all(/run/hive-agent/<name>)` itself, so a root-owned
|
||
# parent would EACCES on the very first agent create on a fresh host
|
||
# (hive-priv only chowns the subdir afterwards, it doesn't make it).
|
||
# /var/lib/hyperhive — hyperhive state dir, created by c0re on
|
||
# first run. Also pre-seed agents.conf with an empty-but-valid
|
||
# header so nginx can start + include the file before c0re writes
|
||
# its first real content (f = create-if-absent, no overwrite).
|
||
systemd.tmpfiles.rules = [
|
||
# Must stay in step with the identical rule hive-priv generates into
|
||
# /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare
|
||
# different owners for this path.
|
||
"d /run/hive-agent 0755 hive-core hive-core - -"
|
||
"d /var/lib/hyperhive 0755 root root - -"
|
||
"d /var/lib/hyperhive/gateway 0755 root root - -"
|
||
"f /var/lib/hyperhive/gateway/agents.conf 0644 root root - # Generated by hive-c0re — do not edit.\n"
|
||
# Pre-create the htpasswd file so nginx can open it even before any
|
||
# users have been added. An empty file causes all auth checks to
|
||
# return 401 (no valid credentials), which is the correct no-users
|
||
# behaviour. `f` = create-if-absent, never overwrite.
|
||
"f /var/lib/hyperhive/gateway/gateway.htpasswd 0644 root root - -"
|
||
];
|
||
|
||
# ⚠️ REMOVED WITH THE CONTAINER, and each one was a workaround for the
|
||
# boundary rather than a thing nginx or dnsmasq needed:
|
||
#
|
||
# - `privateNetwork = false` — the container already shared the host
|
||
# netns, which is why nginx bound host ports and `localhost`
|
||
# upstreams reached hive-c0re. On the host that is simply true.
|
||
# - `additionalCapabilities = [ "CAP_NET_ADMIN" ]` — dnsmasq refuses
|
||
# to start with a `dhcp-range` unless it holds NET_ADMIN, and
|
||
# nspawn's bounding set dropped it for a host-netns container.
|
||
# Host root has it.
|
||
# - `networking.firewall.enable = false` — a container sharing the
|
||
# host netns would run *its* firewall.service against the HOST
|
||
# ruleset, flushing nixos-fw and deleting the nixos-nat-* chains on
|
||
# every boot. With one machine there is one firewall (below).
|
||
# - `networking.resolvconf.enable = false` + the `hive-gateway-resolv`
|
||
# path/service pair — the container's /etc/resolv.conf was a
|
||
# one-shot copy frozen at start, so a host network change left
|
||
# dnsmasq forwarding to a resolver that was gone. The whole
|
||
# watch-copy-reload machine existed to bridge two files. There is
|
||
# now one.
|
||
# - three bind mounts — /run/hive-agent, /run/hive-state, and either
|
||
# /run/hive-tls (operator cert) or /run/hive-ca (self-signed);
|
||
# those last two are mutually exclusive mkIfs, so it was never
|
||
# four. All plain host paths now.
|
||
#
|
||
# See `docs/network.md::Resolver behaviour` for the resolver history.
|
||
# ACME (Let's Encrypt) integration. nginx vhosts set
|
||
# `enableACME = true` via the vhost builder; this provides the
|
||
# shared ACME config (acceptTerms + email).
|
||
security.acme = lib.mkIf cfg.tls.acme.enable {
|
||
acceptTerms = true;
|
||
defaults.email = cfg.tls.acme.email;
|
||
};
|
||
|
||
# Import the hive-CA leaf into nginx's state dir before nginx starts.
|
||
#
|
||
# 🚨 THIS LOOKS LIKE A LEFTOVER OF THE CONTAINER AND IS NOT — do not
|
||
# "simplify" it into pointing nginx at the CA dir. It does TWO jobs:
|
||
#
|
||
# (1) It re-modes the leaf. `hive-tls-ca` writes the key 0600
|
||
# root:root; nginx's pre-start `nginx -t` runs as the nginx
|
||
# *user*, so a 0600 key fails the config test with
|
||
# `BIO_new_file() … Permission denied` and blocks the unit —
|
||
# hence the 0640 root:nginx copy below. That is a file-mode fact,
|
||
# not a namespace one, and it did not go away with the container.
|
||
# (2) It guarantees that **every cert path the nginx config names
|
||
# exists** — which is what the swarm-services fallback at the
|
||
# bottom of the script is for. nginx refuses to load a config
|
||
# naming a missing cert file, so a leaf that never issues takes
|
||
# the whole gateway down rather than one vhost; that has already
|
||
# happened once and it took the forge, dashboard and matrix with
|
||
# it. Removing this unit re-creates it exactly.
|
||
#
|
||
# nginx
|
||
# `Requires=` this via `requiredBy`, so it refuses to start until
|
||
# the copy succeeds. ALWAYS runs (no ConditionPathExists) and is
|
||
# idempotent — necessary to reconcile broken state from prior
|
||
# failed boots (a 0700 dir from a stale UMask, a truncated copy
|
||
# from an interrupted oneshot, etc.). The leaf covers the bare
|
||
# hive domain plus `forge.`, `matrix.` and `*.${hyperhiveDomain}`
|
||
# so all sub-domains validate under the same cert + the hive CA.
|
||
# See `docs/gateway.md` ("Self-signed TLS").
|
||
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
||
description = "Import host-generated TLS leaf for hive-gateway";
|
||
wantedBy = [ "multi-user.target" ];
|
||
before = [ "nginx.service" ];
|
||
requiredBy = [ "nginx.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
# Pin the journal identity (else it's the `script` store-path wrapper).
|
||
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -eu
|
||
mkdir -p ${tlsDir}
|
||
# 0755 on BOTH the cert dir and its parent so the nginx
|
||
# user can traverse the full path. The parent
|
||
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
||
# StateDirectory / mkdir umask depending on which service
|
||
# created it first), which on its own blocks traversal.
|
||
# Re-applied every boot in case a prior run left a tighter
|
||
# mode behind.
|
||
chmod 0755 ${builtins.dirOf tlsDir}
|
||
chmod 0755 ${tlsDir}
|
||
# Copy the host leaf in. `install` writes atomically with the
|
||
# target mode; run as root (container root == host root,
|
||
# privateUsers=false) so the 0600 root:root host key is
|
||
# readable. Key ends up root:nginx 0640 so nginx-pre-start
|
||
# (which runs `nginx -t` as the nginx user, not root) can
|
||
# read it — a 0600 root:root key passes the master load but
|
||
# fails the pre-start config test with `BIO_new_file() …
|
||
# Permission denied`, blocking the unit. Cert is world-read.
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${tlsCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${tlsKey}
|
||
|
||
# The swarm-services leaf, when this host issues one. It is
|
||
# a separate pair rather than more SANs on the one above
|
||
# because no hive CA can sign these names — each is
|
||
# constrained to its own hive's domain and the service
|
||
# names are siblings of it.
|
||
#
|
||
# Absent is a normal state, not a failure: the leaf exists
|
||
# only where the swarm CA is autoconfigured, and issuance
|
||
# can also fail on a host that wants one.
|
||
#
|
||
# ⚠️ When it is absent the HIVE leaf goes to this path
|
||
# anyway, and that fallback is load-bearing rather than
|
||
# tidy. nginx refuses to load a config naming a cert file
|
||
# that does not exist — `cannot load certificate … no such
|
||
# file` fails the pre-start test, so the vhost does not
|
||
# degrade, the ENTIRE gateway dies and takes the forge, the
|
||
# dashboard and matrix with it. Serving the hive leaf on a
|
||
# swarm-service name is a name mismatch: browsers warn,
|
||
# strict clients refuse, everything else keeps working, and
|
||
# the operator gets a bad cert instead of no hive.
|
||
#
|
||
# Measured, not theorised: this exact path took pr1ma's
|
||
# gateway down when the services sub-CA failed to issue.
|
||
if [ -s ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ]; then
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ${svcCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/swarm-services-key.pem ${svcKey}
|
||
else
|
||
echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${svcCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${svcKey}
|
||
fi
|
||
'';
|
||
};
|
||
|
||
# nginx reload is triggered from the HOST side by hive-c0re
|
||
# after each agents.conf write, through hive-priv (c0re is
|
||
# unprivileged and cannot act on a system unit).
|
||
#
|
||
# It stays an explicit trigger rather than a systemd path unit
|
||
# watching the file: the write and the reload belong in one causal
|
||
# chain c0re can retry and report on (see RELOAD_PENDING), not two
|
||
# independent units racing on an inotify event.
|
||
|
||
services.nginx = {
|
||
enable = true;
|
||
recommendedProxySettings = true;
|
||
recommendedTlsSettings = true;
|
||
recommendedGzipSettings = true;
|
||
recommendedOptimisation = true;
|
||
inherit (nginxTree) appendHttpConfig virtualHosts;
|
||
};
|
||
|
||
# dnsmasq is a host service alongside nginx, so it reads the host's
|
||
# /etc/resolv.conf directly and picks up network changes as they
|
||
# happen — no copy to keep in sync.
|
||
services.dnsmasq = import ./dnsmasq.nix {
|
||
inherit
|
||
lib
|
||
networkCfg
|
||
forgeCfg
|
||
matrixCfg
|
||
hyperhiveDomain
|
||
;
|
||
};
|
||
|
||
networking.firewall = lib.mkIf cfg.openFirewall {
|
||
allowedTCPPorts = [
|
||
cfg.port
|
||
# The gateway always terminates TLS (self-signed floor), so
|
||
# `httpsPort` is always opened alongside the plain-http `port`.
|
||
cfg.httpsPort
|
||
];
|
||
};
|
||
|
||
# `/etc/hosts` entries for local dev — bare hive domain + any
|
||
# sub-domain modules that are on. `lib.unique` dedupes if any
|
||
# sub-domain happens to equal another. See `docs/gateway.md`
|
||
# ("Local dev").
|
||
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
||
"127.0.0.1" = lib.unique (
|
||
[ hyperhiveDomain ]
|
||
++ lib.optional (config.services.hyperhive.swarm.forge.behindGateway or false
|
||
) config.services.hyperhive.swarm.forge.domain
|
||
++ lib.optional (matrixCfg.enable && matrixCfg.gatewayHost != null) matrixCfg.gatewayHost
|
||
);
|
||
};
|
||
};
|
||
}
|