c32a9367e4
The `_` vhost was serving the hive's own surface, so every dashboard and agent-UI request matched the default server rather than a named vhost -- and so did a request for any name at all, including a raw IP. Split it: `_` keeps only `return 444`, and the hive surface moves to a vhost named for the hive domain. `_` is `mkDefault` so an operator can claim default_server themselves, plus an assertion for the case where they add one without turning ours off -- nginx refuses to start on a duplicate default_server and nixpkgs asserts nothing, so that would otherwise surface as a gateway outage at rebuild time.
443 lines
21 KiB
Nix
443 lines
21 KiB
Nix
# Single nginx in front of every hyperhive web surface — dashboard,
|
||
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
||
# delegations — plus the hive-internal dnsmasq resolver. Both run on the
|
||
# HOST, next to hive-c0re: nginx binds the host's :80/:443 and dnsmasq
|
||
# answers on the hive bridge, so neither can be confined to a network
|
||
# namespace of its own.
|
||
# Full vhost map + discovery flow + design rationale in
|
||
# `docs/gateway.md`. Layout: ./options.nix (option declarations),
|
||
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
|
||
# (styled static pages), ./dnsmasq.nix (resolver + DHCP config).
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.gateway;
|
||
hyperhiveDomain = config.services.hyperhive.domain;
|
||
# Derived once in ../swarm.nix; the vhosts that get the swarm-services
|
||
# cert are exactly the names that cert is issued for, so both read the
|
||
# same list rather than each deciding what "a swarm service" means.
|
||
swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains;
|
||
matrixCfg = config.services.hyperhive.swarm.matrix;
|
||
networkCfg = config.services.hyperhive.network;
|
||
|
||
# Every vhost claiming `default_server`, ours and the operator's
|
||
# alike. Computed once so the assertion below and the message it
|
||
# prints cannot disagree about what they found.
|
||
defaultVhosts = lib.filter (n: config.services.nginx.virtualHosts.${n}.default or false) (
|
||
lib.attrNames config.services.nginx.virtualHosts
|
||
);
|
||
|
||
# Dashboard SPA dist, static-served by nginx.
|
||
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
||
|
||
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
||
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
||
# (see vhosts.nix's `swaggerUiLocations` and
|
||
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
||
# module (not c0re's) — hive-c0re has no relationship to it.
|
||
swaggerUiTheme = cfg.swaggerUiTheme;
|
||
|
||
# Self-signed TLS is the implicit floor: when neither an operator cert
|
||
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
||
# generates + serves a hive-CA-signed leaf (see hive-tls.nix). There is no
|
||
# explicit toggle and no http-only mode — matrix discovery requires https,
|
||
# so the gateway always terminates TLS.
|
||
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
||
useSelfSigned = cfg.useSelfSigned;
|
||
|
||
# nginx's own state dir. Kept at the historical `/var/lib/hive-gateway`
|
||
# path rather than renamed with the move: it holds the imported leaf
|
||
# across reboots, and renaming it would strand every existing hive's
|
||
# certs for no gain.
|
||
tlsDir = "/var/lib/hive-gateway/tls";
|
||
# TLS cert + key paths.
|
||
# - self-signed (default): the hive-CA-signed leaf, imported into the
|
||
# state dir by `hive-gateway-self-signed-cert` below.
|
||
# - tls.certDir set: the operator's own cert dir, read directly.
|
||
tlsCert =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
||
tlsKey =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
||
# The swarm-services pair, used only by the vhosts whose names this
|
||
# hive's CA cannot sign. Self-signed mode only: with an operator cert
|
||
# or ACME the operator owns every name and there is no second issuer.
|
||
svcCert = "${tlsDir}/swarm-services.pem";
|
||
svcKey = "${tlsDir}/swarm-services-key.pem";
|
||
|
||
# Styled static error pages. Built once here and reached two ways:
|
||
# directly by ./vhosts.nix, and via the published kit by any service
|
||
# module that aims an `error_page` at one.
|
||
errorPages = import ./error-pages.nix { inherit pkgs; };
|
||
|
||
# The vhost construction kit (listen set / per-name TLS attrs /
|
||
# security headers / error pages). Computed here, published as
|
||
# `cfg.lib` below, and handed to ./vhosts.nix **as the published
|
||
# value** — so the tree the gateway renders and the kit a service
|
||
# module gets are the same object by construction, not by two call
|
||
# sites agreeing.
|
||
vhostLib = import ./vhost-lib.nix {
|
||
inherit
|
||
lib
|
||
cfg
|
||
tlsCert
|
||
tlsKey
|
||
svcCert
|
||
svcKey
|
||
swarmServiceDomains
|
||
errorPages
|
||
;
|
||
};
|
||
|
||
nginxTree = import ./vhosts.nix {
|
||
gwLib = cfg.lib;
|
||
inherit
|
||
lib
|
||
cfg
|
||
errorPages
|
||
matrixCfg
|
||
hyperhiveDomain
|
||
dashboardDist
|
||
swaggerUiTheme
|
||
;
|
||
};
|
||
in
|
||
{
|
||
imports = [ ./options.nix ];
|
||
|
||
config = lib.mkIf config.services.hyperhive.enable {
|
||
# Publish the kit. Defined here rather than as an option `default`
|
||
# so it stays a plain value computed once from resolved cert paths —
|
||
# `tlsFor` closes over `svcCert`/`svcKey`, which are derived in this
|
||
# file's `let` and are not option surface.
|
||
services.hyperhive.gateway.lib = vhostLib;
|
||
|
||
assertions = [
|
||
{
|
||
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true and
|
||
tls.certDir are mutually exclusive. Pick one TLS mode.
|
||
'';
|
||
}
|
||
{
|
||
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true requires
|
||
services.hyperhive.gateway.tls.acme.email to be set —
|
||
Let's Encrypt needs a contact address for the ACME account.
|
||
'';
|
||
}
|
||
{
|
||
# Two modules claiming one hostname is a real possibility now
|
||
# that each service contributes its own name, and dnsmasq would
|
||
# not complain: duplicate `address=` rules resolve by precedence,
|
||
# so the loser simply stops being served with no error anywhere.
|
||
# Fail the build instead — a name is owned by exactly one module.
|
||
assertion = lib.length (lib.unique cfg.localNames) == lib.length cfg.localNames;
|
||
message = ''
|
||
services.hyperhive.gateway.localNames contains a duplicate:
|
||
${lib.concatStringsSep ", " (
|
||
lib.unique (lib.filter (n: lib.count (m: m == n) cfg.localNames > 1) cfg.localNames)
|
||
)}
|
||
|
||
Each hostname the hive resolver answers for is contributed by
|
||
exactly one module. Two modules claiming the same name means
|
||
two services believe they serve it — resolve which one does
|
||
rather than letting dnsmasq pick.
|
||
'';
|
||
}
|
||
{
|
||
# nginx refuses to start with two `default_server`s on one
|
||
# address ("a duplicate default server for 0.0.0.0:<port>",
|
||
# exit 1) and nixpkgs asserts nothing — `vhost.default` is a
|
||
# plain bool rendered straight into the listen line. So without
|
||
# this, an operator adding their own default vhost gets a
|
||
# gateway that fails its config test at rebuild time, which
|
||
# takes the forge, dashboard, matrix and swarm UI with it, and
|
||
# reports a port rather than a cause.
|
||
#
|
||
# Not covered by our `mkDefault`: an operator's own vhost is a
|
||
# different option path, so nothing merges and nothing
|
||
# conflicts — priority only helps someone who already knows
|
||
# ours exists. Fail at eval and name both, so the fix
|
||
# (`services.nginx.virtualHosts.<ours>.default = false`) is
|
||
# readable from the error.
|
||
assertion = lib.length defaultVhosts <= 1;
|
||
message = ''
|
||
More than one nginx virtual host is marked `default = true`:
|
||
${lib.concatStringsSep ", " defaultVhosts}
|
||
|
||
nginx allows exactly one default server per listen address
|
||
and refuses to start otherwise, so this would fail at
|
||
service start rather than here — taking every site behind
|
||
the gateway down with it.
|
||
|
||
The gateway's own catch-all (`_`, which returns 444) is set
|
||
with `mkDefault`, so to make yours the default server turn
|
||
ours off explicitly:
|
||
|
||
services.nginx.virtualHosts."_".default = false;
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Ensure the gateway state dirs exist at host boot, before anything
|
||
# reads or writes them: these rules cover the fresh-boot window
|
||
# before c0re has run, and pin owner + mode rather than leaving it
|
||
# to whoever creates the path first.
|
||
#
|
||
# /run/hive-agent — per-agent UDS socket dir, written by c0re's
|
||
# set_nspawn_flags when agents start. Owned by `hive-core` (the
|
||
# unprivileged coordinator user): c0re does the
|
||
# `create_dir_all(/run/hive-agent/<name>)` itself, so a root-owned
|
||
# parent would EACCES on the very first agent create on a fresh host
|
||
# (hive-priv only chowns the subdir afterwards, it doesn't make it).
|
||
#
|
||
# ⚠️ There is deliberately NO rule for /var/lib/hyperhive here. One
|
||
# used to declare it `0755 root root` and could never win:
|
||
# `hive-c0re.service` sets `StateDirectory = "hyperhive"` with
|
||
# `StateDirectoryMode = "0750"`, which systemd re-applies on every
|
||
# start. Two mechanisms owning one path, and the loser still read as
|
||
# a guarantee — it is why the resulting outage was first misdiagnosed
|
||
# as someone having changed the mode. c0re's own unit owns that dir;
|
||
# this module no longer has an opinion about it.
|
||
systemd.tmpfiles.rules = [
|
||
# Must stay in step with the identical rule hive-priv generates into
|
||
# /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare
|
||
# different owners for this path.
|
||
"d /run/hive-agent 0755 hive-core hive-core - -"
|
||
# The gateway's own config dir — NOT under /var/lib/hyperhive. c0re
|
||
# writes here, nginx reads here, and neither needs any access to the
|
||
# other's tree: no shared parent to traverse means no group
|
||
# membership handing nginx c0re's broker db and everything else
|
||
# beside it. Sibling of `tls/`, which already lived under this root.
|
||
#
|
||
# Owned by hive-core because c0re is the writer; 0755 so the
|
||
# unprivileged nginx user can traverse and read. This is now the
|
||
# ONLY declaration of these paths' modes — nothing re-applies a
|
||
# different owner on top the way `StateDirectory=` does for
|
||
# /var/lib/hyperhive.
|
||
"d /var/lib/hive-gateway 0755 root root - -"
|
||
"d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -"
|
||
"f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n"
|
||
# Pre-create both files so nginx's config test can open them before
|
||
# c0re has ever written: nginx names them, and an `include` of a
|
||
# missing file is a fatal config error, not an empty one. tmpfiles
|
||
# runs before services, which is the whole ordering guarantee —
|
||
# content arrives when c0re writes and reloads, which it does on
|
||
# every topology change. `f` = create-if-absent, never overwrite.
|
||
#
|
||
# An empty htpasswd causes all auth checks to return 401 (no valid
|
||
# credentials), which is the correct no-users behaviour.
|
||
"f /var/lib/hive-gateway/conf/gateway.htpasswd 0644 hive-core hive-core - -"
|
||
];
|
||
|
||
# The host asks the hive's own resolver, at the BRIDGE IP.
|
||
#
|
||
# Every container inherits a COPY of this host's `/etc/resolv.conf`
|
||
# at start (`nixos-containers.nix`: `cp --remove-destination`, one
|
||
# shot, not a bind-mount) — so whatever address is written here is
|
||
# the address every container will try, in its own netns.
|
||
#
|
||
# 🚨 That is why this is the bridge IP and not `127.0.0.1`, and the
|
||
# distinction is load-bearing rather than stylistic:
|
||
#
|
||
# value host host-netns containers bridged containers
|
||
# 127.0.0.1 ok ok THEIR OWN loopback
|
||
# bridge IP ok ok ok
|
||
#
|
||
# dnsmasq binds both `lo` and the bridge (./dnsmasq.nix), so the
|
||
# bridge IP is reachable from the host too — it is the only value
|
||
# correct on both sides of a netns boundary. `resolveLocalQueries`
|
||
# publishes loopback by default, hence both overrides here; the
|
||
# flag stays on for its `resolv-file` plumbing, which is what keeps
|
||
# dnsmasq's own upstreams out of the file we are pointing at it.
|
||
#
|
||
# Cost, stated because it is real: the host's DNS now depends on
|
||
# dnsmasq being up. Every container already did.
|
||
networking.nameservers = lib.mkForce [ networkCfg.bridgeIp ];
|
||
networking.resolvconf.useLocalResolver = lib.mkForce false;
|
||
|
||
# ACME (Let's Encrypt) integration. nginx vhosts set
|
||
# `enableACME = true` via the vhost builder; this provides the
|
||
# shared ACME config (acceptTerms + email).
|
||
security.acme = lib.mkIf cfg.tls.acme.enable {
|
||
acceptTerms = true;
|
||
defaults.email = cfg.tls.acme.email;
|
||
};
|
||
|
||
# Import the hive-CA leaf into nginx's state dir before nginx starts.
|
||
#
|
||
# 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It
|
||
# does TWO jobs:
|
||
#
|
||
# (1) It re-modes the leaf. `hive-tls-ca` writes the key 0600
|
||
# root:root; nginx's pre-start `nginx -t` runs as the nginx
|
||
# *user*, so a 0600 key fails the config test with
|
||
# `BIO_new_file() … Permission denied` and blocks the unit —
|
||
# hence the 0640 root:nginx copy below.
|
||
# (2) It guarantees that **every cert path the nginx config names
|
||
# exists** — which is what the swarm-services fallback at the
|
||
# bottom of the script is for. nginx refuses to load a config
|
||
# naming a missing cert file, so a leaf that never issues takes
|
||
# the whole gateway down rather than one vhost; that has already
|
||
# happened once and it took the forge, dashboard and matrix with
|
||
# it. Removing this unit re-creates it exactly.
|
||
#
|
||
# nginx
|
||
# `Requires=` this via `requiredBy`, so it refuses to start until
|
||
# the copy succeeds. ALWAYS runs (no ConditionPathExists) and is
|
||
# idempotent — necessary to reconcile broken state from prior
|
||
# failed boots (a 0700 dir from a stale UMask, a truncated copy
|
||
# from an interrupted oneshot, etc.). The leaf covers the bare
|
||
# hive domain plus `forge.`, `matrix.` and `*.${hyperhiveDomain}`
|
||
# so all sub-domains validate under the same cert + the hive CA.
|
||
# See `docs/gateway.md` ("Self-signed TLS").
|
||
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
||
description = "Import host-generated TLS leaf for hive-gateway";
|
||
wantedBy = [ "multi-user.target" ];
|
||
before = [ "nginx.service" ];
|
||
requiredBy = [ "nginx.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
# Pin the journal identity (else it's the `script` store-path wrapper).
|
||
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -eu
|
||
mkdir -p ${tlsDir}
|
||
# 0755 on BOTH the cert dir and its parent so the nginx
|
||
# user can traverse the full path. The parent
|
||
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
||
# StateDirectory / mkdir umask depending on which service
|
||
# created it first), which on its own blocks traversal.
|
||
# Re-applied every boot in case a prior run left a tighter
|
||
# mode behind.
|
||
chmod 0755 ${builtins.dirOf tlsDir}
|
||
chmod 0755 ${tlsDir}
|
||
# Copy the leaf in. `install` writes atomically with the
|
||
# target mode; runs as root so the 0600 root:root key written
|
||
# by hive-tls-ca is readable. Key ends up root:nginx 0640 so
|
||
# nginx-pre-start
|
||
# (which runs `nginx -t` as the nginx user, not root) can
|
||
# read it — a 0600 root:root key passes the master load but
|
||
# fails the pre-start config test with `BIO_new_file() …
|
||
# Permission denied`, blocking the unit. Cert is world-read.
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${tlsCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${tlsKey}
|
||
|
||
# The swarm-services leaf, when this host issues one. It is
|
||
# a separate pair rather than more SANs on the one above
|
||
# because no hive CA can sign these names — each is
|
||
# constrained to its own hive's domain and the service
|
||
# names are siblings of it.
|
||
#
|
||
# Absent is a normal state, not a failure: the leaf exists
|
||
# only where the swarm CA is autoconfigured, and issuance
|
||
# can also fail on a host that wants one.
|
||
#
|
||
# ⚠️ When it is absent the HIVE leaf goes to this path
|
||
# anyway, and that fallback is load-bearing rather than
|
||
# tidy. nginx refuses to load a config naming a cert file
|
||
# that does not exist — `cannot load certificate … no such
|
||
# file` fails the pre-start test, so the vhost does not
|
||
# degrade, the ENTIRE gateway dies and takes the forge, the
|
||
# dashboard and matrix with it. Serving the hive leaf on a
|
||
# swarm-service name is a name mismatch: browsers warn,
|
||
# strict clients refuse, everything else keeps working, and
|
||
# the operator gets a bad cert instead of no hive.
|
||
#
|
||
# Measured, not theorised: this exact path took pr1ma's
|
||
# gateway down when the services sub-CA failed to issue.
|
||
if [ -s ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ]; then
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ${svcCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/swarm-services-key.pem ${svcKey}
|
||
else
|
||
echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2
|
||
install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${svcCert}
|
||
install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${svcKey}
|
||
fi
|
||
'';
|
||
};
|
||
|
||
# nginx reload is triggered from the HOST side by hive-c0re
|
||
# after each agents.conf write, through hive-priv (c0re is
|
||
# unprivileged and cannot act on a system unit).
|
||
#
|
||
# It stays an explicit trigger rather than a systemd path unit
|
||
# watching the file: the write and the reload belong in one causal
|
||
# chain c0re can retry and report on (see RELOAD_PENDING), not two
|
||
# independent units racing on an inotify event.
|
||
|
||
services.nginx = {
|
||
enable = true;
|
||
recommendedProxySettings = true;
|
||
recommendedTlsSettings = true;
|
||
recommendedGzipSettings = true;
|
||
recommendedOptimisation = true;
|
||
inherit (nginxTree) virtualHosts;
|
||
};
|
||
|
||
# ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
|
||
# absence is load-bearing rather than an omission.
|
||
#
|
||
# It used to be here, to let nginx traverse `/var/lib/hyperhive` and
|
||
# reach the config fragments that lived inside: `hive-c0re.service`
|
||
# declares `StateDirectory = "hyperhive"` with `StateDirectoryMode =
|
||
# "0750"` owned by `hive-core`, and systemd re-applies that on every
|
||
# c0re start — beating this module's own tmpfiles rule for the same
|
||
# path. Without the group, nginx failed its config test and never
|
||
# started (`nginx: [emerg] open() ".../agents.conf" failed (13:
|
||
# Permission denied)`), taking the gateway and every hive domain
|
||
# behind it down.
|
||
#
|
||
# The group fixed the symptom and paid for it: it also gave nginx
|
||
# read access to everything ELSE group-readable under that dir,
|
||
# including the broker sqlite — i.e. every message between every
|
||
# agent, reachable by the process whose entire job is parsing
|
||
# untrusted network input. Moving the fragments to the gateway's own
|
||
# dir removes the need and the exposure together. Re-adding this line
|
||
# would restore both.
|
||
|
||
# dnsmasq is a host service alongside nginx, so it reads the host's
|
||
# /etc/resolv.conf directly and picks up network changes as they
|
||
# happen — no copy to keep in sync.
|
||
services.dnsmasq = import ./dnsmasq.nix {
|
||
inherit
|
||
lib
|
||
cfg
|
||
networkCfg
|
||
hyperhiveDomain
|
||
;
|
||
};
|
||
|
||
networking.firewall = lib.mkIf cfg.openFirewall {
|
||
allowedTCPPorts = [
|
||
cfg.port
|
||
# The gateway always terminates TLS (self-signed floor), so
|
||
# `httpsPort` is always opened alongside the plain-http `port`.
|
||
cfg.httpsPort
|
||
];
|
||
};
|
||
|
||
# `/etc/hosts` entries for local dev — the bare hive domain plus
|
||
# every name a service module contributed. See `docs/gateway.md`
|
||
# ("Local dev").
|
||
#
|
||
# This used to restate the per-service list a THIRD time (after the
|
||
# vhosts and the dnsmasq records), with its own copy of each
|
||
# service's guard. It is the same question — "which names does this
|
||
# host answer for" — so it reads the same answer; a service added
|
||
# later lands here with no edit, and cannot land here with a
|
||
# different condition than it used for DNS.
|
||
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
||
"127.0.0.1" = lib.unique ([ hyperhiveDomain ] ++ cfg.localNames);
|
||
};
|
||
};
|
||
}
|