1c90619fbb
Rewrite comments and option descriptions under nix/ that narrate how the
code got here ("used to", "new name", "sat at the TOP", "no longer",
renamed/moved narration, attributions) to say what the code does now and
why. Comments and description strings only; no logic changes.
Files in open PR #4970's diff are left alone. Refs #3902
425 lines
20 KiB
Nix
425 lines
20 KiB
Nix
# This host's nginx fronts the hyperhive web surfaces on it — dashboard,
|
||
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
||
# delegations — plus the hive-internal dnsmasq resolver. Both run on
|
||
# this host, next to hive-c0re: nginx binds :80/:443 and dnsmasq
|
||
# answers on the hive bridge, so neither can be confined to a network
|
||
# namespace of its own.
|
||
# Full vhost map + discovery flow + design rationale in
|
||
# `docs/networking/gateway.md`. Layout: ./options.nix (option declarations),
|
||
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
|
||
# (styled static pages), ./dns.nix + ./dnsmasq.nix (the resolver, which
|
||
# has an enable of its own because a host can need hive names to resolve
|
||
# without serving a vhost).
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.gateway;
|
||
hyperhiveDomain = config.services.hyperhive.domain;
|
||
# Derived once in ../swarm.nix; the vhosts that get the swarm-services
|
||
# cert are exactly the names that cert is issued for, so both read the
|
||
# same list rather than each deciding what "a swarm service" means.
|
||
swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains;
|
||
matrixCfg = config.services.hyperhive.swarm.matrix;
|
||
matrixDeployCfg = config.services.hyperhive.deploy.matrix;
|
||
tlsStateDir = config.services.hyperhive.deploy.hive-controller.tls.stateDir;
|
||
# The hive leaf exists only where a hive runs; ../hive-tls.nix issues it
|
||
# under that same condition.
|
||
runsHive = config.services.hyperhive.deploy.hive-controller.enable;
|
||
|
||
# Every vhost claiming `default_server`, ours and the operator's
|
||
# alike. Computed once so the assertion below and the message it
|
||
# prints cannot disagree about what they found.
|
||
defaultVhosts = lib.filter (n: config.services.nginx.virtualHosts.${n}.default or false) (
|
||
lib.attrNames config.services.nginx.virtualHosts
|
||
);
|
||
|
||
# Dashboard SPA dist, static-served by nginx.
|
||
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
||
|
||
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
||
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
||
# (see vhosts.nix's `swaggerUiLocations` and
|
||
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
||
# module (not c0re's) — hive-c0re has no relationship to it.
|
||
swaggerUiTheme = cfg.swaggerUiTheme;
|
||
|
||
# Self-signed TLS is the implicit floor: when neither an operator cert
|
||
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
||
# serves a leaf the secret store issues (see hive-tls.nix). There is no
|
||
# explicit toggle and no http-only mode — matrix discovery requires https,
|
||
# so the gateway always terminates TLS.
|
||
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
||
useSelfSigned = cfg.useSelfSigned;
|
||
|
||
# nginx's own state dir, fixed at `/var/lib/hive-gateway` independent of
|
||
# this module's own path: it holds the imported leaf across reboots, and
|
||
# a different path would strand every existing hive's certs.
|
||
tlsDir = "/var/lib/hive-gateway/tls";
|
||
# TLS cert + key paths.
|
||
# - self-signed (default): the hive leaf the store issues (the
|
||
# swarm-services leaf on a host with no hive), imported into the state
|
||
# dir by `hive-gateway-self-signed-cert` below.
|
||
# - tls.certDir set: the operator's own cert dir, read directly.
|
||
tlsCert =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
||
tlsKey =
|
||
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
||
# The swarm-services pair, used only by the vhosts whose names the
|
||
# hive leaf cannot carry. Self-signed mode only: with an operator cert
|
||
# or ACME the operator owns every name and there is no second issuer.
|
||
svcCert = "${tlsDir}/swarm-services.pem";
|
||
svcKey = "${tlsDir}/swarm-services-key.pem";
|
||
|
||
# Styled static error pages. Built once here and reached two ways:
|
||
# directly by ./vhosts.nix, and via the published kit by any service
|
||
# module that aims an `error_page` at one.
|
||
errorPages = import ./error-pages.nix { inherit pkgs; };
|
||
|
||
# The vhost construction kit (listen set / per-name TLS attrs /
|
||
# security headers / error pages). Computed here, published as
|
||
# `cfg.lib` below, and handed to ./vhosts.nix **as the published
|
||
# value** — so the tree the gateway renders and the kit a service
|
||
# module gets are the same object by construction, not by two call
|
||
# sites agreeing.
|
||
vhostLib = import ./vhost-lib.nix {
|
||
inherit
|
||
lib
|
||
cfg
|
||
tlsCert
|
||
tlsKey
|
||
svcCert
|
||
svcKey
|
||
swarmServiceDomains
|
||
errorPages
|
||
;
|
||
# Same path `lib/hive-ca-trust.nix` publishes as `caHostPath`. Named
|
||
# from the tls option here rather than importing that helper: this
|
||
# is the HOST's own bundle, and the helper's job is assembling a
|
||
# per-container copy.
|
||
caBundle = "${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/trust-bundle.pem";
|
||
};
|
||
|
||
nginxTree = import ./vhosts.nix {
|
||
gwLib = cfg.lib;
|
||
inherit
|
||
lib
|
||
cfg
|
||
errorPages
|
||
matrixCfg
|
||
matrixDeployCfg
|
||
hyperhiveDomain
|
||
runsHive
|
||
dashboardDist
|
||
swaggerUiTheme
|
||
;
|
||
};
|
||
in
|
||
{
|
||
imports = [
|
||
./options.nix
|
||
./dns.nix
|
||
];
|
||
|
||
config = lib.mkIf cfg.enable {
|
||
# Publish the kit. Defined here rather than as an option `default`
|
||
# so it stays a plain value computed once from resolved cert paths —
|
||
# `tlsFor` closes over `svcCert`/`svcKey`, which are derived in this
|
||
# file's `let` and are not option surface.
|
||
services.hyperhive.gateway.lib = vhostLib;
|
||
|
||
assertions = [
|
||
{
|
||
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true and
|
||
tls.certDir are mutually exclusive. Pick one TLS mode.
|
||
'';
|
||
}
|
||
{
|
||
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
||
message = ''
|
||
services.hyperhive.gateway.tls.acme.enable = true requires
|
||
services.hyperhive.gateway.tls.acme.email to be set —
|
||
Let's Encrypt needs a contact address for the ACME account.
|
||
'';
|
||
}
|
||
{
|
||
# Two modules claiming one hostname is a real possibility now
|
||
# that each service contributes its own name, and dnsmasq would
|
||
# not complain: duplicate `address=` rules resolve by precedence,
|
||
# so the loser simply stops being served with no error anywhere.
|
||
# Fail the build instead — a name is owned by exactly one module.
|
||
assertion = lib.length (lib.unique cfg.localNames) == lib.length cfg.localNames;
|
||
message = ''
|
||
services.hyperhive.gateway.localNames contains a duplicate:
|
||
${lib.concatStringsSep ", " (
|
||
lib.unique (lib.filter (n: lib.count (m: m == n) cfg.localNames > 1) cfg.localNames)
|
||
)}
|
||
|
||
Each hostname the hive resolver answers for is contributed by
|
||
exactly one module. Two modules claiming the same name means
|
||
two services believe they serve it — resolve which one does
|
||
rather than letting dnsmasq pick.
|
||
'';
|
||
}
|
||
{
|
||
# nginx refuses to start with two `default_server`s on one
|
||
# address ("a duplicate default server for 0.0.0.0:<port>",
|
||
# exit 1) and nixpkgs asserts nothing — `vhost.default` is a
|
||
# plain bool rendered straight into the listen line. So without
|
||
# this, an operator adding their own default vhost gets a
|
||
# gateway that fails its config test at rebuild time, which
|
||
# takes the forge, dashboard, matrix and swarm UI with it, and
|
||
# reports a port rather than a cause.
|
||
#
|
||
# Not covered by our `mkDefault`: an operator's own vhost is a
|
||
# different option path, so nothing merges and nothing
|
||
# conflicts — priority only helps someone who already knows
|
||
# ours exists. Fail at eval and name both, so the fix
|
||
# (`services.nginx.virtualHosts.<ours>.default = false`) is
|
||
# readable from the error.
|
||
assertion = lib.length defaultVhosts <= 1;
|
||
message = ''
|
||
More than one nginx virtual host is marked `default = true`:
|
||
${lib.concatStringsSep ", " defaultVhosts}
|
||
|
||
nginx allows exactly one default server per listen address
|
||
and refuses to start otherwise, so this would fail at
|
||
service start rather than here — taking every site behind
|
||
the gateway down with it.
|
||
|
||
The gateway's own catch-all (`_`, which returns 444) is set
|
||
with `mkDefault`, so to make yours the default server turn
|
||
ours off explicitly:
|
||
|
||
services.nginx.virtualHosts."_".default = false;
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Ensure the gateway state dirs exist at host boot, before anything
|
||
# reads or writes them: these rules cover the fresh-boot window
|
||
# before c0re has run, and pin owner + mode rather than leaving it
|
||
# to whoever creates the path first.
|
||
#
|
||
# ⚠️ There is deliberately NO rule for /var/lib/hyperhive here.
|
||
# `hive-c0re.service` sets `StateDirectory = "hyperhive"` with
|
||
# `StateDirectoryMode = "0750"`, which systemd re-applies on every
|
||
# start, so a tmpfiles rule for the same path could never win: two
|
||
# mechanisms owning one path, the loser still reading as a guarantee.
|
||
# c0re's own unit owns that dir; this module has no opinion about it.
|
||
systemd.tmpfiles.rules = [
|
||
# The gateway's own config dir — NOT under /var/lib/hyperhive. c0re
|
||
# writes here, nginx reads here, and neither needs any access to the
|
||
# other's tree: no shared parent to traverse means no group
|
||
# membership handing nginx c0re's broker db and everything else
|
||
# beside it. Sibling of `tls/`, which already lived under this root.
|
||
#
|
||
# Owned by hive-core because c0re is the writer; 0755 so the
|
||
# unprivileged nginx user can traverse and read. This is now the
|
||
# ONLY declaration of these paths' modes — nothing re-applies a
|
||
# different owner on top the way `StateDirectory=` does for
|
||
# /var/lib/hyperhive.
|
||
"d /var/lib/hive-gateway 0755 root root - -"
|
||
"d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -"
|
||
"f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n"
|
||
# Pre-create both files so nginx's config test can open them before
|
||
# c0re has ever written: nginx names them, and an `include` of a
|
||
# missing file is a fatal config error, not an empty one. tmpfiles
|
||
# runs before services, which is the whole ordering guarantee —
|
||
# content arrives when c0re writes and reloads, which it does on
|
||
# every topology change. `f` = create-if-absent, never overwrite.
|
||
#
|
||
# An empty htpasswd causes all auth checks to return 401 (no valid
|
||
# credentials), which is the correct no-users behaviour.
|
||
"f /var/lib/hive-gateway/conf/gateway.htpasswd 0644 hive-core hive-core - -"
|
||
];
|
||
|
||
# ACME (Let's Encrypt) integration. nginx vhosts set
|
||
# `enableACME = true` via the vhost builder; this provides the
|
||
# shared ACME config (acceptTerms + email).
|
||
security.acme = lib.mkIf cfg.tls.acme.enable {
|
||
acceptTerms = true;
|
||
defaults.email = cfg.tls.acme.email;
|
||
};
|
||
|
||
# Import the gateway leaves into nginx's state dir before nginx starts.
|
||
#
|
||
# 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It does
|
||
# TWO jobs: it re-modes the leaf to the 0640 root:nginx copy below (a
|
||
# 0600 key fails nginx's pre-start config test and blocks the unit),
|
||
# and it guarantees every cert path the config names exists — which is
|
||
# what the swarm-services fallback at the bottom of the script is for.
|
||
# A leaf that never issues therefore takes the WHOLE gateway down, not
|
||
# one vhost. Both mechanisms, and what it cost the time this was
|
||
# collapsed: `docs/networking/gateway.md::Self-signed TLS (default)`.
|
||
#
|
||
# ALWAYS runs (no ConditionPathExists) and is idempotent, because it
|
||
# has to reconcile broken state from earlier failed boots — a 0700 dir
|
||
# from a stale UMask, a truncated copy from an interrupted oneshot.
|
||
# nginx `Requires=` it via `requiredBy`, so nginx refuses to start
|
||
# until the copy succeeds.
|
||
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
||
description = "Import host-generated TLS leaf for hive-gateway";
|
||
wantedBy = [ "multi-user.target" ];
|
||
before = [ "nginx.service" ];
|
||
requiredBy = [ "nginx.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
# Pin the journal identity (else it's the `script` store-path wrapper).
|
||
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
||
};
|
||
# openssl for the throwaway pair a host without a hive starts on.
|
||
path = [ pkgs.coreutils ] ++ lib.optional (!runsHive) pkgs.openssl;
|
||
script = ''
|
||
set -eu
|
||
mkdir -p ${tlsDir}
|
||
# 0755 on BOTH the cert dir and its parent so the nginx
|
||
# user can traverse the full path. The parent
|
||
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
||
# StateDirectory / mkdir umask depending on which service
|
||
# created it first), which on its own blocks traversal.
|
||
# Re-applied every boot in case a prior run left a tighter
|
||
# mode behind.
|
||
chmod 0755 ${builtins.dirOf tlsDir}
|
||
chmod 0755 ${tlsDir}
|
||
# Copy the leaf in. `install` writes atomically with the
|
||
# target mode; runs as root so the 0600 root:root key written
|
||
# by hive-tls-resign is readable. Key ends up root:nginx 0640 so
|
||
# nginx-pre-start
|
||
# (which runs `nginx -t` as the nginx user, not root) can
|
||
# read it — a 0600 root:root key passes the master load but
|
||
# fails the pre-start config test with `BIO_new_file() …
|
||
# Permission denied`, blocking the unit. Cert is world-read.
|
||
${
|
||
if runsHive then
|
||
''
|
||
install -m 0644 ${tlsStateDir}/gateway.pem ${tlsCert}
|
||
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey}
|
||
|
||
# The swarm-services leaf, when this host issues one. It is
|
||
# a separate pair rather than more SANs on the one above
|
||
# because the hive leaf's role allows only its own hive's
|
||
# domain, and the service names are siblings of it.
|
||
#
|
||
# Absent is a normal state, not a failure: a host fronting
|
||
# no swarm name has none, and issuance can also fail on a
|
||
# host that wants one.
|
||
#
|
||
# ⚠️ When it is absent the HIVE leaf goes to this path
|
||
# anyway, and that fallback is load-bearing rather than
|
||
# tidy. nginx refuses to load a config naming a cert file
|
||
# that does not exist — `cannot load certificate … no such
|
||
# file` fails the pre-start test, so the vhost does not
|
||
# degrade, the ENTIRE gateway dies and takes the forge, the
|
||
# dashboard and matrix with it. Serving the hive leaf on a
|
||
# swarm-service name is a name mismatch: browsers warn,
|
||
# strict clients refuse, everything else keeps working, and
|
||
# the operator gets a bad cert instead of no hive.
|
||
#
|
||
# Measured, not theorised: this exact path took pr1ma's
|
||
# gateway down when the services sub-CA failed to issue.
|
||
if [ -s ${tlsStateDir}/swarm-services.pem ]; then
|
||
install -m 0644 ${tlsStateDir}/swarm-services.pem ${svcCert}
|
||
install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${svcKey}
|
||
else
|
||
echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2
|
||
install -m 0644 ${tlsStateDir}/gateway.pem ${svcCert}
|
||
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${svcKey}
|
||
fi''
|
||
else
|
||
''
|
||
# No hive here, so no hive leaf: every vhost, the catch-all
|
||
# included, serves the swarm-services leaf. Until it lands a
|
||
# throwaway self-signed pair stands in, because nginx refuses
|
||
# a config naming a missing cert file. A store-only host does
|
||
# not wait for the leaf, so its stream passthrough comes up with
|
||
# the store sealed. `swarm-services-cert` re-runs this unit
|
||
# once the leaf lands.
|
||
if [ -s ${tlsStateDir}/swarm-services.pem ] && [ -s ${tlsStateDir}/swarm-services-key.pem ]; then
|
||
cert=${tlsStateDir}/swarm-services.pem
|
||
key=${tlsStateDir}/swarm-services-key.pem
|
||
else
|
||
echo "no swarm-services leaf yet — serving a throwaway self-signed certificate until it lands" >&2
|
||
cert=${tlsDir}/fallback.pem
|
||
key=${tlsDir}/fallback-key.pem
|
||
(umask 077; openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
|
||
-subj "/CN=hyperhive-gateway-fallback" -keyout "$key" -out "$cert")
|
||
fi
|
||
install -m 0644 "$cert" ${tlsCert}
|
||
install -m 0640 -g nginx "$key" ${tlsKey}
|
||
install -m 0644 "$cert" ${svcCert}
|
||
install -m 0640 -g nginx "$key" ${svcKey}''
|
||
}
|
||
'';
|
||
};
|
||
|
||
# Nothing here reloads nginx: hive-c0re drives that from the host side
|
||
# through hive-priv after each agents.conf write, deliberately as an
|
||
# explicit trigger rather than a path unit. Why, and what hive-priv
|
||
# dispatches per unit state:
|
||
# `docs/networking/gateway.md::Per-agent unix-socket upstream`.
|
||
|
||
services.nginx = {
|
||
enable = true;
|
||
recommendedProxySettings = true;
|
||
recommendedTlsSettings = true;
|
||
recommendedGzipSettings = true;
|
||
recommendedOptimisation = true;
|
||
inherit (nginxTree) virtualHosts;
|
||
|
||
# Route the error log through syslog so each line keeps its own
|
||
# severity. The module default is `stderr`, and systemd files that
|
||
# whole stream at one priority regardless of what each line says —
|
||
# so `journalctl -p err -u nginx` returns nothing while nginx is
|
||
# logging `[error]` continuously. That failure is inverted rather
|
||
# than merely missing: a severity filter is how you ask "is anything
|
||
# wrong", and it answers "no" *because* the errors are misfiled.
|
||
#
|
||
# Transport only — no level suffix, so nginx keeps its default
|
||
# `error` threshold and this changes where lines are filed, not
|
||
# which lines exist.
|
||
logError = "syslog:server=unix:/dev/log,nohostname,tag=nginx";
|
||
};
|
||
|
||
# ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
|
||
# absence is load-bearing rather than an omission.
|
||
#
|
||
# `hive-c0re.service` declares `StateDirectory = "hyperhive"` with
|
||
# `StateDirectoryMode = "0750"` owned by `hive-core`, and systemd
|
||
# re-applies that on every c0re start. The group would let nginx
|
||
# traverse that dir, but it also gives nginx read access to everything
|
||
# ELSE group-readable under it, including the broker sqlite — i.e.
|
||
# every message between every agent, reachable by the process whose
|
||
# entire job is parsing untrusted network input. The config fragments
|
||
# nginx reads live in the gateway's own dir instead, so nginx needs no
|
||
# access to c0re's tree. Adding this line would reopen that exposure.
|
||
|
||
networking.firewall = lib.mkIf cfg.openFirewall {
|
||
allowedTCPPorts = [
|
||
cfg.port
|
||
# The gateway always terminates TLS (self-signed floor), so
|
||
# `httpsPort` is always opened alongside the plain-http `port`.
|
||
cfg.httpsPort
|
||
];
|
||
};
|
||
|
||
# `/etc/hosts` entries for local dev — the bare hive domain plus
|
||
# every name a service module contributed. See `docs/networking/gateway.md`
|
||
# ("Local dev").
|
||
#
|
||
# This does not restate the per-service list (the vhosts and the
|
||
# dnsmasq records already carry it). It is the same question —
|
||
# "which names does this host answer for" — so it reads the same
|
||
# answer; a service added later lands here with no edit, and cannot
|
||
# land here with a different condition than it used for DNS.
|
||
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
||
"127.0.0.1" = lib.unique ([ hyperhiveDomain ] ++ cfg.localNames);
|
||
};
|
||
};
|
||
}
|