68e30957e2
The catch-all, the dashboard and the hive-domain vhost were built in vhosts.nix with the listen set, TLS attrs and headers applied by hand. They are declared under services.hyperhive.gateway.vhosts now, so every vhost behind the gateway goes through declared-vhosts.nix. Two gateway-side rules cover what the catch-all needed: a vhost can set securityHeaders = false (the catch-all only returns 444), and `default` is rendered at mkDefault priority so an operator's own default vhost still wins. The matrix discovery locations are unchanged. The rendered nginx vhosts are unchanged.
466 lines
22 KiB
Nix
466 lines
22 KiB
Nix
# This host's nginx fronts the hyperhive web surfaces on it — dashboard,
|
|
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
|
# delegations — plus the hive-internal dnsmasq resolver. Both run on
|
|
# this host, next to hive-c0re: nginx binds :80/:443 and dnsmasq
|
|
# answers on the hive bridge, so neither can be confined to a network
|
|
# namespace of its own.
|
|
# Full vhost map + discovery flow + design rationale in
|
|
# `docs/networking/gateway.md`. Layout: ./options.nix (option declarations),
|
|
# ./vhosts.nix (the gateway's own nginx virtual hosts), ./declared-vhosts.nix
|
|
# (renders the vhosts service modules declare under `gateway.vhosts`), ./error-pages.nix
|
|
# (styled static pages), ./dns.nix + ./dnsmasq.nix (the resolver, which
|
|
# has an enable of its own because a host can need hive names to resolve
|
|
# without serving a vhost).
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.gateway;
|
|
hyperhiveDomain = config.services.hyperhive.domain;
|
|
# Derived once in ../swarm.nix; the vhosts that get the swarm-services
|
|
# cert are exactly the names that cert is issued for, so both read the
|
|
# same list rather than each deciding what "a swarm service" means.
|
|
swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains;
|
|
matrixCfg = config.services.hyperhive.swarm.matrix;
|
|
matrixDeployCfg = config.services.hyperhive.deploy.matrix;
|
|
tlsStateDir = config.services.hyperhive.deploy.hive-controller.tls.stateDir;
|
|
# The hive leaf exists only where a hive runs; ../hive-tls.nix issues it
|
|
# under that same condition.
|
|
runsHive = config.services.hyperhive.deploy.hive-controller.enable;
|
|
# Whether the cert import `Requires=` the services leaf's issuing unit
|
|
# (../hive-tls.nix decides). Where it does, a missing leaf fails the
|
|
# import rather than anything standing in for it.
|
|
svcLeafRequired = lib.elem "hive-gateway-self-signed-cert.service" (
|
|
config.systemd.services.swarm-services-cert.requiredBy or [ ]
|
|
);
|
|
svcLeafMissing = ''
|
|
echo "no swarm-services leaf at ${tlsStateDir}/swarm-services.pem, and this gateway serves swarm names only with it." >&2
|
|
echo "swarm-services-cert.service issues it from the secret store's pki mount; its journal says why it has not." >&2
|
|
exit 1'';
|
|
|
|
# Every vhost claiming `default_server`, ours and the operator's
|
|
# alike. Computed once so the assertion below and the message it
|
|
# prints cannot disagree about what they found.
|
|
defaultVhosts = lib.filter (n: config.services.nginx.virtualHosts.${n}.default or false) (
|
|
lib.attrNames config.services.nginx.virtualHosts
|
|
);
|
|
|
|
# Dashboard SPA dist, static-served by nginx.
|
|
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
|
|
|
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
|
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
|
# (see vhosts.nix's `swaggerUiLocations` and
|
|
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
|
# module (not c0re's) — hive-c0re has no relationship to it.
|
|
swaggerUiTheme = cfg.swaggerUiTheme;
|
|
|
|
# Self-signed TLS is the implicit floor: when neither an operator cert
|
|
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
|
# serves a leaf the secret store issues (see hive-tls.nix). There is no
|
|
# explicit toggle and no http-only mode — matrix discovery requires https,
|
|
# so the gateway always terminates TLS.
|
|
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
|
useSelfSigned = cfg.useSelfSigned;
|
|
|
|
# nginx's own state dir, fixed at `/var/lib/hive-gateway` independent of
|
|
# this module's own path: it holds the imported leaf across reboots, and
|
|
# a different path would strand every existing hive's certs.
|
|
tlsDir = "/var/lib/hive-gateway/tls";
|
|
# TLS cert + key paths.
|
|
# - self-signed (default): the hive leaf the store issues (the
|
|
# swarm-services leaf on a host with no hive), imported into the state
|
|
# dir by `hive-gateway-self-signed-cert` below.
|
|
# - tls.certDir set: the operator's own cert dir, read directly.
|
|
tlsCert =
|
|
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
|
tlsKey =
|
|
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
|
# The swarm-services pair, used only by the vhosts whose names the
|
|
# hive leaf cannot carry. Self-signed mode only: with an operator cert
|
|
# or ACME the operator owns every name and there is no second issuer.
|
|
svcCert = "${tlsDir}/swarm-services.pem";
|
|
svcKey = "${tlsDir}/swarm-services-key.pem";
|
|
|
|
# Styled static error pages. Built once here and reached two ways:
|
|
# directly by ./vhosts.nix, and via the published kit by any service
|
|
# module that aims an `error_page` at one.
|
|
errorPages = import ./error-pages.nix { inherit pkgs; };
|
|
|
|
# The vhost construction kit (listen set / per-name TLS attrs /
|
|
# security headers / error pages). Computed here, published as
|
|
# `cfg.lib` below, and handed to ./vhosts.nix **as the published
|
|
# value** — so the tree the gateway renders and the kit a service
|
|
# module gets are the same object by construction, not by two call
|
|
# sites agreeing.
|
|
vhostLib = import ./vhost-lib.nix {
|
|
inherit
|
|
lib
|
|
cfg
|
|
tlsCert
|
|
tlsKey
|
|
svcCert
|
|
svcKey
|
|
swarmServiceDomains
|
|
errorPages
|
|
;
|
|
# Same path `lib/hive-ca-trust.nix` publishes as `caHostPath`. Named
|
|
# from the tls option here rather than importing that helper: this
|
|
# is the HOST's own bundle, and the helper's job is assembling a
|
|
# per-container copy.
|
|
caBundle = "${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/trust-bundle.pem";
|
|
};
|
|
|
|
ownVhosts = import ./vhosts.nix {
|
|
gwLib = cfg.lib;
|
|
inherit (config.services.hyperhive.swarm) dashboardDomain;
|
|
autheliaDomain = config.services.hyperhive.swarm.authelia.domain;
|
|
inherit
|
|
lib
|
|
cfg
|
|
errorPages
|
|
matrixCfg
|
|
matrixDeployCfg
|
|
hyperhiveDomain
|
|
runsHive
|
|
dashboardDist
|
|
swaggerUiTheme
|
|
;
|
|
};
|
|
in
|
|
{
|
|
imports = [
|
|
./options.nix
|
|
./dns.nix
|
|
./declared-vhosts.nix
|
|
];
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# Publish the kit. Defined here rather than as an option `default`
|
|
# so it stays a plain value computed once from resolved cert paths —
|
|
# `tlsFor` closes over `svcCert`/`svcKey`, which are derived in this
|
|
# file's `let` and are not option surface.
|
|
services.hyperhive.gateway.lib = vhostLib;
|
|
|
|
# The gateway's own vhosts, rendered the way every service's are.
|
|
services.hyperhive.gateway.vhosts = ownVhosts.vhosts;
|
|
|
|
assertions = [
|
|
{
|
|
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true and
|
|
tls.certDir are mutually exclusive. Pick one TLS mode.
|
|
'';
|
|
}
|
|
{
|
|
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true requires
|
|
services.hyperhive.gateway.tls.acme.email to be set —
|
|
Let's Encrypt needs a contact address for the ACME account.
|
|
'';
|
|
}
|
|
{
|
|
# Two modules claiming one hostname is a real possibility now
|
|
# that each service contributes its own name, and dnsmasq would
|
|
# not complain: duplicate `address=` rules resolve by precedence,
|
|
# so the loser simply stops being served with no error anywhere.
|
|
# Fail the build instead — a name is owned by exactly one module.
|
|
assertion = lib.length (lib.unique cfg.localNames) == lib.length cfg.localNames;
|
|
message = ''
|
|
services.hyperhive.gateway.localNames contains a duplicate:
|
|
${lib.concatStringsSep ", " (
|
|
lib.unique (lib.filter (n: lib.count (m: m == n) cfg.localNames > 1) cfg.localNames)
|
|
)}
|
|
|
|
Each hostname the hive resolver answers for is contributed by
|
|
exactly one module. Two modules claiming the same name means
|
|
two services believe they serve it — resolve which one does
|
|
rather than letting dnsmasq pick.
|
|
'';
|
|
}
|
|
{
|
|
# nginx refuses to start with two `default_server`s on one
|
|
# address ("a duplicate default server for 0.0.0.0:<port>",
|
|
# exit 1) and nixpkgs asserts nothing — `vhost.default` is a
|
|
# plain bool rendered straight into the listen line. So without
|
|
# this, an operator adding their own default vhost gets a
|
|
# gateway that fails its config test at rebuild time, which
|
|
# takes the forge, dashboard, matrix and swarm UI with it, and
|
|
# reports a port rather than a cause.
|
|
#
|
|
# Not covered by our `mkDefault`: an operator's own vhost is a
|
|
# different option path, so nothing merges and nothing
|
|
# conflicts — priority only helps someone who already knows
|
|
# ours exists. Fail at eval and name both, so the fix
|
|
# (`services.nginx.virtualHosts.<ours>.default = false`) is
|
|
# readable from the error.
|
|
assertion = lib.length defaultVhosts <= 1;
|
|
message = ''
|
|
More than one nginx virtual host is marked `default = true`:
|
|
${lib.concatStringsSep ", " defaultVhosts}
|
|
|
|
nginx allows exactly one default server per listen address
|
|
and refuses to start otherwise, so this would fail at
|
|
service start rather than here — taking every site behind
|
|
the gateway down with it.
|
|
|
|
The gateway's own catch-all (`_`, which returns 444) is set
|
|
with `mkDefault`, so to make yours the default server turn
|
|
ours off explicitly:
|
|
|
|
services.nginx.virtualHosts."_".default = false;
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Ensure the gateway state dirs exist at host boot, before anything
|
|
# reads or writes them: these rules cover the fresh-boot window
|
|
# before c0re has run, and pin owner + mode rather than leaving it
|
|
# to whoever creates the path first.
|
|
#
|
|
# ⚠️ There is deliberately NO rule for /var/lib/hyperhive here.
|
|
# `hive-c0re.service` sets `StateDirectory = "hyperhive"` with
|
|
# `StateDirectoryMode = "0750"`, which systemd re-applies on every
|
|
# start, so a tmpfiles rule for the same path could never win: two
|
|
# mechanisms owning one path, the loser still reading as a guarantee.
|
|
# c0re's own unit owns that dir; this module has no opinion about it.
|
|
systemd.tmpfiles.rules = [
|
|
# The gateway's own config dir — NOT under /var/lib/hyperhive. c0re
|
|
# writes here, nginx reads here, and neither needs any access to the
|
|
# other's tree: no shared parent to traverse means no group
|
|
# membership handing nginx c0re's broker db and everything else
|
|
# beside it. Sibling of `tls/`, which already lived under this root.
|
|
#
|
|
# Owned by hive-core because c0re is the writer; 0755 so the
|
|
# unprivileged nginx user can traverse and read. This is now the
|
|
# ONLY declaration of these paths' modes — nothing re-applies a
|
|
# different owner on top the way `StateDirectory=` does for
|
|
# /var/lib/hyperhive.
|
|
"d /var/lib/hive-gateway 0755 root root - -"
|
|
"d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -"
|
|
# Pre-created so nginx's config test can open it before c0re has
|
|
# ever written: nginx names it, and an `include` of a missing file is
|
|
# a fatal config error, not an empty one. tmpfiles runs before
|
|
# services, which is the whole ordering guarantee — content arrives
|
|
# when c0re writes and reloads, which it does on every topology
|
|
# change. `f` = create-if-absent, never overwrite.
|
|
"f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n"
|
|
# Nothing reads or writes this path. A copy left on an existing host
|
|
# holds bcrypt password hashes at 0644; tmpfiles' `--remove` pass, on
|
|
# switch and at boot, deletes it.
|
|
"r /var/lib/hive-gateway/conf/gateway.htpasswd"
|
|
];
|
|
|
|
# The dashboard's name, for agents on the bridge and (with
|
|
# `localHostsEntry`) the host's own browser.
|
|
services.hyperhive.gateway.localNames = lib.optional (
|
|
runsHive && config.services.hyperhive.swarm.dashboardDomain != null
|
|
) config.services.hyperhive.swarm.dashboardDomain;
|
|
|
|
# ACME (Let's Encrypt) integration. nginx vhosts set
|
|
# `enableACME = true` via the vhost builder; this provides the
|
|
# shared ACME config (acceptTerms + email).
|
|
security.acme = lib.mkIf cfg.tls.acme.enable {
|
|
acceptTerms = true;
|
|
defaults.email = cfg.tls.acme.email;
|
|
};
|
|
|
|
# Import the gateway leaves into nginx's state dir before nginx starts.
|
|
#
|
|
# 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It does
|
|
# TWO jobs: it re-modes the leaf to the 0640 root:nginx copy below (a
|
|
# 0600 key fails nginx's pre-start config test and blocks the unit),
|
|
# and it puts a file at every cert path the config names or fails: one
|
|
# nginx cannot find takes the WHOLE gateway down, not one vhost. Both
|
|
# mechanisms, and what it cost the time this was
|
|
# collapsed: `docs/networking/gateway.md::Self-signed TLS (default)`.
|
|
#
|
|
# ALWAYS runs (no ConditionPathExists) and is idempotent, because it
|
|
# has to reconcile broken state from earlier failed boots — a 0700 dir
|
|
# from a stale UMask, a truncated copy from an interrupted oneshot.
|
|
# nginx `Requires=` it via `requiredBy`, so nginx refuses to start
|
|
# until the copy succeeds.
|
|
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
|
description = "Import host-generated TLS leaf for hive-gateway";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "nginx.service" ];
|
|
requiredBy = [ "nginx.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
# Pin the journal identity (else it's the `script` store-path wrapper).
|
|
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
|
};
|
|
# openssl for the throwaway pair a host without a hive starts on.
|
|
path = [ pkgs.coreutils ] ++ lib.optional (!runsHive) pkgs.openssl;
|
|
script = ''
|
|
set -eu
|
|
mkdir -p ${tlsDir}
|
|
# 0755 on BOTH the cert dir and its parent so the nginx
|
|
# user can traverse the full path. The parent
|
|
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
|
# StateDirectory / mkdir umask depending on which service
|
|
# created it first), which on its own blocks traversal.
|
|
# Re-applied every boot in case a prior run left a tighter
|
|
# mode behind.
|
|
chmod 0755 ${builtins.dirOf tlsDir}
|
|
chmod 0755 ${tlsDir}
|
|
# Copy the leaf in. `install` writes atomically with the
|
|
# target mode; runs as root so the 0600 root:root key written
|
|
# by hive-tls-resign is readable. Key ends up root:nginx 0640 so
|
|
# nginx-pre-start
|
|
# (which runs `nginx -t` as the nginx user, not root) can
|
|
# read it — a 0600 root:root key passes the master load but
|
|
# fails the pre-start config test with `BIO_new_file() …
|
|
# Permission denied`, blocking the unit. Cert is world-read.
|
|
${
|
|
if runsHive then
|
|
''
|
|
install -m 0644 ${tlsStateDir}/gateway.pem ${tlsCert}
|
|
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey}
|
|
|
|
# The swarm-services leaf, when this host issues one. It is
|
|
# a separate pair rather than more SANs on the one above
|
|
# because the hive leaf's role allows only its own hive's
|
|
# domain, and the service names are siblings of it.
|
|
#
|
|
# Where the gateway waits for it, a missing leaf fails this
|
|
# unit and nginx with it. Elsewhere — no swarm name here, or
|
|
# only the store's UI, whose host must start its passthrough
|
|
# with the store sealed — the hive leaf stands in, because
|
|
# nginx refuses a config naming a missing cert file, until
|
|
# `swarm-services-cert` issues one and re-runs this unit.
|
|
if [ -s ${tlsStateDir}/swarm-services.pem ]; then
|
|
install -m 0644 ${tlsStateDir}/swarm-services.pem ${svcCert}
|
|
install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${svcKey}
|
|
else
|
|
${
|
|
if svcLeafRequired then
|
|
svcLeafMissing
|
|
else
|
|
''
|
|
echo "no swarm-services leaf yet — serving the hive leaf on those names until swarm-services-cert.service issues one" >&2
|
|
install -m 0644 ${tlsStateDir}/gateway.pem ${svcCert}
|
|
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${svcKey}''
|
|
}
|
|
fi''
|
|
else
|
|
''
|
|
# No hive here, so no hive leaf: every vhost, the catch-all
|
|
# included, serves the swarm-services leaf. Where the gateway
|
|
# waits for it, a missing leaf fails this unit. A store-only
|
|
# host does not wait, so its stream passthrough comes up with
|
|
# the store sealed: a throwaway self-signed pair stands in,
|
|
# because nginx refuses a config naming a missing cert file,
|
|
# until `swarm-services-cert` issues the leaf and re-runs this
|
|
# unit.
|
|
if [ -s ${tlsStateDir}/swarm-services.pem ] && [ -s ${tlsStateDir}/swarm-services-key.pem ]; then
|
|
cert=${tlsStateDir}/swarm-services.pem
|
|
key=${tlsStateDir}/swarm-services-key.pem
|
|
else
|
|
${
|
|
if svcLeafRequired then
|
|
svcLeafMissing
|
|
else
|
|
''
|
|
echo "no swarm-services leaf yet — serving a throwaway self-signed certificate until it lands" >&2
|
|
cert=${tlsDir}/fallback.pem
|
|
key=${tlsDir}/fallback-key.pem
|
|
(umask 077; openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
|
|
-subj "/CN=hyperhive-gateway-fallback" -keyout "$key" -out "$cert")''
|
|
}
|
|
fi
|
|
install -m 0644 "$cert" ${tlsCert}
|
|
install -m 0640 -g nginx "$key" ${tlsKey}
|
|
install -m 0644 "$cert" ${svcCert}
|
|
install -m 0640 -g nginx "$key" ${svcKey}''
|
|
}
|
|
'';
|
|
};
|
|
|
|
# Nothing here reloads nginx: hive-c0re drives that from the host side
|
|
# through hive-priv after each agents.conf write, deliberately as an
|
|
# explicit trigger rather than a path unit. Why, and what hive-priv
|
|
# dispatches per unit state:
|
|
# `docs/networking/gateway.md::Per-agent unix-socket upstream`.
|
|
|
|
services.nginx = {
|
|
enable = true;
|
|
recommendedProxySettings = true;
|
|
recommendedTlsSettings = true;
|
|
recommendedGzipSettings = true;
|
|
recommendedOptimisation = true;
|
|
|
|
# Route the error log through syslog so each line keeps its own
|
|
# severity. The module default is `stderr`, and systemd files that
|
|
# whole stream at one priority regardless of what each line says —
|
|
# so `journalctl -p err -u nginx` returns nothing while nginx is
|
|
# logging `[error]` continuously. That failure is inverted rather
|
|
# than merely missing: a severity filter is how you ask "is anything
|
|
# wrong", and it answers "no" *because* the errors are misfiled.
|
|
#
|
|
# Transport only — no level suffix, so nginx keeps its default
|
|
# `error` threshold and this changes where lines are filed, not
|
|
# which lines exist.
|
|
logError = "syslog:server=unix:/dev/log,nohostname,tag=nginx";
|
|
|
|
# Read by the hive collector's `nginx` receiver (../otel.nix). Its own
|
|
# loopback server, outside the vhost tree, so it never answers on the
|
|
# gateway's public listeners.
|
|
appendHttpConfig = ''
|
|
server {
|
|
listen 127.0.0.1:${toString cfg.statusPort};
|
|
access_log off;
|
|
location = /stub_status {
|
|
stub_status;
|
|
allow 127.0.0.1;
|
|
deny all;
|
|
}
|
|
location / {
|
|
return 404;
|
|
}
|
|
}
|
|
'';
|
|
};
|
|
|
|
# ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
|
|
# absence is load-bearing rather than an omission.
|
|
#
|
|
# `hive-c0re.service` declares `StateDirectory = "hyperhive"` with
|
|
# `StateDirectoryMode = "0750"` owned by `hive-core`, and systemd
|
|
# re-applies that on every c0re start. The group would let nginx
|
|
# traverse that dir, but it also gives nginx read access to everything
|
|
# ELSE group-readable under it, including the broker sqlite — i.e.
|
|
# every message between every agent, reachable by the process whose
|
|
# entire job is parsing untrusted network input. The config fragments
|
|
# nginx reads live in the gateway's own dir instead, so nginx needs no
|
|
# access to c0re's tree. Adding this line would reopen that exposure.
|
|
|
|
networking.firewall = lib.mkIf cfg.openFirewall {
|
|
allowedTCPPorts = [
|
|
cfg.port
|
|
# The gateway always terminates TLS (self-signed floor), so
|
|
# `httpsPort` is always opened alongside the plain-http `port`.
|
|
cfg.httpsPort
|
|
];
|
|
};
|
|
|
|
# `/etc/hosts` entries for local dev — the bare hive domain plus
|
|
# every name a service module contributed. See `docs/networking/gateway.md`
|
|
# ("Local dev").
|
|
#
|
|
# This does not restate the per-service list (the vhosts and the
|
|
# dnsmasq records already carry it). It is the same question —
|
|
# "which names does this host answer for" — so it reads the same
|
|
# answer; a service added later lands here with no edit, and cannot
|
|
# land here with a different condition than it used for DNS.
|
|
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
|
"127.0.0.1" = lib.unique ([ hyperhiveDomain ] ++ cfg.localNames);
|
|
};
|
|
};
|
|
}
|