f70bbda7d7
The gateway nginx serves stub_status on a loopback server of its own, 127.0.0.1:8085 (services.hyperhive.gateway.statusPort). The hive-tier collector reads it with the contrib nginx receiver, which the pinned otelcol-contrib 0.151.0 ships, so no exporter daemon is added. The receiver dials http:// only (a unix: or http+unix: endpoint fails with "unsupported protocol scheme"), so the listener is loopback TCP with allow 127.0.0.1 / deny all rather than a unix socket. The new gateway dashboard plots nginx.requests, nginx.connections_current by state, and nginx.connections_accepted against handled, per hive. Part of #5342.
462 lines
22 KiB
Nix
462 lines
22 KiB
Nix
# This host's nginx fronts the hyperhive web surfaces on it — dashboard,
|
|
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
|
# delegations — plus the hive-internal dnsmasq resolver. Both run on
|
|
# this host, next to hive-c0re: nginx binds :80/:443 and dnsmasq
|
|
# answers on the hive bridge, so neither can be confined to a network
|
|
# namespace of its own.
|
|
# Full vhost map + discovery flow + design rationale in
|
|
# `docs/networking/gateway.md`. Layout: ./options.nix (option declarations),
|
|
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
|
|
# (styled static pages), ./dns.nix + ./dnsmasq.nix (the resolver, which
|
|
# has an enable of its own because a host can need hive names to resolve
|
|
# without serving a vhost).
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.gateway;
|
|
hyperhiveDomain = config.services.hyperhive.domain;
|
|
# Derived once in ../swarm.nix; the vhosts that get the swarm-services
|
|
# cert are exactly the names that cert is issued for, so both read the
|
|
# same list rather than each deciding what "a swarm service" means.
|
|
swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains;
|
|
matrixCfg = config.services.hyperhive.swarm.matrix;
|
|
matrixDeployCfg = config.services.hyperhive.deploy.matrix;
|
|
tlsStateDir = config.services.hyperhive.deploy.hive-controller.tls.stateDir;
|
|
# The hive leaf exists only where a hive runs; ../hive-tls.nix issues it
|
|
# under that same condition.
|
|
runsHive = config.services.hyperhive.deploy.hive-controller.enable;
|
|
# Whether the cert import `Requires=` the services leaf's issuing unit
|
|
# (../hive-tls.nix decides). Where it does, a missing leaf fails the
|
|
# import rather than anything standing in for it.
|
|
svcLeafRequired = lib.elem "hive-gateway-self-signed-cert.service" (
|
|
config.systemd.services.swarm-services-cert.requiredBy or [ ]
|
|
);
|
|
svcLeafMissing = ''
|
|
echo "no swarm-services leaf at ${tlsStateDir}/swarm-services.pem, and this gateway serves swarm names only with it." >&2
|
|
echo "swarm-services-cert.service issues it from the secret store's pki mount; its journal says why it has not." >&2
|
|
exit 1'';
|
|
|
|
# Every vhost claiming `default_server`, ours and the operator's
|
|
# alike. Computed once so the assertion below and the message it
|
|
# prints cannot disagree about what they found.
|
|
defaultVhosts = lib.filter (n: config.services.nginx.virtualHosts.${n}.default or false) (
|
|
lib.attrNames config.services.nginx.virtualHosts
|
|
);
|
|
|
|
# Dashboard SPA dist, static-served by nginx.
|
|
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
|
|
|
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
|
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
|
# (see vhosts.nix's `swaggerUiLocations` and
|
|
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
|
# module (not c0re's) — hive-c0re has no relationship to it.
|
|
swaggerUiTheme = cfg.swaggerUiTheme;
|
|
|
|
# Self-signed TLS is the implicit floor: when neither an operator cert
|
|
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
|
# serves a leaf the secret store issues (see hive-tls.nix). There is no
|
|
# explicit toggle and no http-only mode — matrix discovery requires https,
|
|
# so the gateway always terminates TLS.
|
|
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
|
useSelfSigned = cfg.useSelfSigned;
|
|
|
|
# nginx's own state dir, fixed at `/var/lib/hive-gateway` independent of
|
|
# this module's own path: it holds the imported leaf across reboots, and
|
|
# a different path would strand every existing hive's certs.
|
|
tlsDir = "/var/lib/hive-gateway/tls";
|
|
# TLS cert + key paths.
|
|
# - self-signed (default): the hive leaf the store issues (the
|
|
# swarm-services leaf on a host with no hive), imported into the state
|
|
# dir by `hive-gateway-self-signed-cert` below.
|
|
# - tls.certDir set: the operator's own cert dir, read directly.
|
|
tlsCert =
|
|
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
|
tlsKey =
|
|
if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
|
# The swarm-services pair, used only by the vhosts whose names the
|
|
# hive leaf cannot carry. Self-signed mode only: with an operator cert
|
|
# or ACME the operator owns every name and there is no second issuer.
|
|
svcCert = "${tlsDir}/swarm-services.pem";
|
|
svcKey = "${tlsDir}/swarm-services-key.pem";
|
|
|
|
# Styled static error pages. Built once here and reached two ways:
|
|
# directly by ./vhosts.nix, and via the published kit by any service
|
|
# module that aims an `error_page` at one.
|
|
errorPages = import ./error-pages.nix { inherit pkgs; };
|
|
|
|
# The vhost construction kit (listen set / per-name TLS attrs /
|
|
# security headers / error pages). Computed here, published as
|
|
# `cfg.lib` below, and handed to ./vhosts.nix **as the published
|
|
# value** — so the tree the gateway renders and the kit a service
|
|
# module gets are the same object by construction, not by two call
|
|
# sites agreeing.
|
|
vhostLib = import ./vhost-lib.nix {
|
|
inherit
|
|
lib
|
|
cfg
|
|
tlsCert
|
|
tlsKey
|
|
svcCert
|
|
svcKey
|
|
swarmServiceDomains
|
|
errorPages
|
|
;
|
|
# Same path `lib/hive-ca-trust.nix` publishes as `caHostPath`. Named
|
|
# from the tls option here rather than importing that helper: this
|
|
# is the HOST's own bundle, and the helper's job is assembling a
|
|
# per-container copy.
|
|
caBundle = "${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/trust-bundle.pem";
|
|
};
|
|
|
|
nginxTree = import ./vhosts.nix {
|
|
gwLib = cfg.lib;
|
|
inherit (config.services.hyperhive.swarm) dashboardDomain;
|
|
autheliaDomain = config.services.hyperhive.swarm.authelia.domain;
|
|
inherit
|
|
lib
|
|
cfg
|
|
errorPages
|
|
matrixCfg
|
|
matrixDeployCfg
|
|
hyperhiveDomain
|
|
runsHive
|
|
dashboardDist
|
|
swaggerUiTheme
|
|
;
|
|
};
|
|
in
|
|
{
|
|
imports = [
|
|
./options.nix
|
|
./dns.nix
|
|
];
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# Publish the kit. Defined here rather than as an option `default`
|
|
# so it stays a plain value computed once from resolved cert paths —
|
|
# `tlsFor` closes over `svcCert`/`svcKey`, which are derived in this
|
|
# file's `let` and are not option surface.
|
|
services.hyperhive.gateway.lib = vhostLib;
|
|
|
|
assertions = [
|
|
{
|
|
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true and
|
|
tls.certDir are mutually exclusive. Pick one TLS mode.
|
|
'';
|
|
}
|
|
{
|
|
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true requires
|
|
services.hyperhive.gateway.tls.acme.email to be set —
|
|
Let's Encrypt needs a contact address for the ACME account.
|
|
'';
|
|
}
|
|
{
|
|
# Two modules claiming one hostname is a real possibility now
|
|
# that each service contributes its own name, and dnsmasq would
|
|
# not complain: duplicate `address=` rules resolve by precedence,
|
|
# so the loser simply stops being served with no error anywhere.
|
|
# Fail the build instead — a name is owned by exactly one module.
|
|
assertion = lib.length (lib.unique cfg.localNames) == lib.length cfg.localNames;
|
|
message = ''
|
|
services.hyperhive.gateway.localNames contains a duplicate:
|
|
${lib.concatStringsSep ", " (
|
|
lib.unique (lib.filter (n: lib.count (m: m == n) cfg.localNames > 1) cfg.localNames)
|
|
)}
|
|
|
|
Each hostname the hive resolver answers for is contributed by
|
|
exactly one module. Two modules claiming the same name means
|
|
two services believe they serve it — resolve which one does
|
|
rather than letting dnsmasq pick.
|
|
'';
|
|
}
|
|
{
|
|
# nginx refuses to start with two `default_server`s on one
|
|
# address ("a duplicate default server for 0.0.0.0:<port>",
|
|
# exit 1) and nixpkgs asserts nothing — `vhost.default` is a
|
|
# plain bool rendered straight into the listen line. So without
|
|
# this, an operator adding their own default vhost gets a
|
|
# gateway that fails its config test at rebuild time, which
|
|
# takes the forge, dashboard, matrix and swarm UI with it, and
|
|
# reports a port rather than a cause.
|
|
#
|
|
# Not covered by our `mkDefault`: an operator's own vhost is a
|
|
# different option path, so nothing merges and nothing
|
|
# conflicts — priority only helps someone who already knows
|
|
# ours exists. Fail at eval and name both, so the fix
|
|
# (`services.nginx.virtualHosts.<ours>.default = false`) is
|
|
# readable from the error.
|
|
assertion = lib.length defaultVhosts <= 1;
|
|
message = ''
|
|
More than one nginx virtual host is marked `default = true`:
|
|
${lib.concatStringsSep ", " defaultVhosts}
|
|
|
|
nginx allows exactly one default server per listen address
|
|
and refuses to start otherwise, so this would fail at
|
|
service start rather than here — taking every site behind
|
|
the gateway down with it.
|
|
|
|
The gateway's own catch-all (`_`, which returns 444) is set
|
|
with `mkDefault`, so to make yours the default server turn
|
|
ours off explicitly:
|
|
|
|
services.nginx.virtualHosts."_".default = false;
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Ensure the gateway state dirs exist at host boot, before anything
|
|
# reads or writes them: these rules cover the fresh-boot window
|
|
# before c0re has run, and pin owner + mode rather than leaving it
|
|
# to whoever creates the path first.
|
|
#
|
|
# ⚠️ There is deliberately NO rule for /var/lib/hyperhive here.
|
|
# `hive-c0re.service` sets `StateDirectory = "hyperhive"` with
|
|
# `StateDirectoryMode = "0750"`, which systemd re-applies on every
|
|
# start, so a tmpfiles rule for the same path could never win: two
|
|
# mechanisms owning one path, the loser still reading as a guarantee.
|
|
# c0re's own unit owns that dir; this module has no opinion about it.
|
|
systemd.tmpfiles.rules = [
|
|
# The gateway's own config dir — NOT under /var/lib/hyperhive. c0re
|
|
# writes here, nginx reads here, and neither needs any access to the
|
|
# other's tree: no shared parent to traverse means no group
|
|
# membership handing nginx c0re's broker db and everything else
|
|
# beside it. Sibling of `tls/`, which already lived under this root.
|
|
#
|
|
# Owned by hive-core because c0re is the writer; 0755 so the
|
|
# unprivileged nginx user can traverse and read. This is now the
|
|
# ONLY declaration of these paths' modes — nothing re-applies a
|
|
# different owner on top the way `StateDirectory=` does for
|
|
# /var/lib/hyperhive.
|
|
"d /var/lib/hive-gateway 0755 root root - -"
|
|
"d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -"
|
|
# Pre-created so nginx's config test can open it before c0re has
|
|
# ever written: nginx names it, and an `include` of a missing file is
|
|
# a fatal config error, not an empty one. tmpfiles runs before
|
|
# services, which is the whole ordering guarantee — content arrives
|
|
# when c0re writes and reloads, which it does on every topology
|
|
# change. `f` = create-if-absent, never overwrite.
|
|
"f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n"
|
|
# Nothing reads or writes this path. A copy left on an existing host
|
|
# holds bcrypt password hashes at 0644; tmpfiles' `--remove` pass, on
|
|
# switch and at boot, deletes it.
|
|
"r /var/lib/hive-gateway/conf/gateway.htpasswd"
|
|
];
|
|
|
|
# The dashboard's name, for agents on the bridge and (with
|
|
# `localHostsEntry`) the host's own browser.
|
|
services.hyperhive.gateway.localNames = lib.optional (
|
|
runsHive && config.services.hyperhive.swarm.dashboardDomain != null
|
|
) config.services.hyperhive.swarm.dashboardDomain;
|
|
|
|
# ACME (Let's Encrypt) integration. nginx vhosts set
|
|
# `enableACME = true` via the vhost builder; this provides the
|
|
# shared ACME config (acceptTerms + email).
|
|
security.acme = lib.mkIf cfg.tls.acme.enable {
|
|
acceptTerms = true;
|
|
defaults.email = cfg.tls.acme.email;
|
|
};
|
|
|
|
# Import the gateway leaves into nginx's state dir before nginx starts.
|
|
#
|
|
# 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It does
|
|
# TWO jobs: it re-modes the leaf to the 0640 root:nginx copy below (a
|
|
# 0600 key fails nginx's pre-start config test and blocks the unit),
|
|
# and it puts a file at every cert path the config names or fails: one
|
|
# nginx cannot find takes the WHOLE gateway down, not one vhost. Both
|
|
# mechanisms, and what it cost the time this was
|
|
# collapsed: `docs/networking/gateway.md::Self-signed TLS (default)`.
|
|
#
|
|
# ALWAYS runs (no ConditionPathExists) and is idempotent, because it
|
|
# has to reconcile broken state from earlier failed boots — a 0700 dir
|
|
# from a stale UMask, a truncated copy from an interrupted oneshot.
|
|
# nginx `Requires=` it via `requiredBy`, so nginx refuses to start
|
|
# until the copy succeeds.
|
|
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
|
description = "Import host-generated TLS leaf for hive-gateway";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "nginx.service" ];
|
|
requiredBy = [ "nginx.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
# Pin the journal identity (else it's the `script` store-path wrapper).
|
|
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
|
};
|
|
# openssl for the throwaway pair a host without a hive starts on.
|
|
path = [ pkgs.coreutils ] ++ lib.optional (!runsHive) pkgs.openssl;
|
|
script = ''
|
|
set -eu
|
|
mkdir -p ${tlsDir}
|
|
# 0755 on BOTH the cert dir and its parent so the nginx
|
|
# user can traverse the full path. The parent
|
|
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
|
# StateDirectory / mkdir umask depending on which service
|
|
# created it first), which on its own blocks traversal.
|
|
# Re-applied every boot in case a prior run left a tighter
|
|
# mode behind.
|
|
chmod 0755 ${builtins.dirOf tlsDir}
|
|
chmod 0755 ${tlsDir}
|
|
# Copy the leaf in. `install` writes atomically with the
|
|
# target mode; runs as root so the 0600 root:root key written
|
|
# by hive-tls-resign is readable. Key ends up root:nginx 0640 so
|
|
# nginx-pre-start
|
|
# (which runs `nginx -t` as the nginx user, not root) can
|
|
# read it — a 0600 root:root key passes the master load but
|
|
# fails the pre-start config test with `BIO_new_file() …
|
|
# Permission denied`, blocking the unit. Cert is world-read.
|
|
${
|
|
if runsHive then
|
|
''
|
|
install -m 0644 ${tlsStateDir}/gateway.pem ${tlsCert}
|
|
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey}
|
|
|
|
# The swarm-services leaf, when this host issues one. It is
|
|
# a separate pair rather than more SANs on the one above
|
|
# because the hive leaf's role allows only its own hive's
|
|
# domain, and the service names are siblings of it.
|
|
#
|
|
# Where the gateway waits for it, a missing leaf fails this
|
|
# unit and nginx with it. Elsewhere — no swarm name here, or
|
|
# only the store's UI, whose host must start its passthrough
|
|
# with the store sealed — the hive leaf stands in, because
|
|
# nginx refuses a config naming a missing cert file, until
|
|
# `swarm-services-cert` issues one and re-runs this unit.
|
|
if [ -s ${tlsStateDir}/swarm-services.pem ]; then
|
|
install -m 0644 ${tlsStateDir}/swarm-services.pem ${svcCert}
|
|
install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${svcKey}
|
|
else
|
|
${
|
|
if svcLeafRequired then
|
|
svcLeafMissing
|
|
else
|
|
''
|
|
echo "no swarm-services leaf yet — serving the hive leaf on those names until swarm-services-cert.service issues one" >&2
|
|
install -m 0644 ${tlsStateDir}/gateway.pem ${svcCert}
|
|
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${svcKey}''
|
|
}
|
|
fi''
|
|
else
|
|
''
|
|
# No hive here, so no hive leaf: every vhost, the catch-all
|
|
# included, serves the swarm-services leaf. Where the gateway
|
|
# waits for it, a missing leaf fails this unit. A store-only
|
|
# host does not wait, so its stream passthrough comes up with
|
|
# the store sealed: a throwaway self-signed pair stands in,
|
|
# because nginx refuses a config naming a missing cert file,
|
|
# until `swarm-services-cert` issues the leaf and re-runs this
|
|
# unit.
|
|
if [ -s ${tlsStateDir}/swarm-services.pem ] && [ -s ${tlsStateDir}/swarm-services-key.pem ]; then
|
|
cert=${tlsStateDir}/swarm-services.pem
|
|
key=${tlsStateDir}/swarm-services-key.pem
|
|
else
|
|
${
|
|
if svcLeafRequired then
|
|
svcLeafMissing
|
|
else
|
|
''
|
|
echo "no swarm-services leaf yet — serving a throwaway self-signed certificate until it lands" >&2
|
|
cert=${tlsDir}/fallback.pem
|
|
key=${tlsDir}/fallback-key.pem
|
|
(umask 077; openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
|
|
-subj "/CN=hyperhive-gateway-fallback" -keyout "$key" -out "$cert")''
|
|
}
|
|
fi
|
|
install -m 0644 "$cert" ${tlsCert}
|
|
install -m 0640 -g nginx "$key" ${tlsKey}
|
|
install -m 0644 "$cert" ${svcCert}
|
|
install -m 0640 -g nginx "$key" ${svcKey}''
|
|
}
|
|
'';
|
|
};
|
|
|
|
# Nothing here reloads nginx: hive-c0re drives that from the host side
|
|
# through hive-priv after each agents.conf write, deliberately as an
|
|
# explicit trigger rather than a path unit. Why, and what hive-priv
|
|
# dispatches per unit state:
|
|
# `docs/networking/gateway.md::Per-agent unix-socket upstream`.
|
|
|
|
services.nginx = {
|
|
enable = true;
|
|
recommendedProxySettings = true;
|
|
recommendedTlsSettings = true;
|
|
recommendedGzipSettings = true;
|
|
recommendedOptimisation = true;
|
|
inherit (nginxTree) virtualHosts;
|
|
|
|
# Route the error log through syslog so each line keeps its own
|
|
# severity. The module default is `stderr`, and systemd files that
|
|
# whole stream at one priority regardless of what each line says —
|
|
# so `journalctl -p err -u nginx` returns nothing while nginx is
|
|
# logging `[error]` continuously. That failure is inverted rather
|
|
# than merely missing: a severity filter is how you ask "is anything
|
|
# wrong", and it answers "no" *because* the errors are misfiled.
|
|
#
|
|
# Transport only — no level suffix, so nginx keeps its default
|
|
# `error` threshold and this changes where lines are filed, not
|
|
# which lines exist.
|
|
logError = "syslog:server=unix:/dev/log,nohostname,tag=nginx";
|
|
|
|
# Read by the hive collector's `nginx` receiver (../otel.nix). Its own
|
|
# loopback server, outside the vhost tree, so it never answers on the
|
|
# gateway's public listeners.
|
|
appendHttpConfig = ''
|
|
server {
|
|
listen 127.0.0.1:${toString cfg.statusPort};
|
|
access_log off;
|
|
location = /stub_status {
|
|
stub_status;
|
|
allow 127.0.0.1;
|
|
deny all;
|
|
}
|
|
location / {
|
|
return 404;
|
|
}
|
|
}
|
|
'';
|
|
};
|
|
|
|
# ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
|
|
# absence is load-bearing rather than an omission.
|
|
#
|
|
# `hive-c0re.service` declares `StateDirectory = "hyperhive"` with
|
|
# `StateDirectoryMode = "0750"` owned by `hive-core`, and systemd
|
|
# re-applies that on every c0re start. The group would let nginx
|
|
# traverse that dir, but it also gives nginx read access to everything
|
|
# ELSE group-readable under it, including the broker sqlite — i.e.
|
|
# every message between every agent, reachable by the process whose
|
|
# entire job is parsing untrusted network input. The config fragments
|
|
# nginx reads live in the gateway's own dir instead, so nginx needs no
|
|
# access to c0re's tree. Adding this line would reopen that exposure.
|
|
|
|
networking.firewall = lib.mkIf cfg.openFirewall {
|
|
allowedTCPPorts = [
|
|
cfg.port
|
|
# The gateway always terminates TLS (self-signed floor), so
|
|
# `httpsPort` is always opened alongside the plain-http `port`.
|
|
cfg.httpsPort
|
|
];
|
|
};
|
|
|
|
# `/etc/hosts` entries for local dev — the bare hive domain plus
|
|
# every name a service module contributed. See `docs/networking/gateway.md`
|
|
# ("Local dev").
|
|
#
|
|
# This does not restate the per-service list (the vhosts and the
|
|
# dnsmasq records already carry it). It is the same question —
|
|
# "which names does this host answer for" — so it reads the same
|
|
# answer; a service added later lands here with no edit, and cannot
|
|
# land here with a different condition than it used for DNS.
|
|
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
|
"127.0.0.1" = lib.unique ([ hyperhiveDomain ] ++ cfg.localNames);
|
|
};
|
|
};
|
|
}
|