hyperhive/nix/host-modules/hive-gateway/default.nix
atlas cf4a60b837 fix(nix): a missing swarm-services leaf must not kill the whole gateway
main is already unborked -- the constraint-syntax fix landed directly in
31e47632 and the hive-tls-ca ordering race in b5066ad0. What is left is
the reason the syntax bug became an OUTAGE rather than a warning, and it
is independent of both.

nginx refuses to load a config naming a cert file that does not exist:
`cannot load certificate ... no such file` fails the pre-start test, so
the vhost does not degrade -- the ENTIRE proxy dies and restart-loops
into start-limit-hit, taking the forge, the dashboard and matrix with
it. Signing tolerated the leaf's absence and the container import
tolerated it; the vhost reference did not, and that is the one that
decides whether anything serves.

Absence stays a normal state here: the leaf only exists where the swarm
CA is autoconfigured. So the import now installs the HIVE leaf at the
services path instead of removing the file. That is a name mismatch on
those vhosts -- browsers warn, strict clients refuse -- and every other
name keeps working. A bad certificate is a bad afternoon; a dead gateway
is a dead hive.

Also keeps the comment explaining why every nameConstraints entry needs
its own `permitted;` prefix, so the next reader doesn't re-derive it
from an outage.
2026-08-06 00:30:22 +02:00

465 lines
23 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Single nginx in front of every hyperhive web surface — dashboard,
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
# delegations — plus the hive-internal dnsmasq resolver, co-located in
# the same `hive-gateway` container (shared host netns, state-free).
# Full vhost map + discovery flow + design rationale in
# `docs/gateway.md`. Layout: ./options.nix (option declarations),
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
# (styled static pages), ./dnsmasq.nix (resolver + DHCP config).
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.gateway;
hyperhiveDomain = config.services.hyperhive.domain;
# Derived once in ../swarm.nix; the vhosts that get the swarm-services
# cert are exactly the names that cert is issued for, so both read the
# same list rather than each deciding what "a swarm service" means.
swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains;
matrixCfg = config.services.hyperhive.swarm.matrix;
forgeCfg = config.services.hyperhive.swarm.forge;
networkCfg = config.services.hyperhive.network;
# Dashboard SPA dist, static-served by nginx. Read in OUTER scope so
# `config` is the host's (inside the container block it'd be the
# container's).
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
# tree straight from the store at /api/docs/, no hive-c0re fallback
# (see vhosts.nix's `swaggerUiLocations` and
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
# module (not c0re's) — hive-c0re has no relationship to it.
swaggerUiTheme = cfg.swaggerUiTheme;
# Self-signed TLS is the implicit floor: when neither an operator cert
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
# generates + serves a hive-CA-signed leaf (see hive-tls.nix). There is no
# explicit toggle and no http-only mode — matrix discovery requires https,
# so the gateway always terminates TLS.
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
useSelfSigned = cfg.useSelfSigned;
in
{
imports = [ ./options.nix ];
config = lib.mkIf config.services.hyperhive.enable {
assertions = [
{
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
message = ''
services.hyperhive.gateway.tls.acme.enable = true and
tls.certDir are mutually exclusive. Pick one TLS mode.
'';
}
{
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
message = ''
services.hyperhive.gateway.tls.acme.enable = true requires
services.hyperhive.gateway.tls.acme.email to be set
Let's Encrypt needs a contact address for the ACME account.
'';
}
];
# Ensure bind-mount sources exist at host boot before the gateway
# container's first start. nspawn would auto-create missing dirs;
# tmpfiles rules make the intent explicit and cover the fresh-boot
# window before c0re has run.
#
# /run/hive-agent — per-agent UDS socket dir, written by c0re's
# set_nspawn_flags when agents start. Owned by `hive-core` (the
# unprivileged coordinator user): c0re does the
# `create_dir_all(/run/hive-agent/<name>)` itself, so a root-owned
# parent would EACCES on the very first agent create on a fresh host
# (hive-priv only chowns the subdir afterwards, it doesn't make it).
# /var/lib/hyperhive — hyperhive state dir, created by c0re on
# first run. Also pre-seed agents.conf with an empty-but-valid
# header so nginx can start + include the file before c0re writes
# its first real content (f = create-if-absent, no overwrite).
systemd.tmpfiles.rules = [
# Must stay in step with the identical rule hive-priv generates into
# /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare
# different owners for this path.
"d /run/hive-agent 0755 hive-core hive-core - -"
"d /var/lib/hyperhive 0755 root root - -"
"d /var/lib/hyperhive/gateway 0755 root root - -"
"f /var/lib/hyperhive/gateway/agents.conf 0644 root root - # Generated by hive-c0re do not edit.\n"
# Pre-create the htpasswd file so nginx can open it even before any
# users have been added. An empty file causes all auth checks to
# return 401 (no valid credentials), which is the correct no-users
# behaviour. `f` = create-if-absent, never overwrite.
"f /var/lib/hyperhive/gateway/gateway.htpasswd 0644 root root - -"
];
# Keep the gateway's `/etc/resolv.conf` in step with the host's.
#
# nixos-container does `cp --remove-destination /etc/resolv.conf
# "$root/etc/resolv.conf"` in its start script, and nspawn's
# `--resolv-conf=auto` copies rather than binds for a writable,
# host-netns container like this one. Both are one-shot: systemd-nspawn(1)
# says outright that "no further propagation of configuration is
# generally done after the one-time early initialization (this is
# because the file is usually updated through copying and renaming)".
#
# So the gateway's copy is frozen at container start. dnsmasq has no
# explicit upstream (see ./dnsmasq.nix) and follows that file, which
# means a host network change — new router, new DHCP lease, laptop
# moving between networks — leaves dnsmasq forwarding to a resolver
# that is gone, and every non-hive lookup from every agent hangs. The
# agents' own resolvers point at the static bridge IP and never go
# stale, which is exactly why the symptom presents as "the gateway
# needs a kick".
#
# Hence: watch on the host, push into the container, reload dnsmasq.
# `reload` is `kill -HUP $MAINPID` (the upstream dnsmasq unit's
# ExecReload), so dnsmasq re-reads its upstream list and drops its
# cache without severing anything — nginx never notices. Why this has
# to run host-side, and why a file bind-mount is worse than the copy:
# `docs/network.md::Resolver behaviour`.
systemd.paths.hive-gateway-resolv = {
description = "Watch the host's /etc/resolv.conf for the hive-gateway container";
wantedBy = [ "multi-user.target" ];
# Arm the watch before anything configures the network, so the very
# first DHCP-driven resolv.conf write of the boot is caught too —
# that's the "gateway came up while DHCP was still settling" case.
# Inert if nothing pulls network-pre.target into the transaction.
before = [ "network-pre.target" ];
pathConfig = {
# PathChanged also watches the parent directory, so openresolv's
# atomic rename-over lands as IN_MOVED_TO on /etc and triggers —
# a watch on the inode alone would die with the replaced file.
PathChanged = "/etc/resolv.conf";
Unit = "hive-gateway-resolv.service";
};
};
systemd.services.hive-gateway-resolv = {
description = "Sync the host's resolvers into hive-gateway and reload dnsmasq";
# Also run once per gateway start, to catch a host resolver change
# that happened while the container was down. Deliberately NOT
# ordered after network-online.target: pulling that target in on
# every resolv.conf change risks blocking the sync behind a
# wait-online timeout on hosts where nothing else reaches it. The
# boot race is closed by arming the path unit early instead.
wantedBy = [ "container@hive-gateway.service" ];
after = [ "container@hive-gateway.service" ];
path = [
pkgs.systemd
pkgs.coreutils
pkgs.gnugrep
];
serviceConfig = {
Type = "oneshot";
SyslogIdentifier = "hive-gateway-resolv";
};
script = ''
set -euo pipefail
src=/etc/resolv.conf
# Marker lives on tmpfs, so the first sync after every host boot
# always goes through rather than trusting a stale comparison.
marker=/run/hive-gateway/resolv.synced
# No nameserver line means either a mid-rewrite snapshot or a host
# with no DNS at all. In both cases the gateway's existing copy is
# the best information available keep it and wait for the next
# event, instead of pushing a file that resolves nothing.
if ! grep -q '^[[:space:]]*nameserver[[:space:]]' "$src"; then
echo "host resolv.conf has no nameserver line keeping the gateway's current copy"
exit 0
fi
if [ -e "$marker" ] && cmp -s "$src" "$marker"; then
echo "host resolvers unchanged since last sync nothing to do"
exit 0
fi
# A stopped gateway needs no push: its next start copies the
# current host file itself.
if ! systemctl is-active --quiet container@hive-gateway.service; then
echo "hive-gateway not running its next start copies the current file itself"
exit 0
fi
# `machinectl copy-to` writes through the container's own mount
# namespace, so this stays correct regardless of how the container
# assembles /etc (e.g. if system.etc.overlay is ever turned on)
# unlike poking at the rootfs path from the host side.
machinectl copy-to hive-gateway "$src" /etc/resolv.conf --force
# Not fatal: dnsmasq polls resolv.conf for mtime changes on its own,
# so a failed reload costs a second of staleness, not correctness.
if ! systemctl -M hive-gateway reload dnsmasq.service; then
echo "dnsmasq reload failed (not up yet?) copy is in place, its own poll will pick it up"
fi
mkdir -p "$(dirname "$marker")"
install -m 0644 "$src" "$marker"
echo "synced host resolvers into hive-gateway and reloaded dnsmasq"
'';
};
containers.hive-gateway = {
autoStart = true;
ephemeral = false;
# Share host netns — nginx then binds host-level ports directly,
# `localhost` upstream resolution reaches hive-c0re without any
# port-forward dance, and the firewall config below is the only
# layer that matters.
privateNetwork = false;
# dnsmasq refuses to start once a dhcp-range is configured unless it
# holds CAP_NET_ADMIN (DNS-only mode doesn't need it). Private-network
# containers retain NET_ADMIN implicitly, but this container shares the
# host netns (above), so nspawn's default bounding set drops it — grant
# it explicitly. Note this is NET_ADMIN over the *host* netns; the
# gateway container is trusted infra (it already terminates TLS and
# fronts every vhost), so no new trust boundary is crossed.
additionalCapabilities = [ "CAP_NET_ADMIN" ];
# Bind-mount the per-agent socket dir so nginx inside the gateway
# container can `connect(2)` to the UDS upstreams.
# Read-only (we just connect; harness writes the socket inside
# the agent's own container). Host-side dir is pre-created by a
# tmpfiles rule so nspawn always finds a source at boot.
bindMounts."/run/hive-agent" = {
hostPath = "/run/hive-agent";
isReadOnly = true;
};
# Bind-mount ONLY the gateway-specific subdir of the hyperhive
# state dir. Scoped to /var/lib/hyperhive/gateway/ rather than
# the whole parent so the gateway container can't read forge
# tokens or other files that may live at the parent level.
# c0re writes agents.conf under this subdir and triggers an nginx
# reload from the host via systemd-run after each write.
# Pre-created by a tmpfiles rule.
bindMounts."/run/hive-state" = {
hostPath = "/var/lib/hyperhive/gateway";
isReadOnly = true;
};
# Operator-provided TLS cert dir (e.g. Let's Encrypt / ACME).
# Only mounted when `tls.certDir` is set; when it is, the self-signed
# floor is off (so the `/run/hive-ca` mount below is absent). nginx
# reads cert + key from `/run/hive-tls/<certName>` and `<keyName>`.
bindMounts."/run/hive-tls" = lib.mkIf (cfg.tls.certDir != null) {
hostPath = cfg.tls.certDir;
isReadOnly = true;
};
# Self-signed mode: the host `hive-tls-ca` service generates a hive
# CA + a leaf signed by it under `services.hyperhive.tls.stateDir`.
# Bind-mount that dir read-only so the in-container import service
# (below) can copy the leaf into nginx's state dir with the right
# owner/mode. Source files: `gateway.pem` + `gateway-key.pem`.
bindMounts."/run/hive-ca" = lib.mkIf useSelfSigned {
hostPath = config.services.hyperhive.tls.stateDir;
isReadOnly = true;
};
config =
{ pkgs, ... }:
let
tlsDir = "/var/lib/hive-gateway/tls";
# TLS cert + key paths inside the container.
# - self-signed (default): imported hive-CA-signed leaf in the
# persistent state dir.
# - tls.certDir set: operator-provided cert bind-mounted at /run/hive-tls.
tlsCert =
if cfg.tls.certDir != null then "/run/hive-tls/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
tlsKey =
if cfg.tls.certDir != null then "/run/hive-tls/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
# The swarm-services pair, used only by the vhosts whose names
# this hive's CA cannot sign. Self-signed mode only: with an
# operator cert or ACME the operator owns every name and there
# is no second issuer in the picture.
svcCert = "${tlsDir}/swarm-services.pem";
svcKey = "${tlsDir}/swarm-services-key.pem";
nginxTree = import ./vhosts.nix {
inherit
lib
cfg
forgeCfg
matrixCfg
hyperhiveDomain
dashboardDist
swaggerUiTheme
tlsCert
tlsKey
svcCert
svcKey
swarmServiceDomains
;
errorPages = import ./error-pages.nix { inherit pkgs; };
};
in
{
system.stateVersion = "26.05";
# This container shares the host netns, so its own
# firewall.service would run against the HOST ruleset: flush
# the nixos-fw chains, rebuild them from this container's
# (empty) port list, and delete the host's nixos-nat-* chains
# — wiping the bridge DHCP/DNS holes and the agents' NAT on
# every container boot. The host firewall owns all filtering;
# never run one in here.
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact. nixos-container
# copies the host's file in at every container start, but
# resolvconf's host-tracking mode then regenerates it — to an
# empty file, since the host file doesn't cross the boundary
# after start (the same failure the matrix container hit).
# With resolvconf off, nothing in here touches the copy: nginx's
# own lookups (ACME) and dnsmasq's follow-the-host upstream
# default (see ./dnsmasq.nix) both read the host's resolvers.
# Keeping it stale-free is the host's job — see the
# `hive-gateway-resolv` path unit above.
networking.resolvconf.enable = false;
# ACME (Let's Encrypt) integration. nginx vhosts set
# `enableACME = true` via the vhost builder; this provides the
# shared ACME config (acceptTerms + email). The gateway
# container has shared host netns so outbound ACME requests
# work without extra routing config. Certs are stored in the
# container's persistent state (`ephemeral = false`).
security.acme = lib.mkIf cfg.tls.acme.enable {
acceptTerms = true;
defaults.email = cfg.tls.acme.email;
};
# Import the host-generated leaf cert before nginx starts.
# The hive CA + gateway leaf are generated on the HOST by
# `hive-tls-ca` (see `hive-tls.nix`) and bind-mounted read-only
# at `/run/hive-ca`; this service copies the leaf into nginx's
# state dir with the owner/mode nginx needs, rather than reading
# the bind-mount directly (the host key is 0600 root:root and a
# cross-namespace bind-mount can't be relaxed in place). nginx
# `Requires=` this via `requiredBy`, so it refuses to start until
# the copy succeeds. ALWAYS runs (no ConditionPathExists) and is
# idempotent — necessary to reconcile broken state from prior
# failed boots (a 0700 dir from a stale UMask, a truncated copy
# from an interrupted oneshot, etc.). The leaf covers the bare
# hive domain plus `forge.`, `matrix.` and `*.${hyperhiveDomain}`
# so all sub-domains validate under the same cert + the hive CA.
# See `docs/gateway.md` ("Self-signed TLS").
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
description = "Import host-generated TLS leaf for hive-gateway";
wantedBy = [ "multi-user.target" ];
before = [ "nginx.service" ];
requiredBy = [ "nginx.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Pin the journal identity (else it's the `script` store-path wrapper).
SyslogIdentifier = "hive-gateway-self-signed-cert";
};
path = [ pkgs.coreutils ];
script = ''
set -eu
mkdir -p ${tlsDir}
# 0755 on BOTH the cert dir and its parent so the nginx
# user can traverse the full path. The parent
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
# StateDirectory / mkdir umask depending on which service
# created it first), which on its own blocks traversal.
# Re-applied every boot in case a prior run left a tighter
# mode behind.
chmod 0755 ${builtins.dirOf tlsDir}
chmod 0755 ${tlsDir}
# Copy the host leaf in. `install` writes atomically with the
# target mode; run as root (container root == host root,
# privateUsers=false) so the 0600 root:root host key is
# readable. Key ends up root:nginx 0640 so nginx-pre-start
# (which runs `nginx -t` as the nginx user, not root) can
# read it a 0600 root:root key passes the master load but
# fails the pre-start config test with `BIO_new_file()
# Permission denied`, blocking the unit. Cert is world-read.
install -m 0644 /run/hive-ca/gateway.pem ${tlsCert}
install -m 0640 -g nginx /run/hive-ca/gateway-key.pem ${tlsKey}
# The swarm-services leaf, when this host issues one. It is
# a separate pair rather than more SANs on the one above
# because no hive CA can sign these names each is
# constrained to its own hive's domain and the service
# names are siblings of it.
#
# Absent is a normal state, not a failure: the leaf exists
# only where the swarm CA is autoconfigured, and issuance
# can also fail on a host that wants one.
#
# When it is absent the HIVE leaf goes to this path
# anyway, and that fallback is load-bearing rather than
# tidy. nginx refuses to load a config naming a cert file
# that does not exist `cannot load certificate no such
# file` fails the pre-start test, so the vhost does not
# degrade, the ENTIRE gateway dies and takes the forge, the
# dashboard and matrix with it. Serving the hive leaf on a
# swarm-service name is a name mismatch: browsers warn,
# strict clients refuse, everything else keeps working, and
# the operator gets a bad cert instead of no hive.
#
# Measured, not theorised: this exact path took pr1ma's
# gateway down when the services sub-CA failed to issue.
if [ -s /run/hive-ca/swarm-services.pem ]; then
install -m 0644 /run/hive-ca/swarm-services.pem ${svcCert}
install -m 0640 -g nginx /run/hive-ca/swarm-services-key.pem ${svcKey}
else
echo "no swarm-services leaf serving the hive leaf on those names (mismatch, not an outage)" >&2
install -m 0644 /run/hive-ca/gateway.pem ${svcCert}
install -m 0640 -g nginx /run/hive-ca/gateway-key.pem ${svcKey}
fi
'';
};
# nginx reload is triggered from the HOST side by hive-c0re
# via `systemctl -M hive-gateway reload nginx` after each
# agents.conf write — letting systemd resolve the nginx binary
# path avoids exit-203 EXEC failures. A path unit watching the
# bind-mounted file inside the container does not work: an
# IN_MOVED_TO from an atomic rename on the host does not
# propagate across the nspawn mount-namespace boundary. The
# host-side trigger is the correct approach.
services.nginx = {
enable = true;
recommendedProxySettings = true;
recommendedTlsSettings = true;
recommendedGzipSettings = true;
recommendedOptimisation = true;
inherit (nginxTree) appendHttpConfig virtualHosts;
};
services.dnsmasq = import ./dnsmasq.nix {
inherit
lib
networkCfg
forgeCfg
matrixCfg
hyperhiveDomain
;
};
};
};
networking.firewall = lib.mkIf cfg.openFirewall {
allowedTCPPorts = [
cfg.port
# The gateway always terminates TLS (self-signed floor), so
# `httpsPort` is always opened alongside the plain-http `port`.
cfg.httpsPort
];
};
# `/etc/hosts` entries for local dev — bare hive domain + any
# sub-domain modules that are on. `lib.unique` dedupes if any
# sub-domain happens to equal another. See `docs/gateway.md`
# ("Local dev").
networking.hosts = lib.mkIf cfg.localHostsEntry {
"127.0.0.1" = lib.unique (
[ hyperhiveDomain ]
++ lib.optional (config.services.hyperhive.swarm.forge.behindGateway or false
) config.services.hyperhive.swarm.forge.domain
++ lib.optional (matrixCfg.enable && matrixCfg.gatewayHost != null) matrixCfg.gatewayHost
);
};
};
}