fcdff04b23
Second slice of the swarm-global service consolidation, same shape as
the forge move: the operator-facing host options become
services.hyperhive.swarm.matrix.*, and one mkRenamedOptionModule on the
namespace carries the whole subtree (nested gui.* included), so there is
no leaf list to forget an entry from.
The rename lives in hive-matrix.nix, the module that declares the
options, so each service's migration stays independent of its siblings.
The per-agent hyperhive.matrix.{enable,url} and hyperhive.matrixAccounts
are a different namespace -- a client pointer at the service, not the
service -- and deliberately do not move.
420 lines
20 KiB
Nix
420 lines
20 KiB
Nix
# Single nginx in front of every hyperhive web surface — dashboard,
|
|
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
|
# delegations — plus the hive-internal dnsmasq resolver, co-located in
|
|
# the same `hive-gateway` container (shared host netns, state-free).
|
|
# Full vhost map + discovery flow + design rationale in
|
|
# `docs/gateway.md`. Layout: ./options.nix (option declarations),
|
|
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
|
|
# (styled static pages), ./dnsmasq.nix (resolver + DHCP config).
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.gateway;
|
|
hyperhiveDomain = config.services.hyperhive.domain;
|
|
matrixCfg = config.services.hyperhive.swarm.matrix;
|
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
|
networkCfg = config.services.hyperhive.network;
|
|
|
|
# Dashboard SPA dist, static-served by nginx. Read in OUTER scope so
|
|
# `config` is the host's (inside the container block it'd be the
|
|
# container's).
|
|
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
|
|
|
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
|
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
|
# (see vhosts.nix's `swaggerUiLocations` and
|
|
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
|
# module (not c0re's) — hive-c0re has no relationship to it.
|
|
swaggerUiTheme = cfg.swaggerUiTheme;
|
|
|
|
# Self-signed TLS is the implicit floor: when neither an operator cert
|
|
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
|
# generates + serves a hive-CA-signed leaf (see hive-tls.nix). There is no
|
|
# explicit toggle and no http-only mode — matrix discovery requires https,
|
|
# so the gateway always terminates TLS.
|
|
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
|
useSelfSigned = cfg.useSelfSigned;
|
|
in
|
|
{
|
|
imports = [ ./options.nix ];
|
|
|
|
config = lib.mkIf config.services.hyperhive.enable {
|
|
assertions = [
|
|
{
|
|
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true and
|
|
tls.certDir are mutually exclusive. Pick one TLS mode.
|
|
'';
|
|
}
|
|
{
|
|
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true requires
|
|
services.hyperhive.gateway.tls.acme.email to be set —
|
|
Let's Encrypt needs a contact address for the ACME account.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Ensure bind-mount sources exist at host boot before the gateway
|
|
# container's first start. nspawn would auto-create missing dirs;
|
|
# tmpfiles rules make the intent explicit and cover the fresh-boot
|
|
# window before c0re has run.
|
|
#
|
|
# /run/hive-agent — per-agent UDS socket dir, written by c0re's
|
|
# set_nspawn_flags when agents start. Owned by `hive-core` (the
|
|
# unprivileged coordinator user): c0re does the
|
|
# `create_dir_all(/run/hive-agent/<name>)` itself, so a root-owned
|
|
# parent would EACCES on the very first agent create on a fresh host
|
|
# (hive-priv only chowns the subdir afterwards, it doesn't make it).
|
|
# /var/lib/hyperhive — hyperhive state dir, created by c0re on
|
|
# first run. Also pre-seed agents.conf with an empty-but-valid
|
|
# header so nginx can start + include the file before c0re writes
|
|
# its first real content (f = create-if-absent, no overwrite).
|
|
systemd.tmpfiles.rules = [
|
|
# Must stay in step with the identical rule hive-priv generates into
|
|
# /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare
|
|
# different owners for this path.
|
|
"d /run/hive-agent 0755 hive-core hive-core - -"
|
|
"d /var/lib/hyperhive 0755 root root - -"
|
|
"d /var/lib/hyperhive/gateway 0755 root root - -"
|
|
"f /var/lib/hyperhive/gateway/agents.conf 0644 root root - # Generated by hive-c0re — do not edit.\n"
|
|
# Pre-create the htpasswd file so nginx can open it even before any
|
|
# users have been added. An empty file causes all auth checks to
|
|
# return 401 (no valid credentials), which is the correct no-users
|
|
# behaviour. `f` = create-if-absent, never overwrite.
|
|
"f /var/lib/hyperhive/gateway/gateway.htpasswd 0644 root root - -"
|
|
];
|
|
|
|
# Keep the gateway's `/etc/resolv.conf` in step with the host's.
|
|
#
|
|
# nixos-container does `cp --remove-destination /etc/resolv.conf
|
|
# "$root/etc/resolv.conf"` in its start script, and nspawn's
|
|
# `--resolv-conf=auto` copies rather than binds for a writable,
|
|
# host-netns container like this one. Both are one-shot: systemd-nspawn(1)
|
|
# says outright that "no further propagation of configuration is
|
|
# generally done after the one-time early initialization (this is
|
|
# because the file is usually updated through copying and renaming)".
|
|
#
|
|
# So the gateway's copy is frozen at container start. dnsmasq has no
|
|
# explicit upstream (see ./dnsmasq.nix) and follows that file, which
|
|
# means a host network change — new router, new DHCP lease, laptop
|
|
# moving between networks — leaves dnsmasq forwarding to a resolver
|
|
# that is gone, and every non-hive lookup from every agent hangs. The
|
|
# agents' own resolvers point at the static bridge IP and never go
|
|
# stale, which is exactly why the symptom presents as "the gateway
|
|
# needs a kick".
|
|
#
|
|
# Hence: watch on the host, push into the container, reload dnsmasq.
|
|
# `reload` is `kill -HUP $MAINPID` (the upstream dnsmasq unit's
|
|
# ExecReload), so dnsmasq re-reads its upstream list and drops its
|
|
# cache without severing anything — nginx never notices. Why this has
|
|
# to run host-side, and why a file bind-mount is worse than the copy:
|
|
# `docs/network.md::Resolver behaviour`.
|
|
systemd.paths.hive-gateway-resolv = {
|
|
description = "Watch the host's /etc/resolv.conf for the hive-gateway container";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Arm the watch before anything configures the network, so the very
|
|
# first DHCP-driven resolv.conf write of the boot is caught too —
|
|
# that's the "gateway came up while DHCP was still settling" case.
|
|
# Inert if nothing pulls network-pre.target into the transaction.
|
|
before = [ "network-pre.target" ];
|
|
pathConfig = {
|
|
# PathChanged also watches the parent directory, so openresolv's
|
|
# atomic rename-over lands as IN_MOVED_TO on /etc and triggers —
|
|
# a watch on the inode alone would die with the replaced file.
|
|
PathChanged = "/etc/resolv.conf";
|
|
Unit = "hive-gateway-resolv.service";
|
|
};
|
|
};
|
|
|
|
systemd.services.hive-gateway-resolv = {
|
|
description = "Sync the host's resolvers into hive-gateway and reload dnsmasq";
|
|
# Also run once per gateway start, to catch a host resolver change
|
|
# that happened while the container was down. Deliberately NOT
|
|
# ordered after network-online.target: pulling that target in on
|
|
# every resolv.conf change risks blocking the sync behind a
|
|
# wait-online timeout on hosts where nothing else reaches it. The
|
|
# boot race is closed by arming the path unit early instead.
|
|
wantedBy = [ "container@hive-gateway.service" ];
|
|
after = [ "container@hive-gateway.service" ];
|
|
path = [
|
|
pkgs.systemd
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
SyslogIdentifier = "hive-gateway-resolv";
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
src=/etc/resolv.conf
|
|
# Marker lives on tmpfs, so the first sync after every host boot
|
|
# always goes through rather than trusting a stale comparison.
|
|
marker=/run/hive-gateway/resolv.synced
|
|
|
|
# No nameserver line means either a mid-rewrite snapshot or a host
|
|
# with no DNS at all. In both cases the gateway's existing copy is
|
|
# the best information available — keep it and wait for the next
|
|
# event, instead of pushing a file that resolves nothing.
|
|
if ! grep -q '^[[:space:]]*nameserver[[:space:]]' "$src"; then
|
|
echo "host resolv.conf has no nameserver line — keeping the gateway's current copy"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -e "$marker" ] && cmp -s "$src" "$marker"; then
|
|
echo "host resolvers unchanged since last sync — nothing to do"
|
|
exit 0
|
|
fi
|
|
|
|
# A stopped gateway needs no push: its next start copies the
|
|
# current host file itself.
|
|
if ! systemctl is-active --quiet container@hive-gateway.service; then
|
|
echo "hive-gateway not running — its next start copies the current file itself"
|
|
exit 0
|
|
fi
|
|
|
|
# `machinectl copy-to` writes through the container's own mount
|
|
# namespace, so this stays correct regardless of how the container
|
|
# assembles /etc (e.g. if system.etc.overlay is ever turned on) —
|
|
# unlike poking at the rootfs path from the host side.
|
|
machinectl copy-to hive-gateway "$src" /etc/resolv.conf --force
|
|
|
|
# Not fatal: dnsmasq polls resolv.conf for mtime changes on its own,
|
|
# so a failed reload costs a second of staleness, not correctness.
|
|
if ! systemctl -M hive-gateway reload dnsmasq.service; then
|
|
echo "dnsmasq reload failed (not up yet?) — copy is in place, its own poll will pick it up"
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$marker")"
|
|
install -m 0644 "$src" "$marker"
|
|
echo "synced host resolvers into hive-gateway and reloaded dnsmasq"
|
|
'';
|
|
};
|
|
|
|
containers.hive-gateway = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Share host netns — nginx then binds host-level ports directly,
|
|
# `localhost` upstream resolution reaches hive-c0re without any
|
|
# port-forward dance, and the firewall config below is the only
|
|
# layer that matters.
|
|
privateNetwork = false;
|
|
# dnsmasq refuses to start once a dhcp-range is configured unless it
|
|
# holds CAP_NET_ADMIN (DNS-only mode doesn't need it). Private-network
|
|
# containers retain NET_ADMIN implicitly, but this container shares the
|
|
# host netns (above), so nspawn's default bounding set drops it — grant
|
|
# it explicitly. Note this is NET_ADMIN over the *host* netns; the
|
|
# gateway container is trusted infra (it already terminates TLS and
|
|
# fronts every vhost), so no new trust boundary is crossed.
|
|
additionalCapabilities = [ "CAP_NET_ADMIN" ];
|
|
# Bind-mount the per-agent socket dir so nginx inside the gateway
|
|
# container can `connect(2)` to the UDS upstreams.
|
|
# Read-only (we just connect; harness writes the socket inside
|
|
# the agent's own container). Host-side dir is pre-created by a
|
|
# tmpfiles rule so nspawn always finds a source at boot.
|
|
bindMounts."/run/hive-agent" = {
|
|
hostPath = "/run/hive-agent";
|
|
isReadOnly = true;
|
|
};
|
|
# Bind-mount ONLY the gateway-specific subdir of the hyperhive
|
|
# state dir. Scoped to /var/lib/hyperhive/gateway/ rather than
|
|
# the whole parent so the gateway container can't read forge
|
|
# tokens or other files that may live at the parent level.
|
|
# c0re writes agents.conf under this subdir and triggers an nginx
|
|
# reload from the host via systemd-run after each write.
|
|
# Pre-created by a tmpfiles rule.
|
|
bindMounts."/run/hive-state" = {
|
|
hostPath = "/var/lib/hyperhive/gateway";
|
|
isReadOnly = true;
|
|
};
|
|
# Operator-provided TLS cert dir (e.g. Let's Encrypt / ACME).
|
|
# Only mounted when `tls.certDir` is set; when it is, the self-signed
|
|
# floor is off (so the `/run/hive-ca` mount below is absent). nginx
|
|
# reads cert + key from `/run/hive-tls/<certName>` and `<keyName>`.
|
|
bindMounts."/run/hive-tls" = lib.mkIf (cfg.tls.certDir != null) {
|
|
hostPath = cfg.tls.certDir;
|
|
isReadOnly = true;
|
|
};
|
|
# Self-signed mode: the host `hive-tls-ca` service generates a hive
|
|
# CA + a leaf signed by it under `services.hyperhive.tls.stateDir`.
|
|
# Bind-mount that dir read-only so the in-container import service
|
|
# (below) can copy the leaf into nginx's state dir with the right
|
|
# owner/mode. Source files: `gateway.pem` + `gateway-key.pem`.
|
|
bindMounts."/run/hive-ca" = lib.mkIf useSelfSigned {
|
|
hostPath = config.services.hyperhive.tls.stateDir;
|
|
isReadOnly = true;
|
|
};
|
|
config =
|
|
{ pkgs, ... }:
|
|
let
|
|
tlsDir = "/var/lib/hive-gateway/tls";
|
|
# TLS cert + key paths inside the container.
|
|
# - self-signed (default): imported hive-CA-signed leaf in the
|
|
# persistent state dir.
|
|
# - tls.certDir set: operator-provided cert bind-mounted at /run/hive-tls.
|
|
tlsCert =
|
|
if cfg.tls.certDir != null then "/run/hive-tls/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
|
tlsKey =
|
|
if cfg.tls.certDir != null then "/run/hive-tls/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
|
nginxTree = import ./vhosts.nix {
|
|
inherit
|
|
lib
|
|
cfg
|
|
forgeCfg
|
|
matrixCfg
|
|
hyperhiveDomain
|
|
dashboardDist
|
|
swaggerUiTheme
|
|
tlsCert
|
|
tlsKey
|
|
;
|
|
errorPages = import ./error-pages.nix { inherit pkgs; };
|
|
};
|
|
in
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# This container shares the host netns, so its own
|
|
# firewall.service would run against the HOST ruleset: flush
|
|
# the nixos-fw chains, rebuild them from this container's
|
|
# (empty) port list, and delete the host's nixos-nat-* chains
|
|
# — wiping the bridge DHCP/DNS holes and the agents' NAT on
|
|
# every container boot. The host firewall owns all filtering;
|
|
# never run one in here.
|
|
networking.firewall.enable = false;
|
|
|
|
# Keep the host-copied /etc/resolv.conf intact. nixos-container
|
|
# copies the host's file in at every container start, but
|
|
# resolvconf's host-tracking mode then regenerates it — to an
|
|
# empty file, since the host file doesn't cross the boundary
|
|
# after start (the same failure the matrix container hit).
|
|
# With resolvconf off, nothing in here touches the copy: nginx's
|
|
# own lookups (ACME) and dnsmasq's follow-the-host upstream
|
|
# default (see ./dnsmasq.nix) both read the host's resolvers.
|
|
# Keeping it stale-free is the host's job — see the
|
|
# `hive-gateway-resolv` path unit above.
|
|
networking.resolvconf.enable = false;
|
|
|
|
# ACME (Let's Encrypt) integration. nginx vhosts set
|
|
# `enableACME = true` via the vhost builder; this provides the
|
|
# shared ACME config (acceptTerms + email). The gateway
|
|
# container has shared host netns so outbound ACME requests
|
|
# work without extra routing config. Certs are stored in the
|
|
# container's persistent state (`ephemeral = false`).
|
|
security.acme = lib.mkIf cfg.tls.acme.enable {
|
|
acceptTerms = true;
|
|
defaults.email = cfg.tls.acme.email;
|
|
};
|
|
|
|
# Import the host-generated leaf cert before nginx starts.
|
|
# The hive CA + gateway leaf are generated on the HOST by
|
|
# `hive-tls-ca` (see `hive-tls.nix`) and bind-mounted read-only
|
|
# at `/run/hive-ca`; this service copies the leaf into nginx's
|
|
# state dir with the owner/mode nginx needs, rather than reading
|
|
# the bind-mount directly (the host key is 0600 root:root and a
|
|
# cross-namespace bind-mount can't be relaxed in place). nginx
|
|
# `Requires=` this via `requiredBy`, so it refuses to start until
|
|
# the copy succeeds. ALWAYS runs (no ConditionPathExists) and is
|
|
# idempotent — necessary to reconcile broken state from prior
|
|
# failed boots (a 0700 dir from a stale UMask, a truncated copy
|
|
# from an interrupted oneshot, etc.). The leaf covers the bare
|
|
# hive domain plus `forge.`, `matrix.` and `*.${hyperhiveDomain}`
|
|
# so all sub-domains validate under the same cert + the hive CA.
|
|
# See `docs/gateway.md` ("Self-signed TLS").
|
|
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
|
description = "Import host-generated TLS leaf for hive-gateway";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "nginx.service" ];
|
|
requiredBy = [ "nginx.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
# Pin the journal identity (else it's the `script` store-path wrapper).
|
|
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
|
};
|
|
path = [ pkgs.coreutils ];
|
|
script = ''
|
|
set -eu
|
|
mkdir -p ${tlsDir}
|
|
# 0755 on BOTH the cert dir and its parent so the nginx
|
|
# user can traverse the full path. The parent
|
|
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
|
# StateDirectory / mkdir umask depending on which service
|
|
# created it first), which on its own blocks traversal.
|
|
# Re-applied every boot in case a prior run left a tighter
|
|
# mode behind.
|
|
chmod 0755 ${builtins.dirOf tlsDir}
|
|
chmod 0755 ${tlsDir}
|
|
# Copy the host leaf in. `install` writes atomically with the
|
|
# target mode; run as root (container root == host root,
|
|
# privateUsers=false) so the 0600 root:root host key is
|
|
# readable. Key ends up root:nginx 0640 so nginx-pre-start
|
|
# (which runs `nginx -t` as the nginx user, not root) can
|
|
# read it — a 0600 root:root key passes the master load but
|
|
# fails the pre-start config test with `BIO_new_file() …
|
|
# Permission denied`, blocking the unit. Cert is world-read.
|
|
install -m 0644 /run/hive-ca/gateway.pem ${tlsCert}
|
|
install -m 0640 -g nginx /run/hive-ca/gateway-key.pem ${tlsKey}
|
|
'';
|
|
};
|
|
|
|
# nginx reload is triggered from the HOST side by hive-c0re
|
|
# via `systemctl -M hive-gateway reload nginx` after each
|
|
# agents.conf write — letting systemd resolve the nginx binary
|
|
# path avoids exit-203 EXEC failures. A path unit watching the
|
|
# bind-mounted file inside the container does not work: an
|
|
# IN_MOVED_TO from an atomic rename on the host does not
|
|
# propagate across the nspawn mount-namespace boundary. The
|
|
# host-side trigger is the correct approach.
|
|
|
|
services.nginx = {
|
|
enable = true;
|
|
recommendedProxySettings = true;
|
|
recommendedTlsSettings = true;
|
|
recommendedGzipSettings = true;
|
|
recommendedOptimisation = true;
|
|
inherit (nginxTree) appendHttpConfig virtualHosts;
|
|
};
|
|
|
|
services.dnsmasq = import ./dnsmasq.nix {
|
|
inherit
|
|
lib
|
|
networkCfg
|
|
forgeCfg
|
|
matrixCfg
|
|
hyperhiveDomain
|
|
;
|
|
};
|
|
};
|
|
};
|
|
|
|
networking.firewall = lib.mkIf cfg.openFirewall {
|
|
allowedTCPPorts = [
|
|
cfg.port
|
|
# The gateway always terminates TLS (self-signed floor), so
|
|
# `httpsPort` is always opened alongside the plain-http `port`.
|
|
cfg.httpsPort
|
|
];
|
|
};
|
|
|
|
# `/etc/hosts` entries for local dev — bare hive domain + any
|
|
# sub-domain modules that are on. `lib.unique` dedupes if any
|
|
# sub-domain happens to equal another. See `docs/gateway.md`
|
|
# ("Local dev").
|
|
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
|
"127.0.0.1" = lib.unique (
|
|
[ hyperhiveDomain ]
|
|
++ lib.optional (config.services.hyperhive.swarm.forge.behindGateway or false
|
|
) config.services.hyperhive.swarm.forge.domain
|
|
++ lib.optional (matrixCfg.enable && matrixCfg.gatewayHost != null) matrixCfg.gatewayHost
|
|
);
|
|
};
|
|
};
|
|
}
|