38c222fadd
Forgejo is a swarm-global service, so its operator-facing host options move to services.hyperhive.swarm.forge (and .swarm.forge.ci) as the first of the namespace consolidation. Existing hive configs keep evaluating: swarm-renames.nix maps every moved leaf with mkRenamedOptionModule, which also emits a deprecation warning naming both the old and new path, so an operator is told what to rename rather than discovering it from a failed eval. The per-agent hyperhive.forge.url does NOT move. It is a client pointer at whatever forge an agent talks to - it shares a word with the service and nothing else, and the two are already documented as separate option surfaces. Verified by evaluating the host module, since no Rust gate evaluates nix: setting the old paths and reading the new ones yields the values (httpPort 3999, ci.concurrency 7), and config.warnings carries the rename notice.
420 lines
20 KiB
Nix
420 lines
20 KiB
Nix
# Single nginx in front of every hyperhive web surface — dashboard,
|
|
# per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known
|
|
# delegations — plus the hive-internal dnsmasq resolver, co-located in
|
|
# the same `hive-gateway` container (shared host netns, state-free).
|
|
# Full vhost map + discovery flow + design rationale in
|
|
# `docs/gateway.md`. Layout: ./options.nix (option declarations),
|
|
# ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix
|
|
# (styled static pages), ./dnsmasq.nix (resolver + DHCP config).
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.gateway;
|
|
hyperhiveDomain = config.services.hyperhive.domain;
|
|
matrixCfg = config.services.hyperhive.matrix;
|
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
|
networkCfg = config.services.hyperhive.network;
|
|
|
|
# Dashboard SPA dist, static-served by nginx. Read in OUTER scope so
|
|
# `config` is the host's (inside the container block it'd be the
|
|
# container's).
|
|
dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard";
|
|
|
|
# Full hyperhive-themed Swagger UI dist — nginx serves this whole
|
|
# tree straight from the store at /api/docs/, no hive-c0re fallback
|
|
# (see vhosts.nix's `swaggerUiLocations` and
|
|
# nix/packages/swagger-ui-{dist,theme}.nix). Own option under this
|
|
# module (not c0re's) — hive-c0re has no relationship to it.
|
|
swaggerUiTheme = cfg.swaggerUiTheme;
|
|
|
|
# Self-signed TLS is the implicit floor: when neither an operator cert
|
|
# (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway
|
|
# generates + serves a hive-CA-signed leaf (see hive-tls.nix). There is no
|
|
# explicit toggle and no http-only mode — matrix discovery requires https,
|
|
# so the gateway always terminates TLS.
|
|
# `cfg.useSelfSigned` (options.nix) is the derived single source of truth.
|
|
useSelfSigned = cfg.useSelfSigned;
|
|
in
|
|
{
|
|
imports = [ ./options.nix ];
|
|
|
|
config = lib.mkIf config.services.hyperhive.enable {
|
|
assertions = [
|
|
{
|
|
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true and
|
|
tls.certDir are mutually exclusive. Pick one TLS mode.
|
|
'';
|
|
}
|
|
{
|
|
assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null;
|
|
message = ''
|
|
services.hyperhive.gateway.tls.acme.enable = true requires
|
|
services.hyperhive.gateway.tls.acme.email to be set —
|
|
Let's Encrypt needs a contact address for the ACME account.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Ensure bind-mount sources exist at host boot before the gateway
|
|
# container's first start. nspawn would auto-create missing dirs;
|
|
# tmpfiles rules make the intent explicit and cover the fresh-boot
|
|
# window before c0re has run.
|
|
#
|
|
# /run/hive-agent — per-agent UDS socket dir, written by c0re's
|
|
# set_nspawn_flags when agents start. Owned by `hive-core` (the
|
|
# unprivileged coordinator user): c0re does the
|
|
# `create_dir_all(/run/hive-agent/<name>)` itself, so a root-owned
|
|
# parent would EACCES on the very first agent create on a fresh host
|
|
# (hive-priv only chowns the subdir afterwards, it doesn't make it).
|
|
# /var/lib/hyperhive — hyperhive state dir, created by c0re on
|
|
# first run. Also pre-seed agents.conf with an empty-but-valid
|
|
# header so nginx can start + include the file before c0re writes
|
|
# its first real content (f = create-if-absent, no overwrite).
|
|
systemd.tmpfiles.rules = [
|
|
# Must stay in step with the identical rule hive-priv generates into
|
|
# /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare
|
|
# different owners for this path.
|
|
"d /run/hive-agent 0755 hive-core hive-core - -"
|
|
"d /var/lib/hyperhive 0755 root root - -"
|
|
"d /var/lib/hyperhive/gateway 0755 root root - -"
|
|
"f /var/lib/hyperhive/gateway/agents.conf 0644 root root - # Generated by hive-c0re — do not edit.\n"
|
|
# Pre-create the htpasswd file so nginx can open it even before any
|
|
# users have been added. An empty file causes all auth checks to
|
|
# return 401 (no valid credentials), which is the correct no-users
|
|
# behaviour. `f` = create-if-absent, never overwrite.
|
|
"f /var/lib/hyperhive/gateway/gateway.htpasswd 0644 root root - -"
|
|
];
|
|
|
|
# Keep the gateway's `/etc/resolv.conf` in step with the host's.
|
|
#
|
|
# nixos-container does `cp --remove-destination /etc/resolv.conf
|
|
# "$root/etc/resolv.conf"` in its start script, and nspawn's
|
|
# `--resolv-conf=auto` copies rather than binds for a writable,
|
|
# host-netns container like this one. Both are one-shot: systemd-nspawn(1)
|
|
# says outright that "no further propagation of configuration is
|
|
# generally done after the one-time early initialization (this is
|
|
# because the file is usually updated through copying and renaming)".
|
|
#
|
|
# So the gateway's copy is frozen at container start. dnsmasq has no
|
|
# explicit upstream (see ./dnsmasq.nix) and follows that file, which
|
|
# means a host network change — new router, new DHCP lease, laptop
|
|
# moving between networks — leaves dnsmasq forwarding to a resolver
|
|
# that is gone, and every non-hive lookup from every agent hangs. The
|
|
# agents' own resolvers point at the static bridge IP and never go
|
|
# stale, which is exactly why the symptom presents as "the gateway
|
|
# needs a kick".
|
|
#
|
|
# Hence: watch on the host, push into the container, reload dnsmasq.
|
|
# `reload` is `kill -HUP $MAINPID` (the upstream dnsmasq unit's
|
|
# ExecReload), so dnsmasq re-reads its upstream list and drops its
|
|
# cache without severing anything — nginx never notices. Why this has
|
|
# to run host-side, and why a file bind-mount is worse than the copy:
|
|
# `docs/network.md::Resolver behaviour`.
|
|
systemd.paths.hive-gateway-resolv = {
|
|
description = "Watch the host's /etc/resolv.conf for the hive-gateway container";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Arm the watch before anything configures the network, so the very
|
|
# first DHCP-driven resolv.conf write of the boot is caught too —
|
|
# that's the "gateway came up while DHCP was still settling" case.
|
|
# Inert if nothing pulls network-pre.target into the transaction.
|
|
before = [ "network-pre.target" ];
|
|
pathConfig = {
|
|
# PathChanged also watches the parent directory, so openresolv's
|
|
# atomic rename-over lands as IN_MOVED_TO on /etc and triggers —
|
|
# a watch on the inode alone would die with the replaced file.
|
|
PathChanged = "/etc/resolv.conf";
|
|
Unit = "hive-gateway-resolv.service";
|
|
};
|
|
};
|
|
|
|
systemd.services.hive-gateway-resolv = {
|
|
description = "Sync the host's resolvers into hive-gateway and reload dnsmasq";
|
|
# Also run once per gateway start, to catch a host resolver change
|
|
# that happened while the container was down. Deliberately NOT
|
|
# ordered after network-online.target: pulling that target in on
|
|
# every resolv.conf change risks blocking the sync behind a
|
|
# wait-online timeout on hosts where nothing else reaches it. The
|
|
# boot race is closed by arming the path unit early instead.
|
|
wantedBy = [ "container@hive-gateway.service" ];
|
|
after = [ "container@hive-gateway.service" ];
|
|
path = [
|
|
pkgs.systemd
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
SyslogIdentifier = "hive-gateway-resolv";
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
src=/etc/resolv.conf
|
|
# Marker lives on tmpfs, so the first sync after every host boot
|
|
# always goes through rather than trusting a stale comparison.
|
|
marker=/run/hive-gateway/resolv.synced
|
|
|
|
# No nameserver line means either a mid-rewrite snapshot or a host
|
|
# with no DNS at all. In both cases the gateway's existing copy is
|
|
# the best information available — keep it and wait for the next
|
|
# event, instead of pushing a file that resolves nothing.
|
|
if ! grep -q '^[[:space:]]*nameserver[[:space:]]' "$src"; then
|
|
echo "host resolv.conf has no nameserver line — keeping the gateway's current copy"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -e "$marker" ] && cmp -s "$src" "$marker"; then
|
|
echo "host resolvers unchanged since last sync — nothing to do"
|
|
exit 0
|
|
fi
|
|
|
|
# A stopped gateway needs no push: its next start copies the
|
|
# current host file itself.
|
|
if ! systemctl is-active --quiet container@hive-gateway.service; then
|
|
echo "hive-gateway not running — its next start copies the current file itself"
|
|
exit 0
|
|
fi
|
|
|
|
# `machinectl copy-to` writes through the container's own mount
|
|
# namespace, so this stays correct regardless of how the container
|
|
# assembles /etc (e.g. if system.etc.overlay is ever turned on) —
|
|
# unlike poking at the rootfs path from the host side.
|
|
machinectl copy-to hive-gateway "$src" /etc/resolv.conf --force
|
|
|
|
# Not fatal: dnsmasq polls resolv.conf for mtime changes on its own,
|
|
# so a failed reload costs a second of staleness, not correctness.
|
|
if ! systemctl -M hive-gateway reload dnsmasq.service; then
|
|
echo "dnsmasq reload failed (not up yet?) — copy is in place, its own poll will pick it up"
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$marker")"
|
|
install -m 0644 "$src" "$marker"
|
|
echo "synced host resolvers into hive-gateway and reloaded dnsmasq"
|
|
'';
|
|
};
|
|
|
|
containers.hive-gateway = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Share host netns — nginx then binds host-level ports directly,
|
|
# `localhost` upstream resolution reaches hive-c0re without any
|
|
# port-forward dance, and the firewall config below is the only
|
|
# layer that matters.
|
|
privateNetwork = false;
|
|
# dnsmasq refuses to start once a dhcp-range is configured unless it
|
|
# holds CAP_NET_ADMIN (DNS-only mode doesn't need it). Private-network
|
|
# containers retain NET_ADMIN implicitly, but this container shares the
|
|
# host netns (above), so nspawn's default bounding set drops it — grant
|
|
# it explicitly. Note this is NET_ADMIN over the *host* netns; the
|
|
# gateway container is trusted infra (it already terminates TLS and
|
|
# fronts every vhost), so no new trust boundary is crossed.
|
|
additionalCapabilities = [ "CAP_NET_ADMIN" ];
|
|
# Bind-mount the per-agent socket dir so nginx inside the gateway
|
|
# container can `connect(2)` to the UDS upstreams.
|
|
# Read-only (we just connect; harness writes the socket inside
|
|
# the agent's own container). Host-side dir is pre-created by a
|
|
# tmpfiles rule so nspawn always finds a source at boot.
|
|
bindMounts."/run/hive-agent" = {
|
|
hostPath = "/run/hive-agent";
|
|
isReadOnly = true;
|
|
};
|
|
# Bind-mount ONLY the gateway-specific subdir of the hyperhive
|
|
# state dir. Scoped to /var/lib/hyperhive/gateway/ rather than
|
|
# the whole parent so the gateway container can't read forge
|
|
# tokens or other files that may live at the parent level.
|
|
# c0re writes agents.conf under this subdir and triggers an nginx
|
|
# reload from the host via systemd-run after each write.
|
|
# Pre-created by a tmpfiles rule.
|
|
bindMounts."/run/hive-state" = {
|
|
hostPath = "/var/lib/hyperhive/gateway";
|
|
isReadOnly = true;
|
|
};
|
|
# Operator-provided TLS cert dir (e.g. Let's Encrypt / ACME).
|
|
# Only mounted when `tls.certDir` is set; when it is, the self-signed
|
|
# floor is off (so the `/run/hive-ca` mount below is absent). nginx
|
|
# reads cert + key from `/run/hive-tls/<certName>` and `<keyName>`.
|
|
bindMounts."/run/hive-tls" = lib.mkIf (cfg.tls.certDir != null) {
|
|
hostPath = cfg.tls.certDir;
|
|
isReadOnly = true;
|
|
};
|
|
# Self-signed mode: the host `hive-tls-ca` service generates a hive
|
|
# CA + a leaf signed by it under `services.hyperhive.tls.stateDir`.
|
|
# Bind-mount that dir read-only so the in-container import service
|
|
# (below) can copy the leaf into nginx's state dir with the right
|
|
# owner/mode. Source files: `gateway.pem` + `gateway-key.pem`.
|
|
bindMounts."/run/hive-ca" = lib.mkIf useSelfSigned {
|
|
hostPath = config.services.hyperhive.tls.stateDir;
|
|
isReadOnly = true;
|
|
};
|
|
config =
|
|
{ pkgs, ... }:
|
|
let
|
|
tlsDir = "/var/lib/hive-gateway/tls";
|
|
# TLS cert + key paths inside the container.
|
|
# - self-signed (default): imported hive-CA-signed leaf in the
|
|
# persistent state dir.
|
|
# - tls.certDir set: operator-provided cert bind-mounted at /run/hive-tls.
|
|
tlsCert =
|
|
if cfg.tls.certDir != null then "/run/hive-tls/${cfg.tls.certName}" else "${tlsDir}/cert.pem";
|
|
tlsKey =
|
|
if cfg.tls.certDir != null then "/run/hive-tls/${cfg.tls.keyName}" else "${tlsDir}/key.pem";
|
|
nginxTree = import ./vhosts.nix {
|
|
inherit
|
|
lib
|
|
cfg
|
|
forgeCfg
|
|
matrixCfg
|
|
hyperhiveDomain
|
|
dashboardDist
|
|
swaggerUiTheme
|
|
tlsCert
|
|
tlsKey
|
|
;
|
|
errorPages = import ./error-pages.nix { inherit pkgs; };
|
|
};
|
|
in
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# This container shares the host netns, so its own
|
|
# firewall.service would run against the HOST ruleset: flush
|
|
# the nixos-fw chains, rebuild them from this container's
|
|
# (empty) port list, and delete the host's nixos-nat-* chains
|
|
# — wiping the bridge DHCP/DNS holes and the agents' NAT on
|
|
# every container boot. The host firewall owns all filtering;
|
|
# never run one in here.
|
|
networking.firewall.enable = false;
|
|
|
|
# Keep the host-copied /etc/resolv.conf intact. nixos-container
|
|
# copies the host's file in at every container start, but
|
|
# resolvconf's host-tracking mode then regenerates it — to an
|
|
# empty file, since the host file doesn't cross the boundary
|
|
# after start (the same failure the matrix container hit).
|
|
# With resolvconf off, nothing in here touches the copy: nginx's
|
|
# own lookups (ACME) and dnsmasq's follow-the-host upstream
|
|
# default (see ./dnsmasq.nix) both read the host's resolvers.
|
|
# Keeping it stale-free is the host's job — see the
|
|
# `hive-gateway-resolv` path unit above.
|
|
networking.resolvconf.enable = false;
|
|
|
|
# ACME (Let's Encrypt) integration. nginx vhosts set
|
|
# `enableACME = true` via the vhost builder; this provides the
|
|
# shared ACME config (acceptTerms + email). The gateway
|
|
# container has shared host netns so outbound ACME requests
|
|
# work without extra routing config. Certs are stored in the
|
|
# container's persistent state (`ephemeral = false`).
|
|
security.acme = lib.mkIf cfg.tls.acme.enable {
|
|
acceptTerms = true;
|
|
defaults.email = cfg.tls.acme.email;
|
|
};
|
|
|
|
# Import the host-generated leaf cert before nginx starts.
|
|
# The hive CA + gateway leaf are generated on the HOST by
|
|
# `hive-tls-ca` (see `hive-tls.nix`) and bind-mounted read-only
|
|
# at `/run/hive-ca`; this service copies the leaf into nginx's
|
|
# state dir with the owner/mode nginx needs, rather than reading
|
|
# the bind-mount directly (the host key is 0600 root:root and a
|
|
# cross-namespace bind-mount can't be relaxed in place). nginx
|
|
# `Requires=` this via `requiredBy`, so it refuses to start until
|
|
# the copy succeeds. ALWAYS runs (no ConditionPathExists) and is
|
|
# idempotent — necessary to reconcile broken state from prior
|
|
# failed boots (a 0700 dir from a stale UMask, a truncated copy
|
|
# from an interrupted oneshot, etc.). The leaf covers the bare
|
|
# hive domain plus `forge.`, `matrix.` and `*.${hyperhiveDomain}`
|
|
# so all sub-domains validate under the same cert + the hive CA.
|
|
# See `docs/gateway.md` ("Self-signed TLS").
|
|
systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned {
|
|
description = "Import host-generated TLS leaf for hive-gateway";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "nginx.service" ];
|
|
requiredBy = [ "nginx.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
# Pin the journal identity (else it's the `script` store-path wrapper).
|
|
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
|
};
|
|
path = [ pkgs.coreutils ];
|
|
script = ''
|
|
set -eu
|
|
mkdir -p ${tlsDir}
|
|
# 0755 on BOTH the cert dir and its parent so the nginx
|
|
# user can traverse the full path. The parent
|
|
# `/var/lib/hive-gateway` lands at 0700 by default (systemd
|
|
# StateDirectory / mkdir umask depending on which service
|
|
# created it first), which on its own blocks traversal.
|
|
# Re-applied every boot in case a prior run left a tighter
|
|
# mode behind.
|
|
chmod 0755 ${builtins.dirOf tlsDir}
|
|
chmod 0755 ${tlsDir}
|
|
# Copy the host leaf in. `install` writes atomically with the
|
|
# target mode; run as root (container root == host root,
|
|
# privateUsers=false) so the 0600 root:root host key is
|
|
# readable. Key ends up root:nginx 0640 so nginx-pre-start
|
|
# (which runs `nginx -t` as the nginx user, not root) can
|
|
# read it — a 0600 root:root key passes the master load but
|
|
# fails the pre-start config test with `BIO_new_file() …
|
|
# Permission denied`, blocking the unit. Cert is world-read.
|
|
install -m 0644 /run/hive-ca/gateway.pem ${tlsCert}
|
|
install -m 0640 -g nginx /run/hive-ca/gateway-key.pem ${tlsKey}
|
|
'';
|
|
};
|
|
|
|
# nginx reload is triggered from the HOST side by hive-c0re
|
|
# via `systemctl -M hive-gateway reload nginx` after each
|
|
# agents.conf write — letting systemd resolve the nginx binary
|
|
# path avoids exit-203 EXEC failures. A path unit watching the
|
|
# bind-mounted file inside the container does not work: an
|
|
# IN_MOVED_TO from an atomic rename on the host does not
|
|
# propagate across the nspawn mount-namespace boundary. The
|
|
# host-side trigger is the correct approach.
|
|
|
|
services.nginx = {
|
|
enable = true;
|
|
recommendedProxySettings = true;
|
|
recommendedTlsSettings = true;
|
|
recommendedGzipSettings = true;
|
|
recommendedOptimisation = true;
|
|
inherit (nginxTree) appendHttpConfig virtualHosts;
|
|
};
|
|
|
|
services.dnsmasq = import ./dnsmasq.nix {
|
|
inherit
|
|
lib
|
|
networkCfg
|
|
forgeCfg
|
|
matrixCfg
|
|
hyperhiveDomain
|
|
;
|
|
};
|
|
};
|
|
};
|
|
|
|
networking.firewall = lib.mkIf cfg.openFirewall {
|
|
allowedTCPPorts = [
|
|
cfg.port
|
|
# The gateway always terminates TLS (self-signed floor), so
|
|
# `httpsPort` is always opened alongside the plain-http `port`.
|
|
cfg.httpsPort
|
|
];
|
|
};
|
|
|
|
# `/etc/hosts` entries for local dev — bare hive domain + any
|
|
# sub-domain modules that are on. `lib.unique` dedupes if any
|
|
# sub-domain happens to equal another. See `docs/gateway.md`
|
|
# ("Local dev").
|
|
networking.hosts = lib.mkIf cfg.localHostsEntry {
|
|
"127.0.0.1" = lib.unique (
|
|
[ hyperhiveDomain ]
|
|
++ lib.optional (config.services.hyperhive.swarm.forge.behindGateway or false
|
|
) config.services.hyperhive.swarm.forge.domain
|
|
++ lib.optional (matrixCfg.enable && matrixCfg.gatewayHost != null) matrixCfg.gatewayHost
|
|
);
|
|
};
|
|
};
|
|
}
|