# Single nginx in front of every hyperhive web surface — dashboard, # per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known # delegations — plus the hive-internal dnsmasq resolver. Both run on the # HOST, next to hive-c0re: nginx binds the host's :80/:443 and dnsmasq # answers on the hive bridge, so neither can be confined to a network # namespace of its own. # Full vhost map + discovery flow + design rationale in # `docs/gateway.md`. Layout: ./options.nix (option declarations), # ./vhosts.nix (the nginx virtual-host tree), ./error-pages.nix # (styled static pages), ./dnsmasq.nix (resolver + DHCP config). { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.gateway; hyperhiveDomain = config.services.hyperhive.domain; # Derived once in ../swarm.nix; the vhosts that get the swarm-services # cert are exactly the names that cert is issued for, so both read the # same list rather than each deciding what "a swarm service" means. swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains; matrixCfg = config.services.hyperhive.swarm.matrix; networkCfg = config.services.hyperhive.network; # Every vhost claiming `default_server`, ours and the operator's # alike. Computed once so the assertion below and the message it # prints cannot disagree about what they found. defaultVhosts = lib.filter (n: config.services.nginx.virtualHosts.${n}.default or false) ( lib.attrNames config.services.nginx.virtualHosts ); # Dashboard SPA dist, static-served by nginx. dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard"; # Full hyperhive-themed Swagger UI dist — nginx serves this whole # tree straight from the store at /api/docs/, no hive-c0re fallback # (see vhosts.nix's `swaggerUiLocations` and # nix/packages/swagger-ui-{dist,theme}.nix). Own option under this # module (not c0re's) — hive-c0re has no relationship to it. swaggerUiTheme = cfg.swaggerUiTheme; # Self-signed TLS is the implicit floor: when neither an operator cert # (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway # generates + serves a hive-CA-signed leaf (see hive-tls.nix). There is no # explicit toggle and no http-only mode — matrix discovery requires https, # so the gateway always terminates TLS. # `cfg.useSelfSigned` (options.nix) is the derived single source of truth. useSelfSigned = cfg.useSelfSigned; # nginx's own state dir. Kept at the historical `/var/lib/hive-gateway` # path rather than renamed with the move: it holds the imported leaf # across reboots, and renaming it would strand every existing hive's # certs for no gain. tlsDir = "/var/lib/hive-gateway/tls"; # TLS cert + key paths. # - self-signed (default): the hive-CA-signed leaf, imported into the # state dir by `hive-gateway-self-signed-cert` below. # - tls.certDir set: the operator's own cert dir, read directly. tlsCert = if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem"; tlsKey = if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem"; # The swarm-services pair, used only by the vhosts whose names this # hive's CA cannot sign. Self-signed mode only: with an operator cert # or ACME the operator owns every name and there is no second issuer. svcCert = "${tlsDir}/swarm-services.pem"; svcKey = "${tlsDir}/swarm-services-key.pem"; # Styled static error pages. Built once here and reached two ways: # directly by ./vhosts.nix, and via the published kit by any service # module that aims an `error_page` at one. errorPages = import ./error-pages.nix { inherit pkgs; }; # The vhost construction kit (listen set / per-name TLS attrs / # security headers / error pages). Computed here, published as # `cfg.lib` below, and handed to ./vhosts.nix **as the published # value** — so the tree the gateway renders and the kit a service # module gets are the same object by construction, not by two call # sites agreeing. vhostLib = import ./vhost-lib.nix { inherit lib cfg tlsCert tlsKey svcCert svcKey swarmServiceDomains errorPages ; }; nginxTree = import ./vhosts.nix { gwLib = cfg.lib; inherit lib cfg errorPages matrixCfg hyperhiveDomain dashboardDist swaggerUiTheme ; }; in { imports = [ ./options.nix ]; config = lib.mkIf config.services.hyperhive.enable { # Publish the kit. Defined here rather than as an option `default` # so it stays a plain value computed once from resolved cert paths — # `tlsFor` closes over `svcCert`/`svcKey`, which are derived in this # file's `let` and are not option surface. services.hyperhive.gateway.lib = vhostLib; # Every request to every hyperhive service passes through here, so this # is the one unit that can say a service was unreachable rather than # merely quiet. Named even on hives that run no swarm collector: the # option is inert unless one is collecting on this host. # # dnsmasq alongside it for the same reason one level down: a name that # stops resolving presents as every client timing out at once. services.hyperhive.swarm.otel.journaldUnits = [ "nginx" "dnsmasq" ]; assertions = [ { assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null); message = '' services.hyperhive.gateway.tls.acme.enable = true and tls.certDir are mutually exclusive. Pick one TLS mode. ''; } { assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null; message = '' services.hyperhive.gateway.tls.acme.enable = true requires services.hyperhive.gateway.tls.acme.email to be set — Let's Encrypt needs a contact address for the ACME account. ''; } { # Two modules claiming one hostname is a real possibility now # that each service contributes its own name, and dnsmasq would # not complain: duplicate `address=` rules resolve by precedence, # so the loser simply stops being served with no error anywhere. # Fail the build instead — a name is owned by exactly one module. assertion = lib.length (lib.unique cfg.localNames) == lib.length cfg.localNames; message = '' services.hyperhive.gateway.localNames contains a duplicate: ${lib.concatStringsSep ", " ( lib.unique (lib.filter (n: lib.count (m: m == n) cfg.localNames > 1) cfg.localNames) )} Each hostname the hive resolver answers for is contributed by exactly one module. Two modules claiming the same name means two services believe they serve it — resolve which one does rather than letting dnsmasq pick. ''; } { # nginx refuses to start with two `default_server`s on one # address ("a duplicate default server for 0.0.0.0:", # exit 1) and nixpkgs asserts nothing — `vhost.default` is a # plain bool rendered straight into the listen line. So without # this, an operator adding their own default vhost gets a # gateway that fails its config test at rebuild time, which # takes the forge, dashboard, matrix and swarm UI with it, and # reports a port rather than a cause. # # Not covered by our `mkDefault`: an operator's own vhost is a # different option path, so nothing merges and nothing # conflicts — priority only helps someone who already knows # ours exists. Fail at eval and name both, so the fix # (`services.nginx.virtualHosts..default = false`) is # readable from the error. assertion = lib.length defaultVhosts <= 1; message = '' More than one nginx virtual host is marked `default = true`: ${lib.concatStringsSep ", " defaultVhosts} nginx allows exactly one default server per listen address and refuses to start otherwise, so this would fail at service start rather than here — taking every site behind the gateway down with it. The gateway's own catch-all (`_`, which returns 444) is set with `mkDefault`, so to make yours the default server turn ours off explicitly: services.nginx.virtualHosts."_".default = false; ''; } ]; # Ensure the gateway state dirs exist at host boot, before anything # reads or writes them: these rules cover the fresh-boot window # before c0re has run, and pin owner + mode rather than leaving it # to whoever creates the path first. # # /run/hive-agent — per-agent UDS socket dir, written by c0re's # set_nspawn_flags when agents start. Owned by `hive-core` (the # unprivileged coordinator user): c0re does the # `create_dir_all(/run/hive-agent/)` itself, so a root-owned # parent would EACCES on the very first agent create on a fresh host # (hive-priv only chowns the subdir afterwards, it doesn't make it). # # ⚠️ There is deliberately NO rule for /var/lib/hyperhive here. One # used to declare it `0755 root root` and could never win: # `hive-c0re.service` sets `StateDirectory = "hyperhive"` with # `StateDirectoryMode = "0750"`, which systemd re-applies on every # start. Two mechanisms owning one path, and the loser still read as # a guarantee — it is why the resulting outage was first misdiagnosed # as someone having changed the mode. c0re's own unit owns that dir; # this module no longer has an opinion about it. systemd.tmpfiles.rules = [ # Must stay in step with the identical rule hive-priv generates into # /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare # different owners for this path. "d /run/hive-agent 0755 hive-core hive-core - -" # The gateway's own config dir — NOT under /var/lib/hyperhive. c0re # writes here, nginx reads here, and neither needs any access to the # other's tree: no shared parent to traverse means no group # membership handing nginx c0re's broker db and everything else # beside it. Sibling of `tls/`, which already lived under this root. # # Owned by hive-core because c0re is the writer; 0755 so the # unprivileged nginx user can traverse and read. This is now the # ONLY declaration of these paths' modes — nothing re-applies a # different owner on top the way `StateDirectory=` does for # /var/lib/hyperhive. "d /var/lib/hive-gateway 0755 root root - -" "d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -" "f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n" # Pre-create both files so nginx's config test can open them before # c0re has ever written: nginx names them, and an `include` of a # missing file is a fatal config error, not an empty one. tmpfiles # runs before services, which is the whole ordering guarantee — # content arrives when c0re writes and reloads, which it does on # every topology change. `f` = create-if-absent, never overwrite. # # An empty htpasswd causes all auth checks to return 401 (no valid # credentials), which is the correct no-users behaviour. "f /var/lib/hive-gateway/conf/gateway.htpasswd 0644 hive-core hive-core - -" ]; # The host asks the hive's own resolver, at the BRIDGE IP. # # Every container inherits a COPY of this host's `/etc/resolv.conf` # at start (`nixos-containers.nix`: `cp --remove-destination`, one # shot, not a bind-mount) — so whatever address is written here is # the address every container will try, in its own netns. # # 🚨 That is why this is the bridge IP and not `127.0.0.1`, and the # distinction is load-bearing rather than stylistic: # # value host host-netns containers bridged containers # 127.0.0.1 ok ok THEIR OWN loopback # bridge IP ok ok ok # # dnsmasq binds both `lo` and the bridge (./dnsmasq.nix), so the # bridge IP is reachable from the host too — it is the only value # correct on both sides of a netns boundary. `resolveLocalQueries` # publishes loopback by default, hence both overrides here; the # flag stays on for its `resolv-file` plumbing, which is what keeps # dnsmasq's own upstreams out of the file we are pointing at it. # # Cost, stated because it is real: the host's DNS now depends on # dnsmasq being up. Every container already did. networking.nameservers = lib.mkForce [ networkCfg.bridgeIp ]; networking.resolvconf.useLocalResolver = lib.mkForce false; # ACME (Let's Encrypt) integration. nginx vhosts set # `enableACME = true` via the vhost builder; this provides the # shared ACME config (acceptTerms + email). security.acme = lib.mkIf cfg.tls.acme.enable { acceptTerms = true; defaults.email = cfg.tls.acme.email; }; # Import the hive-CA leaf into nginx's state dir before nginx starts. # # 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It # does TWO jobs: # # (1) It re-modes the leaf. `hive-tls-ca` writes the key 0600 # root:root; nginx's pre-start `nginx -t` runs as the nginx # *user*, so a 0600 key fails the config test with # `BIO_new_file() … Permission denied` and blocks the unit — # hence the 0640 root:nginx copy below. # (2) It guarantees that **every cert path the nginx config names # exists** — which is what the swarm-services fallback at the # bottom of the script is for. nginx refuses to load a config # naming a missing cert file, so a leaf that never issues takes # the whole gateway down rather than one vhost; that has already # happened once and it took the forge, dashboard and matrix with # it. Removing this unit re-creates it exactly. # # nginx # `Requires=` this via `requiredBy`, so it refuses to start until # the copy succeeds. ALWAYS runs (no ConditionPathExists) and is # idempotent — necessary to reconcile broken state from prior # failed boots (a 0700 dir from a stale UMask, a truncated copy # from an interrupted oneshot, etc.). The leaf covers the bare # hive domain plus `forge.`, `matrix.` and `*.${hyperhiveDomain}` # so all sub-domains validate under the same cert + the hive CA. # See `docs/gateway.md` ("Self-signed TLS"). systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned { description = "Import host-generated TLS leaf for hive-gateway"; wantedBy = [ "multi-user.target" ]; before = [ "nginx.service" ]; requiredBy = [ "nginx.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; # Pin the journal identity (else it's the `script` store-path wrapper). SyslogIdentifier = "hive-gateway-self-signed-cert"; }; path = [ pkgs.coreutils ]; script = '' set -eu mkdir -p ${tlsDir} # 0755 on BOTH the cert dir and its parent so the nginx # user can traverse the full path. The parent # `/var/lib/hive-gateway` lands at 0700 by default (systemd # StateDirectory / mkdir umask depending on which service # created it first), which on its own blocks traversal. # Re-applied every boot in case a prior run left a tighter # mode behind. chmod 0755 ${builtins.dirOf tlsDir} chmod 0755 ${tlsDir} # Copy the leaf in. `install` writes atomically with the # target mode; runs as root so the 0600 root:root key written # by hive-tls-ca is readable. Key ends up root:nginx 0640 so # nginx-pre-start # (which runs `nginx -t` as the nginx user, not root) can # read it — a 0600 root:root key passes the master load but # fails the pre-start config test with `BIO_new_file() … # Permission denied`, blocking the unit. Cert is world-read. install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${tlsCert} install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${tlsKey} # The swarm-services leaf, when this host issues one. It is # a separate pair rather than more SANs on the one above # because no hive CA can sign these names — each is # constrained to its own hive's domain and the service # names are siblings of it. # # Absent is a normal state, not a failure: the leaf exists # only where the swarm CA is autoconfigured, and issuance # can also fail on a host that wants one. # # ⚠️ When it is absent the HIVE leaf goes to this path # anyway, and that fallback is load-bearing rather than # tidy. nginx refuses to load a config naming a cert file # that does not exist — `cannot load certificate … no such # file` fails the pre-start test, so the vhost does not # degrade, the ENTIRE gateway dies and takes the forge, the # dashboard and matrix with it. Serving the hive leaf on a # swarm-service name is a name mismatch: browsers warn, # strict clients refuse, everything else keeps working, and # the operator gets a bad cert instead of no hive. # # Measured, not theorised: this exact path took pr1ma's # gateway down when the services sub-CA failed to issue. if [ -s ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ]; then install -m 0644 ${config.services.hyperhive.tls.stateDir}/swarm-services.pem ${svcCert} install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/swarm-services-key.pem ${svcKey} else echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2 install -m 0644 ${config.services.hyperhive.tls.stateDir}/gateway.pem ${svcCert} install -m 0640 -g nginx ${config.services.hyperhive.tls.stateDir}/gateway-key.pem ${svcKey} fi ''; }; # nginx reload is triggered from the HOST side by hive-c0re # after each agents.conf write, through hive-priv (c0re is # unprivileged and cannot act on a system unit). # # It stays an explicit trigger rather than a systemd path unit # watching the file: the write and the reload belong in one causal # chain c0re can retry and report on (see RELOAD_PENDING), not two # independent units racing on an inotify event. services.nginx = { enable = true; recommendedProxySettings = true; recommendedTlsSettings = true; recommendedGzipSettings = true; recommendedOptimisation = true; inherit (nginxTree) virtualHosts; }; # ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its # absence is load-bearing rather than an omission. # # It used to be here, to let nginx traverse `/var/lib/hyperhive` and # reach the config fragments that lived inside: `hive-c0re.service` # declares `StateDirectory = "hyperhive"` with `StateDirectoryMode = # "0750"` owned by `hive-core`, and systemd re-applies that on every # c0re start — beating this module's own tmpfiles rule for the same # path. Without the group, nginx failed its config test and never # started (`nginx: [emerg] open() ".../agents.conf" failed (13: # Permission denied)`), taking the gateway and every hive domain # behind it down. # # The group fixed the symptom and paid for it: it also gave nginx # read access to everything ELSE group-readable under that dir, # including the broker sqlite — i.e. every message between every # agent, reachable by the process whose entire job is parsing # untrusted network input. Moving the fragments to the gateway's own # dir removes the need and the exposure together. Re-adding this line # would restore both. # dnsmasq is a host service alongside nginx, so it reads the host's # /etc/resolv.conf directly and picks up network changes as they # happen — no copy to keep in sync. services.dnsmasq = import ./dnsmasq.nix { inherit lib cfg networkCfg hyperhiveDomain ; }; networking.firewall = lib.mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port # The gateway always terminates TLS (self-signed floor), so # `httpsPort` is always opened alongside the plain-http `port`. cfg.httpsPort ]; }; # `/etc/hosts` entries for local dev — the bare hive domain plus # every name a service module contributed. See `docs/gateway.md` # ("Local dev"). # # This used to restate the per-service list a THIRD time (after the # vhosts and the dnsmasq records), with its own copy of each # service's guard. It is the same question — "which names does this # host answer for" — so it reads the same answer; a service added # later lands here with no edit, and cannot land here with a # different condition than it used for DNS. networking.hosts = lib.mkIf cfg.localHostsEntry { "127.0.0.1" = lib.unique ([ hyperhiveDomain ] ++ cfg.localNames); }; }; }