# This host's nginx fronts the hyperhive web surfaces on it — dashboard, # per-agent UIs (sub-path), forge + matrix (sub-domain), .well-known # delegations — plus the hive-internal dnsmasq resolver. Both run on # this host, next to hive-c0re: nginx binds :80/:443 and dnsmasq # answers on the hive bridge, so neither can be confined to a network # namespace of its own. # Full vhost map + discovery flow + design rationale in # `docs/networking/gateway.md`. Layout: ./options.nix (option declarations), # ./vhosts.nix (the catch-all `_` vhost), ./declared-vhosts.nix # (renders the vhosts service modules declare under `gateway.vhosts`), ./error-pages.nix # (styled static pages), ./dns.nix + ./dnsmasq.nix (the resolver, which # has an enable of its own because a host can need hive names to resolve # without serving a vhost). { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.gateway; hyperhiveDomain = config.services.hyperhive.domain; # Derived once in ../swarm.nix; the vhosts that get the swarm-services # cert are exactly the names that cert is issued for, so both read the # same list rather than each deciding what "a swarm service" means. swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains; tlsStateDir = config.services.hyperhive.deploy.hive-controller.tls.stateDir; # The hive leaf exists only where a hive runs; ../hive-tls.nix issues it # under that same condition. runsHive = config.services.hyperhive.deploy.hive-controller.enable; # Whether the cert import `Requires=` the services leaf's issuing unit # (../hive-tls.nix decides). Where it does, a missing leaf fails the # import rather than anything standing in for it. svcLeafRequired = lib.elem "hive-gateway-self-signed-cert.service" ( config.systemd.services.swarm-services-cert.requiredBy or [ ] ); svcLeafMissing = '' echo "no swarm-services leaf at ${tlsStateDir}/swarm-services.pem, and this gateway serves swarm names only with it." >&2 echo "swarm-services-cert.service issues it from the secret store's pki mount; its journal says why it has not." >&2 exit 1''; # Every vhost claiming `default_server`, ours and the operator's # alike. Computed once so the assertion below and the message it # prints cannot disagree about what they found. defaultVhosts = lib.filter (n: config.services.nginx.virtualHosts.${n}.default or false) ( lib.attrNames config.services.nginx.virtualHosts ); # Self-signed TLS is the implicit floor: when neither an operator cert # (`tls.certDir`) nor ACME (`tls.acme.enable`) is configured, the gateway # serves a leaf the secret store issues (see hive-tls.nix). There is no # explicit toggle and no http-only mode — matrix discovery requires https, # so the gateway always terminates TLS. # `cfg.useSelfSigned` (options.nix) is the derived single source of truth. useSelfSigned = cfg.useSelfSigned; # nginx's own state dir, fixed at `/var/lib/hive-gateway` independent of # this module's own path: it holds the imported leaf across reboots, and # a different path would strand every existing hive's certs. tlsDir = "/var/lib/hive-gateway/tls"; # TLS cert + key paths. # - self-signed (default): the hive leaf the store issues (the # swarm-services leaf on a host with no hive), imported into the state # dir by `hive-gateway-self-signed-cert` below. # - tls.certDir set: the operator's own cert dir, read directly. tlsCert = if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.certName}" else "${tlsDir}/cert.pem"; tlsKey = if cfg.tls.certDir != null then "${cfg.tls.certDir}/${cfg.tls.keyName}" else "${tlsDir}/key.pem"; # The swarm-services pair, used only by the vhosts whose names the # hive leaf cannot carry. Self-signed mode only: with an operator cert # or ACME the operator owns every name and there is no second issuer. svcCert = "${tlsDir}/swarm-services.pem"; svcKey = "${tlsDir}/swarm-services-key.pem"; # Styled static error pages, reached through the published kit by any # service module that aims an `error_page` at one. errorPages = import ./error-pages.nix { inherit pkgs; }; # The vhost construction kit (listen set / per-name TLS attrs / # security headers / error pages). Computed here and published as # `cfg.lib` below, which ./declared-vhosts.nix and every service module # read, so the tree the gateway renders and the kit a service module # gets are the same object by construction. vhostLib = import ./vhost-lib.nix { inherit lib cfg tlsCert tlsKey svcCert svcKey swarmServiceDomains errorPages ; # Same path `lib/hive-ca-trust.nix` publishes as `caHostPath`. Named # from the tls option here rather than importing that helper: this # is the HOST's own bundle, and the helper's job is assembling a # per-container copy. caBundle = "${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/trust-bundle.pem"; }; in { imports = [ ./options.nix ./dns.nix ./declared-vhosts.nix ]; config = lib.mkIf cfg.enable { # Publish the kit. Defined here rather than as an option `default` # so it stays a plain value computed once from resolved cert paths — # `tlsFor` closes over `svcCert`/`svcKey`, which are derived in this # file's `let` and are not option surface. services.hyperhive.gateway.lib = vhostLib; # The gateway's own vhost, rendered the way every service's are. services.hyperhive.gateway.vhosts = import ./vhosts.nix; assertions = [ { assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null); message = '' services.hyperhive.gateway.tls.acme.enable = true and tls.certDir are mutually exclusive. Pick one TLS mode. ''; } { assertion = !cfg.tls.acme.enable || cfg.tls.acme.email != null; message = '' services.hyperhive.gateway.tls.acme.enable = true requires services.hyperhive.gateway.tls.acme.email to be set — Let's Encrypt needs a contact address for the ACME account. ''; } { # Two modules claiming one hostname is a real possibility now # that each service contributes its own name, and dnsmasq would # not complain: duplicate `address=` rules resolve by precedence, # so the loser simply stops being served with no error anywhere. # Fail the build instead — a name is owned by exactly one module. assertion = lib.length (lib.unique cfg.localNames) == lib.length cfg.localNames; message = '' services.hyperhive.gateway.localNames contains a duplicate: ${lib.concatStringsSep ", " ( lib.unique (lib.filter (n: lib.count (m: m == n) cfg.localNames > 1) cfg.localNames) )} Each hostname the hive resolver answers for is contributed by exactly one module. Two modules claiming the same name means two services believe they serve it — resolve which one does rather than letting dnsmasq pick. ''; } { # nginx refuses to start with two `default_server`s on one # address ("a duplicate default server for 0.0.0.0:", # exit 1) and nixpkgs asserts nothing — `vhost.default` is a # plain bool rendered straight into the listen line. So without # this, an operator adding their own default vhost gets a # gateway that fails its config test at rebuild time, which # takes the forge, dashboard, matrix and swarm UI with it, and # reports a port rather than a cause. # # Not covered by our `mkDefault`: an operator's own vhost is a # different option path, so nothing merges and nothing # conflicts — priority only helps someone who already knows # ours exists. Fail at eval and name both, so the fix # (`services.nginx.virtualHosts..default = false`) is # readable from the error. assertion = lib.length defaultVhosts <= 1; message = '' More than one nginx virtual host is marked `default = true`: ${lib.concatStringsSep ", " defaultVhosts} nginx allows exactly one default server per listen address and refuses to start otherwise, so this would fail at service start rather than here — taking every site behind the gateway down with it. The gateway's own catch-all (`_`, which returns 444) is set with `mkDefault`, so to make yours the default server turn ours off explicitly: services.nginx.virtualHosts."_".default = false; ''; } ]; # Ensure the gateway state dirs exist at host boot, before anything # reads or writes them: these rules cover the fresh-boot window # before c0re has run, and pin owner + mode rather than leaving it # to whoever creates the path first. # # ⚠️ There is deliberately NO rule for /var/lib/hyperhive here. # `hive-c0re.service` sets `StateDirectory = "hyperhive"` with # `StateDirectoryMode = "0750"`, which systemd re-applies on every # start, so a tmpfiles rule for the same path could never win: two # mechanisms owning one path, the loser still reading as a guarantee. # c0re's own unit owns that dir; this module has no opinion about it. systemd.tmpfiles.rules = [ # The gateway's own config dir — NOT under /var/lib/hyperhive. c0re # writes here, nginx reads here, and neither needs any access to the # other's tree: no shared parent to traverse means no group # membership handing nginx c0re's broker db and everything else # beside it. Sibling of `tls/`, which already lived under this root. # # Owned by hive-core because c0re is the writer; 0755 so the # unprivileged nginx user can traverse and read. This is now the # ONLY declaration of these paths' modes — nothing re-applies a # different owner on top the way `StateDirectory=` does for # /var/lib/hyperhive. "d /var/lib/hive-gateway 0755 root root - -" "d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -" # Pre-created so nginx's config test can open it before c0re has # ever written: nginx names it, and an `include` of a missing file is # a fatal config error, not an empty one. tmpfiles runs before # services, which is the whole ordering guarantee — content arrives # when c0re writes and reloads, which it does on every topology # change. `f` = create-if-absent, never overwrite. "f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n" # Nothing reads or writes this path. A copy left on an existing host # holds bcrypt password hashes at 0644; tmpfiles' `--remove` pass, on # switch and at boot, deletes it. "r /var/lib/hive-gateway/conf/gateway.htpasswd" ]; # ACME (Let's Encrypt) integration. nginx vhosts set # `enableACME = true` via the vhost builder; this provides the # shared ACME config (acceptTerms + email). security.acme = lib.mkIf cfg.tls.acme.enable { acceptTerms = true; defaults.email = cfg.tls.acme.email; }; # Import the gateway leaves into nginx's state dir before nginx starts. # # 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It does # TWO jobs: it re-modes the leaf to the 0640 root:nginx copy below (a # 0600 key fails nginx's pre-start config test and blocks the unit), # and it puts a file at every cert path the config names or fails: one # nginx cannot find takes the WHOLE gateway down, not one vhost. Both # mechanisms, and what it cost the time this was # collapsed: `docs/networking/gateway.md::Self-signed TLS (default)`. # # ALWAYS runs (no ConditionPathExists) and is idempotent, because it # has to reconcile broken state from earlier failed boots — a 0700 dir # from a stale UMask, a truncated copy from an interrupted oneshot. # nginx `Requires=` it via `requiredBy`, so nginx refuses to start # until the copy succeeds. systemd.services.hive-gateway-self-signed-cert = lib.mkIf useSelfSigned { description = "Import host-generated TLS leaf for hive-gateway"; wantedBy = [ "multi-user.target" ]; before = [ "nginx.service" ]; requiredBy = [ "nginx.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; # Pin the journal identity (else it's the `script` store-path wrapper). SyslogIdentifier = "hive-gateway-self-signed-cert"; }; # openssl for the throwaway pair a host without a hive starts on. path = [ pkgs.coreutils ] ++ lib.optional (!runsHive) pkgs.openssl; script = '' set -eu mkdir -p ${tlsDir} # 0755 on BOTH the cert dir and its parent so the nginx # user can traverse the full path. The parent # `/var/lib/hive-gateway` lands at 0700 by default (systemd # StateDirectory / mkdir umask depending on which service # created it first), which on its own blocks traversal. # Re-applied every boot in case a prior run left a tighter # mode behind. chmod 0755 ${builtins.dirOf tlsDir} chmod 0755 ${tlsDir} # Copy the leaf in. `install` writes atomically with the # target mode; runs as root so the 0600 root:root key written # by hive-tls-resign is readable. Key ends up root:nginx 0640 so # nginx-pre-start # (which runs `nginx -t` as the nginx user, not root) can # read it — a 0600 root:root key passes the master load but # fails the pre-start config test with `BIO_new_file() … # Permission denied`, blocking the unit. Cert is world-read. ${ if runsHive then '' install -m 0644 ${tlsStateDir}/gateway.pem ${tlsCert} install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey} # The swarm-services leaf, when this host issues one. It is # a separate pair rather than more SANs on the one above # because the hive leaf's role allows only its own hive's # domain, and the service names are siblings of it. # # Where the gateway waits for it, a missing leaf fails this # unit and nginx with it. Elsewhere — no swarm name here, or # only the store's UI, whose host must start its passthrough # with the store sealed — the hive leaf stands in, because # nginx refuses a config naming a missing cert file, until # `swarm-services-cert` issues one and re-runs this unit. if [ -s ${tlsStateDir}/swarm-services.pem ]; then install -m 0644 ${tlsStateDir}/swarm-services.pem ${svcCert} install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${svcKey} else ${ if svcLeafRequired then svcLeafMissing else '' echo "no swarm-services leaf yet — serving the hive leaf on those names until swarm-services-cert.service issues one" >&2 install -m 0644 ${tlsStateDir}/gateway.pem ${svcCert} install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${svcKey}'' } fi'' else '' # No hive here, so no hive leaf: every vhost, the catch-all # included, serves the swarm-services leaf. Where the gateway # waits for it, a missing leaf fails this unit. A store-only # host does not wait, so its stream passthrough comes up with # the store sealed: a throwaway self-signed pair stands in, # because nginx refuses a config naming a missing cert file, # until `swarm-services-cert` issues the leaf and re-runs this # unit. if [ -s ${tlsStateDir}/swarm-services.pem ] && [ -s ${tlsStateDir}/swarm-services-key.pem ]; then cert=${tlsStateDir}/swarm-services.pem key=${tlsStateDir}/swarm-services-key.pem else ${ if svcLeafRequired then svcLeafMissing else '' echo "no swarm-services leaf yet — serving a throwaway self-signed certificate until it lands" >&2 cert=${tlsDir}/fallback.pem key=${tlsDir}/fallback-key.pem (umask 077; openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \ -subj "/CN=hyperhive-gateway-fallback" -keyout "$key" -out "$cert")'' } fi install -m 0644 "$cert" ${tlsCert} install -m 0640 -g nginx "$key" ${tlsKey} install -m 0644 "$cert" ${svcCert} install -m 0640 -g nginx "$key" ${svcKey}'' } ''; }; # Nothing here reloads nginx: hive-c0re drives that from the host side # through hive-priv after each agents.conf write, deliberately as an # explicit trigger rather than a path unit. Why, and what hive-priv # dispatches per unit state: # `docs/networking/gateway.md::Per-agent unix-socket upstream`. services.nginx = { enable = true; recommendedProxySettings = true; recommendedTlsSettings = true; recommendedGzipSettings = true; recommendedOptimisation = true; # Route the error log through syslog so each line keeps its own # severity. The module default is `stderr`, and systemd files that # whole stream at one priority regardless of what each line says — # so `journalctl -p err -u nginx` returns nothing while nginx is # logging `[error]` continuously. That failure is inverted rather # than merely missing: a severity filter is how you ask "is anything # wrong", and it answers "no" *because* the errors are misfiled. # # Transport only — no level suffix, so nginx keeps its default # `error` threshold and this changes where lines are filed, not # which lines exist. logError = "syslog:server=unix:/dev/log,nohostname,tag=nginx"; # Read by the hive collector's `nginx` receiver (../otel.nix). Its own # loopback server, outside the vhost tree, so it never answers on the # gateway's public listeners. appendHttpConfig = '' server { listen 127.0.0.1:${toString cfg.statusPort}; access_log off; location = /stub_status { stub_status; allow 127.0.0.1; deny all; } location / { return 404; } } ''; }; # ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its # absence is load-bearing rather than an omission. # # `hive-c0re.service` declares `StateDirectory = "hyperhive"` with # `StateDirectoryMode = "0750"` owned by `hive-core`, and systemd # re-applies that on every c0re start. The group would let nginx # traverse that dir, but it also gives nginx read access to everything # ELSE group-readable under it, including the broker sqlite — i.e. # every message between every agent, reachable by the process whose # entire job is parsing untrusted network input. The config fragments # nginx reads live in the gateway's own dir instead, so nginx needs no # access to c0re's tree. Adding this line would reopen that exposure. networking.firewall = lib.mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port # The gateway always terminates TLS (self-signed floor), so # `httpsPort` is always opened alongside the plain-http `port`. cfg.httpsPort ]; }; # `/etc/hosts` entries for local dev — the bare hive domain plus # every name a service module contributed. See `docs/networking/gateway.md` # ("Local dev"). # # This does not restate the per-service list (the vhosts and the # dnsmasq records already carry it). It is the same question — # "which names does this host answer for" — so it reads the same # answer; a service added later lands here with no edit, and cannot # land here with a different condition than it used for DNS. networking.hosts = lib.mkIf cfg.localHostsEntry { "127.0.0.1" = lib.unique ([ hyperhiveDomain ] ++ cfg.localNames); }; }; }