diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 453e7cca..3046f46c 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -33,9 +33,8 @@ in # gateway_nginx.rs uses this to generate split location # blocks in agents.conf — static HTML/CSS/JS served from the # nix store directly; dynamic API paths still proxied to the - # agent daemon. The nix store is shared across nspawn - # containers, so this path is reachable from inside the - # gateway container's nginx. + # agent daemon. nginx runs on the host, which is where this + # store path already is, so it is reachable as written. HIVE_AGENT_FRONTEND_DIR = "${cfg.servedFrontend}/agent"; # Path to the static runtime asset tree (branding + claude # prompts). `hive_sh4re::assets::*` reads paths underneath. diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index 0b76cef0..f8122f13 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -26,9 +26,7 @@ let forgeCfg = config.services.hyperhive.swarm.forge; networkCfg = config.services.hyperhive.network; - # Dashboard SPA dist, static-served by nginx. Read in OUTER scope so - # `config` is the host's (inside the container block it'd be the - # container's). + # Dashboard SPA dist, static-served by nginx. dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard"; # Full hyperhive-themed Swagger UI dist — nginx serves this whole @@ -157,32 +155,6 @@ in "f /var/lib/hive-gateway/conf/gateway.htpasswd 0644 hive-core hive-core - -" ]; - # ⚠️ REMOVED WITH THE CONTAINER, and each one was a workaround for the - # boundary rather than a thing nginx or dnsmasq needed: - # - # - `privateNetwork = false` — the container already shared the host - # netns, which is why nginx bound host ports and `localhost` - # upstreams reached hive-c0re. On the host that is simply true. - # - `additionalCapabilities = [ "CAP_NET_ADMIN" ]` — dnsmasq refuses - # to start with a `dhcp-range` unless it holds NET_ADMIN, and - # nspawn's bounding set dropped it for a host-netns container. - # Host root has it. - # - `networking.firewall.enable = false` — a container sharing the - # host netns would run *its* firewall.service against the HOST - # ruleset, flushing nixos-fw and deleting the nixos-nat-* chains on - # every boot. With one machine there is one firewall (below). - # - `networking.resolvconf.enable = false` + the `hive-gateway-resolv` - # path/service pair — the container's /etc/resolv.conf was a - # one-shot copy frozen at start, so a host network change left - # dnsmasq forwarding to a resolver that was gone. The whole - # watch-copy-reload machine existed to bridge two files. There is - # now one. - # - three bind mounts — /run/hive-agent, /run/hive-state, and either - # /run/hive-tls (operator cert) or /run/hive-ca (self-signed); - # those last two are mutually exclusive mkIfs, so it was never - # four. All plain host paths now. - # - # See `docs/network.md::Resolver behaviour` for the resolver history. # ACME (Let's Encrypt) integration. nginx vhosts set # `enableACME = true` via the vhost builder; this provides the # shared ACME config (acceptTerms + email). @@ -193,15 +165,14 @@ in # Import the hive-CA leaf into nginx's state dir before nginx starts. # - # 🚨 THIS LOOKS LIKE A LEFTOVER OF THE CONTAINER AND IS NOT — do not - # "simplify" it into pointing nginx at the CA dir. It does TWO jobs: + # 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It + # does TWO jobs: # # (1) It re-modes the leaf. `hive-tls-ca` writes the key 0600 # root:root; nginx's pre-start `nginx -t` runs as the nginx # *user*, so a 0600 key fails the config test with # `BIO_new_file() … Permission denied` and blocks the unit — - # hence the 0640 root:nginx copy below. That is a file-mode fact, - # not a namespace one, and it did not go away with the container. + # hence the 0640 root:nginx copy below. # (2) It guarantees that **every cert path the nginx config names # exists** — which is what the swarm-services fallback at the # bottom of the script is for. nginx refuses to load a config @@ -243,10 +214,10 @@ in # mode behind. chmod 0755 ${builtins.dirOf tlsDir} chmod 0755 ${tlsDir} - # Copy the host leaf in. `install` writes atomically with the - # target mode; run as root (container root == host root, - # privateUsers=false) so the 0600 root:root host key is - # readable. Key ends up root:nginx 0640 so nginx-pre-start + # Copy the leaf in. `install` writes atomically with the + # target mode; runs as root so the 0600 root:root key written + # by hive-tls-ca is readable. Key ends up root:nginx 0640 so + # nginx-pre-start # (which runs `nginx -t` as the nginx user, not root) can # read it — a 0600 root:root key passes the master load but # fails the pre-start config test with `BIO_new_file() … diff --git a/nix/host-modules/hive-gateway/dnsmasq.nix b/nix/host-modules/hive-gateway/dnsmasq.nix index 42912679..d5096188 100644 --- a/nix/host-modules/hive-gateway/dnsmasq.nix +++ b/nix/host-modules/hive-gateway/dnsmasq.nix @@ -1,10 +1,10 @@ -# Hive-internal DNS resolver + DHCP, co-located in the gateway -# container — single front-door for both DNS and HTTP, saves a -# sibling container. Listens on the bridge interface from +# Hive-internal DNS resolver + DHCP, running on the host alongside the +# gateway's nginx — single front-door for both DNS and HTTP, and no +# container of its own. Listens on the bridge interface from # `services.hyperhive.network`; authoritative for the hive domain + # sub-domains, forwards everything else upstream. Returns the -# `services.dnsmasq` value for the container config (see -# ./default.nix); the DHCP pool bounds are computed by hive-network. +# `services.dnsmasq` value (see ./default.nix); the DHCP pool bounds +# are computed by hive-network. { lib, networkCfg, @@ -68,14 +68,10 @@ dhcp-range = "${networkCfg.dhcpRangeStart},${networkCfg.dhcpRangeEnd},1h"; dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; # No explicit upstream: non-hive queries follow dnsmasq's - # resolv.conf default — the gateway container's `/etc/resolv.conf`, - # which nixos-container copies from the host at every start, so the - # hive always uses the host's resolvers. resolvconf is disabled in - # the container (see ./default.nix) so nothing regenerates that - # copy; the host-side `hive-gateway-resolv` path unit (also in - # ./default.nix) pushes in a fresh copy and reloads dnsmasq whenever - # the host's resolvers change, so the copy can't go stale under a - # network switch. Deliberately no fallback `server=`: dnsmasq queries + # resolv.conf default — the host's own `/etc/resolv.conf`, so the + # hive always uses the host's resolvers and follows them live with + # no copy to go stale. Deliberately no fallback + # `server=`: dnsmasq queries # all known upstreams in parallel, so a hardcoded public resolver # would take a share of *normal* traffic, not just fill in when the # host file is empty. diff --git a/nix/host-modules/hive-gateway/options.nix b/nix/host-modules/hive-gateway/options.nix index 30c38227..ecdcc5c3 100644 --- a/nix/host-modules/hive-gateway/options.nix +++ b/nix/host-modules/hive-gateway/options.nix @@ -28,10 +28,10 @@ in example = 8080; description = '' TCP port the gateway listens on. Default 80 (canonical web - port). nginx inside the container binds <1024 because the - container's init runs as root; if 80 is already taken on the - host (existing nginx, traefik, etc.) override to an unused - port like 8080 or move the conflicting service. + port). nginx runs on the host as root, so it can bind <1024; + if 80 is already taken on the host (existing nginx, traefik, + etc.) override to an unused port like 8080 or move the + conflicting service. ''; }; @@ -40,8 +40,8 @@ in default = "127.0.0.1"; description = '' Host the gateway proxies non-static requests to. Defaults to - `127.0.0.1` because the gateway container shares the host - netns, so loopback resolves directly to hive-c0re. + `127.0.0.1` because nginx runs on the host itself, so loopback + resolves directly to hive-c0re. ''; }; @@ -180,11 +180,11 @@ in default = false; example = true; description = '' - Let nginx inside the gateway container obtain and renew TLS - certificates automatically via ACME (Let's Encrypt). When - enabled, each vhost calls out to Let's Encrypt using the - HTTP-01 challenge on `port` (default 80) and stores certs - inside the gateway container's persistent state dir. + Let the gateway's nginx obtain and renew TLS certificates + automatically via ACME (Let's Encrypt). When enabled, each + vhost calls out to Let's Encrypt using the HTTP-01 + challenge on `port` (default 80) and stores certs in the + gateway's persistent state dir on the host. Requirements: - `services.hyperhive.domain` must be set and publicly diff --git a/nix/host-modules/hive-gateway/vhosts.nix b/nix/host-modules/hive-gateway/vhosts.nix index 969d8c3a..28acb246 100644 --- a/nix/host-modules/hive-gateway/vhosts.nix +++ b/nix/host-modules/hive-gateway/vhosts.nix @@ -1,9 +1,9 @@ -# nginx virtual-host tree for the gateway container: the `_` default -# server (dashboard, per-agent routing, matrix discovery), the forge -# and matrix sub-domain vhosts, and the Accept-header SPA map for the -# matrix GUI. Pure function — called from ./default.nix inside the -# container config with the outer-scope config values as arguments; -# returns `{ virtualHosts, appendHttpConfig }`. +# nginx virtual-host tree for the gateway: the `_` default server +# (dashboard, per-agent routing, matrix discovery), the forge, matrix +# and authelia sub-domain vhosts, and the Accept-header SPA map for the +# matrix GUI. Pure function — called from ./default.nix with the +# outer-scope config values as arguments; returns +# `{ virtualHosts, appendHttpConfig }`. { lib, cfg, # services.hyperhive.gateway diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index add0ba73..89780930 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -581,25 +581,22 @@ in ]; }; - # The matrix container's resolver is the dnsmasq that runs in the - # gateway container (bound at `bridgeIp`). Order the matrix - # container start after the gateway container so the resolver is up - # before tuwunel's first federation lookups. tuwunel boots fine - # without this — it configures the resolver from `/etc/resolv.conf` - # at startup and only queries on-demand (the boot failure this - # module guards against is an *empty* resolv.conf, a parse error, - # not a connectivity one) — so this is robustness, not a boot - # requirement. Soft `after` ordering (not `requires`) keeps the - # matrix container's lifecycle decoupled from the gateway's. The - # gateway always runs alongside hyperhive, so the gateway container - # unit always exists here. (Declarative `containers.` → - # `container@.service` — the nspawn template NixOS generates.) + # The matrix container's resolver is the hive's dnsmasq (bound at + # `bridgeIp`). Order the matrix container start after it so the + # resolver is up before tuwunel's first federation lookups. tuwunel + # boots fine without this — it configures the resolver from + # `/etc/resolv.conf` at startup and only queries on-demand (the boot + # failure this module guards against is an *empty* resolv.conf, a + # parse error, not a connectivity one) — so this is robustness, not a + # boot requirement. Soft `after` ordering (not `requires`) keeps the + # matrix container's lifecycle decoupled from the resolver's. + # # `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also # sets `after`/`requires` (so the bound trust bundle exists before # nspawn wires the mount up), and two plain assignments to the same # unit would conflict rather than combine. systemd.services."container@hive-matrix" = lib.mkMerge [ - { after = [ "container@hive-gateway.service" ]; } + { after = [ "dnsmasq.service" ]; } caTrust.containerOrdering ]; }; diff --git a/nix/host-modules/hive-network.nix b/nix/host-modules/hive-network.nix index 4a1702e8..6d4bc871 100644 --- a/nix/host-modules/hive-network.nix +++ b/nix/host-modules/hive-network.nix @@ -46,9 +46,8 @@ in '') (lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "upstreamDns" ] '' The hive resolver always follows the host's resolvers now - (dnsmasq reads the gateway container's /etc/resolv.conf, the - host copy made at container start). Configure upstream DNS on - the host itself instead. + (dnsmasq runs on the host and reads its /etc/resolv.conf + directly). Configure upstream DNS on the host itself instead. '') ]; @@ -71,8 +70,8 @@ in example = "172.30.0.1"; description = '' IPv4 address assigned to the bridge interface on the host - side. Agents use this address as their DNS server (dnsmasq - in the gateway container binds here). Default `10.42.0.1` + side. Agents use this address as their DNS server (the hive's + dnsmasq binds here). Default `10.42.0.1` is in RFC 1918 space and unlikely to clash with operator's existing setup; override if a different range is already in use. @@ -123,7 +122,7 @@ in # before broadcast). All containers — agents and service # containers alike — receive their IPs dynamically from this pool; # there are no hash-derived static assignments. Consumed by the - # dnsmasq that runs in the gateway container (hive-gateway module). + # hive's dnsmasq (hive-gateway module, host-side). dhcpRangeStart = lib.mkOption { type = lib.types.str; internal = true; @@ -212,7 +211,7 @@ in # is unconditional now). networking.bridges.${cfg.bridgeName}.interfaces = [ ]; - # Bridge IP — dnsmasq (in the gateway container) binds here. + # Bridge IP — the hive's dnsmasq binds here. networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [ { address = cfg.bridgeIp; diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 5d6a6db5..2199b169 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -307,17 +307,19 @@ in matrix.gatewayHost, authelia.domain}) or install the sub-CA. ''; - # Generate (and rotate) the hive CA + gateway leaf before the gateway - # container starts. Idempotent: the CA is created once and reused; the - # leaf is re-signed on expiry under the same CA so the anchor is stable. + # Generate (and rotate) the hive CA + gateway leaf before anything + # serves it. Idempotent: the CA is created once and reused; the leaf + # is re-signed on expiry under the same CA so the anchor is stable. systemd.services.hive-tls-ca = { description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)"; wantedBy = [ "multi-user.target" ]; - # Gateway nginx reads the leaf from the bind-mount, so the cert must - # exist before the container starts. Declarative nixos-containers are - # instances of the `container@.service` template. - before = [ "container@hive-gateway.service" ]; - requiredBy = [ "container@hive-gateway.service" ]; + # The consumer is `hive-gateway-self-signed-cert`, which copies the + # leaf into the gateway's state dir at the mode nginx can read, and + # which nginx in turn `Requires=`. So this must run first or that + # copy fails under `set -eu` and takes nginx down with it — order + # against the unit that reads the file. + before = [ "hive-gateway-self-signed-cert.service" ]; + requiredBy = [ "hive-gateway-self-signed-cert.service" ]; # The issuance below needs the swarm root key on disk, and (for the # services leaf) the services sub-CA it signs under. When this host # generates them (single-host swarm) both units must have run first; @@ -466,21 +468,19 @@ in # `hive-tls-ca` only re-signs at service activation (boot/rebuild); a # long-uptime host would otherwise let a 30-day leaf lapse silently. # This service re-signs the leaf directly (not by bouncing hive-tls-ca) - # and propagates the new leaf into the running gateway container when - # the file actually changed. + # and propagates the new leaf to nginx when the file actually changed. # - # Propagation mechanism: nginx in the gateway container serves a *copy* - # of the leaf written by `hive-gateway-self-signed-cert` (which runs at - # container start). A host-side `systemctl -M hive-gateway` call - # triggers the re-import + reload, mirroring how hive-c0re reloads the - # gateway after each agents.conf write. A path unit *inside* the - # container cannot do this: IN_MOVED_TO from an atomic rename on the - # host does not propagate across the nspawn mount-namespace boundary. + # Propagation mechanism: nginx serves a *copy* of the leaf, written by + # `hive-gateway-self-signed-cert` at the mode nginx can read. Re-signing + # the source therefore changes nothing on its own — the copy has to be + # remade and nginx reloaded, which is what the two calls below do. # - # `|| true` on propagation so a stopped gateway never fails the unit — - # its next boot will import the already-rotated leaf anyway. + # ⚠️ Neither call is `|| true`: a swallowed propagation failure means + # the leaf rotates on disk while nginx keeps serving the old copy + # until it expires, with this unit reporting success the whole time. + # A failure here must fail the timer. systemd.services.hive-tls-resign = { - description = "Re-sign the gateway TLS leaf and propagate it into the gateway container"; + description = "Re-sign the gateway TLS leaf and reload nginx"; # hive-tls-ca must have run first so the CA key exists before we try # to re-sign under it. On first boot `Persistent=true` on the weekly # timer fires immediately; without this ordering the resign could race @@ -534,9 +534,9 @@ in after="$(sha256sum "$leaf" "$svcleaf" 2>/dev/null || true)" if [ "$before" != "$after" ]; then - echo "gateway leaf rotated — propagating into hive-gateway" - systemctl -M hive-gateway restart hive-gateway-self-signed-cert.service || true - systemctl -M hive-gateway reload nginx.service || true + echo "gateway leaf rotated — re-importing and reloading nginx" + systemctl restart hive-gateway-self-signed-cert.service + systemctl reload nginx.service else echo "gateway leaf unchanged (already up to date)" fi diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index a848c448..8cf0280d 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -4,9 +4,9 @@ # the agents on one host, this owns what is true across hives. # # Serves HTTP over a unix socket rather than a TCP port: the gateway's -# nginx is the only intended client, it reaches the socket through a -# bind-mount, and a socket that is never bound to an address cannot be -# reached from off-host by mistake. +# nginx is the only intended client, it runs on this same host and so +# reaches the socket by path, and a socket that is never bound to an +# address cannot be reached from off-host by mistake. { pkgs, lib, @@ -103,13 +103,13 @@ in the socket itself is `0666` (nginx runs as another user, and `connect(2)` needs write), exactly as hive-c0re publishes the per-agent sockets. What keeps that safe is that the directory holds - one socket and is bind-mounted into one container. Moving this path - under a directory that carries anything else — `/run/hyperhive`, - which holds the host admin socket, above all — hands whatever else - lives there to every consumer that mounts it. + exactly one socket and is traverse-only (`0751`) for everyone else. + Moving this path under a directory that carries anything else — + `/run/hyperhive`, which holds the host admin socket, above all — + exposes whatever else lives there to the same reachability. - Changing this therefore means re-checking the gateway bind-mount, - not just the daemon. + Changing this therefore means re-checking what else lives in the + new directory, not just the daemon. ''; }; }; @@ -147,9 +147,9 @@ in # the socket path without being able to list the directory. Same # shape (and same reason) as hive-c0re's runtime dir. RuntimeDirectoryMode = "0751"; - # Preserved across restarts so the bind-mount source never vanishes - # from under a running gateway container. The daemon unlinks a stale - # socket on start, which is what makes preservation safe. + # Preserved across restarts so the path never vanishes from under a + # running nginx. The daemon unlinks a stale socket on start, which + # is what makes preservation safe. RuntimeDirectoryPreserve = "yes"; StateDirectory = "swarm-controller";