From d70b1e0a97289e115f6635524330f3539710dbea Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 12 Aug 2026 12:18:13 +0200 Subject: [PATCH 1/3] fix(3191): order gateway TLS against the units that read it, not a dead container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the gateway container retired two things that were still depended on: an ordering edge and a delivery path. - hive-tls-ca ordered itself before/requiredBy container@hive-gateway.service. That unit no longer exists, so the CA was sequenced against nothing while its real consumer, hive-gateway-self-signed-cert (which nginx Requires=), could win the race and fail its copy under set -eu, blocking nginx. - hive-tls-resign propagated a rotated leaf with `systemctl -M hive-gateway … || true`. The machine is gone, so both calls failed and both failures were swallowed: the unit logged "propagating" and exited 0 while nginx served the stale copy until it expired. Host units now, so no -M and no || true — a failed propagation fails the timer. - container@hive-matrix ordered after the gateway container to get the resolver up first. dnsmasq is a host service now, so it orders after dnsmasq.service, which is what the edge always meant. --- nix/host-modules/hive-matrix.nix | 33 ++++++++++-------- nix/host-modules/hive-tls.nix | 59 +++++++++++++++++++------------- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index add0ba73..d27441ae 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -581,25 +581,30 @@ in ]; }; - # The matrix container's resolver is the dnsmasq that runs in the - # gateway container (bound at `bridgeIp`). Order the matrix - # container start after the gateway container so the resolver is up - # before tuwunel's first federation lookups. tuwunel boots fine - # without this — it configures the resolver from `/etc/resolv.conf` - # at startup and only queries on-demand (the boot failure this - # module guards against is an *empty* resolv.conf, a parse error, - # not a connectivity one) — so this is robustness, not a boot - # requirement. Soft `after` ordering (not `requires`) keeps the - # matrix container's lifecycle decoupled from the gateway's. The - # gateway always runs alongside hyperhive, so the gateway container - # unit always exists here. (Declarative `containers.` → - # `container@.service` — the nspawn template NixOS generates.) + # The matrix container's resolver is the hive's dnsmasq (bound at + # `bridgeIp`). Order the matrix container start after it so the + # resolver is up before tuwunel's first federation lookups. tuwunel + # boots fine without this — it configures the resolver from + # `/etc/resolv.conf` at startup and only queries on-demand (the boot + # failure this module guards against is an *empty* resolv.conf, a + # parse error, not a connectivity one) — so this is robustness, not a + # boot requirement. Soft `after` ordering (not `requires`) keeps the + # matrix container's lifecycle decoupled from the resolver's. + # + # ⚠️ This named `container@hive-gateway.service` until the gateway + # moved onto the host: dnsmasq lived in that container, so ordering + # after the container was how you ordered after the resolver. The + # container is gone and dnsmasq is a plain host `services.dnsmasq`, + # so the ordering now names the resolver directly — which is what it + # always meant. Naming the *container* was already indirection; it + # just happened to be correct while the container existed. + # # `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also # sets `after`/`requires` (so the bound trust bundle exists before # nspawn wires the mount up), and two plain assignments to the same # unit would conflict rather than combine. systemd.services."container@hive-matrix" = lib.mkMerge [ - { after = [ "container@hive-gateway.service" ]; } + { after = [ "dnsmasq.service" ]; } caTrust.containerOrdering ]; }; diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 5d6a6db5..7f1715fe 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -307,17 +307,26 @@ in matrix.gatewayHost, authelia.domain}) or install the sub-CA. ''; - # Generate (and rotate) the hive CA + gateway leaf before the gateway - # container starts. Idempotent: the CA is created once and reused; the - # leaf is re-signed on expiry under the same CA so the anchor is stable. + # Generate (and rotate) the hive CA + gateway leaf before anything + # serves it. Idempotent: the CA is created once and reused; the leaf + # is re-signed on expiry under the same CA so the anchor is stable. systemd.services.hive-tls-ca = { description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)"; wantedBy = [ "multi-user.target" ]; - # Gateway nginx reads the leaf from the bind-mount, so the cert must - # exist before the container starts. Declarative nixos-containers are - # instances of the `container@.service` template. - before = [ "container@hive-gateway.service" ]; - requiredBy = [ "container@hive-gateway.service" ]; + # The consumer is `hive-gateway-self-signed-cert`, which copies the + # leaf into the gateway's state dir at the mode nginx can read, and + # which nginx in turn `Requires=`. So this must run first or that + # copy fails under `set -eu` and takes nginx down with it. + # + # ⚠️ This used to name `container@hive-gateway.service` — ordering + # ran through the container, because the copy happened *at container + # start*. Moving the gateway onto the host retired the container + # without retiring the dependency: both units became plain host + # units with nothing sequencing them, and the copy could win the + # race on a fast disk. Order against the unit that reads the file, + # not against the thing that used to host it. + before = [ "hive-gateway-self-signed-cert.service" ]; + requiredBy = [ "hive-gateway-self-signed-cert.service" ]; # The issuance below needs the swarm root key on disk, and (for the # services leaf) the services sub-CA it signs under. When this host # generates them (single-host swarm) both units must have run first; @@ -466,21 +475,25 @@ in # `hive-tls-ca` only re-signs at service activation (boot/rebuild); a # long-uptime host would otherwise let a 30-day leaf lapse silently. # This service re-signs the leaf directly (not by bouncing hive-tls-ca) - # and propagates the new leaf into the running gateway container when - # the file actually changed. + # and propagates the new leaf to nginx when the file actually changed. # - # Propagation mechanism: nginx in the gateway container serves a *copy* - # of the leaf written by `hive-gateway-self-signed-cert` (which runs at - # container start). A host-side `systemctl -M hive-gateway` call - # triggers the re-import + reload, mirroring how hive-c0re reloads the - # gateway after each agents.conf write. A path unit *inside* the - # container cannot do this: IN_MOVED_TO from an atomic rename on the - # host does not propagate across the nspawn mount-namespace boundary. + # Propagation mechanism: nginx serves a *copy* of the leaf, written by + # `hive-gateway-self-signed-cert` at the mode nginx can read. Re-signing + # the source therefore changes nothing on its own — the copy has to be + # remade and nginx reloaded, which is what the two calls below do. # - # `|| true` on propagation so a stopped gateway never fails the unit — - # its next boot will import the already-rotated leaf anyway. + # ⚠️ Both calls used to be `systemctl -M hive-gateway … || true`, from + # when nginx lived in a container: `-M` entered the machine, and the + # `|| true` covered a *stopped* container, which was a normal state + # ("its next boot imports the rotated leaf anyway"). The container is + # gone, so `-M hive-gateway` names a machine nothing creates — both + # calls failed, both failures were swallowed, and the unit logged + # "propagating" and exited 0 while nginx kept serving the stale copy + # until it expired. A host unit failing is NOT a normal state: no + # `-M`, and no `|| true` either, so a broken propagation fails the + # timer loudly instead of reporting success. systemd.services.hive-tls-resign = { - description = "Re-sign the gateway TLS leaf and propagate it into the gateway container"; + description = "Re-sign the gateway TLS leaf and reload nginx"; # hive-tls-ca must have run first so the CA key exists before we try # to re-sign under it. On first boot `Persistent=true` on the weekly # timer fires immediately; without this ordering the resign could race @@ -534,9 +547,9 @@ in after="$(sha256sum "$leaf" "$svcleaf" 2>/dev/null || true)" if [ "$before" != "$after" ]; then - echo "gateway leaf rotated — propagating into hive-gateway" - systemctl -M hive-gateway restart hive-gateway-self-signed-cert.service || true - systemctl -M hive-gateway reload nginx.service || true + echo "gateway leaf rotated — re-importing and reloading nginx" + systemctl restart hive-gateway-self-signed-cert.service + systemctl reload nginx.service else echo "gateway leaf unchanged (already up to date)" fi From f85724a63875d593e9ce3ac4edbc9ef2415d6f40 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 12 Aug 2026 12:20:28 +0200 Subject: [PATCH 2/3] docs(3191): the gateway's comments describe a host service, not a container Prose-only sweep of every remaining claim that nginx or dnsmasq lives in a container: the port comment (root in a container -> root on the host), upstreamHost's netns rationale, the ACME state dir, the store path reachability note, the vhost tree header, dnsmasq's resolv.conf paragraph (there is no copy and no path unit watching it any more), the two hive-network bridge comments, and swarm-controller's socket access-control note, which described a bind-mount that no longer exists. No behaviour change; all of it was describing a mechanism that was deleted. --- nix/host-modules/hive-c0re/environment.nix | 5 ++--- nix/host-modules/hive-gateway/default.nix | 12 +++++------ nix/host-modules/hive-gateway/dnsmasq.nix | 25 +++++++++++----------- nix/host-modules/hive-gateway/options.nix | 22 +++++++++---------- nix/host-modules/hive-gateway/vhosts.nix | 12 +++++------ nix/host-modules/hive-network.nix | 13 ++++++----- nix/host-modules/swarm-controller.nix | 24 ++++++++++----------- 7 files changed, 54 insertions(+), 59 deletions(-) diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 453e7cca..3046f46c 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -33,9 +33,8 @@ in # gateway_nginx.rs uses this to generate split location # blocks in agents.conf — static HTML/CSS/JS served from the # nix store directly; dynamic API paths still proxied to the - # agent daemon. The nix store is shared across nspawn - # containers, so this path is reachable from inside the - # gateway container's nginx. + # agent daemon. nginx runs on the host, which is where this + # store path already is, so it is reachable as written. HIVE_AGENT_FRONTEND_DIR = "${cfg.servedFrontend}/agent"; # Path to the static runtime asset tree (branding + claude # prompts). `hive_sh4re::assets::*` reads paths underneath. diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index 0b76cef0..5e95ad53 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -26,9 +26,7 @@ let forgeCfg = config.services.hyperhive.swarm.forge; networkCfg = config.services.hyperhive.network; - # Dashboard SPA dist, static-served by nginx. Read in OUTER scope so - # `config` is the host's (inside the container block it'd be the - # container's). + # Dashboard SPA dist, static-served by nginx. dashboardDist = "${config.services.hyperhive.c0re.servedFrontend}/dashboard"; # Full hyperhive-themed Swagger UI dist — nginx serves this whole @@ -243,10 +241,10 @@ in # mode behind. chmod 0755 ${builtins.dirOf tlsDir} chmod 0755 ${tlsDir} - # Copy the host leaf in. `install` writes atomically with the - # target mode; run as root (container root == host root, - # privateUsers=false) so the 0600 root:root host key is - # readable. Key ends up root:nginx 0640 so nginx-pre-start + # Copy the leaf in. `install` writes atomically with the + # target mode; runs as root so the 0600 root:root key written + # by hive-tls-ca is readable. Key ends up root:nginx 0640 so + # nginx-pre-start # (which runs `nginx -t` as the nginx user, not root) can # read it — a 0600 root:root key passes the master load but # fails the pre-start config test with `BIO_new_file() … diff --git a/nix/host-modules/hive-gateway/dnsmasq.nix b/nix/host-modules/hive-gateway/dnsmasq.nix index 42912679..bc63e9dd 100644 --- a/nix/host-modules/hive-gateway/dnsmasq.nix +++ b/nix/host-modules/hive-gateway/dnsmasq.nix @@ -1,10 +1,10 @@ -# Hive-internal DNS resolver + DHCP, co-located in the gateway -# container — single front-door for both DNS and HTTP, saves a -# sibling container. Listens on the bridge interface from +# Hive-internal DNS resolver + DHCP, running on the host alongside the +# gateway's nginx — single front-door for both DNS and HTTP, and no +# container of its own. Listens on the bridge interface from # `services.hyperhive.network`; authoritative for the hive domain + # sub-domains, forwards everything else upstream. Returns the -# `services.dnsmasq` value for the container config (see -# ./default.nix); the DHCP pool bounds are computed by hive-network. +# `services.dnsmasq` value (see ./default.nix); the DHCP pool bounds +# are computed by hive-network. { lib, networkCfg, @@ -68,14 +68,13 @@ dhcp-range = "${networkCfg.dhcpRangeStart},${networkCfg.dhcpRangeEnd},1h"; dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; # No explicit upstream: non-hive queries follow dnsmasq's - # resolv.conf default — the gateway container's `/etc/resolv.conf`, - # which nixos-container copies from the host at every start, so the - # hive always uses the host's resolvers. resolvconf is disabled in - # the container (see ./default.nix) so nothing regenerates that - # copy; the host-side `hive-gateway-resolv` path unit (also in - # ./default.nix) pushes in a fresh copy and reloads dnsmasq whenever - # the host's resolvers change, so the copy can't go stale under a - # network switch. Deliberately no fallback `server=`: dnsmasq queries + # resolv.conf default — the host's own `/etc/resolv.conf`, so the + # hive always uses the host's resolvers and follows them live. This + # used to be a *copy* nixos-container made at container start, kept + # fresh by a host-side path unit that pushed in a new one and + # reloaded dnsmasq; running on the host deleted both the copy and + # the machinery that watched it. Deliberately no fallback + # `server=`: dnsmasq queries # all known upstreams in parallel, so a hardcoded public resolver # would take a share of *normal* traffic, not just fill in when the # host file is empty. diff --git a/nix/host-modules/hive-gateway/options.nix b/nix/host-modules/hive-gateway/options.nix index 30c38227..ecdcc5c3 100644 --- a/nix/host-modules/hive-gateway/options.nix +++ b/nix/host-modules/hive-gateway/options.nix @@ -28,10 +28,10 @@ in example = 8080; description = '' TCP port the gateway listens on. Default 80 (canonical web - port). nginx inside the container binds <1024 because the - container's init runs as root; if 80 is already taken on the - host (existing nginx, traefik, etc.) override to an unused - port like 8080 or move the conflicting service. + port). nginx runs on the host as root, so it can bind <1024; + if 80 is already taken on the host (existing nginx, traefik, + etc.) override to an unused port like 8080 or move the + conflicting service. ''; }; @@ -40,8 +40,8 @@ in default = "127.0.0.1"; description = '' Host the gateway proxies non-static requests to. Defaults to - `127.0.0.1` because the gateway container shares the host - netns, so loopback resolves directly to hive-c0re. + `127.0.0.1` because nginx runs on the host itself, so loopback + resolves directly to hive-c0re. ''; }; @@ -180,11 +180,11 @@ in default = false; example = true; description = '' - Let nginx inside the gateway container obtain and renew TLS - certificates automatically via ACME (Let's Encrypt). When - enabled, each vhost calls out to Let's Encrypt using the - HTTP-01 challenge on `port` (default 80) and stores certs - inside the gateway container's persistent state dir. + Let the gateway's nginx obtain and renew TLS certificates + automatically via ACME (Let's Encrypt). When enabled, each + vhost calls out to Let's Encrypt using the HTTP-01 + challenge on `port` (default 80) and stores certs in the + gateway's persistent state dir on the host. Requirements: - `services.hyperhive.domain` must be set and publicly diff --git a/nix/host-modules/hive-gateway/vhosts.nix b/nix/host-modules/hive-gateway/vhosts.nix index 969d8c3a..28acb246 100644 --- a/nix/host-modules/hive-gateway/vhosts.nix +++ b/nix/host-modules/hive-gateway/vhosts.nix @@ -1,9 +1,9 @@ -# nginx virtual-host tree for the gateway container: the `_` default -# server (dashboard, per-agent routing, matrix discovery), the forge -# and matrix sub-domain vhosts, and the Accept-header SPA map for the -# matrix GUI. Pure function — called from ./default.nix inside the -# container config with the outer-scope config values as arguments; -# returns `{ virtualHosts, appendHttpConfig }`. +# nginx virtual-host tree for the gateway: the `_` default server +# (dashboard, per-agent routing, matrix discovery), the forge, matrix +# and authelia sub-domain vhosts, and the Accept-header SPA map for the +# matrix GUI. Pure function — called from ./default.nix with the +# outer-scope config values as arguments; returns +# `{ virtualHosts, appendHttpConfig }`. { lib, cfg, # services.hyperhive.gateway diff --git a/nix/host-modules/hive-network.nix b/nix/host-modules/hive-network.nix index 4a1702e8..6d4bc871 100644 --- a/nix/host-modules/hive-network.nix +++ b/nix/host-modules/hive-network.nix @@ -46,9 +46,8 @@ in '') (lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "upstreamDns" ] '' The hive resolver always follows the host's resolvers now - (dnsmasq reads the gateway container's /etc/resolv.conf, the - host copy made at container start). Configure upstream DNS on - the host itself instead. + (dnsmasq runs on the host and reads its /etc/resolv.conf + directly). Configure upstream DNS on the host itself instead. '') ]; @@ -71,8 +70,8 @@ in example = "172.30.0.1"; description = '' IPv4 address assigned to the bridge interface on the host - side. Agents use this address as their DNS server (dnsmasq - in the gateway container binds here). Default `10.42.0.1` + side. Agents use this address as their DNS server (the hive's + dnsmasq binds here). Default `10.42.0.1` is in RFC 1918 space and unlikely to clash with operator's existing setup; override if a different range is already in use. @@ -123,7 +122,7 @@ in # before broadcast). All containers — agents and service # containers alike — receive their IPs dynamically from this pool; # there are no hash-derived static assignments. Consumed by the - # dnsmasq that runs in the gateway container (hive-gateway module). + # hive's dnsmasq (hive-gateway module, host-side). dhcpRangeStart = lib.mkOption { type = lib.types.str; internal = true; @@ -212,7 +211,7 @@ in # is unconditional now). networking.bridges.${cfg.bridgeName}.interfaces = [ ]; - # Bridge IP — dnsmasq (in the gateway container) binds here. + # Bridge IP — the hive's dnsmasq binds here. networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [ { address = cfg.bridgeIp; diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index a848c448..8cf0280d 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -4,9 +4,9 @@ # the agents on one host, this owns what is true across hives. # # Serves HTTP over a unix socket rather than a TCP port: the gateway's -# nginx is the only intended client, it reaches the socket through a -# bind-mount, and a socket that is never bound to an address cannot be -# reached from off-host by mistake. +# nginx is the only intended client, it runs on this same host and so +# reaches the socket by path, and a socket that is never bound to an +# address cannot be reached from off-host by mistake. { pkgs, lib, @@ -103,13 +103,13 @@ in the socket itself is `0666` (nginx runs as another user, and `connect(2)` needs write), exactly as hive-c0re publishes the per-agent sockets. What keeps that safe is that the directory holds - one socket and is bind-mounted into one container. Moving this path - under a directory that carries anything else — `/run/hyperhive`, - which holds the host admin socket, above all — hands whatever else - lives there to every consumer that mounts it. + exactly one socket and is traverse-only (`0751`) for everyone else. + Moving this path under a directory that carries anything else — + `/run/hyperhive`, which holds the host admin socket, above all — + exposes whatever else lives there to the same reachability. - Changing this therefore means re-checking the gateway bind-mount, - not just the daemon. + Changing this therefore means re-checking what else lives in the + new directory, not just the daemon. ''; }; }; @@ -147,9 +147,9 @@ in # the socket path without being able to list the directory. Same # shape (and same reason) as hive-c0re's runtime dir. RuntimeDirectoryMode = "0751"; - # Preserved across restarts so the bind-mount source never vanishes - # from under a running gateway container. The daemon unlinks a stale - # socket on start, which is what makes preservation safe. + # Preserved across restarts so the path never vanishes from under a + # running nginx. The daemon unlinks a stale socket on start, which + # is what makes preservation safe. RuntimeDirectoryPreserve = "yes"; StateDirectory = "swarm-controller"; From b12be26f20d5e5fd7da9b3dedd9c2de1f370197e Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 12 Aug 2026 13:26:58 +0200 Subject: [PATCH 3/3] docs(3191): drop the migration history from the gateway comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per mara on the PR: how a thing used to be deployed stops being relevant the moment this is merged and the hives are rebuilt, so the comments state the current constraint and nothing about the container that used to hold it. Removes the two ⚠️ blocks this PR added (hive-tls-ca's ordering, the resign propagation), the matrix ordering's, dnsmasq's resolv.conf paragraph, and the pre-existing 'REMOVED WITH THE CONTAINER' inventory in the gateway module. The 🚨 do-not-simplify warning on the cert-copy unit keeps both its reasons and loses the container framing. Kept deliberately: the two operator-facing 'this used to default to true' option notes (migration information for someone upgrading), and the SupplementaryGroups block, which documents why an absence is load-bearing rather than how a deployment used to look. --- nix/host-modules/hive-gateway/default.nix | 33 +++-------------------- nix/host-modules/hive-gateway/dnsmasq.nix | 7 ++--- nix/host-modules/hive-matrix.nix | 8 ------ nix/host-modules/hive-tls.nix | 25 +++++------------ 4 files changed, 11 insertions(+), 62 deletions(-) diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index 5e95ad53..f8122f13 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -155,32 +155,6 @@ in "f /var/lib/hive-gateway/conf/gateway.htpasswd 0644 hive-core hive-core - -" ]; - # ⚠️ REMOVED WITH THE CONTAINER, and each one was a workaround for the - # boundary rather than a thing nginx or dnsmasq needed: - # - # - `privateNetwork = false` — the container already shared the host - # netns, which is why nginx bound host ports and `localhost` - # upstreams reached hive-c0re. On the host that is simply true. - # - `additionalCapabilities = [ "CAP_NET_ADMIN" ]` — dnsmasq refuses - # to start with a `dhcp-range` unless it holds NET_ADMIN, and - # nspawn's bounding set dropped it for a host-netns container. - # Host root has it. - # - `networking.firewall.enable = false` — a container sharing the - # host netns would run *its* firewall.service against the HOST - # ruleset, flushing nixos-fw and deleting the nixos-nat-* chains on - # every boot. With one machine there is one firewall (below). - # - `networking.resolvconf.enable = false` + the `hive-gateway-resolv` - # path/service pair — the container's /etc/resolv.conf was a - # one-shot copy frozen at start, so a host network change left - # dnsmasq forwarding to a resolver that was gone. The whole - # watch-copy-reload machine existed to bridge two files. There is - # now one. - # - three bind mounts — /run/hive-agent, /run/hive-state, and either - # /run/hive-tls (operator cert) or /run/hive-ca (self-signed); - # those last two are mutually exclusive mkIfs, so it was never - # four. All plain host paths now. - # - # See `docs/network.md::Resolver behaviour` for the resolver history. # ACME (Let's Encrypt) integration. nginx vhosts set # `enableACME = true` via the vhost builder; this provides the # shared ACME config (acceptTerms + email). @@ -191,15 +165,14 @@ in # Import the hive-CA leaf into nginx's state dir before nginx starts. # - # 🚨 THIS LOOKS LIKE A LEFTOVER OF THE CONTAINER AND IS NOT — do not - # "simplify" it into pointing nginx at the CA dir. It does TWO jobs: + # 🚨 DO NOT "simplify" this into pointing nginx at the CA dir. It + # does TWO jobs: # # (1) It re-modes the leaf. `hive-tls-ca` writes the key 0600 # root:root; nginx's pre-start `nginx -t` runs as the nginx # *user*, so a 0600 key fails the config test with # `BIO_new_file() … Permission denied` and blocks the unit — - # hence the 0640 root:nginx copy below. That is a file-mode fact, - # not a namespace one, and it did not go away with the container. + # hence the 0640 root:nginx copy below. # (2) It guarantees that **every cert path the nginx config names # exists** — which is what the swarm-services fallback at the # bottom of the script is for. nginx refuses to load a config diff --git a/nix/host-modules/hive-gateway/dnsmasq.nix b/nix/host-modules/hive-gateway/dnsmasq.nix index bc63e9dd..d5096188 100644 --- a/nix/host-modules/hive-gateway/dnsmasq.nix +++ b/nix/host-modules/hive-gateway/dnsmasq.nix @@ -69,11 +69,8 @@ dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; # No explicit upstream: non-hive queries follow dnsmasq's # resolv.conf default — the host's own `/etc/resolv.conf`, so the - # hive always uses the host's resolvers and follows them live. This - # used to be a *copy* nixos-container made at container start, kept - # fresh by a host-side path unit that pushed in a new one and - # reloaded dnsmasq; running on the host deleted both the copy and - # the machinery that watched it. Deliberately no fallback + # hive always uses the host's resolvers and follows them live with + # no copy to go stale. Deliberately no fallback # `server=`: dnsmasq queries # all known upstreams in parallel, so a hardcoded public resolver # would take a share of *normal* traffic, not just fill in when the diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index d27441ae..89780930 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -591,14 +591,6 @@ in # boot requirement. Soft `after` ordering (not `requires`) keeps the # matrix container's lifecycle decoupled from the resolver's. # - # ⚠️ This named `container@hive-gateway.service` until the gateway - # moved onto the host: dnsmasq lived in that container, so ordering - # after the container was how you ordered after the resolver. The - # container is gone and dnsmasq is a plain host `services.dnsmasq`, - # so the ordering now names the resolver directly — which is what it - # always meant. Naming the *container* was already indirection; it - # just happened to be correct while the container existed. - # # `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also # sets `after`/`requires` (so the bound trust bundle exists before # nspawn wires the mount up), and two plain assignments to the same diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 7f1715fe..2199b169 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -316,15 +316,8 @@ in # The consumer is `hive-gateway-self-signed-cert`, which copies the # leaf into the gateway's state dir at the mode nginx can read, and # which nginx in turn `Requires=`. So this must run first or that - # copy fails under `set -eu` and takes nginx down with it. - # - # ⚠️ This used to name `container@hive-gateway.service` — ordering - # ran through the container, because the copy happened *at container - # start*. Moving the gateway onto the host retired the container - # without retiring the dependency: both units became plain host - # units with nothing sequencing them, and the copy could win the - # race on a fast disk. Order against the unit that reads the file, - # not against the thing that used to host it. + # copy fails under `set -eu` and takes nginx down with it — order + # against the unit that reads the file. before = [ "hive-gateway-self-signed-cert.service" ]; requiredBy = [ "hive-gateway-self-signed-cert.service" ]; # The issuance below needs the swarm root key on disk, and (for the @@ -482,16 +475,10 @@ in # the source therefore changes nothing on its own — the copy has to be # remade and nginx reloaded, which is what the two calls below do. # - # ⚠️ Both calls used to be `systemctl -M hive-gateway … || true`, from - # when nginx lived in a container: `-M` entered the machine, and the - # `|| true` covered a *stopped* container, which was a normal state - # ("its next boot imports the rotated leaf anyway"). The container is - # gone, so `-M hive-gateway` names a machine nothing creates — both - # calls failed, both failures were swallowed, and the unit logged - # "propagating" and exited 0 while nginx kept serving the stale copy - # until it expired. A host unit failing is NOT a normal state: no - # `-M`, and no `|| true` either, so a broken propagation fails the - # timer loudly instead of reporting success. + # ⚠️ Neither call is `|| true`: a swallowed propagation failure means + # the leaf rotates on disk while nginx keeps serving the old copy + # until it expires, with this unit reporting success the whole time. + # A failure here must fail the timer. systemd.services.hive-tls-resign = { description = "Re-sign the gateway TLS leaf and reload nginx"; # hive-tls-ca must have run first so the CA key exists before we try