From d70b1e0a97289e115f6635524330f3539710dbea Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 12 Aug 2026 12:18:13 +0200 Subject: [PATCH] fix(3191): order gateway TLS against the units that read it, not a dead container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the gateway container retired two things that were still depended on: an ordering edge and a delivery path. - hive-tls-ca ordered itself before/requiredBy container@hive-gateway.service. That unit no longer exists, so the CA was sequenced against nothing while its real consumer, hive-gateway-self-signed-cert (which nginx Requires=), could win the race and fail its copy under set -eu, blocking nginx. - hive-tls-resign propagated a rotated leaf with `systemctl -M hive-gateway … || true`. The machine is gone, so both calls failed and both failures were swallowed: the unit logged "propagating" and exited 0 while nginx served the stale copy until it expired. Host units now, so no -M and no || true — a failed propagation fails the timer. - container@hive-matrix ordered after the gateway container to get the resolver up first. dnsmasq is a host service now, so it orders after dnsmasq.service, which is what the edge always meant. --- nix/host-modules/hive-matrix.nix | 33 ++++++++++-------- nix/host-modules/hive-tls.nix | 59 +++++++++++++++++++------------- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index add0ba73..d27441ae 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -581,25 +581,30 @@ in ]; }; - # The matrix container's resolver is the dnsmasq that runs in the - # gateway container (bound at `bridgeIp`). Order the matrix - # container start after the gateway container so the resolver is up - # before tuwunel's first federation lookups. tuwunel boots fine - # without this — it configures the resolver from `/etc/resolv.conf` - # at startup and only queries on-demand (the boot failure this - # module guards against is an *empty* resolv.conf, a parse error, - # not a connectivity one) — so this is robustness, not a boot - # requirement. Soft `after` ordering (not `requires`) keeps the - # matrix container's lifecycle decoupled from the gateway's. The - # gateway always runs alongside hyperhive, so the gateway container - # unit always exists here. (Declarative `containers.` → - # `container@.service` — the nspawn template NixOS generates.) + # The matrix container's resolver is the hive's dnsmasq (bound at + # `bridgeIp`). Order the matrix container start after it so the + # resolver is up before tuwunel's first federation lookups. tuwunel + # boots fine without this — it configures the resolver from + # `/etc/resolv.conf` at startup and only queries on-demand (the boot + # failure this module guards against is an *empty* resolv.conf, a + # parse error, not a connectivity one) — so this is robustness, not a + # boot requirement. Soft `after` ordering (not `requires`) keeps the + # matrix container's lifecycle decoupled from the resolver's. + # + # ⚠️ This named `container@hive-gateway.service` until the gateway + # moved onto the host: dnsmasq lived in that container, so ordering + # after the container was how you ordered after the resolver. The + # container is gone and dnsmasq is a plain host `services.dnsmasq`, + # so the ordering now names the resolver directly — which is what it + # always meant. Naming the *container* was already indirection; it + # just happened to be correct while the container existed. + # # `mkMerge`, not a bare assignment: `caTrust.containerOrdering` also # sets `after`/`requires` (so the bound trust bundle exists before # nspawn wires the mount up), and two plain assignments to the same # unit would conflict rather than combine. systemd.services."container@hive-matrix" = lib.mkMerge [ - { after = [ "container@hive-gateway.service" ]; } + { after = [ "dnsmasq.service" ]; } caTrust.containerOrdering ]; }; diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 5d6a6db5..7f1715fe 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -307,17 +307,26 @@ in matrix.gatewayHost, authelia.domain}) or install the sub-CA. ''; - # Generate (and rotate) the hive CA + gateway leaf before the gateway - # container starts. Idempotent: the CA is created once and reused; the - # leaf is re-signed on expiry under the same CA so the anchor is stable. + # Generate (and rotate) the hive CA + gateway leaf before anything + # serves it. Idempotent: the CA is created once and reused; the leaf + # is re-signed on expiry under the same CA so the anchor is stable. systemd.services.hive-tls-ca = { description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)"; wantedBy = [ "multi-user.target" ]; - # Gateway nginx reads the leaf from the bind-mount, so the cert must - # exist before the container starts. Declarative nixos-containers are - # instances of the `container@.service` template. - before = [ "container@hive-gateway.service" ]; - requiredBy = [ "container@hive-gateway.service" ]; + # The consumer is `hive-gateway-self-signed-cert`, which copies the + # leaf into the gateway's state dir at the mode nginx can read, and + # which nginx in turn `Requires=`. So this must run first or that + # copy fails under `set -eu` and takes nginx down with it. + # + # ⚠️ This used to name `container@hive-gateway.service` — ordering + # ran through the container, because the copy happened *at container + # start*. Moving the gateway onto the host retired the container + # without retiring the dependency: both units became plain host + # units with nothing sequencing them, and the copy could win the + # race on a fast disk. Order against the unit that reads the file, + # not against the thing that used to host it. + before = [ "hive-gateway-self-signed-cert.service" ]; + requiredBy = [ "hive-gateway-self-signed-cert.service" ]; # The issuance below needs the swarm root key on disk, and (for the # services leaf) the services sub-CA it signs under. When this host # generates them (single-host swarm) both units must have run first; @@ -466,21 +475,25 @@ in # `hive-tls-ca` only re-signs at service activation (boot/rebuild); a # long-uptime host would otherwise let a 30-day leaf lapse silently. # This service re-signs the leaf directly (not by bouncing hive-tls-ca) - # and propagates the new leaf into the running gateway container when - # the file actually changed. + # and propagates the new leaf to nginx when the file actually changed. # - # Propagation mechanism: nginx in the gateway container serves a *copy* - # of the leaf written by `hive-gateway-self-signed-cert` (which runs at - # container start). A host-side `systemctl -M hive-gateway` call - # triggers the re-import + reload, mirroring how hive-c0re reloads the - # gateway after each agents.conf write. A path unit *inside* the - # container cannot do this: IN_MOVED_TO from an atomic rename on the - # host does not propagate across the nspawn mount-namespace boundary. + # Propagation mechanism: nginx serves a *copy* of the leaf, written by + # `hive-gateway-self-signed-cert` at the mode nginx can read. Re-signing + # the source therefore changes nothing on its own — the copy has to be + # remade and nginx reloaded, which is what the two calls below do. # - # `|| true` on propagation so a stopped gateway never fails the unit — - # its next boot will import the already-rotated leaf anyway. + # ⚠️ Both calls used to be `systemctl -M hive-gateway … || true`, from + # when nginx lived in a container: `-M` entered the machine, and the + # `|| true` covered a *stopped* container, which was a normal state + # ("its next boot imports the rotated leaf anyway"). The container is + # gone, so `-M hive-gateway` names a machine nothing creates — both + # calls failed, both failures were swallowed, and the unit logged + # "propagating" and exited 0 while nginx kept serving the stale copy + # until it expired. A host unit failing is NOT a normal state: no + # `-M`, and no `|| true` either, so a broken propagation fails the + # timer loudly instead of reporting success. systemd.services.hive-tls-resign = { - description = "Re-sign the gateway TLS leaf and propagate it into the gateway container"; + description = "Re-sign the gateway TLS leaf and reload nginx"; # hive-tls-ca must have run first so the CA key exists before we try # to re-sign under it. On first boot `Persistent=true` on the weekly # timer fires immediately; without this ordering the resign could race @@ -534,9 +547,9 @@ in after="$(sha256sum "$leaf" "$svcleaf" 2>/dev/null || true)" if [ "$before" != "$after" ]; then - echo "gateway leaf rotated — propagating into hive-gateway" - systemctl -M hive-gateway restart hive-gateway-self-signed-cert.service || true - systemctl -M hive-gateway reload nginx.service || true + echo "gateway leaf rotated — re-importing and reloading nginx" + systemctl restart hive-gateway-self-signed-cert.service + systemctl reload nginx.service else echo "gateway leaf unchanged (already up to date)" fi