diff --git a/docs/network.md b/docs/network.md index aa183df8..66aa5fd7 100644 --- a/docs/network.md +++ b/docs/network.md @@ -112,11 +112,48 @@ dnsmasq is **authoritative** for the hive's own zones — answers queries with the bridge IP (where nginx is reachable). Everything else is forwarded to the host's own resolvers: dnsmasq reads the gateway container's `/etc/resolv.conf`, the host copy nixos-container -makes at each container start — a host resolver change is picked up -on the next gateway restart. Containers don't need to know the +makes at each container start. Containers don't need to know the upstream — they query the bridge IP and dnsmasq does the right thing per-name. +That copy is one-shot — systemd-nspawn(1) is explicit that nothing +propagates into it after early init, because resolv.conf is normally +updated by rename rather than in place. Left alone, a host network +change (new router, new lease, laptop moving networks) would strand +dnsmasq on a resolver that no longer answers, and every non-hive +lookup from every agent would hang until someone restarted the +gateway. The host-side **`hive-gateway-resolv`** path unit closes +that: it watches `/etc/resolv.conf`, `machinectl copy-to`s it into +the container, and reloads dnsmasq (`SIGHUP` — re-read upstreams + +flush cache, nothing dropped). The watch is armed before +`network-pre.target` so the boot's first DHCP write is caught as well, +and the sync also runs once per gateway start to pick up a resolver +change that happened while the container was down. A host file with no +`nameserver` line is +skipped rather than pushed, so a mid-rewrite snapshot can't blank the +hive's DNS. There is deliberately no fallback `server=`: dnsmasq +queries all known upstreams in parallel, so a hardcoded public +resolver would take a share of normal traffic, not just cover the gap. + +Two alternatives that look simpler and aren't: + +- **A path unit inside the container.** The host replaces + `/etc/resolv.conf` by rename, and that `IN_MOVED_TO` doesn't cross + the nspawn mount namespace — the same reason hive-c0re reloads nginx + from the host side after each `agents.conf` write. +- **Bind-mounting the host's `/etc/resolv.conf` into the container.** + openresolv writes a temp file and renames over the target, so the + bind mount would pin the *first* inode for the container's whole + lifetime — strictly worse than the copy, which at least a restart + clears. (Reachability is not the problem here: the gateway runs with + `privateNetwork = false`, so it shares the host's netns and can reach + anything the host can.) + +`machinectl copy-to` is used rather than writing to the container's +rootfs from the host, so the push goes through the container's own +mount namespace and stays correct if `/etc` is ever assembled +differently (e.g. `system.etc.overlay`). + `bind-interfaces` + `interface = [ bridgeName "lo" ]` means the listener only accepts queries from the bridge interface (plus lo for container health-checks). External hosts can't reach it — no diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index e8a6b55b..d23de50d 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -81,6 +81,113 @@ in "f /var/lib/hyperhive/gateway/gateway.htpasswd 0644 root root - -" ]; + # Keep the gateway's `/etc/resolv.conf` in step with the host's. + # + # nixos-container does `cp --remove-destination /etc/resolv.conf + # "$root/etc/resolv.conf"` in its start script, and nspawn's + # `--resolv-conf=auto` copies rather than binds for a writable, + # host-netns container like this one. Both are one-shot: systemd-nspawn(1) + # says outright that "no further propagation of configuration is + # generally done after the one-time early initialization (this is + # because the file is usually updated through copying and renaming)". + # + # So the gateway's copy is frozen at container start. dnsmasq has no + # explicit upstream (see ./dnsmasq.nix) and follows that file, which + # means a host network change — new router, new DHCP lease, laptop + # moving between networks — leaves dnsmasq forwarding to a resolver + # that is gone, and every non-hive lookup from every agent hangs. The + # agents' own resolvers point at the static bridge IP and never go + # stale, which is exactly why the symptom presents as "the gateway + # needs a kick". + # + # Hence: watch on the host, push into the container, reload dnsmasq. + # `reload` is `kill -HUP $MAINPID` (the upstream dnsmasq unit's + # ExecReload), so dnsmasq re-reads its upstream list and drops its + # cache without severing anything — nginx never notices. Why this has + # to run host-side, and why a file bind-mount is worse than the copy: + # `docs/network.md::Resolver behaviour`. + systemd.paths.hive-gateway-resolv = { + description = "Watch the host's /etc/resolv.conf for the hive-gateway container"; + wantedBy = [ "multi-user.target" ]; + # Arm the watch before anything configures the network, so the very + # first DHCP-driven resolv.conf write of the boot is caught too — + # that's the "gateway came up while DHCP was still settling" case. + # Inert if nothing pulls network-pre.target into the transaction. + before = [ "network-pre.target" ]; + pathConfig = { + # PathChanged also watches the parent directory, so openresolv's + # atomic rename-over lands as IN_MOVED_TO on /etc and triggers — + # a watch on the inode alone would die with the replaced file. + PathChanged = "/etc/resolv.conf"; + Unit = "hive-gateway-resolv.service"; + }; + }; + + systemd.services.hive-gateway-resolv = { + description = "Sync the host's resolvers into hive-gateway and reload dnsmasq"; + # Also run once per gateway start, to catch a host resolver change + # that happened while the container was down. Deliberately NOT + # ordered after network-online.target: pulling that target in on + # every resolv.conf change risks blocking the sync behind a + # wait-online timeout on hosts where nothing else reaches it. The + # boot race is closed by arming the path unit early instead. + wantedBy = [ "container@hive-gateway.service" ]; + after = [ "container@hive-gateway.service" ]; + path = [ + pkgs.systemd + pkgs.coreutils + pkgs.gnugrep + ]; + serviceConfig = { + Type = "oneshot"; + SyslogIdentifier = "hive-gateway-resolv"; + }; + script = '' + set -euo pipefail + src=/etc/resolv.conf + # Marker lives on tmpfs, so the first sync after every host boot + # always goes through rather than trusting a stale comparison. + marker=/run/hive-gateway/resolv.synced + + # No nameserver line means either a mid-rewrite snapshot or a host + # with no DNS at all. In both cases the gateway's existing copy is + # the best information available — keep it and wait for the next + # event, instead of pushing a file that resolves nothing. + if ! grep -q '^[[:space:]]*nameserver[[:space:]]' "$src"; then + echo "host resolv.conf has no nameserver line — keeping the gateway's current copy" + exit 0 + fi + + if [ -e "$marker" ] && cmp -s "$src" "$marker"; then + echo "host resolvers unchanged since last sync — nothing to do" + exit 0 + fi + + # A stopped gateway needs no push: its next start copies the + # current host file itself. + if ! systemctl is-active --quiet container@hive-gateway.service; then + echo "hive-gateway not running — its next start copies the current file itself" + exit 0 + fi + + # `machinectl copy-to` writes through the container's own mount + # namespace, so this stays correct regardless of how the container + # assembles /etc (e.g. if system.etc.overlay is ever turned on) — + # unlike poking at the rootfs path from the host side. + machinectl copy-to hive-gateway "$src" /etc/resolv.conf --force + + # Not fatal: dnsmasq polls resolv.conf for mtime changes on its own, + # so a failed reload costs a second of staleness, not correctness. + if ! systemctl -M hive-gateway reload dnsmasq.service; then + echo "dnsmasq reload failed (not up yet?) — copy is in place, its own poll will pick it up" + fi + + mkdir -p "$(dirname "$marker")" + install -m 0644 "$src" "$marker" + echo "synced host resolvers into hive-gateway and reloaded dnsmasq" + ''; + }; + containers.hive-gateway = { autoStart = true; ephemeral = false; @@ -177,9 +284,11 @@ in # resolvconf's host-tracking mode then regenerates it — to an # empty file, since the host file doesn't cross the boundary # after start (the same failure the matrix container hit). - # With resolvconf off, nothing touches the copy: nginx's own - # lookups (ACME) and dnsmasq's follow-the-host upstream + # With resolvconf off, nothing in here touches the copy: nginx's + # own lookups (ACME) and dnsmasq's follow-the-host upstream # default (see ./dnsmasq.nix) both read the host's resolvers. + # Keeping it stale-free is the host's job — see the + # `hive-gateway-resolv` path unit above. networking.resolvconf.enable = false; # ACME (Let's Encrypt) integration. nginx vhosts set diff --git a/nix/host-modules/hive-gateway/dnsmasq.nix b/nix/host-modules/hive-gateway/dnsmasq.nix index 6852df5a..7a136ca8 100644 --- a/nix/host-modules/hive-gateway/dnsmasq.nix +++ b/nix/host-modules/hive-gateway/dnsmasq.nix @@ -56,6 +56,12 @@ # which nixos-container copies from the host at every start, so the # hive always uses the host's resolvers. resolvconf is disabled in # the container (see ./default.nix) so nothing regenerates that - # copy. + # copy; the host-side `hive-gateway-resolv` path unit (also in + # ./default.nix) pushes in a fresh copy and reloads dnsmasq whenever + # the host's resolvers change, so the copy can't go stale under a + # network switch. Deliberately no fallback `server=`: dnsmasq queries + # all known upstreams in parallel, so a hardcoded public resolver + # would take a share of *normal* traffic, not just fill in when the + # host file is empty. }; }