diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 812e80b0..1ff0ea7d 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -36,6 +36,7 @@ }: let cfg = config.services.hyperhive.swarm.authelia; + networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; hyperhiveDomain = hyperhiveCfg.domain; @@ -740,6 +741,13 @@ in config = { ... }: { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "authelia-${instance}.service" ]; + }) + ]; + system.stateVersion = "26.05"; # The authelia binary itself, so an operator who gets a shell @@ -753,9 +761,10 @@ in # firewall.service would rewrite the HOST ruleset at every # boot. The host firewall owns all filtering. networking.firewall.enable = false; - # Keep the host-copied /etc/resolv.conf intact — resolvconf's - # host-tracking would regenerate it to an empty file, since - # the host's copy doesn't cross the boundary after start. + # resolvconf stays off because the resolver unit imported above + # owns /etc/resolv.conf. Leaving it on would let host-tracking + # regenerate the file empty, since the host's copy doesn't + # cross the boundary after start. networking.resolvconf.enable = lib.mkForce false; # authelia's own secrets, generated in-container on first diff --git a/nix/host-modules/swarm-container-resolver.nix b/nix/host-modules/swarm-container-resolver.nix new file mode 100644 index 00000000..d82d9ca9 --- /dev/null +++ b/nix/host-modules/swarm-container-resolver.nix @@ -0,0 +1,61 @@ +# The resolver file a swarm service container writes for itself. +# +# Every swarm service container shares the host netns (`privateNetwork = +# false`) and force-disables `resolvconf`, so that the `/etc/resolv.conf` +# `nixos-containers` copies in at start is not regenerated empty. That copy +# is a `cp --remove-destination` in the host-side preStart, run ONCE per +# container start — so the container's resolver is a snapshot of the host's +# file at its boot instant, and stays that snapshot for its whole life. +# +# A snapshot is not a resolver. Anything that makes the host's file wrong at +# that one instant — a resolvconf regeneration mid-deploy, a host that has +# not yet pointed itself at the bridge — leaves the container with a resolver +# it can never recover from, and the symptom surfaces arbitrarily far from +# the cause: a queue refusing every client because the auth-callout responder +# cannot look up its IdP. +# +# So the container writes the file itself, on every boot, from the one +# address that is correct on both sides of a netns boundary (the bridge IP — +# see `hive-gateway/default.nix`, which forces the host to the same value). +{ + bridgeIp, + # Units in this container that resolve a name. The caller names them + # because this module cannot know them, and an unordered resolver write + # is a race that only shows up on a cold boot. + dnsConsumers ? [ ], +}: +{ lib, pkgs, ... }: +{ + # ⚠️ `networking.nameservers` CANNOT replace this unit. `resolvconf` is its + # only consumer, and these containers disable it — so setting it renders no + # file and changes no behaviour, while still evaluating and deploying + # perfectly cleanly. It reads like a fix and is a no-op. + # + # ⚠️ Nor can a static `environment.etc."resolv.conf"`: that has to survive + # `etc` activation landing on top of the regular file the host already + # copied there, which is a runtime property no eval can demonstrate. This + # oneshot shape is the one every agent container already uses + # (`nix/agent-modules/network.nix`), so it has runtime evidence behind it. + systemd.services.swarm-bridge-dns = { + description = "point resolv.conf at the hive bridge resolver"; + wantedBy = [ "multi-user.target" ]; + after = [ "local-fs.target" ]; + before = [ "network-online.target" ] ++ dnsConsumers; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # Pin the journal identity; without it systemd derives one from the + # generated script's store path (an opaque `-…-start`). + SyslogIdentifier = "swarm-bridge-dns"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -eu + # `rm` first: this is a regular file the host copied in, not something + # to write through, and a leftover symlink would redirect the write. + rm -f /etc/resolv.conf + printf 'nameserver %s\n' ${lib.escapeShellArg bridgeIp} > /etc/resolv.conf + echo "swarm-bridge-dns: resolv.conf -> nameserver ${bridgeIp}" + ''; + }; +} diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 8332a6ee..42a0f2ce 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -14,6 +14,7 @@ }: let cfg = config.services.hyperhive.swarm.grafana; + networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; tlsCfg = hyperhiveCfg.tls; @@ -351,15 +352,23 @@ in config = { ... }: { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "grafana.service" ]; + }) + ]; + system.stateVersion = "26.05"; # This container shares the host netns, so its own firewall.service # would rewrite the HOST ruleset at every boot. The host firewall # owns all filtering. networking.firewall.enable = false; - # Keep the host-copied /etc/resolv.conf intact — resolvconf's - # host-tracking would regenerate it to an empty file, since the - # host's copy doesn't cross the boundary after start. + # resolvconf stays off because the resolver unit imported above + # owns /etc/resolv.conf. Leaving it on would let host-tracking + # regenerate the file empty, since the host's copy doesn't cross + # the boundary after start. networking.resolvconf.enable = lib.mkForce false; # Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE` diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 6249fc4f..91aa40ee 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -8,6 +8,7 @@ let cfg = config.services.hyperhive.swarm.nats; autheliaCfg = config.services.hyperhive.swarm.authelia; autheliaUrl = autheliaCfg.url; + networkCfg = config.services.hyperhive.network; # The account the callout responder authenticates as, and the account # authorized clients are placed in. Two accounts rather than one: an @@ -436,15 +437,27 @@ in config = { ... }: { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + # The responder introspects authelia BY NAME on every auth + # request, and a responder that cannot resolve it denies + # every client — so it must not start before the resolver + # file exists. + dnsConsumers = [ "swarm-nats-auth.service" ]; + }) + ]; + system.stateVersion = "26.05"; # Shared host netns: this container's own firewall.service # would rewrite the HOST ruleset at every boot. The host # firewall owns all filtering. networking.firewall.enable = false; - # Keep the host-copied /etc/resolv.conf intact — resolvconf's - # host-tracking regenerates it empty, since the host's copy - # does not cross the boundary after start. + # resolvconf stays off because the resolver unit imported above + # owns /etc/resolv.conf. Leaving it on would let host-tracking + # regenerate the file empty, since the host's copy does not + # cross the boundary after start. networking.resolvconf.enable = lib.mkForce false; services.nats = { diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index c93b7cd7..533dff8b 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -17,6 +17,7 @@ }: let cfg = config.services.hyperhive.swarm.victoriametrics; + networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; swarmDomain = hyperhiveCfg.swarm.domain; @@ -130,15 +131,23 @@ in config = { ... }: { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "victoriametrics.service" ]; + }) + ]; + system.stateVersion = "26.05"; # This container shares the host netns, so its own firewall.service # would rewrite the HOST ruleset at every boot. The host firewall # owns all filtering. networking.firewall.enable = false; - # Keep the host-copied /etc/resolv.conf intact — resolvconf's - # host-tracking would regenerate it to an empty file, since the - # host's copy doesn't cross the boundary after start. + # resolvconf stays off because the resolver unit imported above + # owns /etc/resolv.conf. Leaving it on would let host-tracking + # regenerate the file empty, since the host's copy doesn't cross + # the boundary after start. networking.resolvconf.enable = lib.mkForce false; services.victoriametrics = {