From 81a57a9f81e08b5eb40657ea23785a4ad78caaf2 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 11:06:41 +0100 Subject: [PATCH] containers reach network-online only with a dhcp lease, and bao users wait for it --- docs/networking/network.md | 3 +++ nix/agent-modules/bao.nix | 3 ++- nix/agent-modules/hive-ca.nix | 3 ++- nix/agent-modules/network.nix | 9 ++++++- nix/container-modules/swarm-container.nix | 7 ++++++ .../swarm-journal-forwarder.nix | 2 ++ nix/host-modules/hive-matrix.nix | 2 ++ nix/module-eval/agent-queue-bao.nix | 24 ++++++++++++++++++ nix/module-eval/bao-matrix-reader.nix | 2 ++ nix/module-eval/swarm-services-switch.nix | 25 +++++++++++++++++++ 10 files changed, 77 insertions(+), 3 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index 92bd3fd4..19cf03f4 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -56,6 +56,9 @@ The flows, end to end: - **DHCP** — agent `dhcpcd` broadcasts on `eth0` → veth → bridge → host firewall (udp 67 hole) → dnsmasq pool → lease + router option. + `dhcpcd` runs with `noipv4ll` and waits for an IPv4 address, so + `network-online.target` means a lease, never a 169.254.x fallback; units + that look up a swarm name at start order after it. - **DNS** — agents and the service containers query `bridgeIp:53`; dnsmasq answers hive zones authoritatively with the bridge IP, everything else forwards to the host's resolvers (see _Resolver behaviour_ below). Each diff --git a/nix/agent-modules/bao.nix b/nix/agent-modules/bao.nix index 8bafff2d..83206611 100644 --- a/nix/agent-modules/bao.nix +++ b/nix/agent-modules/bao.nix @@ -125,7 +125,8 @@ in config = lib.mkIf configured { systemd.services.${unitName} = { description = "keep this agent's swarm secret store identity current, and prove it works"; - after = [ "network.target" ]; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; path = [ pkgs.openbao diff --git a/nix/agent-modules/hive-ca.nix b/nix/agent-modules/hive-ca.nix index 94effef1..56db8980 100644 --- a/nix/agent-modules/hive-ca.nix +++ b/nix/agent-modules/hive-ca.nix @@ -85,9 +85,10 @@ in ${unitName} = { description = "fetch the swarm root certificate from the secret store"; after = [ - "network.target" + "network-online.target" "hive-agent-bao-identity.service" ]; + wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; path = [ pkgs.coreutils diff --git a/nix/agent-modules/network.nix b/nix/agent-modules/network.nix index e6ad054c..29834fcf 100644 --- a/nix/agent-modules/network.nix +++ b/nix/agent-modules/network.nix @@ -25,7 +25,14 @@ config = { networking.useDHCP = true; networking.resolvconf.enable = false; - networking.dhcpcd.extraConfig = "nohook resolv.conf"; + # `noipv4ll` and `wait = "ipv4"`: dhcpcd counts an IPv4LL address as an + # address, so by default `network-online.target` is reached on 169.254.x + # before the lease arrives. With both set it waits for the lease. + networking.dhcpcd.extraConfig = '' + nohook resolv.conf + noipv4ll + ''; + networking.dhcpcd.wait = "ipv4"; # A unicast DHCP renewal reply otherwise reaches dhcpcd only by # matching the firewall's ESTABLISHED,RELATED conntrack rule against diff --git a/nix/container-modules/swarm-container.nix b/nix/container-modules/swarm-container.nix index 6a247e00..cbfecd35 100644 --- a/nix/container-modules/swarm-container.nix +++ b/nix/container-modules/swarm-container.nix @@ -45,6 +45,13 @@ in (lib.mkIf (!cfg.privateNetwork) { networking.firewall.enable = false; }) + # A private-netns container with an interface takes its address from the + # hive's DHCP. dhcpcd counts an IPv4LL address as an address, so without + # these `network-online.target` is reached on 169.254.x before the lease. + (lib.mkIf cfg.privateNetwork { + networking.dhcpcd.extraConfig = "noipv4ll"; + networking.dhcpcd.wait = "ipv4"; + }) (lib.mkIf cfg.writesOwnResolvConf { networking.resolvconf.enable = lib.mkForce false; }) diff --git a/nix/container-modules/swarm-journal-forwarder.nix b/nix/container-modules/swarm-journal-forwarder.nix index 1348d00d..60d03177 100644 --- a/nix/container-modules/swarm-journal-forwarder.nix +++ b/nix/container-modules/swarm-journal-forwarder.nix @@ -285,6 +285,8 @@ in # (../host-modules/lib/store-retry.nix), and a collector that restarts until the file lands. systemd.services.${fetchUnit} = lib.mkIf fetchesHere { description = "fetch this container's journal forwarder OIDC client secret from the swarm secret store"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; before = [ "opentelemetry-collector.service" ]; path = [ diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index dbec19be..878914d7 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -1066,6 +1066,8 @@ in # directory: neither unit's stop may remove it from under the other. systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive { description = "render the swarm's appservice registration"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; before = [ "tuwunel.service" ]; wantedBy = [ "tuwunel.service" ]; inherit (storeRetry) startLimitBurst startLimitIntervalSec; diff --git a/nix/module-eval/agent-queue-bao.nix b/nix/module-eval/agent-queue-bao.nix index 3f1151b1..38580968 100644 --- a/nix/module-eval/agent-queue-bao.nix +++ b/nix/module-eval/agent-queue-bao.nix @@ -223,6 +223,30 @@ let in lib.hasInfix "bao kv get -format=json ${arg} >\"$raw\"" (agentBaoIdentity m).script; } + { + # dhcpcd without IPv4LL waiting for IPv4 makes `network-online` a lease, + # and the two units that dial the store at start order after it. The + # trust bundle's order after the identity check and `nohook` are the + # controls that the lookups find something. + name = "an agent reaches network-online only with a lease, and its store identity and trust bundle wait for it"; + ok = + let + d = agentBao.networking.dhcpcd; + after = n: agentBao.systemd.services.${n}.after; + wants = n: agentBao.systemd.services.${n}.wants; + units = [ + "hive-agent-bao-identity" + "hive-agent-trust-bundle" + ]; + in + lib.hasInfix "noipv4ll" d.extraConfig + && lib.hasInfix "nohook resolv.conf" d.extraConfig + && d.wait == "ipv4" + && lib.elem "hive-agent-bao-identity.service" (after "hive-agent-trust-bundle") + && lib.all ( + n: lib.elem "network-online.target" (after n) && lib.elem "network-online.target" (wants n) + ) units; + } { # The absence arm, and what makes the four above able to fail. An agent # whose swarm never minted an identity has nothing to log in with, and a diff --git a/nix/module-eval/bao-matrix-reader.nix b/nix/module-eval/bao-matrix-reader.nix index 553b1384..494692c8 100644 --- a/nix/module-eval/bao-matrix-reader.nix +++ b/nix/module-eval/bao-matrix-reader.nix @@ -276,6 +276,8 @@ let in lib.elem "tuwunel.service" r.before && lib.elem "tuwunel.service" r.wantedBy + && lib.elem "network-online.target" r.after + && lib.elem "network-online.target" r.wants && !(lib.elem "tuwunel.service" (r.requiredBy or [ ])) && baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.serviceConfig.Restart diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 2f52c9ce..fc534688 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -295,6 +295,31 @@ let && !(sc.DynamicUser or false) && swarmServicesHere.services.hyperhive.deploy.nats.enable; } + { + # dhcpcd without IPv4LL, waiting for IPv4: `network-online` is a lease. + # The router sets no `nohook` itself and authelia does; `nohook` is the + # control that the lookup finds `extraConfig`. The queue shares the host + # netns and keeps the default. + name = "private-netns service containers wait for a DHCP lease and the forwarder fetch waits for it"; + ok = + let + cfgOf = name: swarmServicesHere.containers.${name}.config; + leased = + name: + let + d = (cfgOf name).networking.dhcpcd; + in + lib.hasInfix "noipv4ll" d.extraConfig && d.wait == "ipv4"; + fetch = (cfgOf "swarm-builder").systemd.services.swarm-journal-forwarder-oidc; + in + leased "swarm-notify-router" + && leased "swarm-authelia" + && lib.hasInfix "nohook resolv.conf" (cfgOf "swarm-authelia").networking.dhcpcd.extraConfig + && !(lib.hasInfix "noipv4ll" (cfgOf "swarm-nats").networking.dhcpcd.extraConfig) + && (cfgOf "swarm-nats").networking.dhcpcd.wait == "any" + && lib.elem "network-online.target" fetch.after + && lib.elem "network-online.target" fetch.wants; + } { name = "the notification router stays off where the shared services are hosted without the store"; ok =