From 4285faf07c48d5406b59a48c3d5bfa4e1aaf1429 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 15 Oct 2026 03:15:43 +0200 Subject: [PATCH] swarm snapshot store: the receiver socket binds the mesh address even when wg-hive comes up after sockets.target The socket is wanted by sockets.target and nothing orders the wireguard unit before it, so the mesh address can be missing at bind time (EADDRNOTAVAIL) and the receiver never listens. FreeBind lets the bind succeed regardless, and also covers the interface coming back after a flap. Also assert the mesh address is not a wildcard, which the comment above the socket already claimed, and cover the rendered socket in a module-eval case. --- docs/networking/snapshot-store.md | 13 ++-- nix/checks.nix | 4 ++ nix/host-modules/swarm-snapshot-store.nix | 22 ++++++ nix/module-eval/snapshot-store.nix | 85 +++++++++++++++++++++++ 4 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 nix/module-eval/snapshot-store.nix diff --git a/docs/networking/snapshot-store.md b/docs/networking/snapshot-store.md index 9e3353ee..8281427a 100644 --- a/docs/networking/snapshot-store.md +++ b/docs/networking/snapshot-store.md @@ -123,10 +123,15 @@ somebody already thought of. ## Reachability The receiver is socket-activated, and the socket binds **this host's -mesh address**, never a wildcard. The module asserts two things: the -mesh is on, and the address isn't empty --- neither is just documentation: bound to -`0.0.0.0` this socket is an unauthenticated remote write into agent -state. +mesh address**, never a wildcard. The module asserts three things: the +mesh is on, the address isn't empty, and it isn't `0.0.0.0` or `::` --- +none is just documentation: bound to a wildcard this socket is an +unauthenticated remote write into agent state. + +The socket sets `FreeBind`, so the bind succeeds before `wg-hive` has +its address. Nothing orders the wireguard unit before `sockets.target`, +and without it the bind fails with `EADDRNOTAVAIL` at boot and the +receiver never listens. Binding isn't sufficient on its own. NixOS's firewall is default-deny and filters in netfilter, _before_ a packet reaches a bound socket, so diff --git a/nix/checks.nix b/nix/checks.nix index 1bcf7fc3..34fff7ad 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -150,6 +150,10 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; + module-eval-snapshot-store = import ./module-eval/snapshot-store.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/swarm-snapshot-store.nix b/nix/host-modules/swarm-snapshot-store.nix index 4ad556ff..f8906890 100644 --- a/nix/host-modules/swarm-snapshot-store.nix +++ b/nix/host-modules/swarm-snapshot-store.nix @@ -154,6 +154,21 @@ in fall back to a wildcard. ''; } + { + assertion = + !(lib.elem meshAddress [ + "" + "0.0.0.0" + "::" + ]); + message = '' + The snapshot store refuses a wildcard listen address. + services.hyperhive.deploy.wireguard.address is "${wgCfg.address}", + which would make the receiver bind "${meshAddress}" on every + interface. Set it to this host's address on the mesh + (e.g. "10.100.0.1/24"). + ''; + } ]; # The store root must exist before the first connection arrives --- @@ -185,6 +200,12 @@ in # into agent state, so the binding IS the access control and is # asserted above rather than left to a comment. # + # FreeBind lets the bind succeed before `wg-hive` has its address: + # sockets.target is reached before the wireguard unit, which + # nothing orders ahead of it, so without it the bind fails with + # EADDRNOTAVAIL and the receiver never listens. It also covers the + # interface coming back after a flap. + # # Accept=yes gives one service instance per connection and sets # $REMOTE_ADDR for the handler --- which is how the receiver knows # which peer it is talking to. @@ -194,6 +215,7 @@ in socketConfig = { ListenStream = "${meshAddress}:${toString cfg.port}"; Accept = "yes"; + FreeBind = true; }; }; diff --git a/nix/module-eval/snapshot-store.nix b/nix/module-eval/snapshot-store.nix new file mode 100644 index 00000000..5dc4d6c0 --- /dev/null +++ b/nix/module-eval/snapshot-store.nix @@ -0,0 +1,85 @@ +# `checks.module-eval-snapshot-store` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates not +# executes"). +# +# The snapshot store's receiver socket: where it listens, that it can bind +# before the mesh interface is up, and that it refuses a wildcard. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # Bare hive: the store and the mesh it needs, nothing else. + store = hive { + deploy.wireguard.enable = true; + deploy.wireguard.address = "10.100.0.1/24"; + snapshotStore.enable = true; + snapshotStore.port = 51822; + }; + + # The same with a wildcard where the mesh address goes. + storeOnWildcard = hive { + deploy.wireguard.enable = true; + deploy.wireguard.address = "0.0.0.0/24"; + snapshotStore.enable = true; + }; + + # The IPv6 wildcard. + storeOnWildcardV6 = hive { + deploy.wireguard.enable = true; + deploy.wireguard.address = "::/64"; + snapshotStore.enable = true; + }; + + socket = store.systemd.sockets.swarm-snapshot-store.socketConfig; + + refusedAsWildcard = + m: + lib.any ( + a: !a.assertion && lib.hasInfix "refuses a wildcard listen address" a.message + ) m.assertions; + + cases = [ + { + # The wg-hive address does not exist when sockets.target binds, so + # without FreeBind the bind fails with EADDRNOTAVAIL and the receiver + # never listens. The unit still renders and evaluates either way, which + # is why only a rendered-config check can see it. + name = "the receiver socket may bind before the mesh interface has its address"; + ok = socket.FreeBind or false == true; + } + { + # The bare IP and the configured port: the prefix length on + # `deploy.wireguard.address` must not leak into the listen address. + name = "the receiver socket listens on the mesh address and the configured port"; + ok = socket.ListenStream == "10.100.0.1:51822"; + } + { + # The control for the case below: it is the wildcard that trips the + # assertion, not the store being on. + name = "a mesh address on the store raises no wildcard refusal"; + ok = !(refusedAsWildcard store); + } + { + # A wildcard bind is an unauthenticated remote write into agent state. + name = "a wildcard mesh address, IPv4 or IPv6, is refused"; + ok = refusedAsWildcard storeOnWildcard && refusedAsWildcard storeOnWildcardV6; + } + ]; +in +runGroup "snapshot-store" cases