diff --git a/docs/networking/snapshot-store.md b/docs/networking/snapshot-store.md index 9e3353ee..8281427a 100644 --- a/docs/networking/snapshot-store.md +++ b/docs/networking/snapshot-store.md @@ -123,10 +123,15 @@ somebody already thought of. ## Reachability The receiver is socket-activated, and the socket binds **this host's -mesh address**, never a wildcard. The module asserts two things: the -mesh is on, and the address isn't empty --- neither is just documentation: bound to -`0.0.0.0` this socket is an unauthenticated remote write into agent -state. +mesh address**, never a wildcard. The module asserts three things: the +mesh is on, the address isn't empty, and it isn't `0.0.0.0` or `::` --- +none is just documentation: bound to a wildcard this socket is an +unauthenticated remote write into agent state. + +The socket sets `FreeBind`, so the bind succeeds before `wg-hive` has +its address. Nothing orders the wireguard unit before `sockets.target`, +and without it the bind fails with `EADDRNOTAVAIL` at boot and the +receiver never listens. Binding isn't sufficient on its own. NixOS's firewall is default-deny and filters in netfilter, _before_ a packet reaches a bound socket, so diff --git a/nix/checks.nix b/nix/checks.nix index 1bcf7fc3..34fff7ad 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -150,6 +150,10 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; + module-eval-snapshot-store = import ./module-eval/snapshot-store.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/swarm-snapshot-store.nix b/nix/host-modules/swarm-snapshot-store.nix index 4ad556ff..f8906890 100644 --- a/nix/host-modules/swarm-snapshot-store.nix +++ b/nix/host-modules/swarm-snapshot-store.nix @@ -154,6 +154,21 @@ in fall back to a wildcard. ''; } + { + assertion = + !(lib.elem meshAddress [ + "" + "0.0.0.0" + "::" + ]); + message = '' + The snapshot store refuses a wildcard listen address. + services.hyperhive.deploy.wireguard.address is "${wgCfg.address}", + which would make the receiver bind "${meshAddress}" on every + interface. Set it to this host's address on the mesh + (e.g. "10.100.0.1/24"). + ''; + } ]; # The store root must exist before the first connection arrives --- @@ -185,6 +200,12 @@ in # into agent state, so the binding IS the access control and is # asserted above rather than left to a comment. # + # FreeBind lets the bind succeed before `wg-hive` has its address: + # sockets.target is reached before the wireguard unit, which + # nothing orders ahead of it, so without it the bind fails with + # EADDRNOTAVAIL and the receiver never listens. It also covers the + # interface coming back after a flap. + # # Accept=yes gives one service instance per connection and sets # $REMOTE_ADDR for the handler --- which is how the receiver knows # which peer it is talking to. @@ -194,6 +215,7 @@ in socketConfig = { ListenStream = "${meshAddress}:${toString cfg.port}"; Accept = "yes"; + FreeBind = true; }; }; diff --git a/nix/module-eval/snapshot-store.nix b/nix/module-eval/snapshot-store.nix new file mode 100644 index 00000000..5dc4d6c0 --- /dev/null +++ b/nix/module-eval/snapshot-store.nix @@ -0,0 +1,85 @@ +# `checks.module-eval-snapshot-store` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates not +# executes"). +# +# The snapshot store's receiver socket: where it listens, that it can bind +# before the mesh interface is up, and that it refuses a wildcard. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # Bare hive: the store and the mesh it needs, nothing else. + store = hive { + deploy.wireguard.enable = true; + deploy.wireguard.address = "10.100.0.1/24"; + snapshotStore.enable = true; + snapshotStore.port = 51822; + }; + + # The same with a wildcard where the mesh address goes. + storeOnWildcard = hive { + deploy.wireguard.enable = true; + deploy.wireguard.address = "0.0.0.0/24"; + snapshotStore.enable = true; + }; + + # The IPv6 wildcard. + storeOnWildcardV6 = hive { + deploy.wireguard.enable = true; + deploy.wireguard.address = "::/64"; + snapshotStore.enable = true; + }; + + socket = store.systemd.sockets.swarm-snapshot-store.socketConfig; + + refusedAsWildcard = + m: + lib.any ( + a: !a.assertion && lib.hasInfix "refuses a wildcard listen address" a.message + ) m.assertions; + + cases = [ + { + # The wg-hive address does not exist when sockets.target binds, so + # without FreeBind the bind fails with EADDRNOTAVAIL and the receiver + # never listens. The unit still renders and evaluates either way, which + # is why only a rendered-config check can see it. + name = "the receiver socket may bind before the mesh interface has its address"; + ok = socket.FreeBind or false == true; + } + { + # The bare IP and the configured port: the prefix length on + # `deploy.wireguard.address` must not leak into the listen address. + name = "the receiver socket listens on the mesh address and the configured port"; + ok = socket.ListenStream == "10.100.0.1:51822"; + } + { + # The control for the case below: it is the wildcard that trips the + # assertion, not the store being on. + name = "a mesh address on the store raises no wildcard refusal"; + ok = !(refusedAsWildcard store); + } + { + # A wildcard bind is an unauthenticated remote write into agent state. + name = "a wildcard mesh address, IPv4 or IPv6, is refused"; + ok = refusedAsWildcard storeOnWildcard && refusedAsWildcard storeOnWildcardV6; + } + ]; +in +runGroup "snapshot-store" cases