# swarm-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push # agent snapshots here over the existing WireGuard mesh; a destination # hive later pulls one back to complete a migration. Only the receive # half exists today — the pull side needs an authorisation model for # "which hive may fetch which agent's state", which lands with the # swarm controller. # # This is NOT the swarm controller and does not depend on it: a btrfs # subvolume tree, a socket-activated receiver, and the `wg-hive` # interface `swarm.nix` already brings up. Deliberately no WireGuard # config of its own --- the mesh's cryptokey routing # (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's # source address to its public key, so the mesh IS the authentication # and adding certs here would authenticate the same fact twice. # # Confinement is a property of the DEPLOYMENT, not of this unit: in a # real swarm the store is its own small VM (the machine is the # boundary); in the all-local case it's a container on the c0re host. # The module hardcodes neither. docs/snapshot-store.md covers what the # deployment is expected to provide. { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.snapshotStore; wgCfg = config.services.hyperhive.swarm.wireguard; # `swarm.wireguard.address` carries a prefix ("10.100.0.1/24") because # it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen # address must be the bare IP, so strip it. meshAddress = lib.head (lib.splitString "/" wgCfg.address); # The receiver. Socket-activated with Accept=yes, so stdin IS the # accepted connection and systemd hands us the peer address in # $REMOTE_ADDR --- which, on this interface, is a cryptographically # authenticated statement about which hive is talking (see the # cryptokey-routing note above). # # PROTOCOL: one `agent \n` header line, then the raw `btrfs # send` stream. The header exists because a btrfs stream does not # carry the sending hive's notion of *which agent* it is --- the # subvolume name inside the stream is chosen by the sender. # # ⚠️ The security rule, stated precisely, because the absolute form # ("the sender never names its destination") is not achievable with # btrfs send/receive: the RECEIVER owns the destination ROOT, and any # sender-supplied component is VALIDATED, never used as a path. The # name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no # traversal and no absolute path can survive it. The root is ours; # the leaf is checked against a whitelist charset before it is joined. receiveScript = pkgs.writeShellScript "swarm-snapshot-receive" '' set -euo pipefail # Read exactly the header line, leaving the byte stream untouched # for btrfs receive. `read` stops at the newline and does not # buffer ahead, which is why the header is a line and not a # fixed-width record. if ! read -r keyword agent; then echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2 exit 1 fi if [ "$keyword" != "agent" ]; then echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2 exit 1 fi # Validate rather than trust. Anything outside this charset is # rejected outright --- this is the check that makes the joined # path below safe, so it must stay a whitelist, never a blocklist # of bad characters. case "$agent" in "" | *[!A-Za-z0-9_-]*) echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2 exit 1 ;; esac dest="${cfg.path}/$agent" # One subvolume tree per AGENT, not per hive: after a migration the # same agent's next incremental send arrives from a DIFFERENT hive, # and a per-hive prefix would split its snapshot chain in two and # break the incremental parent lookup --- exactly the case this # store exists to serve. mkdir -p "$dest" echo "swarm-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2 exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest" ''; in { options.services.hyperhive.snapshotStore = { enable = lib.mkOption { type = lib.types.bool; default = false; description = '' Run the swarm snapshot store on this host: a `btrfs receive` endpoint that hives push agent snapshots to over the WireGuard mesh. Off by default --- it is a distinct deployment role, not part of a hive. Requires `services.hyperhive.swarm.wireguard.enable`: the mesh is both the transport and the authentication, so there is no meaningful configuration without it. ''; }; path = lib.mkOption { type = lib.types.path; default = "/var/lib/hyperhive-snapshots"; description = '' Root of the snapshot tree. Must be on a btrfs filesystem --- `btrfs receive` fails otherwise. One subvolume directory per agent is created beneath it, so an agent's incremental chain stays contiguous across a migration between hives. ''; }; port = lib.mkOption { type = lib.types.port; default = 51821; description = '' TCP port the receiver listens on. Bound to this host's WireGuard mesh address only --- never a wildcard --- so it is reachable exactly by mesh peers and by nothing else. ''; }; }; config = lib.mkIf cfg.enable { assertions = [ { assertion = wgCfg.enable; message = '' services.hyperhive.snapshotStore.enable requires services.hyperhive.swarm.wireguard.enable --- the mesh is the store's transport AND its authentication (cryptokey routing binds a peer's source address to its public key). Without it there is nothing to bind the listener to and no way to tell which hive is pushing. ''; } { assertion = wgCfg.address != ""; message = '' services.hyperhive.snapshotStore.enable requires services.hyperhive.swarm.wireguard.address to be set --- the receiver binds to this host's mesh address, and refuses to fall back to a wildcard. ''; } ]; # The store root must exist before the first connection arrives --- # the receiver runs on demand and should not be the thing that # creates its own tree lazily. systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ]; # Open the receiver's port, scoped to the mesh interface. # # ⚠️ Binding the socket to the mesh address is NOT sufficient on its # own: NixOS's firewall is default-deny and filters in netfilter, # before a packet ever reaches a bound socket. The bind chooses # WHICH address accepts connections; it does not open the port. The # mesh's own UDP port is opened the same explicit way in swarm.nix. # # Interface-scoped rather than host-wide so the reachability # property stays exactly what the option docs claim --- mesh peers # and nobody else. A global `allowedTCPPorts` would open the port on # every interface, including whatever public NIC the box has, and # only the socket's bind address would still be standing between # the internet and a root `btrfs receive`. networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ cfg.port ]; # Socket-activated on purpose: no long-running root daemon, and the # unit exists only while a transfer does. # # ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a # wildcard this socket would be an unauthenticated remote write # into agent state, so the binding IS the access control and is # asserted above rather than left to a comment. # # Accept=yes gives one service instance per connection and sets # $REMOTE_ADDR for the handler --- which is how the receiver knows # which peer it is talking to. systemd.sockets.swarm-snapshot-store = { description = "hyperhive swarm snapshot store receiver socket"; wantedBy = [ "sockets.target" ]; socketConfig = { ListenStream = "${meshAddress}:${toString cfg.port}"; Accept = "yes"; }; }; # `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by # nature. The hardening below is defence in depth and NOT a # boundary: a process holding CAP_SYS_ADMIN can call mount(2) and # undo the namespace these directives set up. The real boundary is # the deployment --- see docs/snapshot-store.md. systemd.services."swarm-snapshot-store@" = { description = "hyperhive swarm snapshot store receiver"; after = [ "swarm-snapshot-store.socket" ]; requires = [ "swarm-snapshot-store.socket" ]; serviceConfig = { ExecStart = receiveScript; SyslogIdentifier = "swarm-snapshot-store"; # StandardInput=socket wires the accepted connection to stdin, # which is what the handler reads the header + stream from. StandardInput = "socket"; StandardError = "journal"; User = "root"; PrivateTmp = true; ProtectHome = true; ProtectSystem = "strict"; ReadWritePaths = [ cfg.path ]; }; }; }; }