# hive-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push # agent snapshots here over the existing WireGuard mesh; a destination # hive later pulls one back to complete a migration. Only the receive # half exists today — the pull side needs an authorisation model for # "which hive may fetch which agent's state", which lands with the # swarm controller. # # This is NOT the swarm controller and does not depend on it: a btrfs # subvolume tree, a socket-activated receiver, and the `wg-hive` # interface `swarm.nix` already brings up. Deliberately no WireGuard # config of its own --- the mesh's cryptokey routing # (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's # source address to its public key, so the mesh IS the authentication # and adding certs here would authenticate the same fact twice. # # Confinement is a property of the DEPLOYMENT, not of this unit: in a # real swarm the store is its own small VM (the machine is the # boundary); in the all-local case it's a container on the c0re host. # The module therefore hardcodes neither --- see `dedicated` below. { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.snapshotStore; wgCfg = config.services.hyperhive.swarm.wireguard; # `swarm.wireguard.address` carries a prefix ("10.100.0.1/24") because # it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen # address must be the bare IP, so strip it. meshAddress = lib.head (lib.splitString "/" wgCfg.address); # The receiver. Socket-activated with Accept=yes, so stdin IS the # accepted connection and systemd hands us the peer address in # $REMOTE_ADDR --- which, on this interface, is a cryptographically # authenticated statement about which hive is talking (see the # cryptokey-routing note above). # # PROTOCOL: one `agent \n` header line, then the raw `btrfs # send` stream. The header exists because a btrfs stream does not # carry the sending hive's notion of *which agent* it is --- the # subvolume name inside the stream is chosen by the sender. # # ⚠️ The security rule, stated precisely, because the absolute form # ("the sender never names its destination") is not achievable with # btrfs send/receive: the RECEIVER owns the destination ROOT, and any # sender-supplied component is VALIDATED, never used as a path. The # name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no # traversal and no absolute path can survive it. The root is ours; # the leaf is checked against a whitelist charset before it is joined. receiveScript = pkgs.writeShellScript "hive-snapshot-receive" '' set -euo pipefail # Read exactly the header line, leaving the byte stream untouched # for btrfs receive. `read` stops at the newline and does not # buffer ahead, which is why the header is a line and not a # fixed-width record. if ! read -r keyword agent; then echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2 exit 1 fi if [ "$keyword" != "agent" ]; then echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2 exit 1 fi # Validate rather than trust. Anything outside this charset is # rejected outright --- this is the check that makes the joined # path below safe, so it must stay a whitelist, never a blocklist # of bad characters. case "$agent" in "" | *[!A-Za-z0-9_-]*) echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2 exit 1 ;; esac dest="${cfg.path}/$agent" # One subvolume tree per AGENT, not per hive: after a migration the # same agent's next incremental send arrives from a DIFFERENT hive, # and a per-hive prefix would split its snapshot chain in two and # break the incremental parent lookup --- exactly the case this # store exists to serve. mkdir -p "$dest" echo "hive-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2 exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest" ''; in { options.services.hyperhive.snapshotStore = { enable = lib.mkOption { type = lib.types.bool; default = false; description = '' Run the swarm snapshot store on this host: a `btrfs receive` endpoint that hives push agent snapshots to over the WireGuard mesh. Off by default --- it is a distinct deployment role, not part of a hive. Requires `services.hyperhive.swarm.wireguard.enable`: the mesh is both the transport and the authentication, so there is no meaningful configuration without it. ''; }; path = lib.mkOption { type = lib.types.path; default = "/var/lib/hyperhive-snapshots"; description = '' Root of the snapshot tree. Must be on a btrfs filesystem --- `btrfs receive` fails otherwise. One subvolume directory per agent is created beneath it, so an agent's incremental chain stays contiguous across a migration between hives. ''; }; port = lib.mkOption { type = lib.types.port; default = 51821; description = '' TCP port the receiver listens on. Bound to this host's WireGuard mesh address only --- never a wildcard --- so it is reachable exactly by mesh peers and by nothing else. ''; }; dedicated = lib.mkOption { type = lib.types.bool; default = true; description = '' Assert that this host runs no other hyperhive role. The store aggregates every agent's state from every hive in the swarm, so the intended deployment is a dedicated machine (or a container in the all-local case) where the machine itself is the security boundary. That assumption is true on day one and silently false the day someone notices the box has spare disk. This option makes it a thing the build checks rather than a thing the deployer remembers. Set to `false` to co-locate deliberately --- the point is that it becomes a decision, not an accident. ''; }; }; config = lib.mkIf cfg.enable { assertions = [ { assertion = wgCfg.enable; message = '' services.hyperhive.snapshotStore.enable requires services.hyperhive.swarm.wireguard.enable --- the mesh is the store's transport AND its authentication (cryptokey routing binds a peer's source address to its public key). Without it there is nothing to bind the listener to and no way to tell which hive is pushing. ''; } { assertion = wgCfg.address != ""; message = '' services.hyperhive.snapshotStore.enable requires services.hyperhive.swarm.wireguard.address to be set --- the receiver binds to this host's mesh address, and refuses to fall back to a wildcard. ''; } { assertion = !cfg.dedicated || !config.services.hyperhive.c0re.enable; message = '' services.hyperhive.snapshotStore is enabled alongside services.hyperhive.c0re on the same host. The store holds every agent's state from every hive, so it is meant to run on a machine of its own where the machine is the boundary. If the co-location is deliberate (the all-local single-host deployment, where the store runs as a container), set services.hyperhive.snapshotStore.dedicated = false to record that decision explicitly. ''; } ]; # The store root must exist before the first connection arrives --- # the receiver runs on demand and should not be the thing that # creates its own tree lazily. systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ]; # Socket-activated on purpose: no long-running root daemon, and the # unit exists only while a transfer does. # # ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a # wildcard this socket would be an unauthenticated remote write # into agent state, so the binding IS the access control and is # asserted above rather than left to a comment. # # Accept=yes gives one service instance per connection and sets # $REMOTE_ADDR for the handler --- which is how the receiver knows # which peer it is talking to. systemd.sockets.hive-snapshot-store = { description = "hyperhive swarm snapshot store receiver socket"; wantedBy = [ "sockets.target" ]; socketConfig = { ListenStream = "${meshAddress}:${toString cfg.port}"; Accept = "yes"; }; }; # `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by # nature. The hardening below is defence in depth and NOT a # boundary: a process holding CAP_SYS_ADMIN can call mount(2) and # undo the namespace these directives set up. The real boundary is # the deployment (dedicated host / container) --- see `dedicated`. systemd.services."hive-snapshot-store@" = { description = "hyperhive swarm snapshot store receiver"; after = [ "hive-snapshot-store.socket" ]; requires = [ "hive-snapshot-store.socket" ]; serviceConfig = { ExecStart = receiveScript; SyslogIdentifier = "hive-snapshot-store"; # StandardInput=socket wires the accepted connection to stdin, # which is what the handler reads the header + stream from. StandardInput = "socket"; StandardError = "journal"; User = "root"; PrivateTmp = true; ProtectHome = true; ProtectSystem = "strict"; ReadWritePaths = [ cfg.path ]; }; }; }; }