hyperhive/nix/host-modules/hive-snapshot-store.nix

# hive-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push
# agent snapshots here over the existing WireGuard mesh; a destination
# hive later pulls one back to complete a migration. Only the receive
# half exists today — the pull side needs an authorisation model for
# "which hive may fetch which agent's state", which lands with the
# swarm controller.
#
# This is NOT the swarm controller and does not depend on it: a btrfs
# subvolume tree, a socket-activated receiver, and the `wg-hive`
# interface `swarm.nix` already brings up. Deliberately no WireGuard
# config of its own --- the mesh's cryptokey routing
# (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's
# source address to its public key, so the mesh IS the authentication
# and adding certs here would authenticate the same fact twice.
#
# Confinement is a property of the DEPLOYMENT, not of this unit: in a
# real swarm the store is its own small VM (the machine is the
# boundary); in the all-local case it's a container on the c0re host.
# The module hardcodes neither. docs/snapshot-store.md covers what the
# deployment is expected to provide.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.snapshotStore;
  wgCfg = config.services.hyperhive.swarm.wireguard;

  # `swarm.wireguard.address` carries a prefix ("10.100.0.1/24") because
  # it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen
  # address must be the bare IP, so strip it.
  meshAddress = lib.head (lib.splitString "/" wgCfg.address);

  # The receiver. Socket-activated with Accept=yes, so stdin IS the
  # accepted connection and systemd hands us the peer address in
  # $REMOTE_ADDR --- which, on this interface, is a cryptographically
  # authenticated statement about which hive is talking (see the
  # cryptokey-routing note above).
  #
  # PROTOCOL: one `agent <name>\n` header line, then the raw `btrfs
  # send` stream. The header exists because a btrfs stream does not
  # carry the sending hive's notion of *which agent* it is --- the
  # subvolume name inside the stream is chosen by the sender.
  #
  # ⚠️ The security rule, stated precisely, because the absolute form
  # ("the sender never names its destination") is not achievable with
  # btrfs send/receive: the RECEIVER owns the destination ROOT, and any
  # sender-supplied component is VALIDATED, never used as a path. The
  # name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no
  # traversal and no absolute path can survive it. The root is ours;
  # the leaf is checked against a whitelist charset before it is joined.
  receiveScript = pkgs.writeShellScript "hive-snapshot-receive" ''
    set -euo pipefail

    # Read exactly the header line, leaving the byte stream untouched
    # for btrfs receive. `read` stops at the newline and does not
    # buffer ahead, which is why the header is a line and not a
    # fixed-width record.
    if ! read -r keyword agent; then
      echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2
      exit 1
    fi

    if [ "$keyword" != "agent" ]; then
      echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2
      exit 1
    fi

    # Validate rather than trust. Anything outside this charset is
    # rejected outright --- this is the check that makes the joined
    # path below safe, so it must stay a whitelist, never a blocklist
    # of bad characters.
    case "$agent" in
      "" | *[!A-Za-z0-9_-]*)
        echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2
        exit 1
        ;;
    esac

    dest="${cfg.path}/$agent"

    # One subvolume tree per AGENT, not per hive: after a migration the
    # same agent's next incremental send arrives from a DIFFERENT hive,
    # and a per-hive prefix would split its snapshot chain in two and
    # break the incremental parent lookup --- exactly the case this
    # store exists to serve.
    mkdir -p "$dest"

    echo "hive-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2
    exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest"
  '';
in
{
  options.services.hyperhive.snapshotStore = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Run the swarm snapshot store on this host: a `btrfs receive`
        endpoint that hives push agent snapshots to over the WireGuard
        mesh. Off by default --- it is a distinct deployment role, not
        part of a hive.

        Requires `services.hyperhive.swarm.wireguard.enable`: the mesh
        is both the transport and the authentication, so there is no
        meaningful configuration without it.
      '';
    };

    path = lib.mkOption {
      type = lib.types.path;
      default = "/var/lib/hyperhive-snapshots";
      description = ''
        Root of the snapshot tree. Must be on a btrfs filesystem ---
        `btrfs receive` fails otherwise. One subvolume directory per
        agent is created beneath it, so an agent's incremental chain
        stays contiguous across a migration between hives.
      '';
    };

    port = lib.mkOption {
      type = lib.types.port;
      default = 51821;
      description = ''
        TCP port the receiver listens on. Bound to this host's
        WireGuard mesh address only --- never a wildcard --- so it is
        reachable exactly by mesh peers and by nothing else.
      '';
    };

  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = wgCfg.enable;
        message = ''
          services.hyperhive.snapshotStore.enable requires
          services.hyperhive.swarm.wireguard.enable --- the mesh is the
          store's transport AND its authentication (cryptokey routing
          binds a peer's source address to its public key). Without it
          there is nothing to bind the listener to and no way to tell
          which hive is pushing.
        '';
      }
      {
        assertion = wgCfg.address != "";
        message = ''
          services.hyperhive.snapshotStore.enable requires
          services.hyperhive.swarm.wireguard.address to be set --- the
          receiver binds to this host's mesh address, and refuses to
          fall back to a wildcard.
        '';
      }
    ];

    # The store root must exist before the first connection arrives ---
    # the receiver runs on demand and should not be the thing that
    # creates its own tree lazily.
    systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ];

    # Open the receiver's port, scoped to the mesh interface.
    #
    # ⚠️ Binding the socket to the mesh address is NOT sufficient on its
    # own: NixOS's firewall is default-deny and filters in netfilter,
    # before a packet ever reaches a bound socket. The bind chooses
    # WHICH address accepts connections; it does not open the port. The
    # mesh's own UDP port is opened the same explicit way in swarm.nix.
    #
    # Interface-scoped rather than host-wide so the reachability
    # property stays exactly what the option docs claim --- mesh peers
    # and nobody else. A global `allowedTCPPorts` would open the port on
    # every interface, including whatever public NIC the box has, and
    # only the socket's bind address would still be standing between
    # the internet and a root `btrfs receive`.
    networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ cfg.port ];

    # Socket-activated on purpose: no long-running root daemon, and the
    # unit exists only while a transfer does.
    #
    # ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a
    # wildcard this socket would be an unauthenticated remote write
    # into agent state, so the binding IS the access control and is
    # asserted above rather than left to a comment.
    #
    # Accept=yes gives one service instance per connection and sets
    # $REMOTE_ADDR for the handler --- which is how the receiver knows
    # which peer it is talking to.
    systemd.sockets.hive-snapshot-store = {
      description = "hyperhive swarm snapshot store receiver socket";
      wantedBy = [ "sockets.target" ];
      socketConfig = {
        ListenStream = "${meshAddress}:${toString cfg.port}";
        Accept = "yes";
      };
    };

    # `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by
    # nature. The hardening below is defence in depth and NOT a
    # boundary: a process holding CAP_SYS_ADMIN can call mount(2) and
    # undo the namespace these directives set up. The real boundary is
    # the deployment --- see docs/snapshot-store.md.
    systemd.services."hive-snapshot-store@" = {
      description = "hyperhive swarm snapshot store receiver";
      after = [ "hive-snapshot-store.socket" ];
      requires = [ "hive-snapshot-store.socket" ];
      serviceConfig = {
        ExecStart = receiveScript;
        SyslogIdentifier = "hive-snapshot-store";
        # StandardInput=socket wires the accepted connection to stdin,
        # which is what the handler reads the header + stream from.
        StandardInput = "socket";
        StandardError = "journal";
        User = "root";
        PrivateTmp = true;
        ProtectHome = true;
        ProtectSystem = "strict";
        ReadWritePaths = [ cfg.path ];
      };
    };
  };
}