diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 72451000..79647d04 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -19,6 +19,7 @@ ./hive-matrix.nix ./hive-network.nix ./hive-priv.nix + ./hive-snapshot-store.nix ./hive-tls.nix ./otel.nix ./swarm.nix diff --git a/nix/host-modules/hive-snapshot-store.nix b/nix/host-modules/hive-snapshot-store.nix new file mode 100644 index 00000000..fa149656 --- /dev/null +++ b/nix/host-modules/hive-snapshot-store.nix @@ -0,0 +1,238 @@ +# hive-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push +# agent snapshots here over the existing WireGuard mesh; a destination +# hive later pulls one back to complete a migration. Only the receive +# half exists today — the pull side needs an authorisation model for +# "which hive may fetch which agent's state", which lands with the +# swarm controller. +# +# This is NOT the swarm controller and does not depend on it: a btrfs +# subvolume tree, a socket-activated receiver, and the `wg-hive` +# interface `swarm.nix` already brings up. Deliberately no WireGuard +# config of its own --- the mesh's cryptokey routing +# (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's +# source address to its public key, so the mesh IS the authentication +# and adding certs here would authenticate the same fact twice. +# +# Confinement is a property of the DEPLOYMENT, not of this unit: in a +# real swarm the store is its own small VM (the machine is the +# boundary); in the all-local case it's a container on the c0re host. +# The module therefore hardcodes neither --- see `dedicated` below. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.snapshotStore; + wgCfg = config.services.hyperhive.swarm.wireguard; + + # `swarm.wireguard.address` carries a prefix ("10.100.0.1/24") because + # it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen + # address must be the bare IP, so strip it. + meshAddress = lib.head (lib.splitString "/" wgCfg.address); + + # The receiver. Socket-activated with Accept=yes, so stdin IS the + # accepted connection and systemd hands us the peer address in + # $REMOTE_ADDR --- which, on this interface, is a cryptographically + # authenticated statement about which hive is talking (see the + # cryptokey-routing note above). + # + # PROTOCOL: one `agent \n` header line, then the raw `btrfs + # send` stream. The header exists because a btrfs stream does not + # carry the sending hive's notion of *which agent* it is --- the + # subvolume name inside the stream is chosen by the sender. + # + # ⚠️ The security rule, stated precisely, because the absolute form + # ("the sender never names its destination") is not achievable with + # btrfs send/receive: the RECEIVER owns the destination ROOT, and any + # sender-supplied component is VALIDATED, never used as a path. The + # name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no + # traversal and no absolute path can survive it. The root is ours; + # the leaf is checked against a whitelist charset before it is joined. + receiveScript = pkgs.writeShellScript "hive-snapshot-receive" '' + set -euo pipefail + + # Read exactly the header line, leaving the byte stream untouched + # for btrfs receive. `read` stops at the newline and does not + # buffer ahead, which is why the header is a line and not a + # fixed-width record. + if ! read -r keyword agent; then + echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2 + exit 1 + fi + + if [ "$keyword" != "agent" ]; then + echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2 + exit 1 + fi + + # Validate rather than trust. Anything outside this charset is + # rejected outright --- this is the check that makes the joined + # path below safe, so it must stay a whitelist, never a blocklist + # of bad characters. + case "$agent" in + "" | *[!A-Za-z0-9_-]*) + echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2 + exit 1 + ;; + esac + + dest="${cfg.path}/$agent" + + # One subvolume tree per AGENT, not per hive: after a migration the + # same agent's next incremental send arrives from a DIFFERENT hive, + # and a per-hive prefix would split its snapshot chain in two and + # break the incremental parent lookup --- exactly the case this + # store exists to serve. + mkdir -p "$dest" + + echo "hive-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2 + exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest" + ''; +in +{ + options.services.hyperhive.snapshotStore = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm snapshot store on this host: a `btrfs receive` + endpoint that hives push agent snapshots to over the WireGuard + mesh. Off by default --- it is a distinct deployment role, not + part of a hive. + + Requires `services.hyperhive.swarm.wireguard.enable`: the mesh + is both the transport and the authentication, so there is no + meaningful configuration without it. + ''; + }; + + path = lib.mkOption { + type = lib.types.path; + default = "/var/lib/hyperhive-snapshots"; + description = '' + Root of the snapshot tree. Must be on a btrfs filesystem --- + `btrfs receive` fails otherwise. One subvolume directory per + agent is created beneath it, so an agent's incremental chain + stays contiguous across a migration between hives. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 51821; + description = '' + TCP port the receiver listens on. Bound to this host's + WireGuard mesh address only --- never a wildcard --- so it is + reachable exactly by mesh peers and by nothing else. + ''; + }; + + dedicated = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Assert that this host runs no other hyperhive role. The store + aggregates every agent's state from every hive in the swarm, so + the intended deployment is a dedicated machine (or a container + in the all-local case) where the machine itself is the security + boundary. + + That assumption is true on day one and silently false the day + someone notices the box has spare disk. This option makes it a + thing the build checks rather than a thing the deployer + remembers. Set to `false` to co-locate deliberately --- the + point is that it becomes a decision, not an accident. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = wgCfg.enable; + message = '' + services.hyperhive.snapshotStore.enable requires + services.hyperhive.swarm.wireguard.enable --- the mesh is the + store's transport AND its authentication (cryptokey routing + binds a peer's source address to its public key). Without it + there is nothing to bind the listener to and no way to tell + which hive is pushing. + ''; + } + { + assertion = wgCfg.address != ""; + message = '' + services.hyperhive.snapshotStore.enable requires + services.hyperhive.swarm.wireguard.address to be set --- the + receiver binds to this host's mesh address, and refuses to + fall back to a wildcard. + ''; + } + { + assertion = !cfg.dedicated || !config.services.hyperhive.c0re.enable; + message = '' + services.hyperhive.snapshotStore is enabled alongside + services.hyperhive.c0re on the same host. The store holds + every agent's state from every hive, so it is meant to run on + a machine of its own where the machine is the boundary. + + If the co-location is deliberate (the all-local single-host + deployment, where the store runs as a container), set + services.hyperhive.snapshotStore.dedicated = false to record + that decision explicitly. + ''; + } + ]; + + # The store root must exist before the first connection arrives --- + # the receiver runs on demand and should not be the thing that + # creates its own tree lazily. + systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ]; + + # Socket-activated on purpose: no long-running root daemon, and the + # unit exists only while a transfer does. + # + # ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a + # wildcard this socket would be an unauthenticated remote write + # into agent state, so the binding IS the access control and is + # asserted above rather than left to a comment. + # + # Accept=yes gives one service instance per connection and sets + # $REMOTE_ADDR for the handler --- which is how the receiver knows + # which peer it is talking to. + systemd.sockets.hive-snapshot-store = { + description = "hyperhive swarm snapshot store receiver socket"; + wantedBy = [ "sockets.target" ]; + socketConfig = { + ListenStream = "${meshAddress}:${toString cfg.port}"; + Accept = "yes"; + }; + }; + + # `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by + # nature. The hardening below is defence in depth and NOT a + # boundary: a process holding CAP_SYS_ADMIN can call mount(2) and + # undo the namespace these directives set up. The real boundary is + # the deployment (dedicated host / container) --- see `dedicated`. + systemd.services."hive-snapshot-store@" = { + description = "hyperhive swarm snapshot store receiver"; + after = [ "hive-snapshot-store.socket" ]; + requires = [ "hive-snapshot-store.socket" ]; + serviceConfig = { + ExecStart = receiveScript; + SyslogIdentifier = "hive-snapshot-store"; + # StandardInput=socket wires the accepted connection to stdin, + # which is what the handler reads the header + stream from. + StandardInput = "socket"; + StandardError = "journal"; + User = "root"; + PrivateTmp = true; + ProtectHome = true; + ProtectSystem = "strict"; + ReadWritePaths = [ cfg.path ]; + }; + }; + }; +} diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index a6750900..36152c4c 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -184,10 +184,15 @@ }; }; - # Gated on the c0re daemon being enabled — the mesh is part of the - # coordinator host's networking. - config = lib.mkIf config.services.hyperhive.c0re.enable { - assertions = lib.optionals config.services.hyperhive.swarm.wireguard.enable [ + # Gated on the mesh itself, NOT on the c0re daemon. The mesh is host + # networking, not a c0re feature: a swarm host that runs no hive — + # the snapshot store, for one — still has to join the mesh, and under + # the old `c0re.enable` gate it silently got no `wg-hive` interface + # at all. Nothing below is c0re-specific; the peer data + # c0re consumes (HYPERHIVE_PEERS / HIVE_PEER_CA_PATHS) is rendered in + # ./hive-c0re and stays gated there. + config = lib.mkIf config.services.hyperhive.swarm.wireguard.enable { + assertions = [ { assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null; message = '' @@ -208,7 +213,7 @@ # WireGuard inter-hive mesh. Brings up a `wg-hive` interface and # connects to each peer that has `wireguardPublicKey` set. - networking.wireguard.interfaces = lib.mkIf config.services.hyperhive.swarm.wireguard.enable ( + networking.wireguard.interfaces = let wgCfg = config.services.hyperhive.swarm.wireguard; meshPeers = lib.filterAttrs ( @@ -234,12 +239,11 @@ } ) meshPeers; }; - } - ); + }; - # Open the WireGuard UDP port on the host firewall when the mesh is - # on (host-level networking — not inside containers). - networking.firewall.allowedUDPPorts = lib.mkIf config.services.hyperhive.swarm.wireguard.enable [ + # Open the WireGuard UDP port on the host firewall (host-level + # networking — not inside containers). + networking.firewall.allowedUDPPorts = [ config.services.hyperhive.swarm.wireguard.listenPort ]; };