4989579270
argus caught it: binding the socket to the mesh address does not open the port. NixOS's firewall is default-deny and filters in netfilter, before a packet reaches a bound socket — the bind chooses which address accepts connections, not whether packets arrive. As shipped the receiver was unreachable. swarm.nix already shows the pattern for exactly this situation: it opens the mesh's UDP port explicitly right after bringing the interface up. Interface-scoped to wg-hive rather than host-wide, so the option's "reachable exactly by mesh peers" claim is actually true. A global allowedTCPPorts would open the port on every interface including the public NIC, leaving only the socket's bind address between the internet and a root btrfs receive.
254 lines
10 KiB
Nix
254 lines
10 KiB
Nix
# hive-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push
|
|
# agent snapshots here over the existing WireGuard mesh; a destination
|
|
# hive later pulls one back to complete a migration. Only the receive
|
|
# half exists today — the pull side needs an authorisation model for
|
|
# "which hive may fetch which agent's state", which lands with the
|
|
# swarm controller.
|
|
#
|
|
# This is NOT the swarm controller and does not depend on it: a btrfs
|
|
# subvolume tree, a socket-activated receiver, and the `wg-hive`
|
|
# interface `swarm.nix` already brings up. Deliberately no WireGuard
|
|
# config of its own --- the mesh's cryptokey routing
|
|
# (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's
|
|
# source address to its public key, so the mesh IS the authentication
|
|
# and adding certs here would authenticate the same fact twice.
|
|
#
|
|
# Confinement is a property of the DEPLOYMENT, not of this unit: in a
|
|
# real swarm the store is its own small VM (the machine is the
|
|
# boundary); in the all-local case it's a container on the c0re host.
|
|
# The module therefore hardcodes neither --- see `dedicated` below.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.snapshotStore;
|
|
wgCfg = config.services.hyperhive.swarm.wireguard;
|
|
|
|
# `swarm.wireguard.address` carries a prefix ("10.100.0.1/24") because
|
|
# it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen
|
|
# address must be the bare IP, so strip it.
|
|
meshAddress = lib.head (lib.splitString "/" wgCfg.address);
|
|
|
|
# The receiver. Socket-activated with Accept=yes, so stdin IS the
|
|
# accepted connection and systemd hands us the peer address in
|
|
# $REMOTE_ADDR --- which, on this interface, is a cryptographically
|
|
# authenticated statement about which hive is talking (see the
|
|
# cryptokey-routing note above).
|
|
#
|
|
# PROTOCOL: one `agent <name>\n` header line, then the raw `btrfs
|
|
# send` stream. The header exists because a btrfs stream does not
|
|
# carry the sending hive's notion of *which agent* it is --- the
|
|
# subvolume name inside the stream is chosen by the sender.
|
|
#
|
|
# ⚠️ The security rule, stated precisely, because the absolute form
|
|
# ("the sender never names its destination") is not achievable with
|
|
# btrfs send/receive: the RECEIVER owns the destination ROOT, and any
|
|
# sender-supplied component is VALIDATED, never used as a path. The
|
|
# name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no
|
|
# traversal and no absolute path can survive it. The root is ours;
|
|
# the leaf is checked against a whitelist charset before it is joined.
|
|
receiveScript = pkgs.writeShellScript "hive-snapshot-receive" ''
|
|
set -euo pipefail
|
|
|
|
# Read exactly the header line, leaving the byte stream untouched
|
|
# for btrfs receive. `read` stops at the newline and does not
|
|
# buffer ahead, which is why the header is a line and not a
|
|
# fixed-width record.
|
|
if ! read -r keyword agent; then
|
|
echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$keyword" != "agent" ]; then
|
|
echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Validate rather than trust. Anything outside this charset is
|
|
# rejected outright --- this is the check that makes the joined
|
|
# path below safe, so it must stay a whitelist, never a blocklist
|
|
# of bad characters.
|
|
case "$agent" in
|
|
"" | *[!A-Za-z0-9_-]*)
|
|
echo "hive-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
dest="${cfg.path}/$agent"
|
|
|
|
# One subvolume tree per AGENT, not per hive: after a migration the
|
|
# same agent's next incremental send arrives from a DIFFERENT hive,
|
|
# and a per-hive prefix would split its snapshot chain in two and
|
|
# break the incremental parent lookup --- exactly the case this
|
|
# store exists to serve.
|
|
mkdir -p "$dest"
|
|
|
|
echo "hive-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2
|
|
exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest"
|
|
'';
|
|
in
|
|
{
|
|
options.services.hyperhive.snapshotStore = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = ''
|
|
Run the swarm snapshot store on this host: a `btrfs receive`
|
|
endpoint that hives push agent snapshots to over the WireGuard
|
|
mesh. Off by default --- it is a distinct deployment role, not
|
|
part of a hive.
|
|
|
|
Requires `services.hyperhive.swarm.wireguard.enable`: the mesh
|
|
is both the transport and the authentication, so there is no
|
|
meaningful configuration without it.
|
|
'';
|
|
};
|
|
|
|
path = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "/var/lib/hyperhive-snapshots";
|
|
description = ''
|
|
Root of the snapshot tree. Must be on a btrfs filesystem ---
|
|
`btrfs receive` fails otherwise. One subvolume directory per
|
|
agent is created beneath it, so an agent's incremental chain
|
|
stays contiguous across a migration between hives.
|
|
'';
|
|
};
|
|
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 51821;
|
|
description = ''
|
|
TCP port the receiver listens on. Bound to this host's
|
|
WireGuard mesh address only --- never a wildcard --- so it is
|
|
reachable exactly by mesh peers and by nothing else.
|
|
'';
|
|
};
|
|
|
|
dedicated = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
description = ''
|
|
Assert that this host runs no other hyperhive role. The store
|
|
aggregates every agent's state from every hive in the swarm, so
|
|
the intended deployment is a dedicated machine (or a container
|
|
in the all-local case) where the machine itself is the security
|
|
boundary.
|
|
|
|
That assumption is true on day one and silently false the day
|
|
someone notices the box has spare disk. This option makes it a
|
|
thing the build checks rather than a thing the deployer
|
|
remembers. Set to `false` to co-locate deliberately --- the
|
|
point is that it becomes a decision, not an accident.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = wgCfg.enable;
|
|
message = ''
|
|
services.hyperhive.snapshotStore.enable requires
|
|
services.hyperhive.swarm.wireguard.enable --- the mesh is the
|
|
store's transport AND its authentication (cryptokey routing
|
|
binds a peer's source address to its public key). Without it
|
|
there is nothing to bind the listener to and no way to tell
|
|
which hive is pushing.
|
|
'';
|
|
}
|
|
{
|
|
assertion = wgCfg.address != "";
|
|
message = ''
|
|
services.hyperhive.snapshotStore.enable requires
|
|
services.hyperhive.swarm.wireguard.address to be set --- the
|
|
receiver binds to this host's mesh address, and refuses to
|
|
fall back to a wildcard.
|
|
'';
|
|
}
|
|
{
|
|
assertion = !cfg.dedicated || !config.services.hyperhive.c0re.enable;
|
|
message = ''
|
|
services.hyperhive.snapshotStore is enabled alongside
|
|
services.hyperhive.c0re on the same host. The store holds
|
|
every agent's state from every hive, so it is meant to run on
|
|
a machine of its own where the machine is the boundary.
|
|
|
|
If the co-location is deliberate (the all-local single-host
|
|
deployment, where the store runs as a container), set
|
|
services.hyperhive.snapshotStore.dedicated = false to record
|
|
that decision explicitly.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The store root must exist before the first connection arrives ---
|
|
# the receiver runs on demand and should not be the thing that
|
|
# creates its own tree lazily.
|
|
systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ];
|
|
|
|
# Open the receiver's port, scoped to the mesh interface.
|
|
#
|
|
# ⚠️ Binding the socket to the mesh address is NOT sufficient on its
|
|
# own: NixOS's firewall is default-deny and filters in netfilter,
|
|
# before a packet ever reaches a bound socket. The bind chooses
|
|
# WHICH address accepts connections; it does not open the port. The
|
|
# mesh's own UDP port is opened the same explicit way in swarm.nix.
|
|
#
|
|
# Interface-scoped rather than host-wide so the reachability
|
|
# property stays exactly what the option docs claim --- mesh peers
|
|
# and nobody else. A global `allowedTCPPorts` would open the port on
|
|
# every interface, including whatever public NIC the box has, and
|
|
# only the socket's bind address would still be standing between
|
|
# the internet and a root `btrfs receive`.
|
|
networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ cfg.port ];
|
|
|
|
# Socket-activated on purpose: no long-running root daemon, and the
|
|
# unit exists only while a transfer does.
|
|
#
|
|
# ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a
|
|
# wildcard this socket would be an unauthenticated remote write
|
|
# into agent state, so the binding IS the access control and is
|
|
# asserted above rather than left to a comment.
|
|
#
|
|
# Accept=yes gives one service instance per connection and sets
|
|
# $REMOTE_ADDR for the handler --- which is how the receiver knows
|
|
# which peer it is talking to.
|
|
systemd.sockets.hive-snapshot-store = {
|
|
description = "hyperhive swarm snapshot store receiver socket";
|
|
wantedBy = [ "sockets.target" ];
|
|
socketConfig = {
|
|
ListenStream = "${meshAddress}:${toString cfg.port}";
|
|
Accept = "yes";
|
|
};
|
|
};
|
|
|
|
# `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by
|
|
# nature. The hardening below is defence in depth and NOT a
|
|
# boundary: a process holding CAP_SYS_ADMIN can call mount(2) and
|
|
# undo the namespace these directives set up. The real boundary is
|
|
# the deployment (dedicated host / container) --- see `dedicated`.
|
|
systemd.services."hive-snapshot-store@" = {
|
|
description = "hyperhive swarm snapshot store receiver";
|
|
after = [ "hive-snapshot-store.socket" ];
|
|
requires = [ "hive-snapshot-store.socket" ];
|
|
serviceConfig = {
|
|
ExecStart = receiveScript;
|
|
SyslogIdentifier = "hive-snapshot-store";
|
|
# StandardInput=socket wires the accepted connection to stdin,
|
|
# which is what the handler reads the header + stream from.
|
|
StandardInput = "socket";
|
|
StandardError = "journal";
|
|
User = "root";
|
|
PrivateTmp = true;
|
|
ProtectHome = true;
|
|
ProtectSystem = "strict";
|
|
ReadWritePaths = [ cfg.path ];
|
|
};
|
|
};
|
|
};
|
|
}
|