6a6266cd5e
Reverting the namespace move from the previous commit — mara's reason is better than mine was. I grouped it with swarm.peers and swarm.wireguard because the module serves the swarm tier. But those two describe THE SWARM: who is in it, how it is meshed. snapshotStore describes THIS HOST'S ROLE. On a standalone store box the operator enables one service, and nesting it under `swarm` implies they are configuring a swarm when they are not. The swarm- prefix on the file and units stands: the name says which tier the component serves, the option path says what you are turning on. Those are different questions and they are allowed different answers.
224 lines
9.1 KiB
Nix
224 lines
9.1 KiB
Nix
# swarm-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push
|
|
# agent snapshots here over the existing WireGuard mesh; a destination
|
|
# hive later pulls one back to complete a migration. Only the receive
|
|
# half exists today — the pull side needs an authorisation model for
|
|
# "which hive may fetch which agent's state", which lands with the
|
|
# swarm controller.
|
|
#
|
|
# This is NOT the swarm controller and does not depend on it: a btrfs
|
|
# subvolume tree, a socket-activated receiver, and the `wg-hive`
|
|
# interface `swarm.nix` already brings up. Deliberately no WireGuard
|
|
# config of its own --- the mesh's cryptokey routing
|
|
# (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's
|
|
# source address to its public key, so the mesh IS the authentication
|
|
# and adding certs here would authenticate the same fact twice.
|
|
#
|
|
# Confinement is a property of the DEPLOYMENT, not of this unit: in a
|
|
# real swarm the store is its own small VM (the machine is the
|
|
# boundary); in the all-local case it's a container on the c0re host.
|
|
# The module hardcodes neither. docs/snapshot-store.md covers what the
|
|
# deployment is expected to provide.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.snapshotStore;
|
|
wgCfg = config.services.hyperhive.swarm.wireguard;
|
|
|
|
# `swarm.wireguard.address` carries a prefix ("10.100.0.1/24") because
|
|
# it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen
|
|
# address must be the bare IP, so strip it.
|
|
meshAddress = lib.head (lib.splitString "/" wgCfg.address);
|
|
|
|
# The receiver. Socket-activated with Accept=yes, so stdin IS the
|
|
# accepted connection and systemd hands us the peer address in
|
|
# $REMOTE_ADDR --- which, on this interface, is a cryptographically
|
|
# authenticated statement about which hive is talking (see the
|
|
# cryptokey-routing note above).
|
|
#
|
|
# PROTOCOL: one `agent <name>\n` header line, then the raw `btrfs
|
|
# send` stream. The header exists because a btrfs stream does not
|
|
# carry the sending hive's notion of *which agent* it is --- the
|
|
# subvolume name inside the stream is chosen by the sender.
|
|
#
|
|
# ⚠️ The security rule, stated precisely, because the absolute form
|
|
# ("the sender never names its destination") is not achievable with
|
|
# btrfs send/receive: the RECEIVER owns the destination ROOT, and any
|
|
# sender-supplied component is VALIDATED, never used as a path. The
|
|
# name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no
|
|
# traversal and no absolute path can survive it. The root is ours;
|
|
# the leaf is checked against a whitelist charset before it is joined.
|
|
receiveScript = pkgs.writeShellScript "swarm-snapshot-receive" ''
|
|
set -euo pipefail
|
|
|
|
# Read exactly the header line, leaving the byte stream untouched
|
|
# for btrfs receive. `read` stops at the newline and does not
|
|
# buffer ahead, which is why the header is a line and not a
|
|
# fixed-width record.
|
|
if ! read -r keyword agent; then
|
|
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$keyword" != "agent" ]; then
|
|
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Validate rather than trust. Anything outside this charset is
|
|
# rejected outright --- this is the check that makes the joined
|
|
# path below safe, so it must stay a whitelist, never a blocklist
|
|
# of bad characters.
|
|
case "$agent" in
|
|
"" | *[!A-Za-z0-9_-]*)
|
|
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
dest="${cfg.path}/$agent"
|
|
|
|
# One subvolume tree per AGENT, not per hive: after a migration the
|
|
# same agent's next incremental send arrives from a DIFFERENT hive,
|
|
# and a per-hive prefix would split its snapshot chain in two and
|
|
# break the incremental parent lookup --- exactly the case this
|
|
# store exists to serve.
|
|
mkdir -p "$dest"
|
|
|
|
echo "swarm-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2
|
|
exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest"
|
|
'';
|
|
in
|
|
{
|
|
options.services.hyperhive.snapshotStore = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = ''
|
|
Run the swarm snapshot store on this host: a `btrfs receive`
|
|
endpoint that hives push agent snapshots to over the WireGuard
|
|
mesh. Off by default --- it is a distinct deployment role, not
|
|
part of a hive.
|
|
|
|
Requires `services.hyperhive.swarm.wireguard.enable`: the mesh
|
|
is both the transport and the authentication, so there is no
|
|
meaningful configuration without it.
|
|
'';
|
|
};
|
|
|
|
path = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "/var/lib/hyperhive-snapshots";
|
|
description = ''
|
|
Root of the snapshot tree. Must be on a btrfs filesystem ---
|
|
`btrfs receive` fails otherwise. One subvolume directory per
|
|
agent is created beneath it, so an agent's incremental chain
|
|
stays contiguous across a migration between hives.
|
|
'';
|
|
};
|
|
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 51821;
|
|
description = ''
|
|
TCP port the receiver listens on. Bound to this host's
|
|
WireGuard mesh address only --- never a wildcard --- so it is
|
|
reachable exactly by mesh peers and by nothing else.
|
|
'';
|
|
};
|
|
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = wgCfg.enable;
|
|
message = ''
|
|
services.hyperhive.snapshotStore.enable requires
|
|
services.hyperhive.swarm.wireguard.enable --- the mesh is the
|
|
store's transport AND its authentication (cryptokey routing
|
|
binds a peer's source address to its public key). Without it
|
|
there is nothing to bind the listener to and no way to tell
|
|
which hive is pushing.
|
|
'';
|
|
}
|
|
{
|
|
assertion = wgCfg.address != "";
|
|
message = ''
|
|
services.hyperhive.snapshotStore.enable requires
|
|
services.hyperhive.swarm.wireguard.address to be set --- the
|
|
receiver binds to this host's mesh address, and refuses to
|
|
fall back to a wildcard.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The store root must exist before the first connection arrives ---
|
|
# the receiver runs on demand and should not be the thing that
|
|
# creates its own tree lazily.
|
|
systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ];
|
|
|
|
# Open the receiver's port, scoped to the mesh interface.
|
|
#
|
|
# ⚠️ Binding the socket to the mesh address is NOT sufficient on its
|
|
# own: NixOS's firewall is default-deny and filters in netfilter,
|
|
# before a packet ever reaches a bound socket. The bind chooses
|
|
# WHICH address accepts connections; it does not open the port. The
|
|
# mesh's own UDP port is opened the same explicit way in swarm.nix.
|
|
#
|
|
# Interface-scoped rather than host-wide so the reachability
|
|
# property stays exactly what the option docs claim --- mesh peers
|
|
# and nobody else. A global `allowedTCPPorts` would open the port on
|
|
# every interface, including whatever public NIC the box has, and
|
|
# only the socket's bind address would still be standing between
|
|
# the internet and a root `btrfs receive`.
|
|
networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ cfg.port ];
|
|
|
|
# Socket-activated on purpose: no long-running root daemon, and the
|
|
# unit exists only while a transfer does.
|
|
#
|
|
# ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a
|
|
# wildcard this socket would be an unauthenticated remote write
|
|
# into agent state, so the binding IS the access control and is
|
|
# asserted above rather than left to a comment.
|
|
#
|
|
# Accept=yes gives one service instance per connection and sets
|
|
# $REMOTE_ADDR for the handler --- which is how the receiver knows
|
|
# which peer it is talking to.
|
|
systemd.sockets.swarm-snapshot-store = {
|
|
description = "hyperhive swarm snapshot store receiver socket";
|
|
wantedBy = [ "sockets.target" ];
|
|
socketConfig = {
|
|
ListenStream = "${meshAddress}:${toString cfg.port}";
|
|
Accept = "yes";
|
|
};
|
|
};
|
|
|
|
# `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by
|
|
# nature. The hardening below is defence in depth and NOT a
|
|
# boundary: a process holding CAP_SYS_ADMIN can call mount(2) and
|
|
# undo the namespace these directives set up. The real boundary is
|
|
# the deployment --- see docs/snapshot-store.md.
|
|
systemd.services."swarm-snapshot-store@" = {
|
|
description = "hyperhive swarm snapshot store receiver";
|
|
after = [ "swarm-snapshot-store.socket" ];
|
|
requires = [ "swarm-snapshot-store.socket" ];
|
|
serviceConfig = {
|
|
ExecStart = receiveScript;
|
|
SyslogIdentifier = "swarm-snapshot-store";
|
|
# StandardInput=socket wires the accepted connection to stdin,
|
|
# which is what the handler reads the header + stream from.
|
|
StandardInput = "socket";
|
|
StandardError = "journal";
|
|
User = "root";
|
|
PrivateTmp = true;
|
|
ProtectHome = true;
|
|
ProtectSystem = "strict";
|
|
ReadWritePaths = [ cfg.path ];
|
|
};
|
|
};
|
|
};
|
|
}
|