b03549fde6
The receiver blocked in read until a peer sent a header, and a stalled btrfs receive was never reaped, so a silent or stuck connection held a root service instance indefinitely. The receiver now gives the header 30 s and the unit has RuntimeMaxSec=6h. The sender had no deadline on the TCP connect or the header write, so an unreachable mesh address held the admin-socket handler until the kernel gave up. Both now time out after 15 s and report it like any other failure. The stream body stays unbounded on the sender: the receiver's RuntimeMaxSec already caps it, and btrfs send writes it outside this process.
265 lines
11 KiB
Nix
265 lines
11 KiB
Nix
# swarm-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push
|
|
# agent snapshots here over the existing WireGuard mesh; a destination
|
|
# hive later pulls one back to complete a migration. Only the receive
|
|
# half exists today — the pull side needs an authorisation model for
|
|
# "which hive may fetch which agent's state", which lands with the
|
|
# swarm controller.
|
|
#
|
|
# This is NOT the swarm controller and does not depend on it: a btrfs
|
|
# subvolume tree, a socket-activated receiver, and the `wg-hive`
|
|
# interface `swarm.nix` already brings up. Deliberately no WireGuard
|
|
# config of its own --- the mesh's cryptokey routing
|
|
# (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's
|
|
# source address to its public key, so the mesh IS the authentication
|
|
# and adding certs here would authenticate the same fact twice.
|
|
#
|
|
# Confinement is a property of the DEPLOYMENT, not of this unit: in a
|
|
# real swarm the store is its own small VM (the machine is the
|
|
# boundary); in the all-local case it's a container on the c0re host.
|
|
# The module hardcodes neither. docs/networking/snapshot-store.md covers what the
|
|
# deployment is expected to provide.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.snapshotStore;
|
|
wgCfg = config.services.hyperhive.deploy.wireguard;
|
|
|
|
# `deploy.wireguard.address` carries a prefix ("10.100.0.1/24") because
|
|
# it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen
|
|
# address must be the bare IP, so strip it.
|
|
meshAddress = lib.head (lib.splitString "/" wgCfg.address);
|
|
|
|
# How long a connected peer may stay silent before sending the header
|
|
# line. The sender writes it immediately after connecting, so this only
|
|
# ever expires for a peer that is not a hive.
|
|
headerTimeoutSeconds = 30;
|
|
|
|
# The receiver. Socket-activated with Accept=yes, so stdin IS the
|
|
# accepted connection and systemd hands us the peer address in
|
|
# $REMOTE_ADDR --- which, on this interface, is a cryptographically
|
|
# authenticated statement about which hive is talking (see the
|
|
# cryptokey-routing note above).
|
|
#
|
|
# PROTOCOL: one `agent <name>\n` header line, then the raw `btrfs
|
|
# send` stream. The header exists because a btrfs stream does not
|
|
# carry the sending hive's notion of *which agent* it is --- the
|
|
# subvolume name inside the stream is chosen by the sender.
|
|
#
|
|
# ⚠️ The security rule, stated precisely, because the absolute form
|
|
# ("the sender never names its destination") is not achievable with
|
|
# btrfs send/receive: the RECEIVER owns the destination ROOT, and any
|
|
# sender-supplied component is VALIDATED, never used as a path. The
|
|
# name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no
|
|
# traversal and no absolute path can survive it. The root is ours;
|
|
# the leaf is checked against a whitelist charset before it is joined.
|
|
receiveScript = pkgs.writeShellScript "swarm-snapshot-receive" ''
|
|
set -euo pipefail
|
|
|
|
# Read exactly the header line, leaving the byte stream untouched
|
|
# for btrfs receive. `read` stops at the newline and does not
|
|
# buffer ahead, which is why the header is a line and not a
|
|
# fixed-width record. `-t` bounds the wait: a peer that connects
|
|
# and stays silent would otherwise hold this root instance forever.
|
|
# `read` exits above 128 on timeout and 1 on EOF.
|
|
rc=0
|
|
read -r -t ${toString headerTimeoutSeconds} keyword agent || rc=$?
|
|
if [ "$rc" -gt 128 ]; then
|
|
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent no header within ${toString headerTimeoutSeconds}s" >&2
|
|
exit 1
|
|
elif [ "$rc" -ne 0 ]; then
|
|
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$keyword" != "agent" ]; then
|
|
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Validate rather than trust. Anything outside this charset is
|
|
# rejected outright --- this is the check that makes the joined
|
|
# path below safe, so it must stay a whitelist, never a blocklist
|
|
# of bad characters.
|
|
case "$agent" in
|
|
"" | *[!A-Za-z0-9_-]*)
|
|
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
dest="${cfg.path}/$agent"
|
|
|
|
# One subvolume tree per AGENT, not per hive: after a migration the
|
|
# same agent's next incremental send arrives from a DIFFERENT hive,
|
|
# and a per-hive prefix would split its snapshot chain in two and
|
|
# break the incremental parent lookup --- exactly the case this
|
|
# store exists to serve.
|
|
mkdir -p "$dest"
|
|
|
|
echo "swarm-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2
|
|
exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest"
|
|
'';
|
|
in
|
|
{
|
|
options.services.hyperhive.snapshotStore = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = ''
|
|
Run the swarm snapshot store on this host: a `btrfs receive`
|
|
endpoint that hives push agent snapshots to over the WireGuard
|
|
mesh. Off by default --- it is a distinct deployment role, not
|
|
part of a hive.
|
|
|
|
Requires `services.hyperhive.deploy.wireguard.enable`: the mesh
|
|
is both the transport and the authentication, so there is no
|
|
meaningful configuration without it.
|
|
'';
|
|
};
|
|
|
|
path = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "/var/lib/hyperhive-snapshots";
|
|
description = ''
|
|
Root of the snapshot tree. Must be on a btrfs filesystem ---
|
|
`btrfs receive` fails otherwise. One subvolume directory per
|
|
agent is created beneath it, so an agent's incremental chain
|
|
stays contiguous across a migration between hives.
|
|
'';
|
|
};
|
|
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 51821;
|
|
description = ''
|
|
TCP port the receiver listens on. Bound to this host's
|
|
WireGuard mesh address only --- never a wildcard --- so it is
|
|
reachable exactly by mesh peers and by nothing else.
|
|
'';
|
|
};
|
|
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = wgCfg.enable;
|
|
message = ''
|
|
services.hyperhive.snapshotStore.enable requires
|
|
services.hyperhive.deploy.wireguard.enable --- the mesh is the
|
|
store's transport AND its authentication (cryptokey routing
|
|
binds a peer's source address to its public key). Without it
|
|
there is nothing to bind the listener to and no way to tell
|
|
which hive is pushing.
|
|
'';
|
|
}
|
|
{
|
|
assertion = wgCfg.address != "";
|
|
message = ''
|
|
services.hyperhive.snapshotStore.enable requires
|
|
services.hyperhive.deploy.wireguard.address to be set --- the
|
|
receiver binds to this host's mesh address, and refuses to
|
|
fall back to a wildcard.
|
|
'';
|
|
}
|
|
{
|
|
assertion =
|
|
!(lib.elem meshAddress [
|
|
""
|
|
"0.0.0.0"
|
|
"::"
|
|
]);
|
|
message = ''
|
|
The snapshot store refuses a wildcard listen address.
|
|
services.hyperhive.deploy.wireguard.address is "${wgCfg.address}",
|
|
which would make the receiver bind "${meshAddress}" on every
|
|
interface. Set it to this host's address on the mesh
|
|
(e.g. "10.100.0.1/24").
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The store root must exist before the first connection arrives ---
|
|
# the receiver runs on demand and should not be the thing that
|
|
# creates its own tree lazily.
|
|
systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ];
|
|
|
|
# Open the receiver's port, scoped to the mesh interface.
|
|
#
|
|
# ⚠️ Binding the socket to the mesh address is NOT sufficient on its
|
|
# own: NixOS's firewall is default-deny and filters in netfilter,
|
|
# before a packet ever reaches a bound socket. The bind chooses
|
|
# WHICH address accepts connections; it does not open the port. The
|
|
# mesh's own UDP port is opened the same explicit way in swarm.nix.
|
|
#
|
|
# Interface-scoped rather than host-wide so the reachability
|
|
# property stays exactly what the option docs claim --- mesh peers
|
|
# and nobody else. A global `allowedTCPPorts` would open the port on
|
|
# every interface, including whatever public NIC the box has, and
|
|
# only the socket's bind address would still be standing between
|
|
# the internet and a root `btrfs receive`.
|
|
networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ cfg.port ];
|
|
|
|
# Socket-activated on purpose: no long-running root daemon, and the
|
|
# unit exists only while a transfer does.
|
|
#
|
|
# ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a
|
|
# wildcard this socket would be an unauthenticated remote write
|
|
# into agent state, so the binding IS the access control and is
|
|
# asserted above rather than left to a comment.
|
|
#
|
|
# FreeBind lets the bind succeed before `wg-hive` has its address:
|
|
# sockets.target is reached before the wireguard unit, which
|
|
# nothing orders ahead of it, so without it the bind fails with
|
|
# EADDRNOTAVAIL and the receiver never listens. It also covers the
|
|
# interface coming back after a flap.
|
|
#
|
|
# Accept=yes gives one service instance per connection and sets
|
|
# $REMOTE_ADDR for the handler --- which is how the receiver knows
|
|
# which peer it is talking to.
|
|
systemd.sockets.swarm-snapshot-store = {
|
|
description = "hyperhive swarm snapshot store receiver socket";
|
|
wantedBy = [ "sockets.target" ];
|
|
socketConfig = {
|
|
ListenStream = "${meshAddress}:${toString cfg.port}";
|
|
Accept = "yes";
|
|
FreeBind = true;
|
|
};
|
|
};
|
|
|
|
# `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by
|
|
# nature. The hardening below is defence in depth and NOT a
|
|
# boundary: a process holding CAP_SYS_ADMIN can call mount(2) and
|
|
# undo the namespace these directives set up. The real boundary is
|
|
# the deployment --- see docs/networking/snapshot-store.md.
|
|
systemd.services."swarm-snapshot-store@" = {
|
|
description = "hyperhive swarm snapshot store receiver";
|
|
after = [ "swarm-snapshot-store.socket" ];
|
|
requires = [ "swarm-snapshot-store.socket" ];
|
|
serviceConfig = {
|
|
ExecStart = receiveScript;
|
|
SyslogIdentifier = "swarm-snapshot-store";
|
|
# StandardInput=socket wires the accepted connection to stdin,
|
|
# which is what the handler reads the header + stream from.
|
|
StandardInput = "socket";
|
|
StandardError = "journal";
|
|
User = "root";
|
|
# Upper bound on one whole transfer, header to the last byte of
|
|
# `btrfs receive`. Snapshots are whole agent state roots and run to
|
|
# multiple gigabytes (hive-priv `send_agent_snapshot_to_fd`); no
|
|
# tighter bound is documented, so this is sized for ~100 GB at
|
|
# 5 MB/s (about 5.6 h), not measured. A stalled receive is killed
|
|
# here instead of holding a root instance indefinitely.
|
|
RuntimeMaxSec = "6h";
|
|
PrivateTmp = true;
|
|
ProtectHome = true;
|
|
ProtectSystem = "strict";
|
|
ReadWritePaths = [ cfg.path ];
|
|
};
|
|
};
|
|
};
|
|
}
|