mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

remove the old snapshot store; tombstone its options

Browse source
This commit is contained in:
atlas 2026-11-09 23:13:18 +01:00 • committed by mara
commit 20d286f06a
11 changed files with 97 additions and 1308 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/checks.nix
View file

@ -203,10 +203,6 @@ in
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
module-eval-snapshot-store = import ./module-eval/snapshot-store.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;

1
nix/host-modules/default.nix
View file

@ -85,7 +85,6 @@
./swarm-grafana.nix
./swarm-otel-service.nix
./swarm-otel.nix
./swarm-snapshot-store.nix
./swarm-ui.nix
./swarm-victorialogs-service.nix
./swarm-victorialogs.nix

38
nix/host-modules/deploy.nix
View file

@ -68,6 +68,12 @@ let
there is no agent-base closure to pre-fetch into the host store. Remove
this definition.
'';
# Shared by the removed options of the snapshot store.
snapshotStoreRemoved = ''
The snapshot store was removed; agent-state backups are being rebuilt as
swarm-archive. Remove this definition.
'';
in
{
imports = [
@ -143,6 +149,38 @@ in
"queue"
"agentCredentialDir"
] queueAgentCredentialRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"swarm"
"snapshotStore"
"address"
] snapshotStoreRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"swarm"
"snapshotStore"
"port"
] snapshotStoreRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"snapshotStore"
"enable"
] snapshotStoreRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"snapshotStore"
"path"
] snapshotStoreRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"snapshotStore"
"port"
] snapshotStoreRemoved)
# Same type, same meaning, new path — so a rename carries it exactly
# and existing configs keep evaluating with one warning naming both

12
nix/host-modules/hive-c0re/environment.nix
View file

@ -199,18 +199,6 @@ in
# set (no browser-reachable matrix vhost).
HIVE_MATRIX_PUBLIC_URL = "https://${config.services.hyperhive.swarm.matrix.gatewayHost}/";
}
// lib.optionalAttrs (config.services.hyperhive.swarm.snapshotStore.address != null) {
# `host:port` of the swarm's single snapshot store, for pushing agent
# snapshots (hive-c0re::snapshot_push). One per swarm, not one per
# peer — the receiver keys destinations by agent so a migrating agent
# keeps one incremental chain. Absent when no store is configured, and
# a push then fails naming the option rather than guessing.
HYPERHIVE_SNAPSHOT_STORE =
let
s = config.services.hyperhive.swarm.snapshotStore;
in
"${s.address}:${toString s.port}";
}
//
# Swarm-queue coordinates for offering this hive's status upward
# (hive-c0re::swarm_status). All three together or none: a half-set

1
nix/host-modules/hive-network.nix
View file

@ -55,7 +55,6 @@ let
h.gateway.enable
h.gateway.dns.enable
h.network.enable
h.snapshotStore.enable
];
in
{

4
nix/host-modules/hyperhive.nix
View file

@ -120,8 +120,8 @@ in
hive in the swarm states an address at all.
**Required** on every host that runs a hyperhive service (a
`services.hyperhive.deploy.*` service, the gateway, the bridge,
`otel` or the snapshot store), with the same value on every host
`services.hyperhive.deploy.*` service, the gateway, the bridge or
`otel`), with the same value on every host
of the swarm, and deliberately not
defaulted. A guessed swarm domain is a wrong hostname that
evaluates cleanly and deploys, which is worse than an eval

265
nix/host-modules/swarm-snapshot-store.nix
View file

@ -1,265 +0,0 @@
# swarm-snapshot-store — the swarm's `btrfs receive` endpoint. Hives push
# agent snapshots here over the existing WireGuard mesh; a destination
# hive later pulls one back to complete a migration. Only the receive
# half exists today — the pull side needs an authorisation model for
# "which hive may fetch which agent's state", which lands with the
# swarm controller.
#
# This is NOT the swarm controller and does not depend on it: a btrfs
# subvolume tree, a socket-activated receiver, and the `wg-hive`
# interface `swarm.nix` already brings up. Deliberately no WireGuard
# config of its own --- the mesh's cryptokey routing
# (`allowedIPs = [ peer.wireguardAddress ]`) already binds a peer's
# source address to its public key, so the mesh IS the authentication
# and adding certs here would authenticate the same fact twice.
#
# Confinement is a property of the DEPLOYMENT, not of this unit: in a
# real swarm the store is its own small VM (the machine is the
# boundary); in the all-local case it's a container on the c0re host.
# The module hardcodes neither. docs/4-implementation/networking/snapshot-store.md covers what the
# deployment is expected to provide.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.snapshotStore;
wgCfg = config.services.hyperhive.deploy.wireguard;
# `deploy.wireguard.address` carries a prefix ("10.100.0.1/24") because
# it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen
# address must be the bare IP, so strip it.
meshAddress = lib.head (lib.splitString "/" wgCfg.address);
# How long a connected peer may stay silent before sending the header
# line. The sender writes it immediately after connecting, so this only
# ever expires for a peer that is not a hive.
headerTimeoutSeconds = 30;
# The receiver. Socket-activated with Accept=yes, so stdin IS the
# accepted connection and systemd hands us the peer address in
# $REMOTE_ADDR --- which, on this interface, is a cryptographically
# authenticated statement about which hive is talking (see the
# cryptokey-routing note above).
#
# PROTOCOL: one `agent <name>\n` header line, then the raw `btrfs
# send` stream. The header exists because a btrfs stream does not
# carry the sending hive's notion of *which agent* it is --- the
# subvolume name inside the stream is chosen by the sender.
#
# ⚠️ The security rule, stated precisely, because the absolute form
# ("the sender never names its destination") is not achievable with
# btrfs send/receive: the RECEIVER owns the destination ROOT, and any
# sender-supplied component is VALIDATED, never used as a path. The
# name must match [A-Za-z0-9_-]+ exactly --- no slash, no dot, so no
# traversal and no absolute path can survive it. The root is ours;
# the leaf is checked against a whitelist charset before it is joined.
receiveScript = pkgs.writeShellScript "swarm-snapshot-receive" ''
set -euo pipefail
# Read exactly the header line, leaving the byte stream untouched
# for btrfs receive. `read` stops at the newline and does not
# buffer ahead, which is why the header is a line and not a
# fixed-width record. `-t` bounds the wait: a peer that connects
# and stays silent would otherwise hold this root instance forever.
# `read` exits above 128 on timeout and 1 on EOF.
rc=0
read -r -t ${toString headerTimeoutSeconds} keyword agent || rc=$?
if [ "$rc" -gt 128 ]; then
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent no header within ${toString headerTimeoutSeconds}s" >&2
exit 1
elif [ "$rc" -ne 0 ]; then
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} closed before sending a header" >&2
exit 1
fi
if [ "$keyword" != "agent" ]; then
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent a bad header keyword" >&2
exit 1
fi
# Validate rather than trust. Anything outside this charset is
# rejected outright --- this is the check that makes the joined
# path below safe, so it must stay a whitelist, never a blocklist
# of bad characters.
case "$agent" in
"" | *[!A-Za-z0-9_-]*)
echo "swarm-snapshot-store: peer ''${REMOTE_ADDR:-?} sent an invalid agent name" >&2
exit 1
;;
esac
dest="${cfg.path}/$agent"
# One subvolume tree per AGENT, not per hive: after a migration the
# same agent's next incremental send arrives from a DIFFERENT hive,
# and a per-hive prefix would split its snapshot chain in two and
# break the incremental parent lookup --- exactly the case this
# store exists to serve.
mkdir -p "$dest"
echo "swarm-snapshot-store: receiving agent=$agent from ''${REMOTE_ADDR:-?}" >&2
exec ${pkgs.btrfs-progs}/bin/btrfs receive "$dest"
'';
in
{
options.services.hyperhive.snapshotStore = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Run the swarm snapshot store on this host: a `btrfs receive`
endpoint that hives push agent snapshots to over the WireGuard
mesh. Off by default --- it is a distinct deployment role, not
part of a hive.
Requires `services.hyperhive.deploy.wireguard.enable`: the mesh
is both the transport and the authentication, so there is no
meaningful configuration without it.
'';
};
path = lib.mkOption {
type = lib.types.path;
default = "/var/lib/hyperhive-snapshots";
description = ''
Root of the snapshot tree. Must be on a btrfs filesystem ---
`btrfs receive` fails otherwise. One subvolume directory per
agent is created beneath it, so an agent's incremental chain
stays contiguous across a migration between hives.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 51821;
description = ''
TCP port the receiver listens on. Bound to this host's
WireGuard mesh address only --- never a wildcard --- so it is
reachable exactly by mesh peers and by nothing else.
'';
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = wgCfg.enable;
message = ''
services.hyperhive.snapshotStore.enable requires
services.hyperhive.deploy.wireguard.enable --- the mesh is the
store's transport AND its authentication (cryptokey routing
binds a peer's source address to its public key). Without it
there is nothing to bind the listener to and no way to tell
which hive is pushing.
'';
}
{
assertion = wgCfg.address != "";
message = ''
services.hyperhive.snapshotStore.enable requires
services.hyperhive.deploy.wireguard.address to be set --- the
receiver binds to this host's mesh address, and refuses to
fall back to a wildcard.
'';
}
{
assertion =
!(lib.elem meshAddress [
""
"0.0.0.0"
"::"
]);
message = ''
The snapshot store refuses a wildcard listen address.
services.hyperhive.deploy.wireguard.address is "${wgCfg.address}",
which would make the receiver bind "${meshAddress}" on every
interface. Set it to this host's address on the mesh
(e.g. "10.100.0.1/24").
'';
}
];
# The store root must exist before the first connection arrives ---
# the receiver runs on demand and should not be the thing that
# creates its own tree lazily.
systemd.tmpfiles.rules = [ "d ${cfg.path} 0700 root root -" ];
# Open the receiver's port, scoped to the mesh interface.
#
# ⚠️ Binding the socket to the mesh address is NOT sufficient on its
# own: NixOS's firewall is default-deny and filters in netfilter,
# before a packet ever reaches a bound socket. The bind chooses
# WHICH address accepts connections; it does not open the port. The
# mesh's own UDP port is opened the same explicit way in swarm.nix.
#
# Interface-scoped rather than host-wide so the reachability
# property stays exactly what the option docs claim --- mesh peers
# and nobody else. A global `allowedTCPPorts` would open the port on
# every interface, including whatever public NIC the box has, and
# only the socket's bind address would still be standing between
# the internet and a root `btrfs receive`.
networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ cfg.port ];
# Socket-activated on purpose: no long-running root daemon, and the
# unit exists only while a transfer does.
#
# ⚠️ ListenStream is the mesh address, never 0.0.0.0. Bound to a
# wildcard this socket would be an unauthenticated remote write
# into agent state, so the binding IS the access control and is
# asserted above rather than left to a comment.
#
# FreeBind lets the bind succeed before `wg-hive` has its address:
# sockets.target is reached before the wireguard unit, which
# nothing orders ahead of it, so without it the bind fails with
# EADDRNOTAVAIL and the receiver never listens. It also covers the
# interface coming back after a flap.
#
# Accept=yes gives one service instance per connection and sets
# $REMOTE_ADDR for the handler --- which is how the receiver knows
# which peer it is talking to.
systemd.sockets.swarm-snapshot-store = {
description = "hyperhive swarm snapshot store receiver socket";
wantedBy = [ "sockets.target" ];
socketConfig = {
ListenStream = "${meshAddress}:${toString cfg.port}";
Accept = "yes";
FreeBind = true;
};
};
# `btrfs receive` needs CAP_SYS_ADMIN, so this runs as root by
# nature. The hardening below is defence in depth and NOT a
# boundary: a process holding CAP_SYS_ADMIN can call mount(2) and
# undo the namespace these directives set up. The real boundary is
# the deployment --- see docs/4-implementation/networking/snapshot-store.md.
systemd.services."swarm-snapshot-store@" = {
description = "hyperhive swarm snapshot store receiver";
after = [ "swarm-snapshot-store.socket" ];
requires = [ "swarm-snapshot-store.socket" ];
serviceConfig = {
ExecStart = receiveScript;
SyslogIdentifier = "swarm-snapshot-store";
# StandardInput=socket wires the accepted connection to stdin,
# which is what the handler reads the header + stream from.
StandardInput = "socket";
StandardError = "journal";
User = "root";
# Upper bound on one whole transfer, header to the last byte of
# `btrfs receive`. Snapshots are whole agent state roots and run to
# multiple gigabytes (hive-priv `send_agent_snapshot_to_fd`); no
# tighter bound is documented, so this is sized for ~100 GB at
# 5 MB/s (about 5.6 h), not measured. A stalled receive is killed
# here instead of holding a root instance indefinitely.
RuntimeMaxSec = "6h";
PrivateTmp = true;
ProtectHome = true;
ProtectSystem = "strict";
ReadWritePaths = [ cfg.path ];
};
};
};
}

5
nix/host-modules/swarm-wireguard.nix
View file

@ -2,8 +2,7 @@
# ./swarm.nix because the two are different concerns with different
# audiences: that file declares WHO the peers are (consumed by
# swarm-controller's hive roster and, here, the mesh), while this one
# is plain host networking that a machine which runs no hive at all
# --- the snapshot store, for one --- still needs.
# is plain host networking that a machine which runs no hive can also enable.
#
# The two stay coupled by data, not by structure: the per-peer
# `wireguard*` fields live on the peer submodule in ./swarm.nix, since
@ -86,7 +85,7 @@
# Gated on the mesh itself, NOT on `deploy.hive-controller.enable`. The
# mesh is host networking, not a c0re feature: a swarm host that runs no
# hive — the snapshot store, for one — still has to join the mesh.
# hive can join the mesh too.
# Nothing below is c0re-specific; the peer data c0re consumes
# (HIVE_PEER_CA_PATHS) is rendered in ./hive-c0re and stays gated there.
config = lib.mkIf config.services.hyperhive.deploy.wireguard.enable {

37
nix/host-modules/swarm.nix
View file

@ -332,43 +332,6 @@ in
# deployment-shape switch rather than swarm bookkeeping, so it lives
# with its consequences instead of here.
options.services.hyperhive.swarm.snapshotStore = {
address = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "10.100.0.1";
description = ''
Mesh address of the swarm's snapshot store — the single
`btrfs receive` endpoint every hive in this swarm pushes agent
snapshots to. Bare IP, no prefix.
There is exactly **one** store per swarm, not one per peer: the
receiver keys destinations by *agent*, so an agent that migrates
between hives keeps a single unbroken incremental chain. Per-hive
stores would split that chain in two, which is the case the store
exists to serve.
Null means this swarm has no store configured, and pushing fails
saying so rather than guessing an address. Set it on every hive
that pushes; the receiving host separately sets
`services.hyperhive.snapshotStore.enable`.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 51821;
description = ''
TCP port the swarm's snapshot store listens on. Must match the
receiving host's `services.hyperhive.snapshotStore.port`.
Defaulted (unlike `address`) because it is a shared convention
both sides read from the same option docs — whereas an address
is deployment-specific and cannot be guessed.
'';
};
};
# How this hive reaches the swarm queue to offer its own status
# (hive-c0re's `swarm_status`): the token endpoint below, which follows
# the swarm's IdP address wherever that is, and the queue's URL under

37
nix/module-eval/core-toggle.nix
View file

@ -2049,6 +2049,43 @@ let
})
&& !(refused allLocal);
}
{
# One host defining all five removed paths. Each message names its own
# path (`services.hyperhive.swarm.snapshotStore.port` does not contain
# `services.hyperhive.snapshotStore.port`), so each lookup finds only
# its option. The control is a host that defines none.
name = "a definition of a removed snapshot store option fails evaluation, naming the option and swarm-archive";
ok =
let
paths = [
"swarm.snapshotStore.address"
"swarm.snapshotStore.port"
"snapshotStore.enable"
"snapshotStore.path"
"snapshotStore.port"
];
refused =
m: path:
lib.any (
a:
!a.assertion
&& lib.hasInfix "services.hyperhive.${path}" a.message
&& lib.hasInfix "agent-state backups are being rebuilt as swarm-archive" a.message
) m.assertions;
old = hive {
swarm.snapshotStore = {
address = "10.100.0.1";
port = 51821;
};
snapshotStore = {
enable = true;
path = "/var/lib/hyperhive-snapshots";
port = 51821;
};
};
in
lib.all (path: refused old path) paths && !(lib.any (path: refused bare path) paths);
}
{
# Refused wherever any hyperhive service runs, the hive or a swarm
# service alone. The host enabling nothing is the control that it is

102
nix/module-eval/snapshot-store.nix
View file

@ -1,102 +0,0 @@
# `checks.module-eval-snapshot-store` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates not
# executes").
#
# The snapshot store's receiver socket: where it listens, that it can bind
# before the mesh interface is up, that it refuses a wildcard, and that a
# silent or stalled transfer is bounded.
{
pkgs,
lib,
self,
nixosSystem,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
runGroup
;
# Bare hive: the store and the mesh it needs, nothing else.
store = hive {
deploy.wireguard.enable = true;
deploy.wireguard.address = "10.100.0.1/24";
snapshotStore.enable = true;
snapshotStore.port = 51822;
};
# The same with a wildcard where the mesh address goes.
storeOnWildcard = hive {
deploy.wireguard.enable = true;
deploy.wireguard.address = "0.0.0.0/24";
snapshotStore.enable = true;
};
# The IPv6 wildcard.
storeOnWildcardV6 = hive {
deploy.wireguard.enable = true;
deploy.wireguard.address = "::/64";
snapshotStore.enable = true;
};
unit = store.systemd.services."swarm-snapshot-store@";
moduleSource = builtins.readFile ../host-modules/swarm-snapshot-store.nix;
socket = store.systemd.sockets.swarm-snapshot-store.socketConfig;
refusedAsWildcard =
m:
lib.any (
a: !a.assertion && lib.hasInfix "refuses a wildcard listen address" a.message
) m.assertions;
cases = [
{
# The wg-hive address does not exist when sockets.target binds, so
# without FreeBind the bind fails with EADDRNOTAVAIL and the receiver
# never listens. The unit still renders and evaluates either way, which
# is why only a rendered-config check can see it.
name = "the receiver socket may bind before the mesh interface has its address";
ok = socket.FreeBind or false == true;
}
{
# The bare IP and the configured port: the prefix length on
# `deploy.wireguard.address` must not leak into the listen address.
name = "the receiver socket listens on the mesh address and the configured port";
ok = socket.ListenStream == "10.100.0.1:51822";
}
{
# The control for the case below: it is the wildcard that trips the
# assertion, not the store being on.
name = "a mesh address on the store raises no wildcard refusal";
ok = !(refusedAsWildcard store);
}
{
# A wildcard bind is an unauthenticated remote write into agent state.
name = "a wildcard mesh address, IPv4 or IPv6, is refused";
ok = refusedAsWildcard storeOnWildcard && refusedAsWildcard storeOnWildcardV6;
}
{
# Without it a stalled `btrfs receive` holds a root instance forever.
name = "the receiver unit has a runtime ceiling";
ok = unit.serviceConfig.RuntimeMaxSec or null == "6h";
}
{
# The script is a store path, so its text is not evaluable without
# building it; the module source is the nearest thing to read. A
# silent peer must not hold the root instance past the header read.
name = "the receiver script bounds its header read";
ok = lib.hasInfix "read -r -t \${toString headerTimeoutSeconds} keyword agent" moduleSource;
}
];
in
runGroup "snapshot-store" cases