mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

remove the swarm wireguard mesh; tombstone its options

Browse source
This commit is contained in:
atlas 2026-11-15 02:17:44 +01:00
commit adc61c9043
10 changed files with 330 additions and 1068 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/host-modules/default.nix
View file

@ -50,7 +50,6 @@
./glue-nats-auth-bao-identity.nix
./glue-nats-bao-identity.nix
./glue-nats-oidc-client.nix
./glue-nats-wireguard.nix
./glue-builder-oidc-client.nix
./glue-ci-metrics-otel-oidc-client.nix
./glue-ci-runner-otel-oidc-client.nix
@ -95,7 +94,6 @@
./swarm-victorialogs.nix
./swarm-victoriametrics-service.nix
./swarm-victoriametrics.nix
./swarm-wireguard.nix
./swarm.nix
./hive-options-removed.nix
./host-ca-removed.nix

43
nix/host-modules/deploy.nix
View file

@ -74,6 +74,14 @@ let
The snapshot store was removed; agent-state backups are being rebuilt as
swarm-archive. Remove this definition.
'';
# Shared by the removed options of the WireGuard mesh.
wireguardRemoved = ''
The WireGuard mesh was removed. bao and nats are reached through
their routable listeners (`services.hyperhive.deploy.bao.extraListenAddresses`
with `openFirewall`, and `services.hyperhive.deploy.nats.openFirewall`; both
off by default). Remove this definition.
'';
in
{
imports = [
@ -181,6 +189,41 @@ in
"snapshotStore"
"port"
] snapshotStoreRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"deploy"
"wireguard"
"enable"
] wireguardRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"deploy"
"wireguard"
"privateKeyFile"
] wireguardRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"deploy"
"wireguard"
"address"
] wireguardRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"deploy"
"wireguard"
"listenPort"
] wireguardRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"deploy"
"wireguard"
"persistentKeepalive"
] wireguardRemoved)
# Same type, same meaning, new path — so a rename carries it exactly
# and existing configs keep evaluating with one warning naming both

21
nix/host-modules/glue-nats-wireguard.nix
View file

@ -1,21 +0,0 @@
# Glue: where the queue and the wireguard mesh share a host, the queue's port
# is open on the mesh interface.
#
# Interface-scoped, never host-wide: the firewall is default-deny before a
# packet reaches the listener, so without this no other hive can reach the
# queue. Gated on BOTH being here — the queue's own module gate
# (`deploy.nats.enable`) is restated below; a change there must change here.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
deployCfg = hyperhiveCfg.deploy;
in
{
config = lib.mkIf (deployCfg.nats.enable && deployCfg.wireguard.enable) {
networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ hyperhiveCfg.swarm.nats.port ];
};
}

16
nix/host-modules/hive-egress.nix
View file

@ -40,9 +40,6 @@ let
nft = lib.getExe pkgs.nftables;
br = cfg.bridgeName;
# The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
meshInterface = "wg-hive";
# With `refuseEch`, the redirect of port 443 ends at `hive-priv
# egress-ech-relay`, which refuses a ClientHello that carries the ECH
# extension and sends the rest on to squid's HTTPS intercept port. The relay
@ -143,8 +140,8 @@ let
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
# the interface the route to the destination leaves by: the bridge (another
# container), `lo` (an address of the host: `fib daddr type local`) and the
# mesh interface are not egress. The redirect rewrites the destination to
# container) and `lo` (an address of the host: `fib daddr type local`) are
# not egress. The redirect rewrites the destination to
# the bridge address and the port of squid's intercept listener (for port 443
# with `refuseEch`, of the relay), which reads the original destination from
# conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing.
@ -153,7 +150,7 @@ let
type nat hook prerouting priority dstnat; policy accept;
iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return
iifname "${br}" meta nfproto ipv4 fib daddr type local return
iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return
iifname "${br}" meta nfproto ipv4 fib daddr oifname "${br}" return
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString egressCfg.intercept.httpPort}
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${
toString (if refuseEch then intercept.relayPort else intercept.httpsPort)
@ -171,8 +168,8 @@ let
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
# so it allows nothing those refuse; a `reject` ends the packet. The mesh
# and the MACs in `direct_mac` (the unrestricted agents and the service
# so it allows nothing those refuse; a `reject` ends the packet. The MACs
# in `direct_mac` (the unrestricted agents and the service
# containers, the traffic the intercept chain leaves alone) come first in
# both modes, so a `log` line names what `reject` would refuse. Traffic to
# the host's own addresses (squid, dnsmasq, the gateway) is delivered
@ -183,7 +180,6 @@ let
chain forward {
type filter hook forward priority filter - 10; policy accept;
iifname "${br}" oifname "${meshInterface}" accept
${fromBridge} ether saddr @direct_mac accept
${fromBridge} counter
${lib.concatStringsSep "\n " (
@ -315,7 +311,7 @@ in
default = "log";
description = ''
What the `inet hive_egress` table does with traffic the host forwards
from the agent bridge to anywhere but the bridge and the mesh. In `log` and
from the agent bridge to anywhere but the bridge. In `log` and
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
agents and the service containers) is accepted and never logged.

1
nix/host-modules/hive-network.nix
View file

@ -51,7 +51,6 @@ let
d.victoriametrics.enable
d.victorialogs.enable
d.swarm-secret-publisher.enable
d.wireguard.enable
h.gateway.enable
h.gateway.dns.enable
h.network.enable

4
nix/host-modules/swarm-bao.nix
View file

@ -3113,8 +3113,8 @@ in
store is always one of its readers. Every other address depends on
which network the reading hives share with this one, and that is a
deployment fact no other module's config can be read to infer — a
swarm meshed over wireguard names its mesh address, one on a trusted
LAN names that interface, and an all-local swarm names nothing at all.
swarm on a trusted LAN names that interface, and an all-local swarm
names nothing at all.
Addresses only, no port: a store reachable on two ports is a
misconfiguration rather than a topology.

3
nix/host-modules/swarm-nats-service.nix
View file

@ -56,8 +56,7 @@ in
sits outside hyperhive's claimed ranges (dashboard 7000, forge
3000, every agent in 8100..8999 via FNV-1a hash).
Opened on the bridge interface, for agent containers, and on
`wg-hive` when this host is on the mesh, for other hives. Host-wide
Opened on the bridge interface, for agent containers. Host-wide
only when {option}`services.hyperhive.deploy.nats.openFirewall` is
set. TLS only: a client that does not speak it is refused.

9
nix/host-modules/swarm-nats.nix
View file

@ -522,10 +522,9 @@ in
Open {option}`services.hyperhive.swarm.nats.port` in the host firewall
on every interface, so hives on other hosts can reach the queue on
this host's LAN or public address. Off by default: the port is
otherwise open only on the agent bridge and, when this host is on the
mesh, on `wg-hive`.
otherwise open only on the agent bridge.
⚠️ This exposes the queue beyond the tunnel. The server binds every
⚠️ This exposes the queue beyond the agent bridge. The server binds every
address, so the firewall is the only thing this option changes. What
protects the port is the queue itself: TLS is required, and
`auth_callout` refuses every client it cannot identify. Whether that
@ -846,8 +845,8 @@ in
# TLS required on the client port: a server with a `tls`
# block advertises `tls_required` and refuses a client that
# does not upgrade. No `allow_non_tls`, which would keep the
# plaintext path open across the mesh. The 2s timeout is for
# a handshake that crosses the mesh; upstream's 0.5s is sized
# plaintext path open on the network. The 2s timeout is for
# a handshake that crosses the network; upstream's 0.5s is sized
# for a LAN.
tls = {
cert_file = tlsCertPath;

152
nix/host-modules/swarm-wireguard.nix
View file

@ -1,152 +0,0 @@
# The WireGuard inter-hive mesh for the local host. Split out of
# ./swarm.nix because the two are different concerns with different
# audiences: that file declares WHO the peers are (consumed by
# swarm-controller's hive roster and, here, the mesh), while this one
# is plain host networking that a machine which runs no hive can also enable.
#
# The two stay coupled by data, not by structure: the per-peer
# `wireguard*` fields live on the peer submodule in ./swarm.nix, since
# that is where a peer is described, and this module reads them.
#
# Everything declared here is `deploy.*`, not `swarm.*`: by ./deploy.nix's
# rule, `swarm.*` is what a hive needs to be a *client* of the swarm, and
# none of this is. A peer needs this host's `wireguardEndpoint` (the roster
# entry in ./swarm.nix); what interface this host brings up, on which
# address, with which key, is nobody else's business. `listenPort` included
# — it is what this host binds, while the port a peer dials is the one
# inside `wireguardEndpoint`.
{
lib,
config,
...
}:
{
# WireGuard mesh config for the local host.
# When enabled, a `wg-hive` interface connects to all peers that have
# `wireguardPublicKey` declared. Peers reachable over the mesh are
# preferred for inter-hive traffic (no public TLS round-trip needed);
# peers without a public key still work via normal HTTPS.
options.services.hyperhive.deploy.wireguard = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Enable the WireGuard inter-hive mesh. When true, a `wg-hive`
interface is brought up connecting to all swarm peers that
declare a `wireguardPublicKey`. Requires
`privateKeyFile` to be set.
'';
};
privateKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
example = "/etc/wireguard/hive.key";
description = ''
Path to the host's WireGuard private key file. The file must
be readable by root and should have mode 0400. Generate with
`wg genkey > /etc/wireguard/hive.key`. Required when
`deploy.wireguard.enable = true`.
'';
};
address = lib.mkOption {
type = lib.types.str;
default = "";
example = "10.100.0.1/24";
description = ''
IP address (with prefix) of this host on the WireGuard mesh.
Use a /24 (or broader) prefix so the routing table covers all
peer /32 routes. Example: `"10.100.0.1/24"` for a 256-host mesh.
'';
};
listenPort = lib.mkOption {
type = lib.types.port;
default = 51820;
description = ''
UDP port the local WireGuard interface listens on. Must be
reachable from peer hosts when they initiate the tunnel.
Default: 51820 (standard WireGuard port).
'';
};
persistentKeepalive = lib.mkOption {
type = lib.types.nullOr lib.types.int;
default = 25;
example = 25;
description = ''
Seconds between keepalive packets sent to each peer. Useful
when this host (or a peer) is behind NAT — keeps the UDP hole
open. Set to null to disable. Default: 25 seconds.
'';
};
};
# Gated on the mesh itself, NOT on `deploy.hive-controller.enable`. The
# mesh is host networking, not a c0re feature: a swarm host that runs no
# hive can join the mesh too.
# Nothing below is c0re-specific; the peer data c0re consumes
# (HIVE_PEER_CA_PATHS) is rendered in ./hive-c0re and stays gated there.
config = lib.mkIf config.services.hyperhive.deploy.wireguard.enable {
assertions = [
{
assertion = config.services.hyperhive.deploy.wireguard.privateKeyFile != null;
message = ''
services.hyperhive.deploy.wireguard.enable requires
services.hyperhive.deploy.wireguard.privateKeyFile to be set.
Generate a key: wg genkey > /etc/wireguard/hive.key
'';
}
{
assertion = config.services.hyperhive.deploy.wireguard.address != "";
message = ''
services.hyperhive.deploy.wireguard.enable requires
services.hyperhive.deploy.wireguard.address to be set
(e.g. "10.100.0.1/24").
'';
}
];
# WireGuard inter-hive mesh. Brings up a `wg-hive` interface and
# connects to each peer that has `wireguardPublicKey` set.
networking.wireguard.interfaces =
let
wgCfg = config.services.hyperhive.deploy.wireguard;
# `swarm.hives` minus this host's own hive: a mesh that included
# our own entry would configure a tunnel to ourselves.
meshPeers = lib.filterAttrs (
name: p:
name != config.services.hyperhive.deploy.hive-controller.hiveName
&& p.wireguardPublicKey != null
&& p.wireguardAddress != null
) config.services.hyperhive.swarm.hives;
in
{
wg-hive = {
ips = [ wgCfg.address ];
listenPort = wgCfg.listenPort;
privateKeyFile = wgCfg.privateKeyFile;
peers = lib.mapAttrsToList (
_name: p:
{
publicKey = p.wireguardPublicKey;
allowedIPs = [ p.wireguardAddress ];
}
// lib.optionalAttrs (p.wireguardEndpoint != null) {
endpoint = p.wireguardEndpoint;
}
// lib.optionalAttrs (wgCfg.persistentKeepalive != null) {
persistentKeepalive = wgCfg.persistentKeepalive;
}
) meshPeers;
};
};
# Open the WireGuard UDP port on the host firewall (host-level
# networking — not inside containers).
networking.firewall.allowedUDPPorts = [
config.services.hyperhive.deploy.wireguard.listenPort
];
};
}

92
nix/host-modules/swarm.nix
View file

@ -11,10 +11,7 @@
# detected it. One entry per hive makes that unrepresentable.
#
# Consumed by swarm-controller's own hive directory (its `/api/hives`,
# swarm-controller.nix) and the mesh in ./swarm-wireguard.nix. The mesh
# lives there rather than here because bringing up an interface is
# host networking rather than swarm bookkeeping, and a host that runs
# no hive still needs it.
# swarm-controller.nix).
{
lib,
config,
@ -27,6 +24,30 @@ let
hiveNames = lib.attrNames swarmCfg.hives;
# Per-hive fields of the removed WireGuard mesh. Declared hidden so a
# directory that still sets one reaches the assertion below. A removed
# option cannot be tombstoned with `mkRemovedOptionModule` here: it writes
# top-level `assertions`, which a submodule does not have.
removedMeshFields = [
"wireguardPublicKey"
"wireguardEndpoint"
"wireguardAddress"
];
removedMeshOptions = lib.genAttrs removedMeshFields (
_:
lib.mkOption {
type = lib.types.nullOr lib.types.raw;
default = null;
visible = false;
}
);
meshFieldsSet = lib.concatMap (
name:
map (field: "${name}.${field}") (
lib.filter (f: swarmCfg.hives.${name}.${f} != null) removedMeshFields
)
) hiveNames;
nameGuards = import ./lib/name-guards.nix { inherit lib; };
# Names no hive may BE, and words no hive name may CONTAIN. Both files
@ -166,62 +187,15 @@ in
something else.
'';
};
wireguardPublicKey = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "base64pubkey=";
description = ''
WireGuard public key for this hive's host. Required when
`services.hyperhive.deploy.wireguard.enable = true` and
you want this hive reachable over the mesh. Null = TLS-
only peering (public internet, no mesh tunnel).
'';
};
wireguardEndpoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "203.0.113.1:51820";
description = ''
WireGuard endpoint for this hive in `host:port` form.
Null = this hive has no reachable endpoint, so the tunnel
is initiated from the other side.
Reads like a fact about the relationship and is not: it
says whether *this* hive can be dialled, which every other
hive in the swarm needs the same answer to.
'';
};
wireguardAddress = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "10.100.0.2/32";
description = ''
IP address (with prefix) of this hive's host on the
WireGuard mesh. Used as the `allowedIPs` for its
WireGuard config entry (`./swarm-wireguard.nix`), so
intra-swarm traffic can route over the mesh address
rather than the public domain. Required to include
a hive in the mesh (entries missing this field are
silently excluded from `wg-hive`).
'';
};
};
}
// removedMeshOptions;
}
)
);
default = { };
example = {
pr1ma = {
wireguardAddress = "10.100.0.1/32";
wireguardEndpoint = "203.0.113.1:51820";
};
edge = {
domain = "edge.elsewhere.example";
wireguardAddress = "10.100.0.2/32";
};
pr1ma = { };
edge.domain = "edge.elsewhere.example";
};
description = ''
Every hive in this swarm, keyed by `hiveName` — **including this
@ -289,6 +263,14 @@ in
);
assertions = [
{
assertion = meshFieldsSet == [ ];
message = ''
The WireGuard mesh was removed; bao and nats are reached
through their routable listeners. Remove services.hyperhive.swarm.hives.<name>.wireguard{PublicKey,Endpoint,Address}
from: ${lib.concatStringsSep ", " meshFieldsSet}.
'';
}
(nameGuards.mustBeIdent {
option = "services.hyperhive.swarm.hives";
names = hiveNames;