remove the swarm wireguard mesh; tombstone its options
This commit is contained in:
atlas
parent
fbfc6f392a
commit
adc61c9043
19 changed files with 330 additions and 1068 deletions
|
|
@ -267,8 +267,7 @@ in
|
|||
# nix/packages/default.nix + ./packages.nix) rather than the full
|
||||
# `hyperhive` bundle — that bundle also carries `hivectl` (a
|
||||
# host-admin CLI that dials the *host* admin socket — useless and
|
||||
# unreachable from inside a container — wrapped with
|
||||
# `wireguard-tools` for `hivectl wg`). The daemon/harness/MCP bins
|
||||
# unreachable from inside a container). The daemon/harness/MCP bins
|
||||
# the harness execs (hive-agent{,-mcp}, hive-bash-daemon,
|
||||
# hive-subagent-daemon, hive-matrix-daemon, hive-matrix-mcp) are
|
||||
# wired via their own
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ in
|
|||
environment.systemPackages = [
|
||||
# hive-forge <verb>: CLI wrapping common Forgejo REST API operations
|
||||
# (view, pr, issue, comment, assign, close, labels, branches, etc.).
|
||||
# The per-bin split package — narrow closure, no hivectl/wireguard.
|
||||
# The per-bin split package — narrow closure, no hivectl.
|
||||
config.services.hyperhive.agent.packages.hive-forge
|
||||
]
|
||||
++ lib.optional (config.services.hyperhive.agent.forge.url != null) gitCredHelper;
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@
|
|||
./glue-nats-auth-bao-identity.nix
|
||||
./glue-nats-bao-identity.nix
|
||||
./glue-nats-oidc-client.nix
|
||||
./glue-nats-wireguard.nix
|
||||
./glue-builder-oidc-client.nix
|
||||
./glue-ci-metrics-otel-oidc-client.nix
|
||||
./glue-ci-runner-otel-oidc-client.nix
|
||||
|
|
@ -95,7 +94,6 @@
|
|||
./swarm-victorialogs.nix
|
||||
./swarm-victoriametrics-service.nix
|
||||
./swarm-victoriametrics.nix
|
||||
./swarm-wireguard.nix
|
||||
./swarm.nix
|
||||
./hive-options-removed.nix
|
||||
./host-ca-removed.nix
|
||||
|
|
|
|||
|
|
@ -74,6 +74,14 @@ let
|
|||
The snapshot store was removed; agent-state backups are being rebuilt as
|
||||
swarm-archive. Remove this definition.
|
||||
'';
|
||||
|
||||
# Shared by the removed options of the WireGuard mesh.
|
||||
wireguardRemoved = ''
|
||||
The WireGuard mesh was removed. bao and nats are reached through
|
||||
their routable listeners (`services.hyperhive.deploy.bao.extraListenAddresses`
|
||||
with `openFirewall`, and `services.hyperhive.deploy.nats.openFirewall`; both
|
||||
off by default). Remove this definition.
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
|
|
@ -181,6 +189,41 @@ in
|
|||
"snapshotStore"
|
||||
"port"
|
||||
] snapshotStoreRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"wireguard"
|
||||
"enable"
|
||||
] wireguardRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"wireguard"
|
||||
"privateKeyFile"
|
||||
] wireguardRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"wireguard"
|
||||
"address"
|
||||
] wireguardRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"wireguard"
|
||||
"listenPort"
|
||||
] wireguardRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"wireguard"
|
||||
"persistentKeepalive"
|
||||
] wireguardRemoved)
|
||||
|
||||
# Same type, same meaning, new path — so a rename carries it exactly
|
||||
# and existing configs keep evaluating with one warning naming both
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
# Glue: where the queue and the wireguard mesh share a host, the queue's port
|
||||
# is open on the mesh interface.
|
||||
#
|
||||
# Interface-scoped, never host-wide: the firewall is default-deny before a
|
||||
# packet reaches the listener, so without this no other hive can reach the
|
||||
# queue. Gated on BOTH being here — the queue's own module gate
|
||||
# (`deploy.nats.enable`) is restated below; a change there must change here.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
deployCfg = hyperhiveCfg.deploy;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (deployCfg.nats.enable && deployCfg.wireguard.enable) {
|
||||
networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ hyperhiveCfg.swarm.nats.port ];
|
||||
};
|
||||
}
|
||||
|
|
@ -40,9 +40,6 @@ let
|
|||
nft = lib.getExe pkgs.nftables;
|
||||
br = cfg.bridgeName;
|
||||
|
||||
# The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
|
||||
meshInterface = "wg-hive";
|
||||
|
||||
# With `refuseEch`, the redirect of port 443 ends at `hive-priv
|
||||
# egress-ech-relay`, which refuses a ClientHello that carries the ECH
|
||||
# extension and sends the rest on to squid's HTTPS intercept port. The relay
|
||||
|
|
@ -143,8 +140,8 @@ let
|
|||
|
||||
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
|
||||
# the interface the route to the destination leaves by: the bridge (another
|
||||
# container), `lo` (an address of the host: `fib daddr type local`) and the
|
||||
# mesh interface are not egress. The redirect rewrites the destination to
|
||||
# container) and `lo` (an address of the host: `fib daddr type local`) are
|
||||
# not egress. The redirect rewrites the destination to
|
||||
# the bridge address and the port of squid's intercept listener (for port 443
|
||||
# with `refuseEch`, of the relay), which reads the original destination from
|
||||
# conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing.
|
||||
|
|
@ -153,7 +150,7 @@ let
|
|||
type nat hook prerouting priority dstnat; policy accept;
|
||||
iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return
|
||||
iifname "${br}" meta nfproto ipv4 fib daddr type local return
|
||||
iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return
|
||||
iifname "${br}" meta nfproto ipv4 fib daddr oifname "${br}" return
|
||||
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString egressCfg.intercept.httpPort}
|
||||
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${
|
||||
toString (if refuseEch then intercept.relayPort else intercept.httpsPort)
|
||||
|
|
@ -171,8 +168,8 @@ let
|
|||
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
|
||||
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
|
||||
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
|
||||
# so it allows nothing those refuse; a `reject` ends the packet. The mesh
|
||||
# and the MACs in `direct_mac` (the unrestricted agents and the service
|
||||
# so it allows nothing those refuse; a `reject` ends the packet. The MACs
|
||||
# in `direct_mac` (the unrestricted agents and the service
|
||||
# containers, the traffic the intercept chain leaves alone) come first in
|
||||
# both modes, so a `log` line names what `reject` would refuse. Traffic to
|
||||
# the host's own addresses (squid, dnsmasq, the gateway) is delivered
|
||||
|
|
@ -183,7 +180,6 @@ let
|
|||
|
||||
chain forward {
|
||||
type filter hook forward priority filter - 10; policy accept;
|
||||
iifname "${br}" oifname "${meshInterface}" accept
|
||||
${fromBridge} ether saddr @direct_mac accept
|
||||
${fromBridge} counter
|
||||
${lib.concatStringsSep "\n " (
|
||||
|
|
@ -315,7 +311,7 @@ in
|
|||
default = "log";
|
||||
description = ''
|
||||
What the `inet hive_egress` table does with traffic the host forwards
|
||||
from the agent bridge to anywhere but the bridge and the mesh. In `log` and
|
||||
from the agent bridge to anywhere but the bridge. In `log` and
|
||||
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
|
||||
agents and the service containers) is accepted and never logged.
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@ let
|
|||
d.victoriametrics.enable
|
||||
d.victorialogs.enable
|
||||
d.swarm-secret-publisher.enable
|
||||
d.wireguard.enable
|
||||
h.gateway.enable
|
||||
h.gateway.dns.enable
|
||||
h.network.enable
|
||||
|
|
|
|||
|
|
@ -3113,8 +3113,8 @@ in
|
|||
store is always one of its readers. Every other address depends on
|
||||
which network the reading hives share with this one, and that is a
|
||||
deployment fact no other module's config can be read to infer — a
|
||||
swarm meshed over wireguard names its mesh address, one on a trusted
|
||||
LAN names that interface, and an all-local swarm names nothing at all.
|
||||
swarm on a trusted LAN names that interface, and an all-local swarm
|
||||
names nothing at all.
|
||||
|
||||
Addresses only, no port: a store reachable on two ports is a
|
||||
misconfiguration rather than a topology.
|
||||
|
|
|
|||
|
|
@ -56,8 +56,7 @@ in
|
|||
sits outside hyperhive's claimed ranges (dashboard 7000, forge
|
||||
3000, every agent in 8100..8999 via FNV-1a hash).
|
||||
|
||||
Opened on the bridge interface, for agent containers, and on
|
||||
`wg-hive` when this host is on the mesh, for other hives. Host-wide
|
||||
Opened on the bridge interface, for agent containers. Host-wide
|
||||
only when {option}`services.hyperhive.deploy.nats.openFirewall` is
|
||||
set. TLS only: a client that does not speak it is refused.
|
||||
|
||||
|
|
|
|||
|
|
@ -522,10 +522,9 @@ in
|
|||
Open {option}`services.hyperhive.swarm.nats.port` in the host firewall
|
||||
on every interface, so hives on other hosts can reach the queue on
|
||||
this host's LAN or public address. Off by default: the port is
|
||||
otherwise open only on the agent bridge and, when this host is on the
|
||||
mesh, on `wg-hive`.
|
||||
otherwise open only on the agent bridge.
|
||||
|
||||
⚠️ This exposes the queue beyond the tunnel. The server binds every
|
||||
⚠️ This exposes the queue beyond the agent bridge. The server binds every
|
||||
address, so the firewall is the only thing this option changes. What
|
||||
protects the port is the queue itself: TLS is required, and
|
||||
`auth_callout` refuses every client it cannot identify. Whether that
|
||||
|
|
@ -846,8 +845,8 @@ in
|
|||
# TLS required on the client port: a server with a `tls`
|
||||
# block advertises `tls_required` and refuses a client that
|
||||
# does not upgrade. No `allow_non_tls`, which would keep the
|
||||
# plaintext path open across the mesh. The 2s timeout is for
|
||||
# a handshake that crosses the mesh; upstream's 0.5s is sized
|
||||
# plaintext path open on the network. The 2s timeout is for
|
||||
# a handshake that crosses the network; upstream's 0.5s is sized
|
||||
# for a LAN.
|
||||
tls = {
|
||||
cert_file = tlsCertPath;
|
||||
|
|
|
|||
|
|
@ -1,152 +0,0 @@
|
|||
# The WireGuard inter-hive mesh for the local host. Split out of
|
||||
# ./swarm.nix because the two are different concerns with different
|
||||
# audiences: that file declares WHO the peers are (consumed by
|
||||
# swarm-controller's hive roster and, here, the mesh), while this one
|
||||
# is plain host networking that a machine which runs no hive can also enable.
|
||||
#
|
||||
# The two stay coupled by data, not by structure: the per-peer
|
||||
# `wireguard*` fields live on the peer submodule in ./swarm.nix, since
|
||||
# that is where a peer is described, and this module reads them.
|
||||
#
|
||||
# Everything declared here is `deploy.*`, not `swarm.*`: by ./deploy.nix's
|
||||
# rule, `swarm.*` is what a hive needs to be a *client* of the swarm, and
|
||||
# none of this is. A peer needs this host's `wireguardEndpoint` (the roster
|
||||
# entry in ./swarm.nix); what interface this host brings up, on which
|
||||
# address, with which key, is nobody else's business. `listenPort` included
|
||||
# — it is what this host binds, while the port a peer dials is the one
|
||||
# inside `wireguardEndpoint`.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
# WireGuard mesh config for the local host.
|
||||
# When enabled, a `wg-hive` interface connects to all peers that have
|
||||
# `wireguardPublicKey` declared. Peers reachable over the mesh are
|
||||
# preferred for inter-hive traffic (no public TLS round-trip needed);
|
||||
# peers without a public key still work via normal HTTPS.
|
||||
options.services.hyperhive.deploy.wireguard = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Enable the WireGuard inter-hive mesh. When true, a `wg-hive`
|
||||
interface is brought up connecting to all swarm peers that
|
||||
declare a `wireguardPublicKey`. Requires
|
||||
`privateKeyFile` to be set.
|
||||
'';
|
||||
};
|
||||
|
||||
privateKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
default = null;
|
||||
example = "/etc/wireguard/hive.key";
|
||||
description = ''
|
||||
Path to the host's WireGuard private key file. The file must
|
||||
be readable by root and should have mode 0400. Generate with
|
||||
`wg genkey > /etc/wireguard/hive.key`. Required when
|
||||
`deploy.wireguard.enable = true`.
|
||||
'';
|
||||
};
|
||||
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "10.100.0.1/24";
|
||||
description = ''
|
||||
IP address (with prefix) of this host on the WireGuard mesh.
|
||||
Use a /24 (or broader) prefix so the routing table covers all
|
||||
peer /32 routes. Example: `"10.100.0.1/24"` for a 256-host mesh.
|
||||
'';
|
||||
};
|
||||
|
||||
listenPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 51820;
|
||||
description = ''
|
||||
UDP port the local WireGuard interface listens on. Must be
|
||||
reachable from peer hosts when they initiate the tunnel.
|
||||
Default: 51820 (standard WireGuard port).
|
||||
'';
|
||||
};
|
||||
|
||||
persistentKeepalive = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.int;
|
||||
default = 25;
|
||||
example = 25;
|
||||
description = ''
|
||||
Seconds between keepalive packets sent to each peer. Useful
|
||||
when this host (or a peer) is behind NAT — keeps the UDP hole
|
||||
open. Set to null to disable. Default: 25 seconds.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# Gated on the mesh itself, NOT on `deploy.hive-controller.enable`. The
|
||||
# mesh is host networking, not a c0re feature: a swarm host that runs no
|
||||
# hive can join the mesh too.
|
||||
# Nothing below is c0re-specific; the peer data c0re consumes
|
||||
# (HIVE_PEER_CA_PATHS) is rendered in ./hive-c0re and stays gated there.
|
||||
config = lib.mkIf config.services.hyperhive.deploy.wireguard.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = config.services.hyperhive.deploy.wireguard.privateKeyFile != null;
|
||||
message = ''
|
||||
services.hyperhive.deploy.wireguard.enable requires
|
||||
services.hyperhive.deploy.wireguard.privateKeyFile to be set.
|
||||
Generate a key: wg genkey > /etc/wireguard/hive.key
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = config.services.hyperhive.deploy.wireguard.address != "";
|
||||
message = ''
|
||||
services.hyperhive.deploy.wireguard.enable requires
|
||||
services.hyperhive.deploy.wireguard.address to be set
|
||||
(e.g. "10.100.0.1/24").
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# WireGuard inter-hive mesh. Brings up a `wg-hive` interface and
|
||||
# connects to each peer that has `wireguardPublicKey` set.
|
||||
networking.wireguard.interfaces =
|
||||
let
|
||||
wgCfg = config.services.hyperhive.deploy.wireguard;
|
||||
# `swarm.hives` minus this host's own hive: a mesh that included
|
||||
# our own entry would configure a tunnel to ourselves.
|
||||
meshPeers = lib.filterAttrs (
|
||||
name: p:
|
||||
name != config.services.hyperhive.deploy.hive-controller.hiveName
|
||||
&& p.wireguardPublicKey != null
|
||||
&& p.wireguardAddress != null
|
||||
) config.services.hyperhive.swarm.hives;
|
||||
in
|
||||
{
|
||||
wg-hive = {
|
||||
ips = [ wgCfg.address ];
|
||||
listenPort = wgCfg.listenPort;
|
||||
privateKeyFile = wgCfg.privateKeyFile;
|
||||
peers = lib.mapAttrsToList (
|
||||
_name: p:
|
||||
{
|
||||
publicKey = p.wireguardPublicKey;
|
||||
allowedIPs = [ p.wireguardAddress ];
|
||||
}
|
||||
// lib.optionalAttrs (p.wireguardEndpoint != null) {
|
||||
endpoint = p.wireguardEndpoint;
|
||||
}
|
||||
// lib.optionalAttrs (wgCfg.persistentKeepalive != null) {
|
||||
persistentKeepalive = wgCfg.persistentKeepalive;
|
||||
}
|
||||
) meshPeers;
|
||||
};
|
||||
};
|
||||
|
||||
# Open the WireGuard UDP port on the host firewall (host-level
|
||||
# networking — not inside containers).
|
||||
networking.firewall.allowedUDPPorts = [
|
||||
config.services.hyperhive.deploy.wireguard.listenPort
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -11,10 +11,7 @@
|
|||
# detected it. One entry per hive makes that unrepresentable.
|
||||
#
|
||||
# Consumed by swarm-controller's own hive directory (its `/api/hives`,
|
||||
# swarm-controller.nix) and the mesh in ./swarm-wireguard.nix. The mesh
|
||||
# lives there rather than here because bringing up an interface is
|
||||
# host networking rather than swarm bookkeeping, and a host that runs
|
||||
# no hive still needs it.
|
||||
# swarm-controller.nix).
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
|
|
@ -27,6 +24,30 @@ let
|
|||
|
||||
hiveNames = lib.attrNames swarmCfg.hives;
|
||||
|
||||
# Per-hive fields of the removed WireGuard mesh. Declared hidden so a
|
||||
# directory that still sets one reaches the assertion below. A removed
|
||||
# option cannot be tombstoned with `mkRemovedOptionModule` here: it writes
|
||||
# top-level `assertions`, which a submodule does not have.
|
||||
removedMeshFields = [
|
||||
"wireguardPublicKey"
|
||||
"wireguardEndpoint"
|
||||
"wireguardAddress"
|
||||
];
|
||||
removedMeshOptions = lib.genAttrs removedMeshFields (
|
||||
_:
|
||||
lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.raw;
|
||||
default = null;
|
||||
visible = false;
|
||||
}
|
||||
);
|
||||
meshFieldsSet = lib.concatMap (
|
||||
name:
|
||||
map (field: "${name}.${field}") (
|
||||
lib.filter (f: swarmCfg.hives.${name}.${f} != null) removedMeshFields
|
||||
)
|
||||
) hiveNames;
|
||||
|
||||
nameGuards = import ./lib/name-guards.nix { inherit lib; };
|
||||
|
||||
# Names no hive may BE, and words no hive name may CONTAIN. Both files
|
||||
|
|
@ -166,62 +187,15 @@ in
|
|||
something else.
|
||||
'';
|
||||
};
|
||||
|
||||
wireguardPublicKey = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "base64pubkey=";
|
||||
description = ''
|
||||
WireGuard public key for this hive's host. Required when
|
||||
`services.hyperhive.deploy.wireguard.enable = true` and
|
||||
you want this hive reachable over the mesh. Null = TLS-
|
||||
only peering (public internet, no mesh tunnel).
|
||||
'';
|
||||
};
|
||||
|
||||
wireguardEndpoint = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "203.0.113.1:51820";
|
||||
description = ''
|
||||
WireGuard endpoint for this hive in `host:port` form.
|
||||
Null = this hive has no reachable endpoint, so the tunnel
|
||||
is initiated from the other side.
|
||||
|
||||
Reads like a fact about the relationship and is not: it
|
||||
says whether *this* hive can be dialled, which every other
|
||||
hive in the swarm needs the same answer to.
|
||||
'';
|
||||
};
|
||||
|
||||
wireguardAddress = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "10.100.0.2/32";
|
||||
description = ''
|
||||
IP address (with prefix) of this hive's host on the
|
||||
WireGuard mesh. Used as the `allowedIPs` for its
|
||||
WireGuard config entry (`./swarm-wireguard.nix`), so
|
||||
intra-swarm traffic can route over the mesh address
|
||||
rather than the public domain. Required to include
|
||||
a hive in the mesh (entries missing this field are
|
||||
silently excluded from `wg-hive`).
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
// removedMeshOptions;
|
||||
}
|
||||
)
|
||||
);
|
||||
default = { };
|
||||
example = {
|
||||
pr1ma = {
|
||||
wireguardAddress = "10.100.0.1/32";
|
||||
wireguardEndpoint = "203.0.113.1:51820";
|
||||
};
|
||||
edge = {
|
||||
domain = "edge.elsewhere.example";
|
||||
wireguardAddress = "10.100.0.2/32";
|
||||
};
|
||||
pr1ma = { };
|
||||
edge.domain = "edge.elsewhere.example";
|
||||
};
|
||||
description = ''
|
||||
Every hive in this swarm, keyed by `hiveName` — **including this
|
||||
|
|
@ -289,6 +263,14 @@ in
|
|||
);
|
||||
|
||||
assertions = [
|
||||
{
|
||||
assertion = meshFieldsSet == [ ];
|
||||
message = ''
|
||||
The WireGuard mesh was removed; bao and nats are reached
|
||||
through their routable listeners. Remove services.hyperhive.swarm.hives.<name>.wireguard{PublicKey,Endpoint,Address}
|
||||
from: ${lib.concatStringsSep ", " meshFieldsSet}.
|
||||
'';
|
||||
}
|
||||
(nameGuards.mustBeIdent {
|
||||
option = "services.hyperhive.swarm.hives";
|
||||
names = hiveNames;
|
||||
|
|
|
|||
|
|
@ -1615,12 +1615,11 @@ let
|
|||
&& lib.hasInfix ''iifname "hive-br0" oifname != "hive-br0" ct state new update @log_v4 { ip saddr limit rate'' chain
|
||||
&& lib.hasInfix ''iifname "hive-br0" oifname != "hive-br0" ct state new update @log_v6 { ip6 saddr limit rate'' chain
|
||||
&& lib.hasInfix ''log prefix "hive-egress-would-drop "'' chain
|
||||
&& lib.hasInfix ''iifname "hive-br0" oifname "wg-hive" accept'' chain
|
||||
# The mesh comes first, then the exempt MACs (service containers and
|
||||
# unrestricted agents), then the counter and the logging: a flow of an
|
||||
# exempt MAC is neither counted nor logged.
|
||||
# The exempt MACs (service containers and unrestricted agents) come
|
||||
# first, then the counter and the logging: a flow of an exempt MAC is
|
||||
# neither counted nor logged. No interface is exempt by name.
|
||||
&& lib.hasInfix ''iifname "hive-br0" oifname != "hive-br0" ether saddr @direct_mac accept'' chain
|
||||
&& idx ''oifname "wg-hive" accept'' < idx "ether saddr @direct_mac accept"
|
||||
&& !(lib.hasInfix "wg-hive" text)
|
||||
&& idx "ether saddr @direct_mac accept" < idx "counter"
|
||||
&& idx "counter" < idx "update @log_v4"
|
||||
# The per-source rules accept after logging, and the unmetered rule
|
||||
|
|
@ -1749,11 +1748,11 @@ let
|
|||
{
|
||||
# `reject` logs and refuses in one rule, as a TCP rule (reset) and a
|
||||
# non-TCP rule (ICMP admin-prohibited) for each of the four log/verdict
|
||||
# positions, after the mesh and the exempt MACs. The ruleset is read as
|
||||
# text. The `accept` count (the mesh and the exempt MACs only) and the
|
||||
# positions, after the exempt MACs. The ruleset is read as
|
||||
# text. The `accept` count (the exempt MACs only) and the
|
||||
# `bare` host, whose chain has no refusal, are the controls that the
|
||||
# lookups find what is there.
|
||||
name = "egress.forward reject refuses the bridge's forwarded flows after the mesh and the exempt MACs";
|
||||
name = "egress.forward reject refuses the bridge's forwarded flows after the exempt MACs";
|
||||
ok =
|
||||
let
|
||||
text = lib.head (
|
||||
|
|
@ -1772,10 +1771,10 @@ let
|
|||
) egressForwardReject.assertions)
|
||||
&& lib.hasInfix "type filter hook forward priority filter - 10; policy accept;" chain
|
||||
&& lib.hasInfix "chain input" text
|
||||
&& idx ''oifname "wg-hive" accept'' < idx "ether saddr @direct_mac accept"
|
||||
&& !(lib.hasInfix "wg-hive" text)
|
||||
&& idx "ether saddr @direct_mac accept" < idx "counter"
|
||||
&& idx "counter" < idx "reject"
|
||||
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 2
|
||||
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 1
|
||||
&& lib.length tcp == 4
|
||||
&& lib.length other == 4
|
||||
&& lib.all (lib.hasSuffix " reject with tcp reset") tcp
|
||||
|
|
@ -1818,7 +1817,7 @@ let
|
|||
# The redirect of the agents' web traffic: IPv4 TCP 80 and 443 from the
|
||||
# bridge go to squid's two intercept ports, after the rules that leave
|
||||
# alone the exempt MACs (service containers, unrestricted agents), the
|
||||
# host's own addresses, the bridge and the mesh. The chain is read as the
|
||||
# host's own addresses and the bridge. The chain is read as the
|
||||
# lines between its header and the log sets that follow it. The host
|
||||
# without a controller (`egressForwardAccept`) has the set but no chain.
|
||||
name = "the egress table redirects the bridge's web traffic into squid after its exemptions";
|
||||
|
|
@ -1845,7 +1844,7 @@ let
|
|||
&& lib.hasInfix ''oifname != "hive-br0" ether saddr @direct_mac accept'' inet
|
||||
&& !(lib.hasInfix "ether saddr @direct_mac accept" without)
|
||||
&& idx "fib daddr type local return" < idx "tcp dport 80"
|
||||
&& idx ''fib daddr oifname { "hive-br0", "wg-hive" } return'' < idx "tcp dport 80"
|
||||
&& idx ''fib daddr oifname "hive-br0" return'' < idx "tcp dport 80"
|
||||
&& lib.length redirects == 2
|
||||
&& lib.hasInfix "tcp dport 80 counter redirect to :3129" (lib.head redirects)
|
||||
# Port 443 goes to the ClientHello relay, not to squid.
|
||||
|
|
@ -2578,6 +2577,51 @@ let
|
|||
in
|
||||
lib.all (path: refused old path) paths && !(lib.any (path: refused bare path) paths);
|
||||
}
|
||||
{
|
||||
# One host defining every removed mesh option, the five deploy paths and
|
||||
# the three per-hive fields of one directory entry. Each message names
|
||||
# its own path, so each lookup finds only its option. The control is a
|
||||
# host that defines none: it has to pass the same assertions, which also
|
||||
# shows the lookups find a failing assertion when there is one.
|
||||
name = "a definition of a removed WireGuard mesh option fails evaluation, naming the option and the issue";
|
||||
ok =
|
||||
let
|
||||
deployPaths = map (o: "services.hyperhive.deploy.wireguard.${o}") [
|
||||
"enable"
|
||||
"privateKeyFile"
|
||||
"address"
|
||||
"listenPort"
|
||||
"persistentKeepalive"
|
||||
];
|
||||
hiveFields = map (f: "peer.${f}") [
|
||||
"wireguardPublicKey"
|
||||
"wireguardEndpoint"
|
||||
"wireguardAddress"
|
||||
];
|
||||
failed = m: lib.filter (a: !a.assertion) m.assertions;
|
||||
names =
|
||||
m: path:
|
||||
lib.any (a: lib.hasInfix path a.message && lib.hasInfix "WireGuard mesh was removed" a.message) (
|
||||
failed m
|
||||
);
|
||||
old = hive {
|
||||
deploy.wireguard = {
|
||||
enable = true;
|
||||
privateKeyFile = "/etc/wireguard/hive.key";
|
||||
address = "10.100.0.1/24";
|
||||
listenPort = 51820;
|
||||
persistentKeepalive = 25;
|
||||
};
|
||||
swarm.hives.peer = {
|
||||
wireguardPublicKey = "key";
|
||||
wireguardEndpoint = "203.0.113.1:51820";
|
||||
wireguardAddress = "10.100.0.2/32";
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.all (path: names old path) (deployPaths ++ hiveFields)
|
||||
&& !(lib.any (path: names bare path) (deployPaths ++ hiveFields));
|
||||
}
|
||||
{
|
||||
# Refused wherever any hyperhive service runs, the hive or a swarm
|
||||
# service alone. The host enabling nothing is the control that it is
|
||||
|
|
|
|||
|
|
@ -32,13 +32,6 @@ let
|
|||
# are all here, so the scan below reads each of them.
|
||||
inherit (shared) allLocal;
|
||||
|
||||
# The same host on the mesh.
|
||||
allLocalMesh = hive {
|
||||
deploy.singleHostSwarm = true;
|
||||
deploy.wireguard.enable = true;
|
||||
deploy.wireguard.address = "10.100.0.1/24";
|
||||
};
|
||||
|
||||
# The queue on a host whose store is elsewhere: no local policy unit.
|
||||
queueNoStore = hive {
|
||||
deploy.nats.enable = true;
|
||||
|
|
@ -275,13 +268,6 @@ let
|
|||
ok =
|
||||
!(natsContainer.services.nats.settings ? host) && !(natsContainer.services.nats.settings ? listen);
|
||||
}
|
||||
{
|
||||
name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
|
||||
ok =
|
||||
lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
|
||||
&& !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
|
||||
&& lib.elem 4222 (bridgePorts allLocalMesh);
|
||||
}
|
||||
{
|
||||
# Off is today's config; the bridge arm is the control that the lookup
|
||||
# finds a port that is there. On adds the host-wide port and keeps the
|
||||
|
|
@ -295,14 +281,6 @@ let
|
|||
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts)
|
||||
&& lib.elem 4222 (bridgePorts allLocal);
|
||||
}
|
||||
{
|
||||
name = "4222 is not opened on wg-hive when this host is not on the mesh";
|
||||
ok =
|
||||
!(lib.elem 4222
|
||||
(allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
|
||||
)
|
||||
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
|
||||
}
|
||||
{
|
||||
# Ordering, never a requirement: a policy unit that failed still counts
|
||||
# as done, and the leaf unit's own retries carry it past that.
|
||||
|
|
|
|||
|
|
@ -95,22 +95,18 @@ let
|
|||
install -Dm755 ${workspaceBuild}/bin/${bin} $out/bin/${bin}
|
||||
'';
|
||||
|
||||
# Operator CLI — extractor plus shell completions and the `wg`
|
||||
# wrapper. Suitable for `nix profile install .#hivectl` /
|
||||
# Operator CLI — the binary plus shell completions. Suitable for
|
||||
# `nix profile install .#hivectl` /
|
||||
# `environment.systemPackages = [ …packages.<system>.hivectl ]`
|
||||
# when the operator only wants the admin CLI.
|
||||
# Completions come from the binary's own `completions <shell>` verb
|
||||
# (the single source of truth, so they never drift from the actual
|
||||
# verbs). The wireguard-tools wrap makes `hivectl wg` subcommands
|
||||
# work before `deploy.wireguard.enable` is set (wg init is the very
|
||||
# first setup step). Completion generation runs before wrapProgram
|
||||
# since wrapProgram renames the real binary.
|
||||
# verbs).
|
||||
hivectlPkg =
|
||||
pkgs.runCommand "hivectl"
|
||||
{
|
||||
nativeBuildInputs = [
|
||||
pkgs.installShellFiles
|
||||
pkgs.makeWrapper
|
||||
];
|
||||
meta = {
|
||||
description = "hyperhive operator CLI";
|
||||
|
|
@ -123,16 +119,13 @@ let
|
|||
--bash <("$out/bin/hivectl" completions bash) \
|
||||
--zsh <("$out/bin/hivectl" completions zsh) \
|
||||
--fish <("$out/bin/hivectl" completions fish)
|
||||
wrapProgram "$out/bin/hivectl" \
|
||||
--prefix PATH : ${pkgs.wireguard-tools}/bin
|
||||
'';
|
||||
|
||||
# Per-bin split packages. Agent containers depend on the individual
|
||||
# bins they actually exec/PATH-need (see nix/agent-modules/packages.nix)
|
||||
# instead of the `default` bundle — that keeps `hivectl` (dials the *host*
|
||||
# admin socket, unreachable from inside a container, drags in
|
||||
# `wireguard-tools`) and redundant binaries out of every agent's
|
||||
# closure.
|
||||
# admin socket, unreachable from inside a container) and redundant
|
||||
# binaries out of every agent's closure.
|
||||
binPkgs = lib.mapAttrs mkBinPackage daemonBins // {
|
||||
hivectl = hivectlPkg;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -132,7 +132,6 @@ let
|
|||
swarm-archive
|
||||
agent-ephemeral-rootfs
|
||||
full-setup
|
||||
wg-hive
|
||||
semi-spread
|
||||
swarm-builder-offline
|
||||
;
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
}:
|
||||
{
|
||||
agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; };
|
||||
wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
|
||||
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs hivePriv; };
|
||||
hive-egress-antispoof = import ./vm-tests/hive-egress-antispoof.nix { inherit pkgs hivePriv; };
|
||||
swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; };
|
||||
|
|
|
|||
|
|
@ -896,16 +896,6 @@ pkgs.testers.runNixOSTest {
|
|||
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=fd00:0*98:(0+:)*0*3 DST=fd00:0*99:(0+:)*0*2 .*PROTO=TCP .*DPT=443 '"
|
||||
)
|
||||
|
||||
with checked("a flow to the mesh interface is not logged and one to the uplink is"):
|
||||
proxy.succeed("ip link add wg-hive type dummy && ip link set wg-hive up")
|
||||
proxy.succeed("ip route add 198.51.100.0/24 dev wg-hive")
|
||||
agent.execute("echo x > /dev/udp/198.51.100.7/9; echo x > /dev/udp/203.0.113.77/9")
|
||||
proxy.wait_until_succeeds(
|
||||
"journalctl -k -o cat --no-pager | grep -E "
|
||||
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*DST=203\\.0\\.113\\.77 '"
|
||||
)
|
||||
assert not would_drop(r"DST=198\.51\.100\.7 "), "the mesh interface is not egress"
|
||||
|
||||
with checked("one busy agent address is cut to a burst plus a trickle, the rest still leave a trace, and every flow is logged once at most"):
|
||||
agent.execute("for i in $(seq 1 100); do echo x > /dev/udp/203.0.113.78/$((3000 + i)); done")
|
||||
proxy.wait_until_succeeds(
|
||||
|
|
@ -1176,11 +1166,6 @@ pkgs.testers.runNixOSTest {
|
|||
# ---- Reject mode ----
|
||||
# The node switches to its `reject` specialisation, which reloads the
|
||||
# tables with the refusing forward chain. Nothing is in `direct_mac` yet.
|
||||
def reject_lines(pattern):
|
||||
# The reject chain's log lines in the kernel log that match `pattern`.
|
||||
out = proxy.succeed("journalctl -k -o cat --no-pager | grep hive-egress-reject || true")
|
||||
return [line for line in out.splitlines() if re.search(pattern, line)]
|
||||
|
||||
def wait_reject_line(pattern):
|
||||
proxy.wait_until_succeeds(
|
||||
"journalctl -k -o cat --no-pager | grep -E "
|
||||
|
|
@ -1213,20 +1198,12 @@ pkgs.testers.runNixOSTest {
|
|||
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
|
||||
assert found and int(found.group(1)) > 0, out
|
||||
|
||||
with checked("reject mode: a MAC in direct_mac passes and is refused again once it leaves, and the mesh passes"):
|
||||
with checked("reject mode: a MAC in direct_mac passes and is refused again once it leaves"):
|
||||
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
||||
assert dial_status("203.0.113.2", 8443) == 0, "an exempt MAC (a service container's stand-in) reaches TCP 8443"
|
||||
agent.succeed("ping -c1 -W3 203.0.113.2")
|
||||
assert udp_status("203.0.113.77") == 0, "control: an exempt MAC's datagram is not refused"
|
||||
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
||||
assert dial_status("203.0.113.2", 8443) == 1, "refused again"
|
||||
# The mesh comes first in the chain. A dummy interface answers nothing,
|
||||
# so the proof is that the chain refuses neither the ping nor the UDP
|
||||
# datagram, while the same flows to the uplink are refused.
|
||||
status, out = agent.execute("ping -c1 -W5 198.51.100.7")
|
||||
assert "filtered" not in out.lower(), f"the mesh interface is not egress: {out}"
|
||||
agent.execute("echo x > /dev/udp/198.51.100.7/9; echo x > /dev/udp/203.0.113.80/9")
|
||||
wait_reject_line(r"DST=203\.0\.113\.80 ")
|
||||
assert not reject_lines(r"DST=198\.51\.100\.7 "), "the mesh interface is not egress"
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
# `vmTests.<system>.wg-hive`: two hives peered over `wg-hive`, with real traffic across the tunnel.
|
||||
# Unlike ../module-eval, which only reads the rendered `networking.wireguard`
|
||||
# options, this boots both hosts and sends packets.
|
||||
#
|
||||
# Keys are derived from a fixed per-host seed, so every store builds the
|
||||
# same bytes and nothing is committed. The module takes peer public keys
|
||||
# as eval-time strings (`swarm.hives.<name>.wireguardPublicKey`), so they
|
||||
# are read back from the derivation's output.
|
||||
{
|
||||
pkgs,
|
||||
self,
|
||||
}:
|
||||
let
|
||||
mkKey =
|
||||
name:
|
||||
pkgs.runCommand "wg-hive-test-key-${name}"
|
||||
{
|
||||
nativeBuildInputs = [
|
||||
pkgs.wireguard-tools
|
||||
pkgs.openssl
|
||||
];
|
||||
}
|
||||
''
|
||||
# Test-only key: base64 of sha256(seed); wg clamps the scalar itself.
|
||||
mkdir $out
|
||||
printf '%s' 'hyperhive-wg-hive-test-${name}' | openssl dgst -sha256 -binary | base64 > $out/private
|
||||
wg pubkey < $out/private > $out/public
|
||||
'';
|
||||
keys = {
|
||||
h1 = mkKey "h1";
|
||||
h2 = mkKey "h2";
|
||||
# Never installed on any node; stands in for a peer key that does not
|
||||
# match the real peer's private key.
|
||||
wrong = mkKey "wrong";
|
||||
};
|
||||
pub = name: pkgs.lib.removeSuffix "\n" (builtins.readFile "${keys.${name}}/public");
|
||||
|
||||
# The test driver numbers nodes on the test vlan in alphabetical order.
|
||||
lanAddr = {
|
||||
h1 = "192.168.1.1";
|
||||
h2 = "192.168.1.2";
|
||||
};
|
||||
meshAddr = {
|
||||
h1 = "10.100.0.1";
|
||||
h2 = "10.100.0.2";
|
||||
};
|
||||
|
||||
swarmHives = pkgs.lib.mapAttrs (name: _: {
|
||||
domain = "${name}.t.local";
|
||||
wireguardPublicKey = pub name;
|
||||
wireguardAddress = "${meshAddr.${name}}/32";
|
||||
wireguardEndpoint = "${lanAddr.${name}}:51820";
|
||||
}) lanAddr;
|
||||
|
||||
node = name: {
|
||||
imports = [ self.nixosModules.default ];
|
||||
services.hyperhive = {
|
||||
swarm.domain = "t.local";
|
||||
swarm.hives = swarmHives;
|
||||
deploy.hive-controller.hiveName = name;
|
||||
deploy.wireguard = {
|
||||
enable = true;
|
||||
address = "${meshAddr.${name}}/24";
|
||||
privateKeyFile = "${keys.${name}}/private";
|
||||
persistentKeepalive = 1;
|
||||
};
|
||||
};
|
||||
# The listener the TCP check connects to, open on the mesh only.
|
||||
networking.firewall.interfaces.wg-hive.allowedTCPPorts = [ 9000 ];
|
||||
environment.systemPackages = [
|
||||
pkgs.wireguard-tools
|
||||
pkgs.python3
|
||||
pkgs.curl
|
||||
];
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "wg-hive";
|
||||
|
||||
nodes = {
|
||||
h1 = node "h1";
|
||||
h2 = node "h2";
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
def wg_state(m):
|
||||
return m.execute("wg show wg-hive; ip route; systemctl status 'wireguard-wg-hive*' --no-pager")[1]
|
||||
|
||||
start_all()
|
||||
for m, peer_key in ((h1, "${pub "h2"}"), (h2, "${pub "h1"}")):
|
||||
m.wait_for_unit("wireguard-wg-hive.service")
|
||||
m.succeed("ip link show wg-hive")
|
||||
# The interface unit is active before its peers are installed: each
|
||||
# peer has its own unit, which runs after network-online.target. Until
|
||||
# then the interface has no peer and a send fails with ENOKEY.
|
||||
m.wait_until_succeeds(f"wg show wg-hive allowed-ips | grep -F {peer_key}")
|
||||
|
||||
# Traffic makes the tunnel handshake.
|
||||
for src, addr in ((h1, "${meshAddr.h2}"), (h2, "${meshAddr.h1}")):
|
||||
status, out = src.execute(f"ping -c 3 -W 5 {addr}")
|
||||
assert status == 0, f"ping {addr} failed:\n{out}\n{wg_state(src)}"
|
||||
for m in (h1, h2):
|
||||
stamps = m.succeed("wg show wg-hive latest-handshakes").split()
|
||||
assert len(stamps) == 2 and int(stamps[1]) > 0, f"no handshake: {stamps}\n{wg_state(m)}"
|
||||
|
||||
# The peer's mesh address is routed through the tunnel, its plain
|
||||
# address is not.
|
||||
via_mesh = h1.succeed("ip route get ${meshAddr.h2}")
|
||||
assert "dev wg-hive" in via_mesh, via_mesh
|
||||
via_lan = h1.succeed("ip route get ${lanAddr.h2}")
|
||||
assert "dev wg-hive" not in via_lan, via_lan
|
||||
|
||||
# A TCP connect to a listener on the peer's mesh address, and the
|
||||
# tunnel's own counters moved.
|
||||
h2.succeed(
|
||||
"systemd-run --unit=wg-listener ${pkgs.python3}/bin/python3"
|
||||
" -m http.server 9000 --bind ${meshAddr.h2}"
|
||||
)
|
||||
h2.wait_for_open_port(9000, "${meshAddr.h2}")
|
||||
h1.succeed("${pkgs.curl}/bin/curl -sf --max-time 5 http://${meshAddr.h2}:9000/")
|
||||
transfer = h1.succeed("wg show wg-hive transfer").split()
|
||||
assert int(transfer[1]) > 0 and int(transfer[2]) > 0, transfer
|
||||
|
||||
# Negative control: h2 swaps h1's public key for one that does not match
|
||||
# h1's private key. Same endpoint, same address, so the only difference
|
||||
# from the working peer above is the key, and no handshake may happen.
|
||||
h2.succeed("wg set wg-hive peer ${pub "h1"} remove")
|
||||
h2.succeed(
|
||||
"wg set wg-hive peer ${pub "wrong"} allowed-ips ${meshAddr.h1}/32"
|
||||
" endpoint ${lanAddr.h1}:51820 persistent-keepalive 1"
|
||||
)
|
||||
h2.fail("ping -c 3 -W 2 ${meshAddr.h1}")
|
||||
stamps = h2.succeed("wg show wg-hive latest-handshakes").split()
|
||||
assert stamps == ["${pub "wrong"}", "0"], f"handshake with wrong key: {stamps}"
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue