243 lines
9.5 KiB
Nix
243 lines
9.5 KiB
Nix
{
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.network;
|
|
|
|
# IPv4 helpers for the DHCP-pool computation below — nix integers
|
|
# are 64-bit so all /0-/32 values are safe.
|
|
ipToInt =
|
|
ip:
|
|
builtins.foldl' (acc: x: acc * 256 + x) 0 (
|
|
map lib.strings.toIntBase10 (lib.strings.splitString "." ip)
|
|
);
|
|
intToIp =
|
|
n:
|
|
let
|
|
a = n / 16777216;
|
|
b = (n - a * 16777216) / 65536;
|
|
c = (n - a * 16777216 - b * 65536) / 256;
|
|
d = n - a * 16777216 - b * 65536 - c * 256;
|
|
in
|
|
"${toString a}.${toString b}.${toString c}.${toString d}";
|
|
# 2^n via recursion (nix has no pow builtin).
|
|
pow2 = n: if n == 0 then 1 else 2 * (pow2 (n - 1));
|
|
hostCount = pow2 (32 - cfg.bridgePrefixLength);
|
|
# Mask off host bits to get the network base address.
|
|
networkBase = builtins.bitAnd (ipToInt cfg.bridgeIp) (4294967295 - hostCount + 1);
|
|
in
|
|
{
|
|
# Hive-internal network — host-side bridge + per-agent DNS resolver.
|
|
# Always active when hyperhive is enabled: agent containers run in
|
|
# private netns behind the bridge. Full design: docs/network.md.
|
|
|
|
imports = [
|
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "enable" ] ''
|
|
The hive network (bridge + dnsmasq resolver + private-netns
|
|
isolation) is always on whenever hyperhive is enabled. Remove the
|
|
setting.
|
|
'')
|
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "isolateContainers" ] ''
|
|
Network isolation is the only mode and is always on whenever
|
|
hyperhive is enabled; the shared-netns path was removed. Remove
|
|
the setting.
|
|
'')
|
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "upstreamDns" ] ''
|
|
The hive resolver always follows the host's resolvers now
|
|
(dnsmasq reads the gateway container's /etc/resolv.conf, the
|
|
host copy made at container start). Configure upstream DNS on
|
|
the host itself instead.
|
|
'')
|
|
];
|
|
|
|
options.services.hyperhive.network = {
|
|
bridgeName = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-br0";
|
|
example = "h0";
|
|
description = ''
|
|
Name of the host-side bridge interface the hive uses for
|
|
inter-container traffic. Kept short so it survives the
|
|
IFNAMSIZ (15-char) cap, and prefixed so it's obviously
|
|
hive-managed in `ip link` output.
|
|
'';
|
|
};
|
|
|
|
bridgeIp = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "10.42.0.1";
|
|
example = "172.30.0.1";
|
|
description = ''
|
|
IPv4 address assigned to the bridge interface on the host
|
|
side. Agents use this address as their DNS server (dnsmasq
|
|
in the gateway container binds here). Default `10.42.0.1`
|
|
is in RFC 1918 space and unlikely to clash with operator's
|
|
existing setup; override if a different range is already in
|
|
use.
|
|
'';
|
|
};
|
|
|
|
bridgePrefixLength = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 24;
|
|
example = 16;
|
|
description = ''
|
|
Netmask prefix length for the bridge subnet. Default `/24`
|
|
gives 254 usable per-agent addresses, enough for any
|
|
single-host hive. Operator with a larger swarm or a tighter
|
|
addressing scheme overrides.
|
|
'';
|
|
};
|
|
|
|
exposeHostPorts = lib.mkOption {
|
|
type = lib.types.listOf lib.types.port;
|
|
default = [ ];
|
|
example = [ 4318 ];
|
|
description = ''
|
|
TCP ports on the host that agent containers may reach at the bridge
|
|
IP (`bridgeIp`). Each listed port `P` is opened on the bridge-interface
|
|
firewall, so an agent can connect to `''${bridgeIp}:P` (default
|
|
`10.42.0.1:P`).
|
|
|
|
Use this to let agents reach a host-local service — e.g. an
|
|
OpenTelemetry collector for `services.hyperhive.otel.endpoint` (set
|
|
`endpoint = "http://''${bridgeIp}:P"`).
|
|
|
|
**The host service must bind an address reachable from the bridge** —
|
|
`0.0.0.0` or the bridge IP (`bridgeIp`) — not loopback-only. The
|
|
bridge→`127.0.0.0/8` DROP rule (defence-in-depth) is unchanged: this
|
|
only opens the firewall, it does not bridge loopback. A service that
|
|
binds `127.0.0.1` only is still unreachable; rebind it to `0.0.0.0`.
|
|
|
|
The exposed port is reachable by EVERY agent on the bridge subnet
|
|
(same as DNS/gateway), so only expose services safe for any agent to
|
|
reach.
|
|
'';
|
|
};
|
|
|
|
# DHCP pool covering all usable host addresses on the bridge
|
|
# subnet, computed from bridgeIp/bridgePrefixLength: .2 (first
|
|
# usable after the .1 gateway) to .(hostCount-2) (last usable
|
|
# before broadcast). All containers — agents and service
|
|
# containers alike — receive their IPs dynamically from this pool;
|
|
# there are no hash-derived static assignments. Consumed by the
|
|
# dnsmasq that runs in the gateway container (hive-gateway module).
|
|
dhcpRangeStart = lib.mkOption {
|
|
type = lib.types.str;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = intToIp (networkBase + 2);
|
|
defaultText = lib.literalMD "first usable bridge address after the gateway";
|
|
description = ''
|
|
Read-only computed first address of the bridge DHCP pool.
|
|
'';
|
|
};
|
|
|
|
dhcpRangeEnd = lib.mkOption {
|
|
type = lib.types.str;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = intToIp (networkBase + hostCount - 2);
|
|
defaultText = lib.literalMD "last usable bridge address before broadcast";
|
|
description = ''
|
|
Read-only computed last address of the bridge DHCP pool.
|
|
'';
|
|
};
|
|
|
|
};
|
|
|
|
config = lib.mkMerge [
|
|
# The hive network + container isolation are unconditional whenever
|
|
# hyperhive is enabled: the shared-netns mode was removed, so there
|
|
# is one mode (private netns behind the bridge).
|
|
(lib.mkIf config.services.hyperhive.enable {
|
|
assertions = [
|
|
{
|
|
assertion = config.services.hyperhive.domain != null;
|
|
message = ''
|
|
hyperhive requires services.hyperhive.domain to be set — the
|
|
hive resolver is authoritative for `<hive-domain>` and its
|
|
sub-domains, and agents reach the forge/matrix through the
|
|
gateway by that domain. Pin a hostname
|
|
(`services.hyperhive.domain = "example.com";`).
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Virtual bridge — each agent container attaches a veth pair (isolation
|
|
# is unconditional now).
|
|
networking.bridges.${cfg.bridgeName}.interfaces = [ ];
|
|
|
|
# Bridge IP — dnsmasq (in the gateway container) binds here.
|
|
networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [
|
|
{
|
|
address = cfg.bridgeIp;
|
|
prefixLength = cfg.bridgePrefixLength;
|
|
}
|
|
];
|
|
|
|
# DNS + DHCP on the bridge interface only — no external amplification
|
|
# surface. UDP 67 is required for the dnsmasq DHCP pool: dnsmasq
|
|
# receives DHCPDISCOVER via a regular UDP socket (no netfilter-bypassing
|
|
# raw socket like ISC dhcpd), so without this hole the host INPUT chain
|
|
# drops the broadcasts and every container falls back to IPv4LL.
|
|
networking.firewall.interfaces.${cfg.bridgeName} = {
|
|
allowedUDPPorts = [
|
|
53
|
|
67
|
|
];
|
|
allowedTCPPorts = [ 53 ];
|
|
};
|
|
})
|
|
|
|
# Container isolation overlay — now unconditional (the shared-netns
|
|
# mode was removed). See docs/network.md#container-isolation.
|
|
(lib.mkIf config.services.hyperhive.enable {
|
|
|
|
# Agents route internet traffic via the bridge; NAT masquerades their RFC-1918 IPs.
|
|
boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
|
|
networking.nat = {
|
|
enable = true;
|
|
internalInterfaces = [ cfg.bridgeName ];
|
|
};
|
|
|
|
# Defence-in-depth: DROP bridge→loopback so compromised agents can't
|
|
# reach host-loopback services even via routing table leaks.
|
|
networking.firewall.extraInputRules = ''
|
|
ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop
|
|
'';
|
|
|
|
# Allow isolated agents to reach the gateway (nginx on the host, shared
|
|
# netns). Port 80 covers `http://forge.<domain>`, per-agent UI proxies,
|
|
# and any other HTTP services the gateway fronts. Port 443 for HTTPS.
|
|
# (`exposeHostPorts` opens its own ports in its dedicated block below,
|
|
# co-located with the proxies so the firewall hole + listener can't drift.)
|
|
networking.firewall.interfaces.${cfg.bridgeName}.allowedTCPPorts = [
|
|
80
|
|
443
|
|
];
|
|
|
|
# Tells hive-c0re to pass PRIVATE_NETWORK + bridge settings to each
|
|
# container. HIVE_NETWORK_SUBNET is host-bridge IP/prefix, not canonical
|
|
# network address — the Rust side normalises before subnet arithmetic.
|
|
systemd.services.hive-c0re.environment = {
|
|
HIVE_NETWORK_ISOLATION = "1";
|
|
HIVE_NETWORK_BRIDGE = cfg.bridgeName;
|
|
HIVE_NETWORK_SUBNET = "${cfg.bridgeIp}/${toString cfg.bridgePrefixLength}";
|
|
};
|
|
})
|
|
|
|
# Host port exposure: open each `exposeHostPorts` entry on the bridge
|
|
# firewall so agents can reach a host service at `<bridgeIp>:P`. The host
|
|
# service must bind `0.0.0.0` or the bridge IP (a loopback-only bind stays
|
|
# unreachable — the bridge→127.0.0.0/8 DROP rule above is unchanged). This
|
|
# is firewall-only by design: a host service that binds `0.0.0.0` already
|
|
# serves the bridge IP, so an extra bridge-IP proxy would only collide
|
|
# (EADDRINUSE) with it. Merges with the [ 80 443 ] gateway ports above.
|
|
(lib.mkIf (config.services.hyperhive.enable && cfg.exposeHostPorts != [ ]) {
|
|
networking.firewall.interfaces.${cfg.bridgeName}.allowedTCPPorts = cfg.exposeHostPorts;
|
|
})
|
|
];
|
|
}
|