f85724a638
Prose-only sweep of every remaining claim that nginx or dnsmasq lives in a container: the port comment (root in a container -> root on the host), upstreamHost's netns rationale, the ACME state dir, the store path reachability note, the vhost tree header, dnsmasq's resolv.conf paragraph (there is no copy and no path unit watching it any more), the two hive-network bridge comments, and swarm-controller's socket access-control note, which described a bind-mount that no longer exists. No behaviour change; all of it was describing a mechanism that was deleted.
284 lines
12 KiB
Nix
284 lines
12 KiB
Nix
{
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.network;
|
|
|
|
# IPv4 helpers for the DHCP-pool computation below — nix integers
|
|
# are 64-bit so all /0-/32 values are safe.
|
|
ipToInt =
|
|
ip:
|
|
builtins.foldl' (acc: x: acc * 256 + x) 0 (
|
|
map lib.strings.toIntBase10 (lib.strings.splitString "." ip)
|
|
);
|
|
intToIp =
|
|
n:
|
|
let
|
|
a = n / 16777216;
|
|
b = (n - a * 16777216) / 65536;
|
|
c = (n - a * 16777216 - b * 65536) / 256;
|
|
d = n - a * 16777216 - b * 65536 - c * 256;
|
|
in
|
|
"${toString a}.${toString b}.${toString c}.${toString d}";
|
|
# 2^n via recursion (nix has no pow builtin).
|
|
pow2 = n: if n == 0 then 1 else 2 * (pow2 (n - 1));
|
|
hostCount = pow2 (32 - cfg.bridgePrefixLength);
|
|
# Mask off host bits to get the network base address.
|
|
networkBase = builtins.bitAnd (ipToInt cfg.bridgeIp) (4294967295 - hostCount + 1);
|
|
in
|
|
{
|
|
# Hive-internal network — host-side bridge + per-agent DNS resolver.
|
|
# Always active when hyperhive is enabled: agent containers run in
|
|
# private netns behind the bridge. Full design: docs/network.md.
|
|
|
|
imports = [
|
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "enable" ] ''
|
|
The hive network (bridge + dnsmasq resolver + private-netns
|
|
isolation) is always on whenever hyperhive is enabled. Remove the
|
|
setting.
|
|
'')
|
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "isolateContainers" ] ''
|
|
Network isolation is the only mode and is always on whenever
|
|
hyperhive is enabled; the shared-netns path was removed. Remove
|
|
the setting.
|
|
'')
|
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "network" "upstreamDns" ] ''
|
|
The hive resolver always follows the host's resolvers now
|
|
(dnsmasq runs on the host and reads its /etc/resolv.conf
|
|
directly). Configure upstream DNS on the host itself instead.
|
|
'')
|
|
];
|
|
|
|
options.services.hyperhive.network = {
|
|
bridgeName = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-br0";
|
|
example = "h0";
|
|
description = ''
|
|
Name of the host-side bridge interface the hive uses for
|
|
inter-container traffic. Kept short so it survives the
|
|
IFNAMSIZ (15-char) cap, and prefixed so it's obviously
|
|
hive-managed in `ip link` output.
|
|
'';
|
|
};
|
|
|
|
bridgeIp = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "10.42.0.1";
|
|
example = "172.30.0.1";
|
|
description = ''
|
|
IPv4 address assigned to the bridge interface on the host
|
|
side. Agents use this address as their DNS server (the hive's
|
|
dnsmasq binds here). Default `10.42.0.1`
|
|
is in RFC 1918 space and unlikely to clash with operator's
|
|
existing setup; override if a different range is already in
|
|
use.
|
|
'';
|
|
};
|
|
|
|
bridgePrefixLength = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 24;
|
|
example = 16;
|
|
description = ''
|
|
Netmask prefix length for the bridge subnet. Default `/24`
|
|
gives 254 usable per-agent addresses, enough for any
|
|
single-host hive. Operator with a larger swarm or a tighter
|
|
addressing scheme overrides.
|
|
'';
|
|
};
|
|
|
|
exposeHostPorts = lib.mkOption {
|
|
type = lib.types.listOf lib.types.port;
|
|
default = [ ];
|
|
example = [ 4318 ];
|
|
description = ''
|
|
TCP ports on the host that agent containers may reach at the bridge
|
|
IP (`bridgeIp`). Each listed port `P` is opened on the bridge-interface
|
|
firewall, so an agent can connect to `''${bridgeIp}:P` (default
|
|
`10.42.0.1:P`).
|
|
|
|
Use this to let agents reach a host-local service — e.g. an
|
|
OpenTelemetry collector for `services.hyperhive.otel.endpoint` (set
|
|
`endpoint = "http://''${bridgeIp}:P"`).
|
|
|
|
**The host service must bind an address reachable from the bridge** —
|
|
`0.0.0.0` or the bridge IP (`bridgeIp`) — not loopback-only. The
|
|
bridge→`127.0.0.0/8` DROP rule (defence-in-depth) is unchanged: this
|
|
only opens the firewall, it does not bridge loopback. A service that
|
|
binds `127.0.0.1` only is still unreachable; rebind it to `0.0.0.0`.
|
|
|
|
The exposed port is reachable by EVERY agent on the bridge subnet
|
|
(same as DNS/gateway), so only expose services safe for any agent to
|
|
reach.
|
|
'';
|
|
};
|
|
|
|
# DHCP pool covering all usable host addresses on the bridge
|
|
# subnet, computed from bridgeIp/bridgePrefixLength: .2 (first
|
|
# usable after the .1 gateway) to .(hostCount-2) (last usable
|
|
# before broadcast). All containers — agents and service
|
|
# containers alike — receive their IPs dynamically from this pool;
|
|
# there are no hash-derived static assignments. Consumed by the
|
|
# hive's dnsmasq (hive-gateway module, host-side).
|
|
dhcpRangeStart = lib.mkOption {
|
|
type = lib.types.str;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = intToIp (networkBase + 2);
|
|
defaultText = lib.literalMD "first usable bridge address after the gateway";
|
|
description = ''
|
|
Read-only computed first address of the bridge DHCP pool.
|
|
'';
|
|
};
|
|
|
|
dhcpRangeEnd = lib.mkOption {
|
|
type = lib.types.str;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = intToIp (networkBase + hostCount - 2);
|
|
defaultText = lib.literalMD "last usable bridge address before broadcast";
|
|
description = ''
|
|
Read-only computed last address of the bridge DHCP pool.
|
|
'';
|
|
};
|
|
|
|
};
|
|
|
|
config = lib.mkMerge [
|
|
# The hive network + container isolation are unconditional whenever
|
|
# hyperhive is enabled: the shared-netns mode was removed, so there
|
|
# is one mode (private netns behind the bridge).
|
|
(lib.mkIf config.services.hyperhive.enable {
|
|
# This message is only useful if an operator can actually reach
|
|
# it, and an assertion competes with every eager default that
|
|
# reads the value it guards: option defaults that interpolate the
|
|
# domain (`forge.<domain>`, `matrix.<domain>`) throw while the
|
|
# assertion list is being evaluated, so the operator sees
|
|
# `cannot coerce null to a string` naming an unrelated option
|
|
# instead of the sentence below. Those defaults therefore stay
|
|
# total, falling back to a name under the reserved `.invalid` TLD
|
|
# (RFC 2606) — a value this assertion then refuses to let out the
|
|
# door, and one that fails loudly at resolution rather than
|
|
# quietly working if it somehow did.
|
|
assertions = [
|
|
{
|
|
assertion = config.services.hyperhive.domain != null;
|
|
message = ''
|
|
hyperhive requires services.hyperhive.domain to be set — the
|
|
hive resolver is authoritative for `<hive-domain>` and its
|
|
sub-domains, and agents reach the forge/matrix through the
|
|
gateway by that domain.
|
|
|
|
It is read from this hive's entry in the swarm directory, so
|
|
what is actually missing is that entry:
|
|
|
|
services.hyperhive.swarm.hives."<hiveName>" = { };
|
|
|
|
whose `domain` defaults to `<hiveName>.<swarm.domain>`. The
|
|
assertion in ./swarm.nix names it precisely; this one is the
|
|
backstop.
|
|
'';
|
|
}
|
|
{
|
|
assertion = config.services.hyperhive.swarm.domain != null;
|
|
message = ''
|
|
hyperhive requires services.hyperhive.swarm.domain to be
|
|
set — the DNS domain of the swarm this hive belongs to,
|
|
of which this hive occupies one sub-domain. There is no
|
|
fallback: a guessed value would be a wrong hostname that
|
|
evaluates cleanly and deploys. Set it
|
|
(`services.hyperhive.swarm.domain = "example.com";`) —
|
|
with `hiveName` it also derives
|
|
`services.hyperhive.domain` for you.
|
|
'';
|
|
}
|
|
{
|
|
assertion = config.services.hyperhive.hiveName != null;
|
|
message = ''
|
|
hyperhive requires services.hyperhive.hiveName to be set —
|
|
it is this hive's label within the swarm, and the leftmost
|
|
part of the domain it is addressed by
|
|
(`<hiveName>.<swarm.domain>`), not only a display name.
|
|
Set it (`services.hyperhive.hiveName = "pr1ma";`).
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Virtual bridge — each agent container attaches a veth pair (isolation
|
|
# is unconditional now).
|
|
networking.bridges.${cfg.bridgeName}.interfaces = [ ];
|
|
|
|
# Bridge IP — the hive's dnsmasq binds here.
|
|
networking.interfaces.${cfg.bridgeName}.ipv4.addresses = [
|
|
{
|
|
address = cfg.bridgeIp;
|
|
prefixLength = cfg.bridgePrefixLength;
|
|
}
|
|
];
|
|
|
|
# DNS + DHCP on the bridge interface only — no external amplification
|
|
# surface. UDP 67 is required for the dnsmasq DHCP pool: dnsmasq
|
|
# receives DHCPDISCOVER via a regular UDP socket (no netfilter-bypassing
|
|
# raw socket like ISC dhcpd), so without this hole the host INPUT chain
|
|
# drops the broadcasts and every container falls back to IPv4LL.
|
|
networking.firewall.interfaces.${cfg.bridgeName} = {
|
|
allowedUDPPorts = [
|
|
53
|
|
67
|
|
];
|
|
allowedTCPPorts = [ 53 ];
|
|
};
|
|
})
|
|
|
|
# Container isolation overlay — now unconditional (the shared-netns
|
|
# mode was removed). See docs/network.md#container-isolation.
|
|
(lib.mkIf config.services.hyperhive.enable {
|
|
|
|
# Agents route internet traffic via the bridge; NAT masquerades their RFC-1918 IPs.
|
|
boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
|
|
networking.nat = {
|
|
enable = true;
|
|
internalInterfaces = [ cfg.bridgeName ];
|
|
};
|
|
|
|
# Defence-in-depth: DROP bridge→loopback so compromised agents can't
|
|
# reach host-loopback services even via routing table leaks.
|
|
networking.firewall.extraInputRules = ''
|
|
ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop
|
|
'';
|
|
|
|
# Allow isolated agents to reach the gateway (nginx on the host, shared
|
|
# netns). Port 80 covers `http://forge.<domain>`, per-agent UI proxies,
|
|
# and any other HTTP services the gateway fronts. Port 443 for HTTPS.
|
|
# (`exposeHostPorts` opens its own ports in its dedicated block below,
|
|
# co-located with the proxies so the firewall hole + listener can't drift.)
|
|
networking.firewall.interfaces.${cfg.bridgeName}.allowedTCPPorts = [
|
|
80
|
|
443
|
|
];
|
|
|
|
# Tells hive-c0re to pass PRIVATE_NETWORK + bridge settings to each
|
|
# container. HIVE_NETWORK_SUBNET is host-bridge IP/prefix, not canonical
|
|
# network address — the Rust side normalises before subnet arithmetic.
|
|
systemd.services.hive-c0re.environment = {
|
|
HIVE_NETWORK_ISOLATION = "1";
|
|
HIVE_NETWORK_BRIDGE = cfg.bridgeName;
|
|
HIVE_NETWORK_SUBNET = "${cfg.bridgeIp}/${toString cfg.bridgePrefixLength}";
|
|
};
|
|
})
|
|
|
|
# Host port exposure: open each `exposeHostPorts` entry on the bridge
|
|
# firewall so agents can reach a host service at `<bridgeIp>:P`. The host
|
|
# service must bind `0.0.0.0` or the bridge IP (a loopback-only bind stays
|
|
# unreachable — the bridge→127.0.0.0/8 DROP rule above is unchanged). This
|
|
# is firewall-only by design: a host service that binds `0.0.0.0` already
|
|
# serves the bridge IP, so an extra bridge-IP proxy would only collide
|
|
# (EADDRINUSE) with it. Merges with the [ 80 443 ] gateway ports above.
|
|
(lib.mkIf (config.services.hyperhive.enable && cfg.exposeHostPorts != [ ]) {
|
|
networking.firewall.interfaces.${cfg.bridgeName}.allowedTCPPorts = cfg.exposeHostPorts;
|
|
})
|
|
];
|
|
}
|