mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

load the agent-bridge to loopback drop in its own nft table

Browse source
This commit is contained in:
atlas 2026-11-11 15:45:56 +01:00 • committed by mara
commit eb39a90057
6 changed files with 186 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

21
docs/4-implementation/networking/network.md
View file

@ -214,7 +214,7 @@ The hive's own collector needs none of this: agents dial it by
This is **firewall-only**: the host service must bind an address
reachable from the bridge — `0.0.0.0` or the bridge IP — not loopback
only. The bridge→`127.0.0.0/8` DROP rule (below) is unchanged, so a
only. The bridge→`127.0.0.0/8` DROP rule (below) still applies, so a
service bound to `127.0.0.1` only stays unreachable; rebind it to
`0.0.0.0`.
@ -512,13 +512,13 @@ Each agent container runs in a private network namespace with a dedicated
veth pair attached to the bridge. The following table summarises what
the nix side sets up unconditionally:
| effect | mechanism |
| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
| IP forwarding | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1` |
| Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC |
| Loopback DROP | `networking.firewall.extraInputRules` — drops bridge-subnet → `127.0.0.0/8` traffic; defence-in-depth against routing table leaks |
| Gateway access | `networking.firewall.interfaces.<bridge>.allowedTCPPorts = [ 80 443 ]` — lets isolated agents (private netns, veth on bridge) reach nginx on the host |
| c0re signal | `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` — both **required**; `hive-c0re` refuses to start without them |
| effect | mechanism |
| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| IP forwarding | `boot.kernel.sysctl."net.ipv4.ip_forward" = 1` |
| Internet NAT | `networking.nat { enable = true; internalInterfaces = [ bridgeName ]; }` — MASQUERADE on packets leaving via any external NIC |
| Loopback DROP | `nix/host-modules/hive-egress.nix` — own nftables table `inet hive_egress` (whichever firewall the host runs): drops bridge → `127.0.0.0/8` and `::1` traffic; the kernel default `route_localnet=0` is the first layer |
| Gateway access | `networking.firewall.interfaces.<bridge>.allowedTCPPorts = [ 80 443 ]` — lets isolated agents (private netns, veth on bridge) reach nginx on the host |
| c0re signal | `HIVE_NETWORK_BRIDGE`, `HIVE_NETWORK_SUBNET` in `systemd.services.hive-c0re.environment` — both **required**; `hive-c0re` refuses to start without them |
`HIVE_NETWORK_SUBNET` is the host-side bridge IP + prefix (e.g.
`10.42.0.1/24`), **not** the canonical network address. The Rust side
@ -574,8 +574,9 @@ map](#network-map) above. `PRIVATE_NETWORK=1` has no effect on a path
that never touches the network stack.
The nix side also enables IP forwarding + NAT (agents reach the internet
through the host) and drops bridge-subnet → loopback traffic (defence-in-depth
against a compromised agent reaching the c0re dashboard HTTP at
through the host) and drops bridge-subnet → loopback traffic from its own
nftables table (`inet hive_egress`, `nix/host-modules/hive-egress.nix`;
defence-in-depth against a compromised agent reaching the c0re dashboard HTTP at
`127.0.0.1`). Agents have no legitimate reason to reach the dashboard over
loopback — the hive-c0re admin socket is a UDS, not TCP.

1
nix/host-modules/default.nix
View file

@ -21,6 +21,7 @@
./hive-matrix-service.nix
./hive-matrix.nix
./hive-network.nix
./hive-egress.nix
./hive-priv.nix
./hive-archive-backup.nix
./hive-priv-cache.nix

86
nix/host-modules/hive-egress.nix Normal file
View file

@ -0,0 +1,86 @@
# The hive's own nftables table, `inet hive_egress`, loaded by a systemd
# oneshot. It is independent of the NixOS firewall: `networking.firewall`
# renders only the rules of the backend in use (`iptables` unless
# `networking.nftables.enable` is set), so a rule that has to hold on every
# host lives here, in a table no firewall backend writes. Only
# `networking.nftables.flushRuleset` removes it; the unit re-applies it then.
#
# The table holds one chain: traffic from the agent bridge to host
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on.
{
lib,
config,
pkgs,
...
}:
let
cfg = config.services.hyperhive.network;
nft = lib.getExe pkgs.nftables;
# One transaction: `add` + `delete` drop an old copy of the table and the
# definition recreates it, so the file loads the same on first start, on a
# reload and after something else flushed the ruleset.
ruleset = pkgs.writeText "hive-egress.nft" ''
add table inet hive_egress
delete table inet hive_egress
table inet hive_egress {
chain input {
type filter hook input priority filter - 10; policy accept;
iifname "${cfg.bridgeName}" ip daddr 127.0.0.0/8 counter drop
iifname "${cfg.bridgeName}" ip6 daddr ::1 counter drop
}
}
'';
routeLocalnetKeys = [
"net.ipv4.conf.all.route_localnet"
"net.ipv4.conf.default.route_localnet"
"net.ipv4.conf.${cfg.bridgeName}.route_localnet"
];
isOn =
v:
builtins.elem v [
1
"1"
true
];
in
{
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
message = ''
boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}).
With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8
instead of dropping it as a martian, which leaves the hive_egress
table as the only barrier between an agent and the host's loopback services.
'';
}
];
systemd.services.hive-egress = {
description = "hive nftables table (agent bridge to host loopback drop)";
wantedBy = [ "multi-user.target" ];
wants = [ "network-pre.target" ];
before = [ "network-pre.target" ];
# `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset`
# is on: start after it, follow its restarts and re-apply on its reload.
after = [ "nftables.service" ];
partOf = [ "nftables.service" ];
unitConfig.ReloadPropagatedFrom = [ "nftables.service" ];
reloadIfChanged = true;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${nft} -f ${ruleset}";
ExecReload = "${nft} -f ${ruleset}";
ExecStop = "-${nft} delete table inet hive_egress";
};
};
};
}

10
nix/host-modules/hive-network.nix
View file

@ -122,7 +122,7 @@ in
**The host service must bind an address reachable from the bridge** —
`0.0.0.0` or the bridge IP (`bridgeIp`) — not loopback-only. The
bridge→`127.0.0.0/8` DROP rule (defence-in-depth) is unchanged: this
bridge→`127.0.0.0/8` DROP rule (`hive-egress.nix`) still applies: this
only opens the firewall, it does not bridge loopback. A service that
binds `127.0.0.1` only is still unreachable; rebind it to `0.0.0.0`.
@ -276,12 +276,6 @@ in
internalInterfaces = [ cfg.bridgeName ];
};
# Defence-in-depth: DROP bridge→loopback so compromised agents can't
# reach host-loopback services even via routing table leaks.
networking.firewall.extraInputRules = ''
ip saddr ${cfg.bridgeIp}/${toString cfg.bridgePrefixLength} ip daddr 127.0.0.0/8 drop
'';
# Allow isolated agents to reach the gateway (nginx on the host, shared
# netns). Port 80 covers `http://forge.<domain>`, per-agent UI proxies,
# and any other HTTP services the gateway fronts. Port 443 for HTTPS.
@ -304,7 +298,7 @@ in
# Host port exposure: open each `exposeHostPorts` entry on the bridge
# firewall so agents can reach a host service at `<bridgeIp>:P`. The host
# service must bind `0.0.0.0` or the bridge IP (a loopback-only bind stays
# unreachable — the bridge→127.0.0.0/8 DROP rule above is unchanged). This
# unreachable — the bridge→127.0.0.0/8 DROP rule in hive-egress.nix still applies). This
# is firewall-only by design: a host service that binds `0.0.0.0` already
# serves the bridge IP, so an extra bridge-IP proxy would only collide
# (EADDRINUSE) with it. Merges with the [ 80 443 ] gateway ports above.

34
nix/module-eval/core-toggle.nix
View file

@ -63,6 +63,13 @@ let
inherit (shared) forgeHere;
# A host that turns `route_localnet` on, for the assertion in
# ../host-modules/hive-egress.nix: the kernel then routes a bridge packet
# for 127.0.0.0/8 instead of dropping it, so the config is refused.
routeLocalnetOn = hiveWith [
{ boot.kernel.sysctl."net.ipv4.conf.all.route_localnet" = 1; }
] { network.enable = true; };
# Every hive defaults the agents' queue address to the queue's name, so the
# only hive without one is a hive told to have none. Also the host still
# setting the four removed host options; no case on its queue reads them.
@ -1497,6 +1504,33 @@ let
&& bare.services.dnsmasq.enable
&& bare.networking.bridges ? hive-br0;
}
{
# The bridge→loopback drop is a unit of its own that loads its own nft
# table, so it does not depend on which firewall backend the host runs.
# `bare` is the presence arm; the firewall option the old rule used
# stays empty, so the rule has one home.
name = "the bridge to loopback drop is its own unit where the bridge is up and absent elsewhere";
ok =
let
unit = bare.systemd.services.hive-egress or null;
in
unit != null
&& unit.serviceConfig.Type == "oneshot"
&& lib.hasInfix "hive-egress.nft" unit.serviceConfig.ExecStart
&& lib.hasInfix "delete table inet hive_egress" unit.serviceConfig.ExecStop
&& bare.networking.firewall.extraInputRules == ""
&& !(centralToggleOff.systemd.services ? hive-egress);
}
{
# The host that turns `route_localnet` on is refused by an assertion
# naming it; the ordinary host, which leaves it alone, is not.
name = "route_localnet on the sysctl is refused on a host with the bridge up";
ok =
let
refused = m: lib.any (a: !a.assertion && lib.hasInfix "route_localnet" a.message) m.assertions;
in
refused routeLocalnetOn && !(refused bare);
}
{
# Refused where a hive runs and nowhere else. A name is an identifier
# only for a hive; the hive host is the control that the lookup finds the

53
nix/vm-tests/hive-squid.nix
View file

@ -8,6 +8,9 @@
# `proxy` imports the squid module against stub options for the few hive
# options it reads, so no hive-c0re build is pulled in; the module-eval suite
# covers the real option wiring.
#
# It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the
# agent's dials to the host's loopback run against the table the module installs.
{ pkgs }:
let
bridgeIp = "192.168.1.1";
@ -37,6 +40,10 @@ let
type = lib.types.bool;
default = true;
};
bridgeName = lib.mkOption {
type = lib.types.str;
default = "eth1";
};
};
otel = {
scrapeTargets = lib.mkOption {
@ -114,9 +121,12 @@ pkgs.testers.runNixOSTest {
imports = [
hiveStubs
../host-modules/hive-squid.nix
../host-modules/hive-egress.nix
# Listeners a leaked destination would reach: loopback (`::` lands on `::1`) and the bridge.
(listen443 [
"127.0.0.1"
# An address the agent can route to the proxy: the agent's own lo owns 127.0.0.1 itself.
"127.0.0.5"
"[::1]"
bridgeIp
])
@ -174,7 +184,10 @@ pkgs.testers.runNixOSTest {
# The gateway: nginx answers on the bridge, which agents reach directly.
networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ];
# Renders the users file the way hive-c0re does.
environment.systemPackages = [ pkgs.mkpasswd ];
environment.systemPackages = [
pkgs.mkpasswd
pkgs.nftables
];
services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local";
services.hyperhive.deploy.hive-controller.squid.selfcheck.canary = "cache.t.local";
services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains = [
@ -684,5 +697,43 @@ pkgs.testers.runNixOSTest {
write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U)
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
assert connect("other.t.local", UMA) == "200", "listed, and now logged in"
def dial(addr):
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")[0] == 0
def dropped():
# Packets the table's v4 rule has dropped so far.
out = proxy.succeed("nft list chain inet hive_egress input")
found = re.search(r"ip daddr 127\.0\.0\.0/8 counter packets (\d+)", out)
assert found, out
return int(found.group(1))
with checked("the agent bridge cannot reach the host's loopback, and the table is what stops it"):
assert dial("${bridgeIp}"), "control: a listener on the bridge IP answers"
# The agent has a route for an address the proxy owns on lo, and both
# ends accept loopback addresses on the link, so only the table is left
# between the dial and the listener.
for vm, dev in ((agent, "eth1"), (proxy, "eth1")):
vm.succeed(
"sysctl -w net.ipv4.conf.all.route_localnet=1 net.ipv4.conf.all.rp_filter=0 "
f"net.ipv4.conf.{dev}.route_localnet=1 net.ipv4.conf.{dev}.rp_filter=0"
)
agent.succeed("ip route add 127.0.0.5/32 via ${bridgeIp} table local")
proxy.succeed("nft delete table inet hive_egress")
assert dial("127.0.0.5"), "control: with the table gone the loopback listener answers"
proxy.succeed("systemctl reload hive-egress.service")
before = dropped()
assert not dial("127.0.0.5"), "the table drops the dial"
assert dropped() > before, "the drop rule counted it"
assert dial("${bridgeIp}"), "the bridge IP still answers"
# A firewall reload rewrites the NixOS chains and leaves this table alone.
proxy.succeed("systemctl reload firewall.service")
assert not dial("127.0.0.5"), "still dropped after a firewall reload"
assert dial("${bridgeIp}"), "the bridge IP still answers after the reload"
# With the kernel default back, the dial is refused with no table either.
proxy.succeed("sysctl -w net.ipv4.conf.all.route_localnet=0 net.ipv4.conf.eth1.route_localnet=0")
proxy.succeed("nft delete table inet hive_egress")
assert not dial("127.0.0.5"), "route_localnet=0 refuses it alone"
proxy.succeed("systemctl reload hive-egress.service")
'';
}