mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

egress: forward = reject for intercepted agents, direct_mac exempt in log mode too

Browse source
This commit is contained in:
atlas 2026-11-13 12:22:36 +01:00
commit dad15ca855
1 changed files with 248 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

112
nix/host-modules/hive-egress.nix
View file

@ -9,14 +9,14 @@
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
# the host routes from the agent bridge to anywhere else is logged and
# counted, never blocked. `intercept` (`egress.intercept.enable`, set by
# below keeps the first one on. `forward` (`egress.forward`): traffic from the
# agent bridge to anywhere else is logged and counted (`log`) or logged and
# refused (`reject`). `intercept` (`egress.intercept.enable`, set by
# hive-squid.nix): the web traffic of the bridge's agents is redirected into
# squid's intercept ports (port 443 through the ClientHello relay). The ports in
# `direct_mac` are left alone: the service containers and the unrestricted
# agents. `hive-priv egress-pin` and `egress-load` fill that set from the bridge
# table's bindings, so a MAC in no binding is intercepted, and squid refuses it.
# squid's intercept ports (port 443 through the ClientHello relay). Both leave
# alone the ports in `direct_mac`, the service containers and the unrestricted
# agents. `hive-priv egress-pin` and `egress-load` fill that set from the
# bridge table's bindings, so a MAC in no binding is intercepted and refused.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
@ -79,7 +79,7 @@ let
# Logging is per source address, so one busy agent cannot use up the budget
# of another: a burst of 20 lines, then one a second, and an address is
# forgotten after a minute without a new flow. The rule accepts after it
# forgotten after a minute without a new flow. The rule decides after it
# logs. A flow the rule cannot log falls through to the unmetered rule below
# it: the address was over its rate, or the set is full (`update` then fails
# and the rule stops matching), which a container spoofing source addresses
@ -90,9 +90,55 @@ let
logRate = "1/second burst 20 packets";
unmeteredRate = "5/second burst 50 packets";
fromBridge = ''iifname "${br}" oifname != "${br}"'';
logRule =
# `log` accepts what it logs. `reject` logs and refuses in one rule, so a
# line always names a refused flow. A TCP flow gets a reset, which fails the
# connect at once (to a TCP socket an ICMP error is a soft error, and the
# connect would stall); anything else gets ICMP admin-prohibited. A rule that
# refuses is therefore a pair: one for TCP, one for the rest.
isReject = cfg.egress.forward == "reject";
logPrefix = if isReject then "hive-egress-reject" else "hive-egress-would-drop";
refuse = {
tcp = "meta l4proto tcp";
other = "meta l4proto != tcp";
};
verdict = {
tcp = "reject with tcp reset";
other = "reject with icmpx type admin-prohibited";
};
decide =
statements:
if isReject then
map
(
k:
lib.concatStringsSep " " (
lib.filter (x: x != "") [
fromBridge
refuse.${k}
statements
verdict.${k}
]
)
)
[
"tcp"
"other"
]
else
[ "${fromBridge} ${statements} accept" ];
logRules =
family: addr:
''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept'';
decide ''ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "${logPrefix} "'';
# In `log` mode the unmetered rule has no verdict: the chain's last rule is
# the verdict for every flow that gets past the logging.
unmeteredRules =
let
statements = ''ct state new limit rate ${unmeteredRate} log prefix "${logPrefix}-unmetered "'';
in
if isReject then decide statements else [ "${fromBridge} ${statements}" ];
# What no rule above decided, a packet that is no new flow included.
finalRules = if isReject then decide "" else [ "${fromBridge} accept" ];
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
# the interface the route to the destination leaves by: the bridge (another
@ -124,9 +170,12 @@ let
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
# so the chain neither allows what those refuse nor refuses anything itself.
# Traffic to the host's own addresses (squid, dnsmasq, the gateway) is
# delivered locally and never reaches the forward hook.
# so it allows nothing those refuse; a `reject` ends the packet. The mesh
# and the MACs in `direct_mac` (the unrestricted agents and the service
# containers, the traffic the intercept chain leaves alone) come first in
# both modes, so a `log` line names what `reject` would refuse. Traffic to
# the host's own addresses (squid, dnsmasq, the gateway) is delivered
# locally and never reaches the forward hook.
forwardChain = ''
set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; }
set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; }
@ -134,11 +183,11 @@ let
chain forward {
type filter hook forward priority filter - 10; policy accept;
iifname "${br}" oifname "${meshInterface}" accept
${fromBridge} ether saddr @direct_mac accept
${fromBridge} counter
${logRule "v4" "ip"}
${logRule "v6" "ip6"}
${fromBridge} ct state new limit rate ${unmeteredRate} log prefix "hive-egress-would-drop-unmetered "
${fromBridge} accept
${lib.concatStringsSep "\n " (
logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules
)}
}
'';
@ -232,7 +281,7 @@ let
}
set direct_mac { type ether_addr; }
${lib.optionalString cfg.egress.intercept.enable interceptChain}
${lib.optionalString (cfg.egress.forward == "log") forwardChain}
${lib.optionalString (cfg.egress.forward != "accept") forwardChain}
}
${bridgeTable}
@ -257,20 +306,27 @@ in
type = lib.types.enum [
"accept"
"log"
"drop"
"reject"
];
default = "log";
description = ''
What the `inet hive_egress` table does with traffic the host forwards
from the agent bridge to anywhere but the bridge and the mesh.
from the agent bridge to anywhere but the bridge and the mesh. In `log` and
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
agents and the service containers) is accepted and never logged.
- `accept`: nothing; the table has no forward chain.
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
rate limited per source address, and counted. A flow the per-source limit
cannot log gets a line with the prefix `hive-egress-would-drop-unmetered `
from a limit shared by all addresses. Nothing is blocked. The source
address maps to an agent through the hive's DHCP leases.
- `drop`: not available yet; the evaluation refuses it.
from a limit shared by all addresses. Nothing is blocked. The lines are
the flows `reject` would refuse. The source address maps to an agent
through the hive's DHCP leases.
- `reject`: the same lines with the prefix `hive-egress-reject ` and
`hive-egress-reject-unmetered `, and every other packet from the bridge
is refused as well: a TCP flow with a reset, anything else with ICMP
administratively prohibited. Only the web traffic that the intercept
chain redirects into squid still gets out.
'';
};
@ -409,16 +465,6 @@ in
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.egress.forward != "drop";
message = ''
services.hyperhive.network.egress.forward = "drop" is not available yet:
the traffic squid does not handle (the unrestricted agents, the service
containers, every port but 80 and 443) has no rule that lets it through, so a
drop would cut it.
Use "log" or "accept".
'';
}
{
assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15;
message = ''