mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

egress: forward = reject for intercepted agents, direct_mac exempt in log mode too

Browse source
This commit is contained in:
atlas 2026-11-13 12:22:36 +01:00
commit dad15ca855
3 changed files with 248 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

112
nix/host-modules/hive-egress.nix
View file

@ -9,14 +9,14 @@
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
# the host routes from the agent bridge to anywhere else is logged and
# counted, never blocked. `intercept` (`egress.intercept.enable`, set by
# below keeps the first one on. `forward` (`egress.forward`): traffic from the
# agent bridge to anywhere else is logged and counted (`log`) or logged and
# refused (`reject`). `intercept` (`egress.intercept.enable`, set by
# hive-squid.nix): the web traffic of the bridge's agents is redirected into
# squid's intercept ports (port 443 through the ClientHello relay). The ports in
# `direct_mac` are left alone: the service containers and the unrestricted
# agents. `hive-priv egress-pin` and `egress-load` fill that set from the bridge
# table's bindings, so a MAC in no binding is intercepted, and squid refuses it.
# squid's intercept ports (port 443 through the ClientHello relay). Both leave
# alone the ports in `direct_mac`, the service containers and the unrestricted
# agents. `hive-priv egress-pin` and `egress-load` fill that set from the
# bridge table's bindings, so a MAC in no binding is intercepted and refused.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
@ -79,7 +79,7 @@ let
# Logging is per source address, so one busy agent cannot use up the budget
# of another: a burst of 20 lines, then one a second, and an address is
# forgotten after a minute without a new flow. The rule accepts after it
# forgotten after a minute without a new flow. The rule decides after it
# logs. A flow the rule cannot log falls through to the unmetered rule below
# it: the address was over its rate, or the set is full (`update` then fails
# and the rule stops matching), which a container spoofing source addresses
@ -90,9 +90,55 @@ let
logRate = "1/second burst 20 packets";
unmeteredRate = "5/second burst 50 packets";
fromBridge = ''iifname "${br}" oifname != "${br}"'';
logRule =
# `log` accepts what it logs. `reject` logs and refuses in one rule, so a
# line always names a refused flow. A TCP flow gets a reset, which fails the
# connect at once (to a TCP socket an ICMP error is a soft error, and the
# connect would stall); anything else gets ICMP admin-prohibited. A rule that
# refuses is therefore a pair: one for TCP, one for the rest.
isReject = cfg.egress.forward == "reject";
logPrefix = if isReject then "hive-egress-reject" else "hive-egress-would-drop";
refuse = {
tcp = "meta l4proto tcp";
other = "meta l4proto != tcp";
};
verdict = {
tcp = "reject with tcp reset";
other = "reject with icmpx type admin-prohibited";
};
decide =
statements:
if isReject then
map
(
k:
lib.concatStringsSep " " (
lib.filter (x: x != "") [
fromBridge
refuse.${k}
statements
verdict.${k}
]
)
)
[
"tcp"
"other"
]
else
[ "${fromBridge} ${statements} accept" ];
logRules =
family: addr:
''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept'';
decide ''ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "${logPrefix} "'';
# In `log` mode the unmetered rule has no verdict: the chain's last rule is
# the verdict for every flow that gets past the logging.
unmeteredRules =
let
statements = ''ct state new limit rate ${unmeteredRate} log prefix "${logPrefix}-unmetered "'';
in
if isReject then decide statements else [ "${fromBridge} ${statements}" ];
# What no rule above decided, a packet that is no new flow included.
finalRules = if isReject then decide "" else [ "${fromBridge} accept" ];
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
# the interface the route to the destination leaves by: the bridge (another
@ -124,9 +170,12 @@ let
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
# so the chain neither allows what those refuse nor refuses anything itself.
# Traffic to the host's own addresses (squid, dnsmasq, the gateway) is
# delivered locally and never reaches the forward hook.
# so it allows nothing those refuse; a `reject` ends the packet. The mesh
# and the MACs in `direct_mac` (the unrestricted agents and the service
# containers, the traffic the intercept chain leaves alone) come first in
# both modes, so a `log` line names what `reject` would refuse. Traffic to
# the host's own addresses (squid, dnsmasq, the gateway) is delivered
# locally and never reaches the forward hook.
forwardChain = ''
set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; }
set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; }
@ -134,11 +183,11 @@ let
chain forward {
type filter hook forward priority filter - 10; policy accept;
iifname "${br}" oifname "${meshInterface}" accept
${fromBridge} ether saddr @direct_mac accept
${fromBridge} counter
${logRule "v4" "ip"}
${logRule "v6" "ip6"}
${fromBridge} ct state new limit rate ${unmeteredRate} log prefix "hive-egress-would-drop-unmetered "
${fromBridge} accept
${lib.concatStringsSep "\n " (
logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules
)}
}
'';
@ -232,7 +281,7 @@ let
}
set direct_mac { type ether_addr; }
${lib.optionalString cfg.egress.intercept.enable interceptChain}
${lib.optionalString (cfg.egress.forward == "log") forwardChain}
${lib.optionalString (cfg.egress.forward != "accept") forwardChain}
}
${bridgeTable}
@ -257,20 +306,27 @@ in
type = lib.types.enum [
"accept"
"log"
"drop"
"reject"
];
default = "log";
description = ''
What the `inet hive_egress` table does with traffic the host forwards
from the agent bridge to anywhere but the bridge and the mesh.
from the agent bridge to anywhere but the bridge and the mesh. In `log` and
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
agents and the service containers) is accepted and never logged.
- `accept`: nothing; the table has no forward chain.
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
rate limited per source address, and counted. A flow the per-source limit
cannot log gets a line with the prefix `hive-egress-would-drop-unmetered `
from a limit shared by all addresses. Nothing is blocked. The source
address maps to an agent through the hive's DHCP leases.
- `drop`: not available yet; the evaluation refuses it.
from a limit shared by all addresses. Nothing is blocked. The lines are
the flows `reject` would refuse. The source address maps to an agent
through the hive's DHCP leases.
- `reject`: the same lines with the prefix `hive-egress-reject ` and
`hive-egress-reject-unmetered `, and every other packet from the bridge
is refused as well: a TCP flow with a reset, anything else with ICMP
administratively prohibited. Only the web traffic that the intercept
chain redirects into squid still gets out.
'';
};
@ -409,16 +465,6 @@ in
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.egress.forward != "drop";
message = ''
services.hyperhive.network.egress.forward = "drop" is not available yet:
the traffic squid does not handle (the unrestricted agents, the service
containers, every port but 80 and 443) has no rule that lets it through, so a
drop would cut it.
Use "log" or "accept".
'';
}
{
assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15;
message = ''

73
nix/module-eval/core-toggle.nix
View file

@ -72,17 +72,17 @@ let
# The two settings of `egress.forward` the ordinary host (`log`) does not
# render, for ../host-modules/hive-egress.nix: `accept` leaves the table
# without a forward chain, `drop` is refused by an assertion. The `accept`
# host also sets `egress.antiSpoof = "log"`, the one setting the ordinary
# host (`drop`) does not render.
# without a forward chain, `reject` renders the chain that refuses. The
# `accept` host also sets `egress.antiSpoof = "log"`, the one setting the
# ordinary host (`drop`) does not render.
egressForwardAccept = hive {
network.enable = true;
network.egress.forward = "accept";
network.egress.antiSpoof = "log";
};
egressForwardDrop = hive {
egressForwardReject = hive {
network.enable = true;
network.egress.forward = "drop";
network.egress.forward = "reject";
};
# Every hive defaults the agents' queue address to the queue's name, so the
@ -1561,6 +1561,13 @@ let
&& lib.hasInfix ''iifname "hive-br0" oifname != "hive-br0" ct state new update @log_v6 { ip6 saddr limit rate'' chain
&& lib.hasInfix ''log prefix "hive-egress-would-drop "'' chain
&& lib.hasInfix ''iifname "hive-br0" oifname "wg-hive" accept'' chain
# The mesh comes first, then the exempt MACs (service containers and
# unrestricted agents), then the counter and the logging: a flow of an
# exempt MAC is neither counted nor logged.
&& lib.hasInfix ''iifname "hive-br0" oifname != "hive-br0" ether saddr @direct_mac accept'' chain
&& idx ''oifname "wg-hive" accept'' < idx "ether saddr @direct_mac accept"
&& idx "ether saddr @direct_mac accept" < idx "counter"
&& idx "counter" < idx "update @log_v4"
# The per-source rules accept after logging, and the unmetered rule
# comes after both: a flow gets one line, and one the per-source rules
# cannot log still gets the unmetered one.
@ -1672,22 +1679,48 @@ let
&& !(lib.any (a: !a.assertion) egressForwardAccept.assertions);
}
{
# `drop` is refused by an assertion that names the option and says it is
# not available; `bare` and the `accept` host, which set `log` and
# `accept`, are the controls that the lookup finds nothing for them.
name = "egress.forward drop is refused as not available yet";
# `reject` logs and refuses in one rule, as a TCP rule (reset) and a
# non-TCP rule (ICMP admin-prohibited) for each of the four log/verdict
# positions, after the mesh and the exempt MACs. The ruleset is read as
# text. The `accept` count (the mesh and the exempt MACs only) and the
# `bare` host, whose chain has no refusal, are the controls that the
# lookups find what is there.
name = "egress.forward reject refuses the bridge's forwarded flows after the mesh and the exempt MACs";
ok =
let
refused =
m:
lib.any (
a:
!a.assertion
&& lib.hasInfix "services.hyperhive.network.egress.forward" a.message
&& lib.hasInfix "not available yet" a.message
) m.assertions;
text = lib.head (
lib.splitString "table bridge hive_egress {" egressForwardReject.services.hyperhive.network.egress.ruleset
);
chain = lib.last (lib.splitString "chain forward" text);
rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain));
idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
tcp = lib.filter (lib.hasInfix " meta l4proto tcp ") rules;
other = lib.filter (lib.hasInfix " meta l4proto != tcp") rules;
logged = lib.filter (lib.hasInfix "log prefix") rules;
bareText = bare.services.hyperhive.network.egress.ruleset;
in
refused egressForwardDrop && !(refused bare) && !(refused egressForwardAccept);
!(lib.any (a: !a.assertion) egressForwardReject.assertions)
&& lib.hasInfix "type filter hook forward priority filter - 10; policy accept;" chain
&& lib.hasInfix "chain input" text
&& idx ''oifname "wg-hive" accept'' < idx "ether saddr @direct_mac accept"
&& idx "ether saddr @direct_mac accept" < idx "counter"
&& idx "counter" < idx "reject"
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 2
&& lib.length tcp == 4
&& lib.length other == 4
&& lib.all (lib.hasSuffix " reject with tcp reset") tcp
&& lib.all (lib.hasSuffix " reject with icmpx type admin-prohibited") other
&& lib.length logged == 6
&& lib.all (l: lib.hasInfix ''log prefix "hive-egress-reject'' l) logged
&& !(lib.hasInfix "hive-egress-would-drop" chain)
&& lib.all (
l: lib.hasInfix "update @log_" l || lib.hasInfix "limit rate 5/second burst 50 packets" l
) logged
&&
lib.last rules
== ''iifname "hive-br0" oifname != "hive-br0" meta l4proto != tcp reject with icmpx type admin-prohibited''
&& !(lib.hasInfix "reject with" bareText)
&& lib.hasInfix "would-drop" bareText;
}
{
# The redirect of the agents' web traffic: IPv4 TCP 80 and 443 from the
@ -1715,6 +1748,10 @@ let
lib.hasInfix "type nat hook prerouting priority dstnat; policy accept;" chain
&& lib.hasInfix "set direct_mac { type ether_addr; }" inet
&& idx "ether saddr @direct_mac return" < idx "tcp dport 80"
# The same MACs are exempt in the forward chain, which follows the
# redirect chain in the table.
&& lib.hasInfix ''oifname != "hive-br0" ether saddr @direct_mac accept'' inet
&& !(lib.hasInfix "ether saddr @direct_mac accept" without)
&& idx "fib daddr type local return" < idx "tcp dport 80"
&& idx ''fib daddr oifname { "hive-br0", "wg-hive" } return'' < idx "tcp dport 80"
&& lib.length redirects == 2

72
nix/vm-tests/hive-squid.nix
View file

@ -225,6 +225,10 @@ pkgs.testers.runNixOSTest {
pkgs.mkpasswd
pkgs.nftables
];
# The reject-mode half of the test switches to this at its end. The
# forward mode is fixed when the node evaluates; the switch reloads
# `hive-egress` with the other ruleset.
specialisation.reject.configuration.services.hyperhive.network.egress.forward = "reject";
services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local";
services.hyperhive.deploy.hive-controller.squid.selfcheck.canary = "cache.t.local";
services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains = [
@ -1157,11 +1161,71 @@ pkgs.testers.runNixOSTest {
assert icpt("https://203.0.113.2/") == "200", "an IP literal, directly"
assert agent_lines() == seen, "squid saw nothing of it"
assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"
)
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
assert icpt("https://other.t.local/") == "000", "intercepted again"
# Control: a new flow of the same MAC, now off the set, is logged, and the
# kernel log is in order, so a line of the exempt flows would be before it.
agent.execute("echo x > /dev/udp/203.0.113.79/9")
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*DST=203\\.0\\.113\\.79 '"
)
assert not would_drop(r"DST=203\.0\.113\.2 .*DPT=443 "), "the exempt MAC's flows leave no line"
# ---- Reject mode ----
# The node switches to its `reject` specialisation, which reloads the
# tables with the refusing forward chain. Nothing is in `direct_mac` yet.
def reject_lines(pattern):
# The reject chain's log lines in the kernel log that match `pattern`.
out = proxy.succeed("journalctl -k -o cat --no-pager | grep hive-egress-reject || true")
return [line for line in out.splitlines() if re.search(pattern, line)]
def wait_reject_line(pattern):
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
+ shlex.quote(f"hive-egress-reject(-unmetered)? IN=eth1 OUT=eth2 .*{pattern}")
)
def dial_status(addr, port):
# 0 connected, 1 refused, 124 no answer in 3 seconds.
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/{port}'")[0]
def udp_status(addr):
# A connected UDP socket reports the ICMP error of the first datagram
# on the second write: non-zero when the host refused it. Nothing
# answers a datagram that is let through, so that case is 0.
return agent.execute(f"bash -c 'exec 3<>/dev/udp/{addr}/9; echo x >&3; sleep 1; echo y >&3'")[0]
with checked("reject mode: web still goes through squid and everything else from a restricted agent is refused"):
proxy.succeed("/run/current-system/specialisation/reject/bin/switch-to-configuration test")
out = proxy.succeed("nft list chain inet hive_egress forward")
assert "tcp reset" in out and "admin-prohibited" in out, out
assert icpt("https://forge.t.local/") == "200", "control: web still goes through squid"
assert dial_status("203.0.113.2", 8443) == 1, "TCP 8443 is reset at once, not left to time out"
wait_reject_line(r"SRC=192\.168\.1\.3 DST=203\.0\.113\.2 .*PROTO=TCP .*DPT=8443 ")
status, out = agent.execute("ping -c1 -W3 203.0.113.2")
assert status != 0 and "filtered" in out.lower(), f"ICMP is refused with admin-prohibited: {out}"
wait_reject_line(r"SRC=192\.168\.1\.3 DST=203\.0\.113\.2 .*PROTO=ICMP ")
assert udp_status("203.0.113.77") != 0, "a datagram is refused with an ICMP error"
wait_reject_line(r"DST=203\.0\.113\.77 .*PROTO=UDP ")
out = proxy.succeed("nft list chain inet hive_egress forward")
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
assert found and int(found.group(1)) > 0, out
with checked("reject mode: a MAC in direct_mac passes and is refused again once it leaves, and the mesh passes"):
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
assert dial_status("203.0.113.2", 8443) == 0, "an exempt MAC (a service container's stand-in) reaches TCP 8443"
agent.succeed("ping -c1 -W3 203.0.113.2")
assert udp_status("203.0.113.77") == 0, "control: an exempt MAC's datagram is not refused"
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
assert dial_status("203.0.113.2", 8443) == 1, "refused again"
# The mesh comes first in the chain. A dummy interface answers nothing,
# so the proof is that the chain refuses neither the ping nor the UDP
# datagram, while the same flows to the uplink are refused.
status, out = agent.execute("ping -c1 -W5 198.51.100.7")
assert "filtered" not in out.lower(), f"the mesh interface is not egress: {out}"
agent.execute("echo x > /dev/udp/198.51.100.7/9; echo x > /dev/udp/203.0.113.80/9")
wait_reject_line(r"DST=203\.0\.113\.80 ")
assert not reject_lines(r"DST=198\.51\.100\.7 "), "the mesh interface is not egress"
'';
}