nix: move the egress firewall options under deploy.hive-controller.egress
This commit is contained in:
damocles
mara
parent
1cef99e992
commit
7468aa4cff
7 changed files with 148 additions and 31 deletions
|
|
@ -522,7 +522,7 @@ the `intercept` chain, at `prerouting`, redirects IPv4 TCP port 80 to
|
|||
`<bridgeIp>:3131`, the ClientHello relay (see _ECH_ below), which sends the
|
||||
connection on to squid's other intercept listener, `<bridgeIp>:3130`. Squid reads
|
||||
the original destination from conntrack. The chain exists where the host runs
|
||||
squid (`services.hyperhive.network.egress.intercept.enable`).
|
||||
squid (`services.hyperhive.deploy.hive-controller.egress.intercept.enable`).
|
||||
|
||||
- **Who gets redirected.** Every flow from the bridge to port 80 or 443, except
|
||||
a flow to an address that stays on the bridge, to one of the host's own
|
||||
|
|
@ -725,7 +725,7 @@ loopback — the hive-c0re admin socket is a UDS, not TCP.
|
|||
|
||||
### Agent bridge egress log
|
||||
|
||||
`services.hyperhive.network.egress.forward` sets what the `forward` chain of
|
||||
`services.hyperhive.deploy.hive-controller.egress.forward` sets what the `forward` chain of
|
||||
the `inet hive_egress` table does with traffic the host routes from the agent
|
||||
bridge to anywhere but the bridge and the `wg-hive` mesh interface. It takes
|
||||
`accept`, `log` (the default) or `reject`. In `log` mode the chain blocks
|
||||
|
|
@ -751,7 +751,7 @@ also refuses the flow.
|
|||
`hive-egress-would-drop` lines. Each is a flow that `reject` would refuse; a
|
||||
flow that belongs to a legitimate use needs a fix first (a service container
|
||||
missing from `direct_mac`, an agent that should be `unrestricted`). Then set
|
||||
`services.hyperhive.network.egress.forward = "reject"`. Making it the default
|
||||
`services.hyperhive.deploy.hive-controller.egress.forward = "reject"`. Making it the default
|
||||
is a one-line change of the option's default.
|
||||
- **What `reject` can break.**
|
||||
- A service container with no pin yet isn't in `direct_mac`, so the
|
||||
|
|
@ -801,7 +801,7 @@ also refuses the flow.
|
|||
|
||||
Every container on the agent bridge is root in its own network namespace and
|
||||
shares one L2 segment with the others, so it can send with another container's
|
||||
MAC or address. `services.hyperhive.network.egress.antiSpoof` sets what the
|
||||
MAC or address. `services.hyperhive.deploy.hive-controller.egress.antiSpoof` sets what the
|
||||
`bridge hive_egress` table (`nix/host-modules/hive-egress.nix`) does about it. It
|
||||
takes `drop` (the default) or `log`. Two tables, not one: the bridge carries traffic between two
|
||||
ports, and the `inet` hooks never see it as coming from a port. Only a
|
||||
|
|
|
|||
|
|
@ -305,6 +305,102 @@ in
|
|||
[ "services" "hyperhive" "deploy" "hive-controller" "adminUsers" ]
|
||||
)
|
||||
|
||||
# The host's egress firewall settings sit with the controller's, like the
|
||||
# egress proxy's below. The two read-only outputs, `leaseScript` and
|
||||
# `ruleset`, stay under `network.egress`.
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "forward" ]
|
||||
[ "services" "hyperhive" "deploy" "hive-controller" "egress" "forward" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "antiSpoof" ]
|
||||
[ "services" "hyperhive" "deploy" "hive-controller" "egress" "antiSpoof" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "intercept" "enable" ]
|
||||
[
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"egress"
|
||||
"intercept"
|
||||
"enable"
|
||||
]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "intercept" "httpPort" ]
|
||||
[
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"egress"
|
||||
"intercept"
|
||||
"httpPort"
|
||||
]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "intercept" "httpsPort" ]
|
||||
[
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"egress"
|
||||
"intercept"
|
||||
"httpsPort"
|
||||
]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "intercept" "relayPort" ]
|
||||
[
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"egress"
|
||||
"intercept"
|
||||
"relayPort"
|
||||
]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "intercept" "refuseEch" ]
|
||||
[
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"egress"
|
||||
"intercept"
|
||||
"refuseEch"
|
||||
]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "intercept" "agentAllowFile" ]
|
||||
[
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"egress"
|
||||
"intercept"
|
||||
"agentAllowFile"
|
||||
]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "network" "egress" "intercept" "unrestrictedFile" ]
|
||||
[
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"egress"
|
||||
"intercept"
|
||||
"unrestrictedFile"
|
||||
]
|
||||
)
|
||||
|
||||
# The egress proxy runs only on a hive host, so its settings sit with the
|
||||
# controller's. The read-only ones moved without a forward.
|
||||
(lib.mkRenamedOptionModule
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
}:
|
||||
let
|
||||
cfg = config.services.hyperhive.network;
|
||||
egressCfg = config.services.hyperhive.deploy.hive-controller.egress;
|
||||
nft = lib.getExe pkgs.nftables;
|
||||
br = cfg.bridgeName;
|
||||
|
||||
|
|
@ -49,7 +50,7 @@ let
|
|||
# pool and carrying the mark; the `output` chain DNATs those connections to
|
||||
# squid. `egress_ech.rs` derives the pool address from the agent's and
|
||||
# `egress-whois` maps it back, so the pool is the one defined there.
|
||||
intercept = cfg.egress.intercept;
|
||||
intercept = egressCfg.intercept;
|
||||
refuseEch = intercept.enable && intercept.refuseEch;
|
||||
relayPool = "198.18.0.0/15";
|
||||
relayMark = "0x45434800";
|
||||
|
|
@ -69,8 +70,8 @@ let
|
|||
pinEnvironment = {
|
||||
HIVE_EGRESS_BRIDGE = br;
|
||||
}
|
||||
// lib.optionalAttrs (cfg.egress.intercept.unrestrictedFile != null) {
|
||||
HIVE_EGRESS_UNRESTRICTED = toString cfg.egress.intercept.unrestrictedFile;
|
||||
// lib.optionalAttrs (egressCfg.intercept.unrestrictedFile != null) {
|
||||
HIVE_EGRESS_UNRESTRICTED = toString egressCfg.intercept.unrestrictedFile;
|
||||
};
|
||||
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
|
||||
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
|
||||
|
|
@ -96,7 +97,7 @@ let
|
|||
# connect at once (to a TCP socket an ICMP error is a soft error, and the
|
||||
# connect would stall); anything else gets ICMP admin-prohibited. A rule that
|
||||
# refuses is therefore a pair: one for TCP, one for the rest.
|
||||
isReject = cfg.egress.forward == "reject";
|
||||
isReject = egressCfg.forward == "reject";
|
||||
logPrefix = if isReject then "hive-egress-reject" else "hive-egress-would-drop";
|
||||
refuse = {
|
||||
tcp = "meta l4proto tcp";
|
||||
|
|
@ -153,7 +154,7 @@ let
|
|||
iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return
|
||||
iifname "${br}" meta nfproto ipv4 fib daddr type local return
|
||||
iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return
|
||||
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort}
|
||||
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString egressCfg.intercept.httpPort}
|
||||
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${
|
||||
toString (if refuseEch then intercept.relayPort else intercept.httpsPort)
|
||||
}
|
||||
|
|
@ -220,7 +221,7 @@ let
|
|||
# packet its port cannot log (over its rate, or the set is full) falls to the
|
||||
# unmetered line, which has one budget for all ports. It logs
|
||||
# packets, not flows: the bridge hook has no connection tracking.
|
||||
spoofVerdict = if cfg.egress.antiSpoof == "drop" then "drop" else "accept";
|
||||
spoofVerdict = if egressCfg.antiSpoof == "drop" then "drop" else "accept";
|
||||
bridgeTable = ''
|
||||
table bridge hive_egress {
|
||||
set port_mac { type ifname . ether_addr; }
|
||||
|
|
@ -283,8 +284,8 @@ let
|
|||
${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''}
|
||||
}
|
||||
set direct_mac { type ether_addr; }
|
||||
${lib.optionalString cfg.egress.intercept.enable interceptChain}
|
||||
${lib.optionalString (cfg.egress.forward != "accept") forwardChain}
|
||||
${lib.optionalString egressCfg.intercept.enable interceptChain}
|
||||
${lib.optionalString (egressCfg.forward != "accept") forwardChain}
|
||||
}
|
||||
|
||||
${bridgeTable}
|
||||
|
|
@ -305,7 +306,7 @@ let
|
|||
ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset;
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.network.egress.forward = lib.mkOption {
|
||||
options.services.hyperhive.deploy.hive-controller.egress.forward = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"accept"
|
||||
"log"
|
||||
|
|
@ -333,7 +334,7 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.network.egress.antiSpoof = lib.mkOption {
|
||||
options.services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"drop"
|
||||
"log"
|
||||
|
|
@ -362,7 +363,7 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.network.egress.intercept = {
|
||||
options.services.hyperhive.deploy.hive-controller.egress.intercept = {
|
||||
enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports";
|
||||
|
||||
httpPort = lib.mkOption {
|
||||
|
|
@ -479,10 +480,10 @@ in
|
|||
'';
|
||||
}
|
||||
{
|
||||
assertion = cfg.egress.forward != "reject" || cfg.egress.intercept.enable;
|
||||
assertion = egressCfg.forward != "reject" || egressCfg.intercept.enable;
|
||||
message = ''
|
||||
services.hyperhive.network.egress.forward = "reject" needs
|
||||
services.hyperhive.network.egress.intercept.enable = true:
|
||||
services.hyperhive.deploy.hive-controller.egress.forward = "reject" needs
|
||||
services.hyperhive.deploy.hive-controller.egress.intercept.enable = true:
|
||||
without the intercept nothing redirects the agents' web traffic into
|
||||
squid, and the reject would refuse all of it.
|
||||
Enable the intercept (hive-squid.nix does on a hive controller host),
|
||||
|
|
@ -620,11 +621,11 @@ in
|
|||
|
||||
# A pass also refills `direct_mac`, so a change of the unrestricted list
|
||||
# takes effect at once instead of at the next tick of the timer.
|
||||
systemd.paths.hive-egress-direct = lib.mkIf (cfg.egress.intercept.unrestrictedFile != null) {
|
||||
systemd.paths.hive-egress-direct = lib.mkIf (egressCfg.intercept.unrestrictedFile != null) {
|
||||
description = "Watch the unrestricted agents list for the egress intercept exemptions";
|
||||
wantedBy = [ "paths.target" ];
|
||||
pathConfig = {
|
||||
PathChanged = toString cfg.egress.intercept.unrestrictedFile;
|
||||
PathChanged = toString egressCfg.intercept.unrestrictedFile;
|
||||
Unit = "hive-egress-pin.service";
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ let
|
|||
listen = "${netCfg.bridgeIp}:${toString port}";
|
||||
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
|
||||
exporterListen = "127.0.0.1:9301";
|
||||
interceptCfg = netCfg.egress.intercept;
|
||||
interceptCfg = hyperhiveCfg.deploy.hive-controller.egress.intercept;
|
||||
# The path every platform container's unit runs hive-priv by (hive-priv.nix
|
||||
# points it at the current build). Squid runs `egress-whois` through it.
|
||||
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
|
||||
|
|
@ -551,7 +551,7 @@ in
|
|||
interceptCfg.httpPort
|
||||
interceptCfg.httpsPort
|
||||
];
|
||||
services.hyperhive.network.egress.intercept = {
|
||||
services.hyperhive.deploy.hive-controller.egress.intercept = {
|
||||
enable = lib.mkDefault true;
|
||||
inherit (cfg) unrestrictedFile agentAllowFile;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -76,16 +76,23 @@ let
|
|||
# `accept` host also sets `egress.antiSpoof = "log"`, the one setting the
|
||||
# ordinary host (`drop`) does not render.
|
||||
egressForwardAccept = hive {
|
||||
network.enable = true;
|
||||
deploy.hive-controller.egress.forward = "accept";
|
||||
deploy.hive-controller.egress.antiSpoof = "log";
|
||||
};
|
||||
# The same two settings under their old names, `network.egress.*`, which
|
||||
# `deploy.nix` still renames.
|
||||
egressForwardAcceptLegacy = hive {
|
||||
network.enable = true;
|
||||
network.egress.forward = "accept";
|
||||
network.egress.antiSpoof = "log";
|
||||
};
|
||||
egressForwardReject = hiveWithController {
|
||||
network.egress.forward = "reject";
|
||||
deploy.hive-controller.egress.forward = "reject";
|
||||
};
|
||||
egressRejectNoIntercept = hive {
|
||||
network.enable = true;
|
||||
network.egress.forward = "reject";
|
||||
deploy.hive-controller.egress.forward = "reject";
|
||||
};
|
||||
|
||||
# Every hive defaults the agents' queue address to the queue's name, so the
|
||||
|
|
@ -94,7 +101,7 @@ let
|
|||
# It is also the one controller host with `egress.intercept.refuseEch` off,
|
||||
# which the egress cases read.
|
||||
noAgentQueue = hiveWithController {
|
||||
network.egress.intercept.refuseEch = false;
|
||||
deploy.hive-controller.egress.intercept.refuseEch = false;
|
||||
deploy.hive-controller.queue.agentNatsUrl = null;
|
||||
c0re.operatorPronouns = "they/them";
|
||||
c0re.contextWindowTokens.sonnet = 900000;
|
||||
|
|
@ -1642,7 +1649,7 @@ let
|
|||
spoof = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" (section "spoof")));
|
||||
ip4 = section "l3_v4";
|
||||
in
|
||||
bare.services.hyperhive.network.egress.antiSpoof == "drop"
|
||||
bare.services.hyperhive.deploy.hive-controller.egress.antiSpoof == "drop"
|
||||
&& lib.hasInfix "hook prerouting priority -300" bridgeText
|
||||
&& lib.hasInfix ''meta ibrname != "hive-br0" accept'' bridgeText
|
||||
&& lib.hasInfix "iifname . ether saddr != @port_mac goto spoof" bridgeText
|
||||
|
|
@ -1676,7 +1683,7 @@ let
|
|||
);
|
||||
logged = spoofOf egressForwardAccept;
|
||||
in
|
||||
egressForwardAccept.services.hyperhive.network.egress.antiSpoof == "log"
|
||||
egressForwardAccept.services.hyperhive.deploy.hive-controller.egress.antiSpoof == "log"
|
||||
&& lib.length logged == 3
|
||||
&& lib.all (l: lib.hasSuffix " accept" l) logged
|
||||
&& lib.hasInfix ''log prefix "hive-egress-spoof "'' (textOf egressForwardAccept)
|
||||
|
|
@ -1727,6 +1734,18 @@ let
|
|||
&& !(lib.hasInfix "hive-egress-would-drop" text)
|
||||
&& !(lib.any (a: !a.assertion) egressForwardAccept.assertions);
|
||||
}
|
||||
{
|
||||
# A host still setting the old `network.egress.*` names gets the same
|
||||
# table, and a deprecation warning for each name.
|
||||
name = "the old network.egress names still set the egress options";
|
||||
ok =
|
||||
egressForwardAcceptLegacy.services.hyperhive.deploy.hive-controller.egress.forward == "accept"
|
||||
&& egressForwardAcceptLegacy.services.hyperhive.deploy.hive-controller.egress.antiSpoof == "log"
|
||||
&&
|
||||
egressForwardAcceptLegacy.services.hyperhive.network.egress.ruleset
|
||||
== egressForwardAccept.services.hyperhive.network.egress.ruleset
|
||||
&& lib.any (w: lib.hasInfix "network.egress.forward" w) egressForwardAcceptLegacy.warnings;
|
||||
}
|
||||
{
|
||||
# `reject` logs and refuses in one rule, as a TCP rule (reset) and a
|
||||
# non-TCP rule (ICMP admin-prohibited) for each of the four log/verdict
|
||||
|
|
@ -1749,7 +1768,7 @@ let
|
|||
bareText = bare.services.hyperhive.network.egress.ruleset;
|
||||
in
|
||||
!(lib.any (
|
||||
a: !a.assertion && lib.hasInfix "network.egress" a.message
|
||||
a: !a.assertion && lib.hasInfix "deploy.hive-controller.egress" a.message
|
||||
) egressForwardReject.assertions)
|
||||
&& lib.hasInfix "type filter hook forward priority filter - 10; policy accept;" chain
|
||||
&& lib.hasInfix "chain input" text
|
||||
|
|
@ -1786,7 +1805,7 @@ let
|
|||
lib.filter (
|
||||
a:
|
||||
!a.assertion
|
||||
&& lib.hasInfix "services.hyperhive.network.egress.forward = \"reject\" needs" a.message
|
||||
&& lib.hasInfix "services.hyperhive.deploy.hive-controller.egress.forward = \"reject\" needs" a.message
|
||||
) m.assertions;
|
||||
in
|
||||
lib.length (refused egressRejectNoIntercept) == 1
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ pkgs.testers.runNixOSTest {
|
|||
containers.b = container;
|
||||
|
||||
specialisation.logmode.configuration = {
|
||||
services.hyperhive.network.egress.antiSpoof = lib.mkForce "log";
|
||||
services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkForce "log";
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -228,7 +228,8 @@ pkgs.testers.runNixOSTest {
|
|||
# The reject-mode half of the test switches to this at its end. The
|
||||
# forward mode is fixed when the node evaluates; the switch reloads
|
||||
# `hive-egress` with the other ruleset.
|
||||
specialisation.reject.configuration.services.hyperhive.network.egress.forward = "reject";
|
||||
specialisation.reject.configuration.services.hyperhive.deploy.hive-controller.egress.forward =
|
||||
"reject";
|
||||
services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local";
|
||||
services.hyperhive.deploy.hive-controller.squid.selfcheck.canary = "cache.t.local";
|
||||
services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains = [
|
||||
|
|
|
|||
Loading…
Reference in a new issue