622 lines
27 KiB
Nix
622 lines
27 KiB
Nix
# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`,
|
|
# loaded by a systemd oneshot. They are independent of the NixOS firewall:
|
|
# `networking.firewall` renders only the rules of the backend in use
|
|
# (`iptables` unless `networking.nftables.enable` is set), so a rule that has
|
|
# to hold on every host lives here, in tables no firewall backend writes. Only
|
|
# `networking.nftables.flushRuleset` removes them; the unit re-applies them then.
|
|
#
|
|
# The `inet` table holds three chains. `input`: traffic from the agent bridge to host
|
|
# loopback is dropped. The kernel does the same by default (a packet for
|
|
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
|
|
# `route_localnet` is 0); this rule is the second layer, and the assertion
|
|
# below keeps the first one on. `forward` (`egress.forward`): traffic from the
|
|
# agent bridge to anywhere else is logged and counted (`log`) or logged and
|
|
# refused (`reject`). `intercept` (`egress.intercept.enable`, set by
|
|
# hive-squid.nix): the web traffic of the bridge's agents is redirected into
|
|
# squid's intercept ports (port 443 through the ClientHello relay). Both leave
|
|
# alone the ports in `direct_mac`, the service containers and the unrestricted
|
|
# agents. `hive-priv egress-pin` and `egress-load` fill that set from the
|
|
# bridge table's bindings, so a MAC in no binding is intercepted and refused.
|
|
#
|
|
# The `bridge` table checks the source address of every frame a port of the
|
|
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
|
|
# traffic between two ports is bridged: the `inet` hooks see it as coming from
|
|
# the bridge device, and only a bridge hook knows the port it came in on
|
|
# (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC
|
|
# may use are not known until a container runs, so they are set contents.
|
|
# `hive-priv egress-load` puts the current ones into the same transaction as the
|
|
# tables; `egress-pin` (per new port from udev, and from a timer for the ports
|
|
# still unbound) and dnsmasq's `dhcp-script` (per lease) keep them current.
|
|
# `hive-priv`'s `egress.rs` documents how.
|
|
{
|
|
lib,
|
|
config,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.network;
|
|
nft = lib.getExe pkgs.nftables;
|
|
br = cfg.bridgeName;
|
|
|
|
# The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
|
|
meshInterface = "wg-hive";
|
|
|
|
# With `refuseEch`, the redirect of port 443 ends at `hive-priv
|
|
# egress-ech-relay`, which refuses a ClientHello that carries the ECH
|
|
# extension and sends the rest on to squid's HTTPS intercept port. The relay
|
|
# connects to the address the agent dialed, bound to a source address of the
|
|
# pool and carrying the mark; the `output` chain DNATs those connections to
|
|
# squid. `egress_ech.rs` derives the pool address from the agent's and
|
|
# `egress-whois` maps it back, so the pool is the one defined there.
|
|
intercept = cfg.egress.intercept;
|
|
refuseEch = intercept.enable && intercept.refuseEch;
|
|
relayPool = "198.18.0.0/15";
|
|
relayMark = "0x45434800";
|
|
relayCommand = "${hivePriv} egress-ech-relay ${cfg.bridgeIp}:${toString intercept.relayPort} ${relayMark}";
|
|
|
|
# The sub-commands of hive-priv that fill the bridge table's sets. The path is
|
|
# the one every platform container's unit uses (hive-priv.nix points it at the
|
|
# current build).
|
|
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
|
|
pinCommand = args: lib.trim "${hivePriv} egress-pin ${args}";
|
|
loadCommand = "${hivePriv} egress-load ${ruleset}";
|
|
pinPath = [
|
|
pkgs.nftables
|
|
pkgs.iproute2
|
|
pkgs.util-linux
|
|
];
|
|
pinEnvironment = {
|
|
HIVE_EGRESS_BRIDGE = br;
|
|
}
|
|
// lib.optionalAttrs (cfg.egress.intercept.unrestrictedFile != null) {
|
|
HIVE_EGRESS_UNRESTRICTED = toString cfg.egress.intercept.unrestrictedFile;
|
|
};
|
|
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
|
|
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
|
|
exec ${hivePriv} egress-lease "$@"
|
|
'';
|
|
|
|
# Logging is per source address, so one busy agent cannot use up the budget
|
|
# of another: a burst of 20 lines, then one a second, and an address is
|
|
# forgotten after a minute without a new flow. The rule decides after it
|
|
# logs. A flow the rule cannot log falls through to the unmetered rule below
|
|
# it: the address was over its rate, or the set is full (`update` then fails
|
|
# and the rule stops matching), which a container spoofing source addresses
|
|
# can bring about. Only that case reaches the unmetered rule, so a flow
|
|
# gets one line at most. The unmetered rule has one budget for all
|
|
# addresses, 5 lines a second after a burst of 50, about 100 MB of journal a
|
|
# day when it never rests.
|
|
logRate = "1/second burst 20 packets";
|
|
unmeteredRate = "5/second burst 50 packets";
|
|
fromBridge = ''iifname "${br}" oifname != "${br}"'';
|
|
|
|
# `log` accepts what it logs. `reject` logs and refuses in one rule, so a
|
|
# line always names a refused flow. A TCP flow gets a reset, which fails the
|
|
# connect at once (to a TCP socket an ICMP error is a soft error, and the
|
|
# connect would stall); anything else gets ICMP admin-prohibited. A rule that
|
|
# refuses is therefore a pair: one for TCP, one for the rest.
|
|
isReject = cfg.egress.forward == "reject";
|
|
logPrefix = if isReject then "hive-egress-reject" else "hive-egress-would-drop";
|
|
refuse = {
|
|
tcp = "meta l4proto tcp";
|
|
other = "meta l4proto != tcp";
|
|
};
|
|
verdict = {
|
|
tcp = "reject with tcp reset";
|
|
other = "reject with icmpx type admin-prohibited";
|
|
};
|
|
decide =
|
|
statements:
|
|
if isReject then
|
|
map
|
|
(
|
|
k:
|
|
lib.concatStringsSep " " (
|
|
lib.filter (x: x != "") [
|
|
fromBridge
|
|
refuse.${k}
|
|
statements
|
|
verdict.${k}
|
|
]
|
|
)
|
|
)
|
|
[
|
|
"tcp"
|
|
"other"
|
|
]
|
|
else
|
|
[ "${fromBridge} ${statements} accept" ];
|
|
logRules =
|
|
family: addr:
|
|
decide ''ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "${logPrefix} "'';
|
|
# In `log` mode the unmetered rule has no verdict: the chain's last rule is
|
|
# the verdict for every flow that gets past the logging.
|
|
unmeteredRules =
|
|
let
|
|
statements = ''ct state new limit rate ${unmeteredRate} log prefix "${logPrefix}-unmetered "'';
|
|
in
|
|
if isReject then decide statements else [ "${fromBridge} ${statements}" ];
|
|
# What no rule above decided, a packet that is no new flow included.
|
|
finalRules = if isReject then decide "" else [ "${fromBridge} accept" ];
|
|
|
|
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
|
|
# the interface the route to the destination leaves by: the bridge (another
|
|
# container), `lo` (an address of the host: `fib daddr type local`) and the
|
|
# mesh interface are not egress. The redirect rewrites the destination to
|
|
# the bridge address and the port of squid's intercept listener (for port 443
|
|
# with `refuseEch`, of the relay), which reads the original destination from
|
|
# conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing.
|
|
interceptChain = ''
|
|
chain intercept {
|
|
type nat hook prerouting priority dstnat; policy accept;
|
|
iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return
|
|
iifname "${br}" meta nfproto ipv4 fib daddr type local return
|
|
iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return
|
|
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort}
|
|
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${
|
|
toString (if refuseEch then intercept.relayPort else intercept.httpsPort)
|
|
}
|
|
}
|
|
''
|
|
+ lib.optionalString refuseEch ''
|
|
|
|
chain intercept_relay {
|
|
type nat hook output priority dstnat; policy accept;
|
|
meta nfproto ipv4 ip saddr ${relayPool} meta mark ${relayMark} tcp dport 443 counter dnat ip to ${cfg.bridgeIp}:${toString intercept.httpsPort}
|
|
}
|
|
'';
|
|
|
|
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
|
|
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
|
|
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
|
|
# so it allows nothing those refuse; a `reject` ends the packet. The mesh
|
|
# and the MACs in `direct_mac` (the unrestricted agents and the service
|
|
# containers, the traffic the intercept chain leaves alone) come first in
|
|
# both modes, so a `log` line names what `reject` would refuse. Traffic to
|
|
# the host's own addresses (squid, dnsmasq, the gateway) is delivered
|
|
# locally and never reaches the forward hook.
|
|
forwardChain = ''
|
|
set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; }
|
|
set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; }
|
|
|
|
chain forward {
|
|
type filter hook forward priority filter - 10; policy accept;
|
|
iifname "${br}" oifname "${meshInterface}" accept
|
|
${fromBridge} ether saddr @direct_mac accept
|
|
${fromBridge} counter
|
|
${lib.concatStringsSep "\n " (
|
|
logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules
|
|
)}
|
|
}
|
|
'';
|
|
|
|
# The bridge table. A frame of a port goes through `prerouting` in this
|
|
# order: the port must be bound to the frame's source MAC, then the frame's
|
|
# protocol decides which source address it may carry. Everything else ends in
|
|
# `spoof`, which logs and then drops or, with `antiSpoof = "log"`, accepts.
|
|
#
|
|
# - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second
|
|
# as the 48 bits that sit at the `chaddr` field of a DHCP message. A port
|
|
# with no pair sends nothing, DHCP included.
|
|
# - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its
|
|
# lease.
|
|
# - IPv6 has no leases: a source is link-local, or `::` in the packets that
|
|
# duplicate address detection and multicast listener reports send before an
|
|
# address exists. The bridge carries no other IPv6 traffic.
|
|
# - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`,
|
|
# so a port cannot lease, renew or release another's address; a client with
|
|
# no address yet sends from 0.0.0.0 to the broadcast address.
|
|
# - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or
|
|
# leased to that MAC.
|
|
# - A VLAN tag is refused: a tagged frame would carry its packet past the
|
|
# protocol checks. The bridge uses no VLANs.
|
|
#
|
|
# Logging is per ingress port, in the style of the forward chain: a burst of
|
|
# 20 lines, then one a second, and a port is forgotten after a minute. A
|
|
# packet its port cannot log (over its rate, or the set is full) falls to the
|
|
# unmetered line, which has one budget for all ports. It logs
|
|
# packets, not flows: the bridge hook has no connection tracking.
|
|
spoofVerdict = if cfg.egress.antiSpoof == "drop" then "drop" else "accept";
|
|
bridgeTable = ''
|
|
table bridge hive_egress {
|
|
set port_mac { type ifname . ether_addr; }
|
|
set port_chaddr { typeof iifname . @th,288,48; }
|
|
set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; }
|
|
set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; }
|
|
|
|
chain spoof {
|
|
update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof " ${spoofVerdict}
|
|
limit rate ${unmeteredRate} counter log prefix "hive-egress-spoof-unmetered " ${spoofVerdict}
|
|
counter ${spoofVerdict}
|
|
}
|
|
|
|
chain l3_v4 {
|
|
udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof
|
|
ip saddr 0.0.0.0 ip daddr 255.255.255.255 udp sport 68 udp dport 67 accept
|
|
ether saddr . ip saddr @lease_v4 accept
|
|
goto spoof
|
|
}
|
|
|
|
chain l3_v6 {
|
|
ip6 saddr fe80::/10 accept
|
|
ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept
|
|
goto spoof
|
|
}
|
|
|
|
chain l3_arp {
|
|
iifname . arp saddr ether != @port_mac goto spoof
|
|
arp saddr ip 0.0.0.0 accept
|
|
arp saddr ether . arp saddr ip @lease_v4 accept
|
|
goto spoof
|
|
}
|
|
|
|
chain prerouting {
|
|
type filter hook prerouting priority -300; policy accept;
|
|
meta ibrname != "${br}" accept
|
|
iifname . ether saddr != @port_mac goto spoof
|
|
ether type { 8021q, 8021ad } goto spoof
|
|
ether type ip goto l3_v4
|
|
ether type ip6 goto l3_v6
|
|
ether type arp goto l3_arp
|
|
}
|
|
}
|
|
'';
|
|
|
|
# One transaction: `add` + `delete` drop an old copy of each table and the
|
|
# definitions recreate them, so the file loads the same on first start, on a
|
|
# reload and after something else flushed the ruleset.
|
|
rulesetText = ''
|
|
add table inet hive_egress
|
|
delete table inet hive_egress
|
|
add table bridge hive_egress
|
|
delete table bridge hive_egress
|
|
|
|
table inet hive_egress {
|
|
chain input {
|
|
type filter hook input priority filter - 10; policy accept;
|
|
iifname "${br}" ip daddr 127.0.0.0/8 counter drop
|
|
iifname "${br}" ip6 daddr ::1 counter drop
|
|
${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''}
|
|
}
|
|
set direct_mac { type ether_addr; }
|
|
${lib.optionalString cfg.egress.intercept.enable interceptChain}
|
|
${lib.optionalString (cfg.egress.forward != "accept") forwardChain}
|
|
}
|
|
|
|
${bridgeTable}
|
|
'';
|
|
|
|
routeLocalnetKeys = [
|
|
"net.ipv4.conf.all.route_localnet"
|
|
"net.ipv4.conf.default.route_localnet"
|
|
"net.ipv4.conf.${cfg.bridgeName}.route_localnet"
|
|
];
|
|
isOn =
|
|
v:
|
|
builtins.elem v [
|
|
1
|
|
"1"
|
|
true
|
|
];
|
|
ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset;
|
|
in
|
|
{
|
|
options.services.hyperhive.network.egress.forward = lib.mkOption {
|
|
type = lib.types.enum [
|
|
"accept"
|
|
"log"
|
|
"reject"
|
|
];
|
|
default = "log";
|
|
description = ''
|
|
What the `inet hive_egress` table does with traffic the host forwards
|
|
from the agent bridge to anywhere but the bridge and the mesh. In `log` and
|
|
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
|
|
agents and the service containers) is accepted and never logged.
|
|
|
|
- `accept`: nothing; the table has no forward chain.
|
|
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
|
|
rate limited per source address, and counted. A flow the per-source limit
|
|
cannot log gets a line with the prefix `hive-egress-would-drop-unmetered `
|
|
from a limit shared by all addresses. Nothing is blocked. The lines are
|
|
the flows `reject` would refuse. The source address maps to an agent
|
|
through the hive's DHCP leases.
|
|
- `reject`: the same lines with the prefix `hive-egress-reject ` and
|
|
`hive-egress-reject-unmetered `, and every other packet from the bridge
|
|
is refused as well: a TCP flow with a reset, anything else with ICMP
|
|
administratively prohibited. Only the web traffic that the intercept
|
|
chain redirects into squid still gets out.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.network.egress.antiSpoof = lib.mkOption {
|
|
type = lib.types.enum [
|
|
"drop"
|
|
"log"
|
|
];
|
|
default = "drop";
|
|
description = ''
|
|
What the `bridge hive_egress` table does with a frame whose source is not
|
|
the sending container's own. Every port of the agent bridge is checked:
|
|
the frame's MAC must be the one bound to the port when its container
|
|
started, and its IPv4 address the one leased to that MAC. A port with no
|
|
bound MAC sends nothing. Allowed besides the leased address: DHCP from
|
|
`0.0.0.0` whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local
|
|
sources (and `::` in neighbour discovery and listener reports).
|
|
|
|
- `drop`: the frame is dropped and logged with the prefix
|
|
`hive-egress-spoof `, rate limited per port. A packet the per-port limit
|
|
cannot log gets a line with the prefix `hive-egress-spoof-unmetered `
|
|
from a limit shared by all ports. A container sending from its leased
|
|
address is never affected, so nothing legitimate breaks.
|
|
- `log`: the same lines and counters, and the frame goes through. For
|
|
finding out why a legitimate container is refused on a live hive.
|
|
|
|
With `log`, the identity squid takes from the source of an intercepted
|
|
flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.network.egress.intercept = {
|
|
enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports";
|
|
|
|
httpPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 3129;
|
|
description = ''
|
|
The port on the bridge address where squid listens for the redirected
|
|
TCP port 80 traffic.
|
|
'';
|
|
};
|
|
|
|
httpsPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 3130;
|
|
description = ''
|
|
The port on the bridge address where squid listens for TCP port 443
|
|
traffic: what the ClientHello relay sends on, or with `refuseEch` off
|
|
the redirected traffic itself.
|
|
'';
|
|
};
|
|
|
|
relayPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 3131;
|
|
description = ''
|
|
The port on the bridge address where `hive-priv egress-ech-relay`
|
|
listens for the redirected TCP port 443 traffic (`refuseEch`).
|
|
'';
|
|
};
|
|
|
|
refuseEch = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
description = ''
|
|
Refuse a TLS ClientHello that carries the `encrypted_client_hello`
|
|
extension on the redirected port 443 traffic. Squid 7.6 sees only the
|
|
outer name of such a hello, so without this an agent that holds an ECH
|
|
configuration could dial an allowed name and reach another site behind
|
|
the same address.
|
|
|
|
`hive-priv egress-ech-relay` reads the first TLS record of each
|
|
redirected connection and, if the hello is clean, sends the connection
|
|
on to squid. The extension of a client with no configuration (GREASE,
|
|
which Chrome and Firefox send by default) looks the same as a real one on
|
|
the wire, so it is refused too: those clients fail on the redirected
|
|
path and work through the proxy variables. An agent in `log` mode is
|
|
observed instead: its hello goes on and the relay logs
|
|
`hive-egress-ech-would-refuse`. Turn this off to send port 443
|
|
straight to squid, with ECH unchecked.
|
|
'';
|
|
};
|
|
|
|
agentAllowFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.path;
|
|
default = null;
|
|
description = ''
|
|
The squid include `hive-c0re` renders with each agent's own rules, in
|
|
which an agent in `log` mode has an `acl agent_log_<name> proxy_auth
|
|
<name>` line. The ClientHello relay (`refuseEch`) reads it to tell a
|
|
`log` agent, whose ECH hello is let through and logged as
|
|
`hive-egress-ech-would-refuse`, from a restricted one, whose hello is
|
|
refused. Squid reads the same file for the same split. `null` or an
|
|
unreadable file lists no `log` agent, so every agent's ECH hello is
|
|
refused.
|
|
'';
|
|
};
|
|
|
|
unrestrictedFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.path;
|
|
default = null;
|
|
description = ''
|
|
The file hive-c0re renders with the names of the agents that hold the
|
|
`unrestricted_internet` flag, one per line. The `direct_mac` set of the
|
|
`inet hive_egress` table holds the MACs of the ports of the agents it
|
|
lists and of every container that is not an agent, and the redirect
|
|
skips them. `null` or an unreadable file lists no agent, so every agent's
|
|
web traffic is redirected. The file is read when a port is bound, on
|
|
each pass of the 30 second timer and when the file changes.
|
|
'';
|
|
};
|
|
};
|
|
|
|
options.services.hyperhive.network.egress.leaseScript = lib.mkOption {
|
|
type = lib.types.path;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = leaseScript;
|
|
defaultText = lib.literalMD "the script that loads a lease into the bridge table";
|
|
description = ''
|
|
The program dnsmasq runs on each lease change (its `dhcp-script`).
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.network.egress.ruleset = lib.mkOption {
|
|
type = lib.types.lines;
|
|
internal = true;
|
|
readOnly = true;
|
|
default = rulesetText;
|
|
defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions";
|
|
description = ''
|
|
The nft file the `hive-egress` unit loads, as text.
|
|
'';
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15;
|
|
message = ''
|
|
services.hyperhive.network.bridgePrefixLength must be 15 or more while
|
|
egress.intercept.refuseEch is on: the relay names a flow's source with
|
|
the low 17 bits of the agent's address.
|
|
'';
|
|
}
|
|
{
|
|
assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
|
|
message = ''
|
|
boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}).
|
|
With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8
|
|
instead of dropping it as a martian, which leaves the hive_egress
|
|
table as the only barrier between an agent and the host's loopback services.
|
|
'';
|
|
}
|
|
];
|
|
|
|
services.hyperhive.network.exposeHostPorts = lib.mkIf refuseEch [ intercept.relayPort ];
|
|
|
|
# The relay is not part of the tables: with it down, the redirect of port 443
|
|
# reaches a closed port and the agent's connection is refused.
|
|
systemd.services.hive-egress-ech-relay = lib.mkIf refuseEch {
|
|
description = "refuse TLS hellos with encrypted client hello on the way to squid's HTTPS intercept port";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "hive-egress.service" ];
|
|
after = [
|
|
"hive-egress.service"
|
|
"systemd-tmpfiles-setup.service"
|
|
"systemd-tmpfiles-resetup.service"
|
|
];
|
|
environment = lib.optionalAttrs (intercept.agentAllowFile != null) {
|
|
HIVE_EGRESS_AGENT_ALLOW = toString intercept.agentAllowFile;
|
|
};
|
|
serviceConfig = {
|
|
# The pool is local, so the relay can bind to a source address in it.
|
|
ExecStartPre = "+${pkgs.iproute2}/bin/ip route replace local ${relayPool} dev lo";
|
|
ExecStart = relayCommand;
|
|
ExecStopPost = "+-${pkgs.iproute2}/bin/ip route del local ${relayPool} dev lo";
|
|
# The bridge address may not be assigned yet at first start.
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
# The packet mark needs CAP_NET_ADMIN. The include is `squid`'s group's.
|
|
DynamicUser = true;
|
|
SupplementaryGroups = lib.optional (intercept.agentAllowFile != null) "squid";
|
|
AmbientCapabilities = "CAP_NET_ADMIN";
|
|
CapabilityBoundingSet = "CAP_NET_ADMIN";
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
PrivateDevices = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectKernelModules = true;
|
|
ProtectKernelLogs = true;
|
|
ProtectControlGroups = true;
|
|
RestrictAddressFamilies = "AF_INET";
|
|
RestrictNamespaces = true;
|
|
RestrictRealtime = true;
|
|
RestrictSUIDSGID = true;
|
|
LockPersonality = true;
|
|
SystemCallArchitectures = "native";
|
|
};
|
|
};
|
|
|
|
systemd.services.hive-egress = {
|
|
description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-pre.target" ];
|
|
before = [ "network-pre.target" ];
|
|
# `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset`
|
|
# is on: start after it, follow its restarts and re-apply on its reload.
|
|
# The tmpfiles units link `hivePriv`.
|
|
after = [
|
|
"nftables.service"
|
|
"systemd-tmpfiles-setup.service"
|
|
"systemd-tmpfiles-resetup.service"
|
|
];
|
|
partOf = [ "nftables.service" ];
|
|
unitConfig.ReloadPropagatedFrom = [ "nftables.service" ];
|
|
reloadIfChanged = true;
|
|
path = pinPath;
|
|
environment = pinEnvironment;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
# The tables load with the bindings of the running containers' ports and
|
|
# the leases of dnsmasq's file in the same transaction, so a reload never
|
|
# leaves the bridge with empty sets.
|
|
ExecStart = loadCommand;
|
|
ExecReload = loadCommand;
|
|
ExecStop = [
|
|
"-${nft} delete table inet hive_egress"
|
|
"-${nft} delete table bridge hive_egress"
|
|
];
|
|
};
|
|
};
|
|
|
|
# Binds one port of the bridge once its veth exists: the run waits for the
|
|
# container to register with systemd-machined.
|
|
systemd.services."hive-egress-pin@" = {
|
|
description = "bind agent bridge port %i to its container's MAC";
|
|
wants = [ "hive-egress.service" ];
|
|
after = [ "hive-egress.service" ];
|
|
path = pinPath;
|
|
environment = pinEnvironment;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = pinCommand "%i";
|
|
};
|
|
};
|
|
|
|
# Looks again at the ports still unbound. A port a rival MAC kept unbound
|
|
# sends nothing, so nothing else would tell the hive it can be bound now.
|
|
# Ports with a binding cost a listing of the bridge and no container is read.
|
|
systemd.services.hive-egress-pin = {
|
|
description = "bind the agent bridge ports that have no binding yet";
|
|
wants = [ "hive-egress.service" ];
|
|
after = [ "hive-egress.service" ];
|
|
path = pinPath;
|
|
environment = pinEnvironment;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = pinCommand "";
|
|
};
|
|
};
|
|
systemd.timers.hive-egress-pin = {
|
|
description = "bind the agent bridge ports that have no binding yet";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnBootSec = "1min";
|
|
OnUnitActiveSec = "30s";
|
|
AccuracySec = "5s";
|
|
};
|
|
};
|
|
|
|
# A pass also refills `direct_mac`, so a change of the unrestricted list
|
|
# takes effect at once instead of at the next tick of the timer.
|
|
systemd.paths.hive-egress-direct = lib.mkIf (cfg.egress.intercept.unrestrictedFile != null) {
|
|
description = "Watch the unrestricted agents list for the egress intercept exemptions";
|
|
wantedBy = [ "paths.target" ];
|
|
pathConfig = {
|
|
PathChanged = toString cfg.egress.intercept.unrestrictedFile;
|
|
Unit = "hive-egress-pin.service";
|
|
};
|
|
};
|
|
|
|
# nspawn names the host end of a container's veth `vb-<machine>` when the
|
|
# container joins a bridge.
|
|
services.udev.extraRules = ''
|
|
SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service"
|
|
'';
|
|
};
|
|
}
|