Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/host-modules/hive-egress.nix

707 lines
31 KiB
Nix

# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`,
# loaded by a systemd oneshot. They are independent of the NixOS firewall:
# `networking.firewall` renders only the rules of the backend in use
# (`iptables` unless `networking.nftables.enable` is set), so a rule that has
# to hold on every host lives here, in tables no firewall backend writes. Only
# `networking.nftables.flushRuleset` removes them; the unit re-applies them then.
#
# The `inet` table holds three chains. `input`: traffic from the agent bridge to host
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on. `forward` (`egress.forward`): traffic from the
# agent bridge to anywhere else is logged and counted (`log`) or logged and
# refused (`reject`). `intercept` (`egress.intercept.enable`, set by
# hive-squid.nix): the web traffic of the bridge's agents is redirected into
# squid's intercept ports (port 443 through the ClientHello relay). Both leave
# alone the ports in `direct_mac`, the service containers and the unrestricted
# agents. `hive-priv egress-pin` and `egress-load` fill that set from the
# bridge table's bindings, so a MAC in no binding is intercepted and refused.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
# traffic between two ports is bridged: the `inet` hooks see it as coming from
# the bridge device, and only a bridge hook knows the port it came in on
# (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC
# may use are not known until a container runs, so they are set contents.
# `hive-priv egress-load` puts the current ones into the same transaction as the
# tables; `egress-pin` (per new port from udev, and from a timer for the ports
# still unbound) and dnsmasq's `dhcp-script` (per lease and renewal) keep them
# current. `hive-priv`'s `egress.rs` documents how.
{
lib,
config,
pkgs,
...
}:
let
cfg = config.services.hyperhive.network;
egressCfg = config.services.hyperhive.deploy.hive-controller.egress;
nft = lib.getExe pkgs.nftables;
br = cfg.bridgeName;
# With `refuseEch`, the redirect of port 443 ends at `hive-priv
# egress-ech-relay`, which refuses a ClientHello that carries the ECH
# extension and sends the rest on to squid's HTTPS intercept port. The relay
# connects to the address the agent dialed, bound to a source address of the
# pool and carrying the mark; the `output` chain DNATs those connections to
# squid. `egress_ech.rs` derives the pool address from the agent's and
# `egress-whois` maps it back, so the pool is the one defined there.
intercept = egressCfg.intercept;
refuseEch = intercept.enable && intercept.refuseEch;
relayPool = "198.18.0.0/15";
relayMark = "0x45434800";
relayCommand = "${hivePriv} egress-ech-relay ${cfg.bridgeIp}:${toString intercept.relayPort} ${relayMark}";
# The sub-commands of hive-priv that fill the bridge table's sets. The path is
# the one every platform container's unit uses (hive-priv.nix points it at the
# current build).
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
pinCommand = args: lib.trim "${hivePriv} egress-pin ${args}";
loadCommand = "${hivePriv} egress-load ${ruleset}";
pinPath = [
pkgs.nftables
pkgs.iproute2
pkgs.util-linux
];
# systemd gives a oneshot unit no start timeout unless it is set. A pass takes
# the lock for up to 30 s and runs each child for up to 10 s (hive-priv's
# `egress.rs`); this is above both, and kills a pass that still hangs so the
# timer and udev can start the next.
pinTimeout = "90s";
pinEnvironment = {
HIVE_EGRESS_BRIDGE = br;
}
// lib.optionalAttrs (egressCfg.intercept.unrestrictedFile != null) {
HIVE_EGRESS_UNRESTRICTED = toString egressCfg.intercept.unrestrictedFile;
}
// lib.optionalAttrs (egressCfg.connectionsFile != null) {
HIVE_EGRESS_CONNECTIONS = toString egressCfg.connectionsFile;
};
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
exec ${hivePriv} egress-lease "$@"
'';
# Logging is per source address, so one busy agent cannot use up the budget
# of another: a burst of 20 lines, then one a second, and an address is
# forgotten after a minute without a new flow. The rule decides after it
# logs. A flow the rule cannot log falls through to the unmetered rule below
# it: the address was over its rate, or the set is full (`update` then fails
# and the rule stops matching), which a container spoofing source addresses
# can bring about. Only that case reaches the unmetered rule, so a flow
# gets one line at most. The unmetered rule has one budget for all
# addresses, 5 lines a second after a burst of 50, about 100 MB of journal a
# day when it never rests.
logRate = "1/second burst 20 packets";
unmeteredRate = "5/second burst 50 packets";
fromBridge = ''iifname "${br}" oifname != "${br}"'';
# `log` accepts what it logs. `reject` logs and refuses in one rule, so a
# line always names a refused flow. A TCP flow gets a reset, which fails the
# connect at once (to a TCP socket an ICMP error is a soft error, and the
# connect would stall); anything else gets ICMP admin-prohibited. A rule that
# refuses is therefore a pair: one for TCP, one for the rest.
isReject = egressCfg.forward == "reject";
logPrefix = if isReject then "hive-egress-reject" else "hive-egress-would-drop";
refuse = {
tcp = "meta l4proto tcp";
other = "meta l4proto != tcp";
};
verdict = {
tcp = "reject with tcp reset";
other = "reject with icmpx type admin-prohibited";
};
decide =
statements:
if isReject then
map
(
k:
lib.concatStringsSep " " (
lib.filter (x: x != "") [
fromBridge
refuse.${k}
statements
verdict.${k}
]
)
)
[
"tcp"
"other"
]
else
[ "${fromBridge} ${statements} accept" ];
logRules =
family: addr:
decide ''ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "${logPrefix} "'';
# In `log` mode the unmetered rule has no verdict: the chain's last rule is
# the verdict for every flow that gets past the logging.
unmeteredRules =
let
statements = ''ct state new limit rate ${unmeteredRate} log prefix "${logPrefix}-unmetered "'';
in
if isReject then decide statements else [ "${fromBridge} ${statements}" ];
# What no rule above decided, a packet that is no new flow included.
finalRules = if isReject then decide "" else [ "${fromBridge} accept" ];
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
# the interface the route to the destination leaves by: the bridge (another
# container) and `lo` (an address of the host: `fib daddr type local`) are
# not egress. The redirect rewrites the destination to
# the bridge address and the port of squid's intercept listener (for port 443
# with `refuseEch`, of the relay), which reads the original destination from
# conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing.
interceptChain = ''
chain intercept {
type nat hook prerouting priority dstnat; policy accept;
iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return
iifname "${br}" meta nfproto ipv4 fib daddr type local return
iifname "${br}" meta nfproto ipv4 fib daddr oifname "${br}" return
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString egressCfg.intercept.httpPort}
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${
toString (if refuseEch then intercept.relayPort else intercept.httpsPort)
}
}
''
+ lib.optionalString refuseEch ''
chain intercept_relay {
type nat hook output priority dstnat; policy accept;
meta nfproto ipv4 ip saddr ${relayPool} meta mark ${relayMark} tcp dport 443 counter dnat ip to ${cfg.bridgeIp}:${toString intercept.httpsPort}
}
'';
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
# so it allows nothing those refuse; a `reject` ends the packet. The MACs
# in `direct_mac` (the unrestricted agents and the service
# containers, the traffic the intercept chain leaves alone) come first in
# both modes, so a `log` line names what `reject` would refuse. Traffic to
# the host's own addresses (squid, dnsmasq, the gateway) is delivered
# locally and never reaches the forward hook.
forwardChain = ''
set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; }
set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; }
chain forward {
type filter hook forward priority filter - 10; policy accept;
${fromBridge} ether saddr @direct_mac accept
${fromBridge} meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept
${fromBridge} counter
${lib.concatStringsSep "\n " (
logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules
)}
}
'';
# The bridge table. A frame of a port goes through `prerouting` in this
# order: the port must be bound to the frame's source MAC, then the frame's
# protocol decides which source address it may carry. Everything else logs in
# `spoof_<reason>`, then `spoof` drops it (`antiSpoof = "log"`: accepts).
#
# - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second
# as the 48 bits that sit at the `chaddr` field of a DHCP message. A port
# with no pair sends nothing, DHCP included.
# - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its
# lease, and a renewal moves the timeout.
# - IPv6 has no leases: a source is link-local, or `::` in the packets that
# duplicate address detection and multicast listener reports send before an
# address exists. The bridge carries no other IPv6 traffic.
# - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`,
# so a port cannot lease, renew or release another's address. A client
# message to the broadcast address or the bridge address then passes from
# any source address: a client with no address sends from 0.0.0.0, and one
# whose pair has left `lease_v4` renews (unicast) or rebinds (broadcast)
# from its own address. The reply puts the pair back.
# - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or
# leased to that MAC.
# - A VLAN tag is refused: a tagged frame would carry its packet past the
# protocol checks. The bridge uses no VLANs.
#
# Logging is per ingress port, in the style of the forward chain: a burst of
# 20 lines, then one a second, and a port is forgotten after a minute. A
# packet its port cannot log (over its rate, or the set is full) falls to the
# unmetered line, which has one budget for all ports and reasons: the named
# limit `spoof_unmetered`. It logs packets, not flows: the bridge hook has no
# connection tracking.
spoofVerdict = if egressCfg.antiSpoof == "drop" then "drop" else "accept";
# The log prefix names the check that refused the frame: `mac` (the port is
# not bound to the source MAC), `vlan`, `dhcp` (the `chaddr` is not the
# port's), `v4`, `v6`, and `arp` (the ARP sender's MAC or address is not the
# port's). Each reason has its own chain and counters; `spoof` holds the
# verdict and one counter for every refused frame.
spoofReasons = [
"mac"
"vlan"
"dhcp"
"v4"
"v6"
"arp"
];
spoofChain = reason: ''
chain spoof_${reason} {
update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof ${reason} " goto spoof
limit name "spoof_unmetered" counter log prefix "hive-egress-spoof-unmetered ${reason} " goto spoof
counter goto spoof
}
'';
bridgeTable = ''
table bridge hive_egress {
set port_mac { type ifname . ether_addr; }
set port_chaddr { typeof iifname . @th,288,48; }
set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; }
set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; }
limit spoof_unmetered { rate ${unmeteredRate} }
chain spoof {
counter ${spoofVerdict}
}
${lib.concatMapStringsSep "\n" spoofChain spoofReasons}
chain l3_v4 {
udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof_dhcp
ip daddr { 255.255.255.255, ${cfg.bridgeIp} } udp sport 68 udp dport 67 accept
ether saddr . ip saddr @lease_v4 accept
goto spoof_v4
}
chain l3_v6 {
ip6 saddr fe80::/10 accept
ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept
goto spoof_v6
}
chain l3_arp {
iifname . arp saddr ether != @port_mac goto spoof_arp
arp saddr ip 0.0.0.0 accept
arp saddr ether . arp saddr ip @lease_v4 accept
goto spoof_arp
}
chain prerouting {
type filter hook prerouting priority -300; policy accept;
meta ibrname != "${br}" accept
iifname . ether saddr != @port_mac goto spoof_mac
ether type { 8021q, 8021ad } goto spoof_vlan
ether type ip goto l3_v4
ether type ip6 goto l3_v6
ether type arp goto l3_arp
}
}
'';
# One transaction: `add` + `delete` drop an old copy of each table and the
# definitions recreate them, so the file loads the same on first start, on a
# reload and after something else flushed the ruleset.
rulesetText = ''
add table inet hive_egress
delete table inet hive_egress
add table bridge hive_egress
delete table bridge hive_egress
table inet hive_egress {
chain input {
type filter hook input priority filter - 10; policy accept;
iifname "${br}" ip daddr 127.0.0.0/8 counter drop
iifname "${br}" ip6 daddr ::1 counter drop
${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''}
}
set direct_mac { type ether_addr; }
set agent_allow { type ether_addr . ipv4_addr . inet_service; }
${lib.optionalString egressCfg.intercept.enable interceptChain}
${lib.optionalString (egressCfg.forward != "accept") forwardChain}
}
${bridgeTable}
'';
routeLocalnetKeys = [
"net.ipv4.conf.all.route_localnet"
"net.ipv4.conf.default.route_localnet"
"net.ipv4.conf.${cfg.bridgeName}.route_localnet"
];
isOn =
v:
builtins.elem v [
1
"1"
true
];
ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset;
in
{
options.services.hyperhive.deploy.hive-controller.egress.forward = lib.mkOption {
type = lib.types.enum [
"accept"
"log"
"reject"
];
default = "log";
description = ''
What the `inet hive_egress` table does with traffic the host forwards
from the agent bridge to anywhere but the bridge. In `log` and
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
agents and the service containers) is accepted and never logged, and so
is TCP from an agent to an address and port in the `agent_allow` set
(`connectionsFile`).
- `accept`: nothing; the table has no forward chain.
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
rate limited per source address, and counted. A flow the per-source limit
cannot log gets a line with the prefix `hive-egress-would-drop-unmetered `
from a limit shared by all addresses. Nothing is blocked. The lines are
the flows `reject` would refuse. The source address maps to an agent
through the hive's DHCP leases.
- `reject`: the same lines with the prefix `hive-egress-reject ` and
`hive-egress-reject-unmetered `, and every other packet from the bridge
is refused as well: a TCP flow with a reset, anything else with ICMP
administratively prohibited. Only the web traffic that the intercept
chain redirects into squid still gets out.
'';
};
options.services.hyperhive.deploy.hive-controller.egress.connectionsFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The file hive-c0re renders with the SSH connections each agent may open,
one `<agent> <ipv4> <port>` per line: the addresses behind the `host` of
each connection in the permissions record of an agent whose `ssh` switch
is on. `hive-priv egress-pin` turns each line into elements of the
`agent_allow` set of the `inet hive_egress` table, keyed by the MAC of
the agent's port, and the forward chain accepts TCP to an address and
port in it before it logs or refuses. The file is read when a port is
bound, on each pass of the 30 second timer and when the file changes.
`null` leaves the set alone, empty: no agent has an allowance.
'';
};
options.services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkOption {
type = lib.types.enum [
"drop"
"log"
];
default = "log";
description = ''
What the `bridge hive_egress` table does with a frame whose source is not
the sending container's own. Every port of the agent bridge is checked:
the frame's MAC must be the one bound to the port when its container
started, and its IPv4 address the one leased to that MAC. A port with no
bound MAC sends nothing. Allowed besides the leased address: DHCP from
a DHCP client message to the broadcast or bridge address from any source
address whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local
sources (and `::` in neighbour discovery and listener reports).
- `drop`: the frame is dropped and logged with the prefix
`hive-egress-spoof <reason> `, rate limited per port; the reason is the
check that failed: `mac`, `vlan`, `dhcp`, `v4`, `v6` or `arp`. A packet
the per-port limit cannot log gets a line with the prefix
`hive-egress-spoof-unmetered <reason> `, from one limit shared by all
ports and reasons. A container sending from its leased address is
unaffected while its (MAC, IP) pair is in the lease set; a lost pair,
as after a host suspend/resume, cuts the container off until it leases
again.
- `log` (the default): the same lines and counters, and the frame goes
through. For finding out why a legitimate container is refused on a
live hive. It is the default because the lease set does not yet survive
a host suspend/resume; `drop` returns once a resume test proves it and the
`egress-lease: … skipped (stale)` lines confirm the renewal fix on a live hive.
With `log`, the identity squid takes from the source of an intercepted
flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run.
'';
};
options.services.hyperhive.deploy.hive-controller.egress.intercept = {
enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports";
httpPort = lib.mkOption {
type = lib.types.port;
default = 3129;
description = ''
The port on the bridge address where squid listens for the redirected
TCP port 80 traffic.
'';
};
httpsPort = lib.mkOption {
type = lib.types.port;
default = 3130;
description = ''
The port on the bridge address where squid listens for TCP port 443
traffic: what the ClientHello relay sends on, or with `refuseEch` off
the redirected traffic itself.
'';
};
relayPort = lib.mkOption {
type = lib.types.port;
default = 3131;
description = ''
The port on the bridge address where `hive-priv egress-ech-relay`
listens for the redirected TCP port 443 traffic (`refuseEch`).
'';
};
refuseEch = lib.mkOption {
type = lib.types.bool;
default = true;
description = ''
Refuse a TLS ClientHello that carries the `encrypted_client_hello`
extension on the redirected port 443 traffic. Squid 7.6 sees only the
outer name of such a hello, so without this an agent that holds an ECH
configuration could dial an allowed name and reach another site behind
the same address.
`hive-priv egress-ech-relay` reads the first TLS record of each
redirected connection and, if the hello is clean, sends the connection
on to squid. The extension of a client with no configuration (GREASE,
which Chrome and Firefox send by default) looks the same as a real one on
the wire, so it is refused too: those clients fail on the redirected
path and work through the proxy variables. An agent in `log` mode is
observed instead: its hello goes on and the relay logs
`hive-egress-ech-would-refuse`. Turn this off to send port 443
straight to squid, with ECH unchecked.
'';
};
agentAllowFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The squid include `hive-c0re` renders with each agent's own rules, in
which an agent in `log` mode has an `acl agent_log_<name> proxy_auth
<name>` line. The ClientHello relay (`refuseEch`) reads it to tell a
`log` agent, whose ECH hello is let through and logged as
`hive-egress-ech-would-refuse`, from a restricted one, whose hello is
refused. Squid reads the same file for the same split. `null` or an
unreadable file lists no `log` agent, so every agent's ECH hello is
refused.
'';
};
unrestrictedFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The file hive-c0re renders with the names of the agents that hold the
`unrestricted_internet` flag, one per line. The `direct_mac` set of the
`inet hive_egress` table holds the MACs of the ports of the agents it
lists and of every container that is not an agent, and the redirect
skips them. `null` or an unreadable file lists no agent, so every agent's
web traffic is redirected. The file is read when a port is bound, on
each pass of the 30 second timer and when the file changes.
'';
};
};
options.services.hyperhive.network.egress.leaseScript = lib.mkOption {
type = lib.types.path;
internal = true;
readOnly = true;
default = leaseScript;
defaultText = lib.literalMD "the script that loads a lease into the bridge table";
description = ''
The program dnsmasq runs on each lease change and renewal (its
`dhcp-script` with `script-on-renewal`).
'';
};
options.services.hyperhive.network.egress.ruleset = lib.mkOption {
type = lib.types.lines;
internal = true;
readOnly = true;
default = rulesetText;
defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions";
description = ''
The nft file the `hive-egress` unit loads, as text.
'';
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15;
message = ''
services.hyperhive.network.bridgePrefixLength must be 15 or more while
egress.intercept.refuseEch is on: the relay names a flow's source with
the low 17 bits of the agent's address.
'';
}
{
assertion = egressCfg.forward != "reject" || egressCfg.intercept.enable;
message = ''
services.hyperhive.deploy.hive-controller.egress.forward = "reject" needs
services.hyperhive.deploy.hive-controller.egress.intercept.enable = true:
without the intercept nothing redirects the agents' web traffic into
squid, and the reject would refuse all of it.
Enable the intercept (hive-squid.nix does on a hive controller host),
or keep forward = "log".
'';
}
{
assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
message = ''
boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}).
With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8
instead of dropping it as a martian, which leaves the hive_egress
table as the only barrier between an agent and the host's loopback services.
'';
}
];
services.hyperhive.network.exposeHostPorts = lib.mkIf refuseEch [ intercept.relayPort ];
# The relay is not part of the tables: with it down, the redirect of port 443
# reaches a closed port and the agent's connection is refused.
systemd.services.hive-egress-ech-relay = lib.mkIf refuseEch {
description = "refuse TLS hellos with encrypted client hello on the way to squid's HTTPS intercept port";
wantedBy = [ "multi-user.target" ];
wants = [ "hive-egress.service" ];
after = [
"hive-egress.service"
"systemd-tmpfiles-setup.service"
"systemd-tmpfiles-resetup.service"
];
environment = lib.optionalAttrs (intercept.agentAllowFile != null) {
HIVE_EGRESS_AGENT_ALLOW = toString intercept.agentAllowFile;
};
serviceConfig = {
# The pool is local, so the relay can bind to a source address in it.
ExecStartPre = "+${pkgs.iproute2}/bin/ip route replace local ${relayPool} dev lo";
ExecStart = relayCommand;
ExecStopPost = "+-${pkgs.iproute2}/bin/ip route del local ${relayPool} dev lo";
# The bridge address may not be assigned yet at first start.
Restart = "on-failure";
RestartSec = 5;
# The packet mark needs CAP_NET_ADMIN. The include is `squid`'s group's.
DynamicUser = true;
SupplementaryGroups = lib.optional (intercept.agentAllowFile != null) "squid";
AmbientCapabilities = "CAP_NET_ADMIN";
CapabilityBoundingSet = "CAP_NET_ADMIN";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectControlGroups = true;
RestrictAddressFamilies = "AF_INET";
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
LockPersonality = true;
SystemCallArchitectures = "native";
};
};
systemd.services.hive-egress = {
description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)";
wantedBy = [ "multi-user.target" ];
wants = [ "network-pre.target" ];
before = [ "network-pre.target" ];
# `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset`
# is on: start after it, follow its restarts and re-apply on its reload.
# The tmpfiles units link `hivePriv`.
after = [
"nftables.service"
"systemd-tmpfiles-setup.service"
"systemd-tmpfiles-resetup.service"
];
partOf = [ "nftables.service" ];
unitConfig.ReloadPropagatedFrom = [ "nftables.service" ];
reloadIfChanged = true;
path = pinPath;
environment = pinEnvironment;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# The tables load with the bindings of the running containers' ports and
# the leases of dnsmasq's file in the same transaction, so a reload never
# leaves the bridge with empty sets.
ExecStart = loadCommand;
ExecReload = loadCommand;
ExecStop = [
"-${nft} delete table inet hive_egress"
"-${nft} delete table bridge hive_egress"
];
};
};
# Binds one port of the bridge once its veth exists: the run waits for the
# container to register with systemd-machined.
systemd.services."hive-egress-pin@" = {
description = "bind agent bridge port %i to its container's MAC";
wants = [ "hive-egress.service" ];
after = [ "hive-egress.service" ];
path = pinPath;
environment = pinEnvironment;
serviceConfig = {
Type = "oneshot";
ExecStart = pinCommand "%i";
TimeoutStartSec = pinTimeout;
};
};
# Looks again at the ports still unbound. A port a rival MAC kept unbound
# sends nothing, so nothing else would tell the hive it can be bound now.
# Ports with a binding cost a listing of the bridge and no container is read.
systemd.services.hive-egress-pin = {
description = "bind the agent bridge ports that have no binding yet";
wants = [ "hive-egress.service" ];
after = [ "hive-egress.service" ];
path = pinPath;
environment = pinEnvironment;
serviceConfig = {
Type = "oneshot";
ExecStart = pinCommand "";
TimeoutStartSec = pinTimeout;
};
};
systemd.timers.hive-egress-pin = {
description = "bind the agent bridge ports that have no binding yet";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "1min";
OnUnitActiveSec = "30s";
AccuracySec = "5s";
};
};
# A pass also refills `direct_mac`, so a change of the unrestricted list
# takes effect at once instead of at the next tick of the timer.
systemd.paths.hive-egress-direct = lib.mkIf (egressCfg.intercept.unrestrictedFile != null) {
description = "Watch the unrestricted agents list for the egress intercept exemptions";
wantedBy = [ "paths.target" ];
pathConfig = {
PathChanged = toString egressCfg.intercept.unrestrictedFile;
Unit = "hive-egress-pin.service";
};
};
# The same pass refills `agent_allow`, so a connection an operator declares
# is allowed at once.
systemd.paths.hive-egress-connections = lib.mkIf (egressCfg.connectionsFile != null) {
description = "Watch the agents' allowed connections list for the egress forward chain";
wantedBy = [ "paths.target" ];
pathConfig = {
PathChanged = toString egressCfg.connectionsFile;
Unit = "hive-egress-pin.service";
};
};
# nspawn names the host end of a container's veth `vb-<machine>` when the
# container joins a bridge.
services.udev.extraRules = ''
SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service"
'';
};
}