mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

squid intercepts restricted and log agents' web traffic, named from the bound source address

Browse source
This commit is contained in:
atlas 2026-11-13 05:17:01 +01:00 • committed by mara
commit 712dcf2b25
2 changed files with 1005 additions and 154 deletions
Download patch file Download diff file Expand all files Collapse all files

91
nix/host-modules/hive-egress.nix
View file

@ -5,13 +5,18 @@
# to hold on every host lives here, in tables no firewall backend writes. Only
# `networking.nftables.flushRuleset` removes them; the unit re-applies them then.
#
# The `inet` table holds two chains. `input`: traffic from the agent bridge to host
# The `inet` table holds three chains. `input`: traffic from the agent bridge to host
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
# the host routes from the agent bridge to anywhere else is logged and
# counted, never blocked.
# counted, never blocked. `intercept` (`egress.intercept.enable`, set by
# hive-squid.nix): the web traffic of the bridge's agents is redirected into
# squid's intercept ports. The traffic of the ports in the set `direct_mac` is
# left alone: the service containers and the unrestricted agents. `hive-priv
# egress-pin` and `egress-load` fill that set from the bindings the bridge
# table uses, so a MAC in no binding is intercepted, and squid refuses the flow.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
@ -48,7 +53,12 @@ let
pkgs.iproute2
pkgs.util-linux
];
pinEnvironment.HIVE_EGRESS_BRIDGE = br;
pinEnvironment = {
HIVE_EGRESS_BRIDGE = br;
}
// lib.optionalAttrs (cfg.egress.intercept.unrestrictedFile != null) {
HIVE_EGRESS_UNRESTRICTED = toString cfg.egress.intercept.unrestrictedFile;
};
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
exec ${hivePriv} egress-lease "$@"
@ -71,6 +81,24 @@ let
family: addr:
''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept'';
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
# the interface the route to the destination leaves by: the bridge (another
# container), `lo` (an address of the host: `fib daddr type local`) and the
# mesh interface are not egress. The redirect rewrites the destination to
# the bridge address and the port of squid's intercept listener, which reads
# the original destination from conntrack. IPv6 is left alone: the agent
# bridge has no IPv6 addressing.
interceptChain = ''
chain intercept {
type nat hook prerouting priority dstnat; policy accept;
iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return
iifname "${br}" meta nfproto ipv4 fib daddr type local return
iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort}
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${toString cfg.egress.intercept.httpsPort}
}
'';
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
@ -179,6 +207,8 @@ let
iifname "${br}" ip daddr 127.0.0.0/8 counter drop
iifname "${br}" ip6 daddr ::1 counter drop
}
set direct_mac { type ether_addr; }
${lib.optionalString cfg.egress.intercept.enable interceptChain}
${lib.optionalString (cfg.egress.forward == "log") forwardChain}
}
@ -243,9 +273,48 @@ in
address is never affected, so nothing legitimate breaks.
- `log`: the same lines and counters, and the frame goes through. For
finding out why a legitimate container is refused on a live hive.
With `log`, the identity squid takes from the source of an intercepted
flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run.
'';
};
options.services.hyperhive.network.egress.intercept = {
enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports";
httpPort = lib.mkOption {
type = lib.types.port;
default = 3129;
description = ''
The port on the bridge address where squid listens for the redirected
TCP port 80 traffic.
'';
};
httpsPort = lib.mkOption {
type = lib.types.port;
default = 3130;
description = ''
The port on the bridge address where squid listens for the redirected
TCP port 443 traffic.
'';
};
unrestrictedFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The file hive-c0re renders with the names of the agents that hold the
`unrestricted_internet` flag, one per line. The `direct_mac` set of the
`inet hive_egress` table holds the MACs of the ports of the agents it
lists and of every container that is not an agent, and the redirect
skips them. `null` or an unreadable file lists no agent, so every agent's
web traffic is redirected. The file is read when a port is bound, on
each pass of the 30 second timer and when the file changes.
'';
};
};
options.services.hyperhive.network.egress.leaseScript = lib.mkOption {
type = lib.types.path;
internal = true;
@ -274,8 +343,9 @@ in
assertion = cfg.egress.forward != "drop";
message = ''
services.hyperhive.network.egress.forward = "drop" is not available yet:
the interception that lets an agent's web traffic through squid has not
landed, so a drop would cut every direct connection with no replacement.
the traffic squid does not handle (the unrestricted agents, the service
containers, every port but 80 and 443) has no rule that lets it through, so a
drop would cut it.
Use "log" or "accept".
'';
}
@ -361,6 +431,17 @@ in
};
};
# A pass also refills `direct_mac`, so a change of the unrestricted list
# takes effect at once instead of at the next tick of the timer.
systemd.paths.hive-egress-direct = lib.mkIf (cfg.egress.intercept.unrestrictedFile != null) {
description = "Watch the unrestricted agents list for the egress intercept exemptions";
wantedBy = [ "paths.target" ];
pathConfig = {
PathChanged = toString cfg.egress.intercept.unrestrictedFile;
Unit = "hive-egress-pin.service";
};
};
# nspawn names the host end of a container's veth `vb-<machine>` when the
# container joins a bridge.
services.udev.extraRules = ''

113
nix/host-modules/hive-squid.nix
View file

@ -1,8 +1,9 @@
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
# must carry a per-agent proxy credential, checked against the users file
# hive-c0re renders. A credential's agent either has the swarm's
# plain-HTTP hosts and never decrypts: the one TLS key it loads is generated at
# each start, signs nothing and is trusted by no agent. A request through the
# proxy port must carry a per-agent proxy credential, checked against the users
# file hive-c0re renders. A credential's agent either has the swarm's
# `unrestricted_internet` flag, and may then reach any destination the deny
# rules leave, or reaches only the listed names: the base list, the allow-list
# the swarm keeps plus this host's `allowedDomains` (rendered by hive-c0re
@ -15,6 +16,15 @@
# An agent sends its credential
# once it has fetched it (../agent-modules/egress-proxy.nix); the exporter,
# journal forwarding and the self-check below show the proxy works.
#
# Web traffic that ignores the proxy variables reaches squid anyway: hive-egress.nix
# redirects the TCP 80 and 443 flows of every agent that does not hold the flag
# into two intercept ports. Such a flow carries no credential, so the agent is
# the one whose container holds the flow's source address, which
# `hive-priv egress-whois` reads from the bindings and leases (the external ACL
# `hive_agent`); a source it cannot name is refused. The listed-names and deny
# rules are the same ones, matched on the identity from that helper. HTTPS is
# peeked at for its SNI and spliced: squid checks the name and never decrypts.
{
pkgs,
lib,
@ -30,6 +40,14 @@ let
listen = "${netCfg.bridgeIp}:${toString port}";
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
exporterListen = "127.0.0.1:9301";
interceptCfg = netCfg.egress.intercept;
# The path every platform container's unit runs hive-priv by (hive-priv.nix
# points it at the current build). Squid runs `egress-whois` through it.
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
# Written at each start of the unit by `interceptCert`: `https_port` requires
# a certificate and key, and nothing uses them, since every flow is spliced.
interceptCertFile = "/run/squid/intercept.crt";
interceptKeyFile = "/run/squid/intercept.key";
denyCanary = "deny-canary.invalid";
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
@ -92,11 +110,41 @@ let
# refusal into an allow. The `dst` denies cost an unlisted name a lookup on
# the host's resolver before it is refused.
# The swarm's deny list (`denyFile`) sits right after those destination
# denies and before `deny !authed`, so a denied name is refused before any
# denies and before `deny !icpt !authed`, so a denied name is refused before any
# credential check and before every allow, in every mode: its rules name the
# destination first, and a swarm-wide one names no user at all.
# A flow on an intercept port has no credential: `icpt` is those ports and
# `icpt_agent` the helper's answer for the flow's source, which also sets the
# user that `ext_user` ACLs and the log read. A source the helper cannot name
# is refused first. The first request of an intercepted HTTPS connection is
# the CONNECT squid builds from the destination address before it has read
# the ClientHello (`first_hop`); it passes once the address denies have, so
# squid can peek. The CONNECT it builds next has the SNI as its host and meets
# every rule; with no SNI the host stays an address, which the IP-literal
# denies refuse, and `host_verify_strict` answers 409 for an SNI that does not
# resolve to the destination. `!icpt` guards each rule that reads the proxy
# credential, since squid logs a warning when such an ACL meets an
# intercepted request; the rules for those read `ext_user`.
squidConf = ''
http_port ${listen}
http_port ${netCfg.bridgeIp}:${toString interceptCfg.httpPort} intercept name=icpt_http
https_port ${netCfg.bridgeIp}:${toString interceptCfg.httpsPort} intercept ssl-bump name=icpt_tls tls-cert=${interceptCertFile} tls-key=${interceptKeyFile} generate-host-certificates=off
# Never decrypt: peek at the ClientHello for the SNI, then splice every
# connection. The decision on a name is made by `http_access` on the CONNECT
# that carries it. A connection whose first bytes are not a ClientHello
# (or not an HTTP request, on the other port) is refused.
acl icpt myportname icpt_http icpt_tls
acl first_hop at_step SslBump1
ssl_bump peek first_hop
ssl_bump splice all
host_verify_strict on
# The agent that holds the source address of a flow on an intercept port.
# An answer is kept for 10 seconds, a refusal for 2.
external_acl_type hive_agent ttl=10 negative_ttl=2 children-startup=1 children-max=4 %SRC ${hivePriv} egress-whois
acl icpt_agent external hive_agent
pid_filename /run/squid/squid.pid
cache_log stdio:/var/log/squid/cache.log
coredump_dir /var/cache/squid
@ -161,6 +209,8 @@ let
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny icpt !icpt_agent
http_access allow icpt first_hop CONNECT !to_localhost !to_linklocal !to_bridge !to_private
http_access deny ip_v4_host
http_access deny ip_v6_host
http_access deny to_localhost
@ -168,15 +218,27 @@ let
http_access deny to_bridge
http_access deny to_private
include ${cfg.denyFile}
http_access deny !authed
http_access deny !icpt !authed
${lib.optionalString (base != [ ]) "http_access allow base_dst"}
include ${cfg.agentAllowFile}
http_access allow unrestricted
http_access allow !icpt unrestricted
http_access deny !allowed_dst
http_access allow allowed_dst
http_access deny all
'';
# A throwaway self-signed key and certificate for the intercept HTTPS port,
# written to the two paths given. Squid refuses to start without them and
# loads them to build its TLS context; it never presents them, since every
# connection is spliced.
interceptCert = pkgs.writeShellScript "squid-intercept-cert" ''
set -eu
umask 077
${lib.getExe pkgs.openssl} req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \
-nodes -subj /CN=hive-squid-intercept -days 3650 \
-keyout "$1" -out "$2"
'';
# `squid -k parse` on the configuration with the includes pointing at an empty
# file. The nixpkgs module runs the same parse on the real text, which
# cannot open the includes in the build sandbox.
@ -186,11 +248,13 @@ let
nativeBuildInputs = [ config.services.squid.package ];
}
''
squid -k parse -f ${
${interceptCert} key.pem cert.pem
sed -e "s|${interceptKeyFile}|$PWD/key.pem|; s|${interceptCertFile}|$PWD/cert.pem|" ${
pkgs.writeText "squid.conf" (
builtins.replaceStrings [ cfg.agentAllowFile cfg.denyFile ] [ "/dev/null" "/dev/null" ] squidConf
)
}
} > squid.conf
squid -k parse -f squid.conf
touch $out
'';
@ -385,12 +449,13 @@ in
defaultText = lib.literalMD "`agent-allow.conf`, in the directory of `usersFile`";
description = ''
Squid configuration that hive-c0re generates and the main configuration
includes after every destination deny and `deny !authed`, where the
includes after every destination deny and `deny !icpt !authed`, where the
base list is allowed: for each agent of this hive the swarm lists
names for, an ACL for its user, an ACL for its names and the
`http_access allow` joining the two. After those, for each agent in
`log` mode, an `http_access allow` for names `allowed_dst` lacks that
annotates the request `would_deny=1`. Empty until the controller has a
names for, an ACL for its proxy user, one for the user the intercept
helper names, an ACL for its names and two `http_access allow` rules
joining them, one for the proxy port and one for the intercept ports.
After those, for each agent in `log` mode, the same pair of rules for
names `allowed_dst` lacks, which annotate the request `would_deny=1`. Empty until the controller has a
list or an agent is in `log` mode. Squid is reloaded when it changes. Owned by `hive-core` and
readable by the `squid` group.
'';
@ -404,9 +469,10 @@ in
description = ''
Squid configuration that hive-c0re generates from the swarm's deny list
(`/api/egress/denied-domains`) and the main configuration includes after
the destination denies and before `deny !authed` and every allow: a
`http_access deny` for the swarm-wide names, and one per agent of this
hive the swarm lists names for. A denied name is refused for every user
the destination denies and before `deny !icpt !authed` and every allow: a
`http_access deny` for the swarm-wide names, and two per agent of this
hive the swarm lists names for, one for the proxy port and one for the
intercept ports. A denied name is refused for every user
in every egress mode, whether or not an allow rule covers it. The file
is empty, and squid configured as without it, until the swarm lists a
name. Squid is reloaded when it changes. Owned by `hive-core` and readable
@ -478,7 +544,15 @@ in
config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable {
# Reachable from agent containers and nowhere else: this opens the port on
# the bridge interface only.
services.hyperhive.network.exposeHostPorts = [ port ];
services.hyperhive.network.exposeHostPorts = [
port
interceptCfg.httpPort
interceptCfg.httpsPort
];
services.hyperhive.network.egress.intercept = {
enable = lib.mkDefault true;
inherit (cfg) unrestrictedFile;
};
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
@ -555,6 +629,11 @@ in
];
systemd.services.squid = {
# Before the module's own pre-start, which parses the configuration and
# so loads the certificate.
preStart = lib.mkBefore ''
${interceptCert} ${interceptKeyFile} ${interceptCertFile}
'';
serviceConfig = sandbox // {
User = "squid";
Group = "squid";