mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

squid intercepts restricted and log agents' web traffic, named from the bound source address

Browse source
This commit is contained in:
atlas 2026-11-13 05:17:01 +01:00 • committed by mara
commit 712dcf2b25
6 changed files with 1005 additions and 154 deletions
Download patch file Download diff file Expand all files Collapse all files

91
nix/host-modules/hive-egress.nix
View file

@ -5,13 +5,18 @@
# to hold on every host lives here, in tables no firewall backend writes. Only
# `networking.nftables.flushRuleset` removes them; the unit re-applies them then.
#
# The `inet` table holds two chains. `input`: traffic from the agent bridge to host
# The `inet` table holds three chains. `input`: traffic from the agent bridge to host
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
# the host routes from the agent bridge to anywhere else is logged and
# counted, never blocked.
# counted, never blocked. `intercept` (`egress.intercept.enable`, set by
# hive-squid.nix): the web traffic of the bridge's agents is redirected into
# squid's intercept ports. The traffic of the ports in the set `direct_mac` is
# left alone: the service containers and the unrestricted agents. `hive-priv
# egress-pin` and `egress-load` fill that set from the bindings the bridge
# table uses, so a MAC in no binding is intercepted, and squid refuses the flow.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
@ -48,7 +53,12 @@ let
pkgs.iproute2
pkgs.util-linux
];
pinEnvironment.HIVE_EGRESS_BRIDGE = br;
pinEnvironment = {
HIVE_EGRESS_BRIDGE = br;
}
// lib.optionalAttrs (cfg.egress.intercept.unrestrictedFile != null) {
HIVE_EGRESS_UNRESTRICTED = toString cfg.egress.intercept.unrestrictedFile;
};
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
exec ${hivePriv} egress-lease "$@"
@ -71,6 +81,24 @@ let
family: addr:
''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept'';
# IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is
# the interface the route to the destination leaves by: the bridge (another
# container), `lo` (an address of the host: `fib daddr type local`) and the
# mesh interface are not egress. The redirect rewrites the destination to
# the bridge address and the port of squid's intercept listener, which reads
# the original destination from conntrack. IPv6 is left alone: the agent
# bridge has no IPv6 addressing.
interceptChain = ''
chain intercept {
type nat hook prerouting priority dstnat; policy accept;
iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return
iifname "${br}" meta nfproto ipv4 fib daddr type local return
iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort}
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${toString cfg.egress.intercept.httpsPort}
}
'';
# Priority `filter - 10` runs before the NixOS firewall's forward handling,
# which sits at `filter` on both backends (`FORWARD` of the iptables backend,
# the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
@ -179,6 +207,8 @@ let
iifname "${br}" ip daddr 127.0.0.0/8 counter drop
iifname "${br}" ip6 daddr ::1 counter drop
}
set direct_mac { type ether_addr; }
${lib.optionalString cfg.egress.intercept.enable interceptChain}
${lib.optionalString (cfg.egress.forward == "log") forwardChain}
}
@ -243,9 +273,48 @@ in
address is never affected, so nothing legitimate breaks.
- `log`: the same lines and counters, and the frame goes through. For
finding out why a legitimate container is refused on a live hive.
With `log`, the identity squid takes from the source of an intercepted
flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run.
'';
};
options.services.hyperhive.network.egress.intercept = {
enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports";
httpPort = lib.mkOption {
type = lib.types.port;
default = 3129;
description = ''
The port on the bridge address where squid listens for the redirected
TCP port 80 traffic.
'';
};
httpsPort = lib.mkOption {
type = lib.types.port;
default = 3130;
description = ''
The port on the bridge address where squid listens for the redirected
TCP port 443 traffic.
'';
};
unrestrictedFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The file hive-c0re renders with the names of the agents that hold the
`unrestricted_internet` flag, one per line. The `direct_mac` set of the
`inet hive_egress` table holds the MACs of the ports of the agents it
lists and of every container that is not an agent, and the redirect
skips them. `null` or an unreadable file lists no agent, so every agent's
web traffic is redirected. The file is read when a port is bound, on
each pass of the 30 second timer and when the file changes.
'';
};
};
options.services.hyperhive.network.egress.leaseScript = lib.mkOption {
type = lib.types.path;
internal = true;
@ -274,8 +343,9 @@ in
assertion = cfg.egress.forward != "drop";
message = ''
services.hyperhive.network.egress.forward = "drop" is not available yet:
the interception that lets an agent's web traffic through squid has not
landed, so a drop would cut every direct connection with no replacement.
the traffic squid does not handle (the unrestricted agents, the service
containers, every port but 80 and 443) has no rule that lets it through, so a
drop would cut it.
Use "log" or "accept".
'';
}
@ -361,6 +431,17 @@ in
};
};
# A pass also refills `direct_mac`, so a change of the unrestricted list
# takes effect at once instead of at the next tick of the timer.
systemd.paths.hive-egress-direct = lib.mkIf (cfg.egress.intercept.unrestrictedFile != null) {
description = "Watch the unrestricted agents list for the egress intercept exemptions";
wantedBy = [ "paths.target" ];
pathConfig = {
PathChanged = toString cfg.egress.intercept.unrestrictedFile;
Unit = "hive-egress-pin.service";
};
};
# nspawn names the host end of a container's veth `vb-<machine>` when the
# container joins a bridge.
services.udev.extraRules = ''

113
nix/host-modules/hive-squid.nix
View file

@ -1,8 +1,9 @@
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
# must carry a per-agent proxy credential, checked against the users file
# hive-c0re renders. A credential's agent either has the swarm's
# plain-HTTP hosts and never decrypts: the one TLS key it loads is generated at
# each start, signs nothing and is trusted by no agent. A request through the
# proxy port must carry a per-agent proxy credential, checked against the users
# file hive-c0re renders. A credential's agent either has the swarm's
# `unrestricted_internet` flag, and may then reach any destination the deny
# rules leave, or reaches only the listed names: the base list, the allow-list
# the swarm keeps plus this host's `allowedDomains` (rendered by hive-c0re
@ -15,6 +16,15 @@
# An agent sends its credential
# once it has fetched it (../agent-modules/egress-proxy.nix); the exporter,
# journal forwarding and the self-check below show the proxy works.
#
# Web traffic that ignores the proxy variables reaches squid anyway: hive-egress.nix
# redirects the TCP 80 and 443 flows of every agent that does not hold the flag
# into two intercept ports. Such a flow carries no credential, so the agent is
# the one whose container holds the flow's source address, which
# `hive-priv egress-whois` reads from the bindings and leases (the external ACL
# `hive_agent`); a source it cannot name is refused. The listed-names and deny
# rules are the same ones, matched on the identity from that helper. HTTPS is
# peeked at for its SNI and spliced: squid checks the name and never decrypts.
{
pkgs,
lib,
@ -30,6 +40,14 @@ let
listen = "${netCfg.bridgeIp}:${toString port}";
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
exporterListen = "127.0.0.1:9301";
interceptCfg = netCfg.egress.intercept;
# The path every platform container's unit runs hive-priv by (hive-priv.nix
# points it at the current build). Squid runs `egress-whois` through it.
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
# Written at each start of the unit by `interceptCert`: `https_port` requires
# a certificate and key, and nothing uses them, since every flow is spliced.
interceptCertFile = "/run/squid/intercept.crt";
interceptKeyFile = "/run/squid/intercept.key";
denyCanary = "deny-canary.invalid";
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
@ -92,11 +110,41 @@ let
# refusal into an allow. The `dst` denies cost an unlisted name a lookup on
# the host's resolver before it is refused.
# The swarm's deny list (`denyFile`) sits right after those destination
# denies and before `deny !authed`, so a denied name is refused before any
# denies and before `deny !icpt !authed`, so a denied name is refused before any
# credential check and before every allow, in every mode: its rules name the
# destination first, and a swarm-wide one names no user at all.
# A flow on an intercept port has no credential: `icpt` is those ports and
# `icpt_agent` the helper's answer for the flow's source, which also sets the
# user that `ext_user` ACLs and the log read. A source the helper cannot name
# is refused first. The first request of an intercepted HTTPS connection is
# the CONNECT squid builds from the destination address before it has read
# the ClientHello (`first_hop`); it passes once the address denies have, so
# squid can peek. The CONNECT it builds next has the SNI as its host and meets
# every rule; with no SNI the host stays an address, which the IP-literal
# denies refuse, and `host_verify_strict` answers 409 for an SNI that does not
# resolve to the destination. `!icpt` guards each rule that reads the proxy
# credential, since squid logs a warning when such an ACL meets an
# intercepted request; the rules for those read `ext_user`.
squidConf = ''
http_port ${listen}
http_port ${netCfg.bridgeIp}:${toString interceptCfg.httpPort} intercept name=icpt_http
https_port ${netCfg.bridgeIp}:${toString interceptCfg.httpsPort} intercept ssl-bump name=icpt_tls tls-cert=${interceptCertFile} tls-key=${interceptKeyFile} generate-host-certificates=off
# Never decrypt: peek at the ClientHello for the SNI, then splice every
# connection. The decision on a name is made by `http_access` on the CONNECT
# that carries it. A connection whose first bytes are not a ClientHello
# (or not an HTTP request, on the other port) is refused.
acl icpt myportname icpt_http icpt_tls
acl first_hop at_step SslBump1
ssl_bump peek first_hop
ssl_bump splice all
host_verify_strict on
# The agent that holds the source address of a flow on an intercept port.
# An answer is kept for 10 seconds, a refusal for 2.
external_acl_type hive_agent ttl=10 negative_ttl=2 children-startup=1 children-max=4 %SRC ${hivePriv} egress-whois
acl icpt_agent external hive_agent
pid_filename /run/squid/squid.pid
cache_log stdio:/var/log/squid/cache.log
coredump_dir /var/cache/squid
@ -161,6 +209,8 @@ let
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny icpt !icpt_agent
http_access allow icpt first_hop CONNECT !to_localhost !to_linklocal !to_bridge !to_private
http_access deny ip_v4_host
http_access deny ip_v6_host
http_access deny to_localhost
@ -168,15 +218,27 @@ let
http_access deny to_bridge
http_access deny to_private
include ${cfg.denyFile}
http_access deny !authed
http_access deny !icpt !authed
${lib.optionalString (base != [ ]) "http_access allow base_dst"}
include ${cfg.agentAllowFile}
http_access allow unrestricted
http_access allow !icpt unrestricted
http_access deny !allowed_dst
http_access allow allowed_dst
http_access deny all
'';
# A throwaway self-signed key and certificate for the intercept HTTPS port,
# written to the two paths given. Squid refuses to start without them and
# loads them to build its TLS context; it never presents them, since every
# connection is spliced.
interceptCert = pkgs.writeShellScript "squid-intercept-cert" ''
set -eu
umask 077
${lib.getExe pkgs.openssl} req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \
-nodes -subj /CN=hive-squid-intercept -days 3650 \
-keyout "$1" -out "$2"
'';
# `squid -k parse` on the configuration with the includes pointing at an empty
# file. The nixpkgs module runs the same parse on the real text, which
# cannot open the includes in the build sandbox.
@ -186,11 +248,13 @@ let
nativeBuildInputs = [ config.services.squid.package ];
}
''
squid -k parse -f ${
${interceptCert} key.pem cert.pem
sed -e "s|${interceptKeyFile}|$PWD/key.pem|; s|${interceptCertFile}|$PWD/cert.pem|" ${
pkgs.writeText "squid.conf" (
builtins.replaceStrings [ cfg.agentAllowFile cfg.denyFile ] [ "/dev/null" "/dev/null" ] squidConf
)
}
} > squid.conf
squid -k parse -f squid.conf
touch $out
'';
@ -385,12 +449,13 @@ in
defaultText = lib.literalMD "`agent-allow.conf`, in the directory of `usersFile`";
description = ''
Squid configuration that hive-c0re generates and the main configuration
includes after every destination deny and `deny !authed`, where the
includes after every destination deny and `deny !icpt !authed`, where the
base list is allowed: for each agent of this hive the swarm lists
names for, an ACL for its user, an ACL for its names and the
`http_access allow` joining the two. After those, for each agent in
`log` mode, an `http_access allow` for names `allowed_dst` lacks that
annotates the request `would_deny=1`. Empty until the controller has a
names for, an ACL for its proxy user, one for the user the intercept
helper names, an ACL for its names and two `http_access allow` rules
joining them, one for the proxy port and one for the intercept ports.
After those, for each agent in `log` mode, the same pair of rules for
names `allowed_dst` lacks, which annotate the request `would_deny=1`. Empty until the controller has a
list or an agent is in `log` mode. Squid is reloaded when it changes. Owned by `hive-core` and
readable by the `squid` group.
'';
@ -404,9 +469,10 @@ in
description = ''
Squid configuration that hive-c0re generates from the swarm's deny list
(`/api/egress/denied-domains`) and the main configuration includes after
the destination denies and before `deny !authed` and every allow: a
`http_access deny` for the swarm-wide names, and one per agent of this
hive the swarm lists names for. A denied name is refused for every user
the destination denies and before `deny !icpt !authed` and every allow: a
`http_access deny` for the swarm-wide names, and two per agent of this
hive the swarm lists names for, one for the proxy port and one for the
intercept ports. A denied name is refused for every user
in every egress mode, whether or not an allow rule covers it. The file
is empty, and squid configured as without it, until the swarm lists a
name. Squid is reloaded when it changes. Owned by `hive-core` and readable
@ -478,7 +544,15 @@ in
config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable {
# Reachable from agent containers and nowhere else: this opens the port on
# the bridge interface only.
services.hyperhive.network.exposeHostPorts = [ port ];
services.hyperhive.network.exposeHostPorts = [
port
interceptCfg.httpPort
interceptCfg.httpsPort
];
services.hyperhive.network.egress.intercept = {
enable = lib.mkDefault true;
inherit (cfg) unrestrictedFile;
};
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
@ -555,6 +629,11 @@ in
];
systemd.services.squid = {
# Before the module's own pre-start, which parses the configuration and
# so loads the certificate.
preStart = lib.mkBefore ''
${interceptCert} ${interceptKeyFile} ${interceptCertFile}
'';
serviceConfig = sandbox // {
User = "squid";
Group = "squid";

100
nix/module-eval/core-toggle.nix
View file

@ -1686,6 +1686,104 @@ let
in
refused egressForwardDrop && !(refused bare) && !(refused egressForwardAccept);
}
{
# The redirect of the agents' web traffic: IPv4 TCP 80 and 443 from the
# bridge go to squid's two intercept ports, after the rules that leave
# alone the exempt MACs (service containers, unrestricted agents), the
# host's own addresses, the bridge and the mesh. The chain is read as the
# lines between its header and the log sets that follow it. The host
# without a controller (`egressForwardAccept`) has the set but no chain.
name = "the egress table redirects the bridge's web traffic into squid after its exemptions";
ok =
let
inet = lib.head (
lib.splitString "table bridge hive_egress {" bare.services.hyperhive.network.egress.ruleset
);
chain = lib.head (
lib.splitString "set log_v4" (lib.last (lib.splitString "chain intercept {" inet))
);
rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain));
idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
redirects = lib.filter (lib.hasInfix " redirect to ") rules;
without = egressForwardAccept.services.hyperhive.network.egress.ruleset;
in
lib.hasInfix "type nat hook prerouting priority dstnat; policy accept;" chain
&& lib.hasInfix "set direct_mac { type ether_addr; }" inet
&& idx "ether saddr @direct_mac return" < idx "tcp dport 80"
&& idx "fib daddr type local return" < idx "tcp dport 80"
&& idx ''fib daddr oifname { "hive-br0", "wg-hive" } return'' < idx "tcp dport 80"
&& lib.length redirects == 2
&& lib.hasInfix "tcp dport 80 counter redirect to :3129" (lib.head redirects)
&& lib.hasInfix "tcp dport 443 counter redirect to :3130" (lib.last redirects)
# Only IPv4 that comes in on the bridge is looked at.
&& lib.all (l: lib.hasPrefix ''iifname "hive-br0" meta nfproto ipv4 '' l) (lib.drop 1 rules)
&& lib.hasInfix "set direct_mac { type ether_addr; }" without
&& !(lib.hasInfix "chain intercept" without);
}
{
# Squid listens on the two intercept ports, peeks and splices and never
# decrypts, names the agent of a flow with the helper, refuses a source
# it cannot name before any other rule, lets the CONNECT it builds
# before the ClientHello through only after the address denies, and
# leaves the proxy credential rules to the proxy port.
name = "squid's intercept ports splice, name the agent from the bindings and keep the credential rules for the proxy port";
ok =
let
conf = bare.services.squid.configText;
lines = map lib.trim (lib.splitString "\n" conf);
idx = line: lib.lists.findFirstIndex (l: l == line) 1000 lines;
squidCfg = bare.services.hyperhive.deploy.hive-controller.squid;
in
lib.hasInfix "http_port 10.42.0.1:3129 intercept name=icpt_http" conf
&& lib.hasInfix "https_port 10.42.0.1:3130 intercept ssl-bump name=icpt_tls " conf
&& lib.hasInfix "generate-host-certificates=off" conf
&& lib.elem "ssl_bump peek first_hop" lines
&& lib.elem "ssl_bump splice all" lines
&& !(lib.hasInfix "ssl_bump bump" conf)
&& !(lib.hasInfix "ssl_bump stare" conf)
&& !(lib.hasInfix "sslcrtd_program" conf)
&& lib.elem "host_verify_strict on" lines
&& lib.hasInfix "%SRC /run/hive-rootfs-check egress-whois" conf
&& idx "http_access deny icpt !icpt_agent" < idx "http_access deny ip_v4_host"
&& idx "http_access deny icpt !icpt_agent" < idx "include ${squidCfg.denyFile}"
&&
idx "http_access deny icpt !icpt_agent"
< idx "http_access allow icpt first_hop CONNECT !to_localhost !to_linklocal !to_bridge !to_private"
&&
idx "http_access allow icpt first_hop CONNECT !to_localhost !to_linklocal !to_bridge !to_private"
< idx "http_access deny ip_v4_host"
&& lib.elem "http_access deny !icpt !authed" lines
&& !(lib.elem "http_access deny !authed" lines)
&& lib.elem "http_access allow !icpt unrestricted" lines
&& !(lib.elem "http_access allow unrestricted" lines)
&& lib.all (p: lib.elem p bare.services.hyperhive.network.exposeHostPorts) [
3128
3129
3130
];
}
{
# `hive-priv` fills `direct_mac` from the same bindings as the bridge
# table's, so the units that run it are told the list of unrestricted
# agents, and a change of the list runs a pass. A host without squid has
# neither.
name = "the units that fill the egress exemptions read squid's unrestricted list and a change of it runs a pass";
ok =
let
list = bare.services.hyperhive.deploy.hive-controller.squid.unrestrictedFile;
envOf = m: name: m.systemd.services.${name}.environment;
in
list == "/var/lib/hive-squid/unrestricted"
&& lib.all (name: ((envOf bare name).HIVE_EGRESS_UNRESTRICTED or null) == list) [
"hive-egress"
"hive-egress-pin@"
"hive-egress-pin"
]
&& bare.systemd.paths.hive-egress-direct.pathConfig.PathChanged == list
&& bare.systemd.paths.hive-egress-direct.pathConfig.Unit == "hive-egress-pin.service"
&& !((envOf egressForwardAccept "hive-egress") ? HIVE_EGRESS_UNRESTRICTED)
&& !(egressForwardAccept.systemd.paths ? hive-egress-direct);
}
{
# The host that turns `route_localnet` on is refused by an assertion
# naming it; the ordinary host, which leaves it alone, is not.
@ -2189,7 +2287,7 @@ let
"http_access deny to_linklocal"
"http_access deny to_bridge"
"http_access deny to_private"
"http_access deny !authed"
"http_access deny !icpt !authed"
];
in
e.HIVE_SQUID_PROXY == "${withIdentity.services.hyperhive.network.bridgeIp}:3128"

35
nix/module-eval/hive-otel.nix
View file

@ -513,17 +513,24 @@ let
}
{
# Control for the absence checks: the bridge listener is there, so the
# absences below are not a misspelled lookup.
name = "squid listens on the bridge IP only, without a second listener or ssl_bump";
# absences below are not a misspelled lookup. The two intercept ports are
# on the bridge IP too, and the only TLS directives are the ones that
# peek and splice.
name = "squid listens on the bridge IP only, with one proxy port, two intercept ports and no decryption";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
ip = hiveOtel.services.hyperhive.network.bridgeIp;
listeners = lib.filter (l: lib.hasPrefix "http_port " l || lib.hasPrefix "https_port " l) lines;
in
builtins.elem "http_port ${hiveOtel.services.hyperhive.network.bridgeIp}:3128" lines
&& lib.length (lib.filter (l: lib.hasPrefix "http_port " l) lines) == 1
&& !(lib.hasInfix "ssl_bump" conf)
&& !(lib.hasInfix "ssl-bump" conf)
builtins.elem "http_port ${ip}:3128" lines
&& lib.length listeners == 3
&& lib.all (l: lib.hasPrefix "${lib.head (lib.splitString " " l)} ${ip}:" l) listeners
&& lib.length (lib.filter (lib.hasInfix " intercept ") listeners) == 2
&& lib.elem "ssl_bump peek first_hop" lines
&& lib.elem "ssl_bump splice all" lines
&& lib.length (lib.filter (lib.hasPrefix "ssl_bump ") lines) == 2
&& !(lib.hasInfix "localnet" conf);
}
{
@ -540,8 +547,8 @@ let
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
allow = at "http_access allow allowed_dst";
base = at "http_access allow base_dst";
unrestricted = at "http_access allow unrestricted";
authed = at "http_access deny !authed";
unrestricted = at "http_access allow !icpt unrestricted";
authed = at "http_access deny !icpt !authed";
# The generated per-agent `http_access allow` rules sit in this file.
agentRules = at "include ${hiveOtel.services.hyperhive.deploy.hive-controller.squid.agentAllowFile}";
firstAllow = lib.min agentRules (lib.min base (lib.min unrestricted allow));
@ -631,11 +638,11 @@ let
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
list = hiveOtel.services.hyperhive.deploy.hive-controller.squid.unrestrictedFile;
in
at "http_access allow unrestricted" != null
at "http_access allow !icpt unrestricted" != null
&& at "http_access deny !allowed_dst" != null
&& at "http_access deny !authed" < at "http_access allow base_dst"
&& at "http_access allow base_dst" < at "http_access allow unrestricted"
&& at "http_access allow unrestricted" < at "http_access deny !allowed_dst"
&& at "http_access deny !icpt !authed" < at "http_access allow base_dst"
&& at "http_access allow base_dst" < at "http_access allow !icpt unrestricted"
&& at "http_access allow !icpt unrestricted" < at "http_access deny !allowed_dst"
&& at "http_access deny !allowed_dst" < at "http_access allow allowed_dst"
&& builtins.elem ''acl unrestricted proxy_auth "${list}"'' lines
&& list == "/var/lib/hive-squid/unrestricted";
@ -655,7 +662,7 @@ let
allows = map at [
"http_access allow base_dst"
"include ${sq.agentAllowFile}"
"http_access allow unrestricted"
"http_access allow !icpt unrestricted"
"http_access allow allowed_dst"
];
in
@ -665,7 +672,7 @@ let
&& lib.all (a: deny < a) allows
&& at "http_access deny to_private" < deny
&& at "http_access deny ip_v4_host" < deny
&& deny < at "http_access deny !authed"
&& deny < at "http_access deny !icpt !authed"
# Two includes: this one and the per-agent allow rules.
&& lib.length (lib.filter (l: lib.hasPrefix "include " l) lines) == 2;
}

2
nix/vm-tests.nix
View file

@ -9,7 +9,7 @@
{
agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; };
wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs hivePriv; };
hive-egress-antispoof = import ./vm-tests/hive-egress-antispoof.nix { inherit pkgs hivePriv; };
swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; };
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };

262
nix/vm-tests/hive-squid.nix
View file

@ -12,8 +12,10 @@
# It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the
# agent's dials to the host's loopback, and its direct connects to the uplink
# (`proxy` forwards and masquerades like the hive host), run against the table
# the module installs.
{ pkgs }:
# the module installs. The same table redirects the agents' web traffic into
# squid's intercept ports; the test writes the bindings and leases that
# `hive-priv egress-whois` reads, standing in for `egress-pin` and dnsmasq.
{ pkgs, hivePriv }:
let
bridgeIp = "192.168.1.1";
@ -105,6 +107,14 @@ let
prefixLength = 64;
};
# The certificate `web` answers TLS with. Nothing trusts it: squid splices,
# so the agent's curl sees the origin's own and runs with `-k`.
webCert = pkgs.runCommand "hive-squid-web-cert" { nativeBuildInputs = [ pkgs.openssl ]; } ''
mkdir $out
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
-subj /CN=web.t.local -days 3650 -keyout $out/key.pem -out $out/cert.pem
'';
# Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
# test reads.
listen443 = addrs: {
@ -199,10 +209,13 @@ pkgs.testers.runNixOSTest {
networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ];
# hive-priv.nix links the binary the hive-egress unit loads its tables with;
# `eth1` is no bridge here, so a stub that only loads the ruleset (no
# bindings, no leases) stands in.
# bindings, no leases) stands in, and hands `egress-whois` to the real
# binary.
systemd.tmpfiles.rules = [
"d /var/lib/dnsmasq 0755 root root - -"
"L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" ''
[ "$1" = egress-load ] && exec ${pkgs.nftables}/bin/nft -f "$2"
[ "$1" = egress-whois ] && exec ${hivePriv}/bin/hive-priv "$@"
exit 0
''}"
];
@ -231,12 +244,37 @@ pkgs.testers.runNixOSTest {
web =
{ lib, ... }:
{
imports = [
(listen443 [
"0.0.0.0"
"[::]"
])
];
# TLS on 443 for the intercepted flows (a CONNECT through the proxy is
# answered before any TLS), plain HTTP on 80, and a plain listener on
# 8443 that the proxy never intercepts.
services.nginx = {
enable = true;
virtualHosts.default = {
listen = [
{
addr = "0.0.0.0";
port = 443;
ssl = true;
}
{
addr = "[::]";
port = 443;
ssl = true;
}
{
addr = "0.0.0.0";
port = 80;
}
{
addr = "0.0.0.0";
port = 8443;
}
];
sslCertificate = "${webCert}/cert.pem";
sslCertificateKey = "${webCert}/key.pem";
locations."/".return = "200 web-ok";
};
};
virtualisation.vlans = [ 2 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce (
[
@ -244,6 +282,12 @@ pkgs.testers.runNixOSTest {
address = "203.0.113.2";
prefixLength = 24;
}
# Another address of the same host: where a flow goes whose SNI
# names the first.
{
address = "203.0.113.9";
prefixLength = 24;
}
]
++ privateAddrs 2
);
@ -255,15 +299,27 @@ pkgs.testers.runNixOSTest {
via = "fd00:99::1";
}
];
networking.firewall.allowedTCPPorts = [ 443 ];
networking.firewall.allowedTCPPorts = [
80
443
8443
];
};
agent =
{ lib, ... }:
{ lib, pkgs, ... }:
{
virtualisation.vlans = [ 1 ];
# The bridge listener, by the name the proxy resolves to the same address.
networking.hosts.${bridgeIp} = [ "bridge.t.local" ];
# The names the proxy resolves, for the flows that skip the proxy variables.
networking.hosts."203.0.113.2" = [
"forge.t.local"
"other.t.local"
"agentonly.t.local"
"api.t.local"
];
environment.systemPackages = [ pkgs.openssl ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.3";
@ -637,7 +693,9 @@ pkgs.testers.runNixOSTest {
"# source: swarm\n"
"acl agent_allow_atlas proxy_auth atlas\n"
"acl agent_allow_atlas_dst dstdomain agentonly.t.local priv10.t.local\n"
"http_access allow agent_allow_atlas agent_allow_atlas_dst\n",
"acl agent_allow_atlas_ix ext_user atlas\n"
"http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n"
"http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n",
)
wait_status("swarm.t.local", RUTH, "200")
wait_status("agentonly.t.local", ATLAS, "200")
@ -659,7 +717,9 @@ pkgs.testers.runNixOSTest {
"# source: swarm\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_ruth proxy_auth ruth\n"
"http_access allow agent_log_ruth !allowed_dst log_mark\n",
"acl agent_log_ruth_ix ext_user ruth\n"
"http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n"
"http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n",
)
wait_status("other.t.local", RUTH, "200")
proxy.wait_until_succeeds(
@ -694,7 +754,9 @@ pkgs.testers.runNixOSTest {
"# source: swarm\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_ruth proxy_auth ruth\n"
"http_access allow agent_log_ruth !allowed_dst log_mark\n",
"acl agent_log_ruth_ix ext_user ruth\n"
"http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n"
"http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n",
)
wait_status("other.t.local", RUTH, "200")
write_deny(
@ -703,7 +765,9 @@ pkgs.testers.runNixOSTest {
"http_access deny deny_swarm_dst deny_hit_mark\n"
"acl deny_agent_atlas_dst dstdomain api.t.local\n"
"acl deny_agent_atlas proxy_auth atlas\n"
"http_access deny deny_agent_atlas_dst deny_agent_atlas deny_hit_mark\n"
"acl deny_agent_atlas_ix ext_user atlas\n"
"http_access deny deny_agent_atlas_dst !icpt deny_agent_atlas deny_hit_mark\n"
"http_access deny deny_agent_atlas_dst icpt deny_agent_atlas_ix deny_hit_mark\n"
)
wait_status("forge.t.local", ATLAS, "403")
assert connect("forge.t.local", RUTH) == "403", "a listed name, log mode"
@ -748,8 +812,8 @@ pkgs.testers.runNixOSTest {
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
assert connect("other.t.local", UMA) == "200", "listed, and now logged in"
def dial(addr):
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")[0] == 0
def dial(addr, port=443):
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/{port}'")[0] == 0
def dropped():
# Packets the table's v4 rule has dropped so far.
@ -796,10 +860,11 @@ pkgs.testers.runNixOSTest {
# first: none of them is forwarded traffic, so none may be logged.
assert connect("forge.t.local", ATLAS) == "200", "through squid"
assert dial("${bridgeIp}"), "the gateway on the bridge IP"
assert dial("203.0.113.2"), "a direct connect to an outside IP works"
# Port 8443: a flow to 80 or 443 is redirected into squid and never forwarded.
assert dial("203.0.113.2", 8443), "a direct connect to an outside IP works"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=8443 '"
)
agent.succeed("ping -c1 -W2 203.0.113.2")
proxy.wait_until_succeeds(
@ -850,5 +915,164 @@ pkgs.testers.runNixOSTest {
out = proxy.succeed("nft list chain inet hive_egress forward")
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
assert found and int(found.group(1)) >= 100, out
# ---- The agents' web traffic that skips the proxy variables ----
# The test stands in for `egress-pin` and dnsmasq's lease script: it writes
# the pin file and the lease file that `hive-priv egress-whois` reads, and
# the exemption set that `egress-pin` fills on a hive. The agent node holds
# three addresses, one per agent, and the exemption is by its MAC.
AGENT_MAC = agent.succeed("cat /sys/class/net/eth1/address").strip()
ATLAS_IP, LOGAN_IP, NOBODY_IP = "192.168.1.3", "192.168.1.4", "192.168.1.7"
BOUND = {"atlas": (ATLAS_IP, AGENT_MAC), "logan": (LOGAN_IP, "02:00:00:00:00:04")}
agent.succeed(f"ip addr add {LOGAN_IP}/24 dev eth1; ip addr add {NOBODY_IP}/24 dev eth1")
def bind(*names):
pins = "".join(f"vb-{n} {i + 2} {BOUND[n][1]} h-{n}\n" for i, n in enumerate(names))
leases = "".join(f"0 {BOUND[n][1]} {BOUND[n][0]} {n} *\n" for n in names)
proxy.succeed(
f"printf '%s' {shlex.quote(pins)} > /run/hive-egress-pins "
f"&& printf '%s' {shlex.quote(leases)} > /var/lib/dnsmasq/dnsmasq.leases"
)
def icpt(url, src=ATLAS_IP, extra=""):
# The status of the answer to a request that carries no proxy variables;
# 000 when the connection was closed or the TLS handshake failed.
return agent.succeed(
f"curl -sS -k -m 15 --interface {src} {extra} -o /dev/null -w '%{{http_code}}' {url} 2>/dev/null || true"
).strip()
def wait_icpt(url, want, src=ATLAS_IP, extra=""):
# The reload that follows a list write is asynchronous.
got = None
for _ in range(30):
got = icpt(url, src, extra)
if got == want:
return
time.sleep(1)
raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}")
def journal():
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
def agent_lines():
# Log lines of requests from the agent node's addresses. The self-check
# connects from the bridge address, so its lines are not counted.
return len(re.findall(r" 192\.168\.1\.[347] [A-Z_]+/\d+ ", journal()))
seeded = proxy.succeed(f"cat {ALLOWED}")
bind("atlas", "logan")
# What hive-c0re renders: `atlas` has a name of its own, `logan` is in log mode.
write_allow(
seeded,
"# source: swarm\n"
"acl agent_allow_atlas proxy_auth atlas\n"
"acl agent_allow_atlas_ix ext_user atlas\n"
"acl agent_allow_atlas_dst dstdomain agentonly.t.local\n"
"http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n"
"http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_logan proxy_auth logan\n"
"acl agent_log_logan_ix ext_user logan\n"
"http_access allow !icpt agent_log_logan !allowed_dst log_mark\n"
"http_access allow icpt agent_log_logan_ix !allowed_dst log_mark\n",
)
with checked("a restricted agent's allowed name is tunnelled without the proxy variables, and the log names the agent"):
wait_icpt("https://forge.t.local/", "200")
assert re.search(
r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ", journal()
), "the HTTPS flow, as atlas"
assert icpt("http://forge.t.local/") == "200", "plain HTTP"
assert re.search(r"192\.168\.1\.3 TCP_\w+/200 .* GET http://forge\.t\.local/ atlas ", journal()), "the HTTP flow, as atlas"
assert icpt("https://agentonly.t.local/") == "200", "a name of atlas alone"
assert re.search(r"CONNECT agentonly\.t\.local:443 atlas ", journal()), "as atlas"
# The proxy-aware path is the same as before, and is not given the address's identity.
assert connect("forge.t.local", ATLAS) == "200", "a credential through the proxy port"
assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one"
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
assert icpt("https://other.t.local/") == "000", "HTTPS"
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()), "logged as atlas"
assert icpt("http://other.t.local/") == "403", "plain HTTP"
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
# Control: the same address reaches the listed name, so the refusal is the policy's.
assert icpt("https://forge.t.local/") == "200"
with checked("the swarm's deny list refuses an intercepted name that is otherwise allowed, and logs egress_denied"):
assert icpt("https://api.t.local/") == "200", "control: a base-list name"
write_deny(
"acl deny_hit_mark annotate_transaction egress_deny=1\n"
"acl deny_swarm_dst dstdomain api.t.local\n"
"http_access deny deny_swarm_dst deny_hit_mark\n"
)
wait_icpt("https://api.t.local/", "000")
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
)
assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected"
with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"):
assert icpt("https://other.t.local/", LOGAN_IP) == "200"
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'would_deny user=logan method=CONNECT host=other\\.t\\.local port=443 result=TCP_TUNNEL/200'"
)
assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name"
assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked"
assert icpt("https://api.t.local/", LOGAN_IP) == "000", "the deny list refuses it in log mode too"
assert icpt("https://other.t.local/") == "000", "control: atlas, unmarked and refused"
write_deny("")
wait_icpt("https://api.t.local/", "200")
with checked("a source the bindings do not name is refused, and so is every source when they are gone"):
assert icpt("https://forge.t.local/", NOBODY_IP) == "000", "no lease for the address"
assert re.search(r"192\.168\.1\.7 TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 - ", journal()), "logged without a user"
assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
assert icpt("https://forge.t.local/") == "200", "control: the bound address, the same name"
# A hive whose pin file is missing: the lease names a MAC, no port holds it.
proxy.succeed("rm /run/hive-egress-pins")
time.sleep(12) # the helper's answers are kept for 10 seconds
assert icpt("https://forge.t.local/") == "000", "no pin file"
bind("atlas", "logan")
wait_icpt("https://forge.t.local/", "200")
with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"):
# Control: the right SNI at the right address is the 200 above. The
# same name sent to another address of the origin is not.
assert icpt("https://forge.t.local/", extra="--resolve forge.t.local:443:203.0.113.9") == "000"
assert re.search(r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", journal()), "squid answered 409"
assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too"
hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443"
out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true")
assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back"
out = agent.succeed(f"{hello} -noservername 2>&1 || true")
assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate"
assert icpt("https://203.0.113.2/") == "000", "an IP literal sends no SNI"
assert re.search(r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ", journal()), "logged as atlas"
assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host"
out = agent.succeed(
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
)
assert out.strip() == "", "bytes that are no ClientHello get no answer"
with checked("the traffic that stays on the bridge or is the unrestricted agents' is not intercepted"):
seen = agent_lines()
# The gateway on the bridge address, in the stand-in's port 443.
assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers"
assert agent_lines() == seen, "squid saw nothing of it"
# Unrestricted agents and service containers are the MACs in `direct_mac`.
assert icpt("https://other.t.local/") == "000", "control: an unlisted name is refused"
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
seen = agent_lines()
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
assert icpt("https://203.0.113.2/") == "200", "an IP literal, directly"
assert agent_lines() == seen, "squid saw nothing of it"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"
)
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
assert icpt("https://other.t.local/") == "000", "intercepted again"
'';
}