vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

deny private and reserved destinations for every squid user

Browse source
This commit is contained in:
atlas 2026-10-28 14:23:21 +01:00 • committed by mara
commit 3d9561a9a6
4 changed files with 167 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

28
docs/networking/network.md
View file

@ -238,13 +238,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
(`/var/lib/hive-squid/unrestricted`, `hive-core:squid`, `0640`): the names of
the users whose agent holds the `unrestricted_internet` flag in the
permissions the hive last read from the swarm, one per line, rewritten when
it changes. An agent with no record, or a record that never set the flag,
holds it, the same default as the
`hyperhive.agent.unrestricted_internet` metric. Squid reads the list when it
it changes. The default is permissive: an agent with no record, or a record
that never set the flag, holds it, the same default as the
`hyperhive.agent.unrestricted_internet` metric. While the hive's read of the
permissions is stale or failing, a new agent that already holds a
credential counts as unrestricted. Squid reads the list when it
parses its configuration, so `squid-unrestricted-reload.path` reloads squid
when the file changes. A missing or empty file lists no one, and squid
starts either way. A flag change reaches squid after the hive's next read of
the permissions and its next 30-second pass.
An older `hive-c0re` writes no list, so every credential on that hive is
restricted. Agents must not start sending credentials until every hive runs
the `hive-c0re` that writes it, or a legacy agent loses general internet.
- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the
swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each
@ -253,11 +258,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
- **Unlisted names.** For a user who isn't on the unrestricted list, refused
after the destination checks, which look the name up on the host.
- **Denied before any allow.** Destinations that resolve to loopback,
link-local or the bridge subnet, and IP-literal hosts, for every user
including the unrestricted ones. Squid resolves names on the host, so
without these a name pointing at `127.0.0.1` would walk around the
bridge→loopback DROP rule. Only `CONNECT` to port 443 and
plain HTTP to ports 80 and 443 pass.
link-local, the bridge subnet, a private range (`10.0.0.0/8`,
`172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`), shared address space
(`100.64.0.0/10`), `0.0.0.0/8`, multicast or reserved space, and IP-literal
hosts, for every user including the unrestricted ones and for listed names.
Squid resolves names on the host and connects from it, so without these a
name pointing at `127.0.0.1`, at the host's own LAN address or at a private
neighbour would walk around the bridge→loopback DROP rule and the swarm's
network. Squid checks the address it connects to, so a name that changes its
answer between a check and the connect gains nothing. A listed name must
therefore resolve to a public address on the host's resolver: a split-horizon
name that answers a LAN address gets a 403, and so does the self-check
canary. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass.
- **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on
`127.0.0.1:9301`. The hive collector scrapes it as job `squid` through
`services.hyperhive.otel.scrapeTargets`. `squid_up` and

46
nix/host-modules/hive-squid.nix
View file

@ -31,11 +31,30 @@ let
selfcheckUnrestrictedUser = "selfcheck-unrestricted.squid";
usersDir = builtins.dirOf cfg.usersFile;
# Evaluated top to bottom, first match wins. The three destination denies
# sit before every allow, `allow unrestricted` included: squid resolves
# names on the host, so a name that answers 127.0.0.1, 169.254.169.254 or a
# bridge address would otherwise reach services the bridge firewall keeps
# agents out of, and an unrestricted agent is still an agent. The IP-literal
# Destinations that are not the public internet: loopback, link-local, the
# bridge, and the private, shared (CGNAT), unspecified, multicast and
# reserved ranges. Squid checks the address it connects to, so a name an
# agent controls cannot point past them.
privateRanges = [
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
"100.64.0.0/10"
"0.0.0.0/8"
"224.0.0.0/4"
"240.0.0.0/4"
"fc00::/7"
"ff00::/8"
];
# Evaluated top to bottom, first match wins. The destination denies sit
# before every allow, `allow unrestricted` included: squid resolves names on
# the host and connects from it, so a name that answers 127.0.0.1,
# 169.254.169.254, the host's own LAN address or a private neighbour would
# otherwise reach services the bridge firewall keeps agents out of, and an
# unrestricted agent is still an agent. The same denies bind a listed name,
# so a listed name must resolve to a public address on the host's resolver.
# The IP-literal
# denies come first because `dstdomain` on an IP literal does a reverse
# lookup. The credential check sits right before the first allow, so only a
# request that would otherwise be let through gets the 407 challenge. An
@ -75,6 +94,7 @@ let
acl selfcheck src ${netCfg.bridgeIp}/32
acl to_bridge dst ${netCfg.networkCidr}
acl to_private dst ${lib.concatStringsSep " " privateRanges}
# `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record
# an agent controls could match an allowed name.
acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
@ -95,6 +115,7 @@ let
http_access deny to_localhost
http_access deny to_linklocal
http_access deny to_bridge
http_access deny to_private
http_access deny !authed
http_access allow unrestricted
${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"}
@ -181,8 +202,12 @@ in
];
description = ''
Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
entry matches that exact name. Everything else gets a 403. The
self-check canary is allowed on top of this list.
entry matches that exact name. Everything else gets a 403, except
for an unrestricted user. The self-check canary is allowed on top of
this list. A name must resolve, on this host's resolver, to a public
address: one in a loopback, link-local, private, shared (100.64.0.0/10),
multicast or reserved range, or in the bridge subnet, gets a 403 for
every user.
'';
};
@ -246,9 +271,10 @@ in
description = ''
Hostname the self-check CONNECTs to on port 443 and expects squid to
let through. Squid resolves it with this host's resolver, not the
hive's: a host whose resolver answers a loopback or bridge address
for it (a single-host swarm with the gateway's `localHostsEntry`) gets
a 403, and the self-check reports `ok=false`.
hive's: a host whose resolver answers a loopback, bridge or other
private address for it (a single-host swarm with the gateway's
`localHostsEntry`, or a split-horizon LAN) gets a 403, and the
self-check reports `ok=false`.
'';
};
};

31
nix/module-eval/hive-otel.nix
View file

@ -491,7 +491,7 @@ let
# reaches loopback, link-local and the bridge through the proxy. The two
# IP-literal denies stay ahead of every allow: `dstdomain` on an IP
# literal does a reverse lookup.
name = "squid refuses loopback, link-local and bridge destinations and IP literals, then a missing credential, before any allow";
name = "squid refuses loopback, link-local, bridge and private destinations and IP literals, then a missing credential, before any allow";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
@ -507,22 +507,51 @@ let
&& at "http_access deny to_localhost" != null
&& at "http_access deny to_linklocal" != null
&& at "http_access deny to_bridge" != null
&& at "http_access deny to_private" != null
&& at "http_access deny ip_v4_host" != null
&& at "http_access deny ip_v6_host" != null
&& lib.all (l: at l < firstAllow) [
"http_access deny to_localhost"
"http_access deny to_linklocal"
"http_access deny to_bridge"
"http_access deny to_private"
"http_access deny ip_v4_host"
"http_access deny ip_v6_host"
]
&& at "http_access deny to_localhost" < authed
&& at "http_access deny to_linklocal" < authed
&& at "http_access deny to_bridge" < authed
&& at "http_access deny to_private" < authed
&& authed < firstAllow
&& builtins.elem "cache deny all" lines
&& lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
}
{
# The ranges are what keeps an unrestricted user on the public internet.
# Control: the bridge deny and the ACL line are found, so a range missing
# below is not a lookup of nothing.
name = "squid's private-destination ACL names every private, shared, multicast and reserved range";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
acl = lib.findFirst (l: lib.hasPrefix "acl to_private dst " l) null lines;
ranges = lib.splitString " " (lib.removePrefix "acl to_private dst " acl);
in
acl != null
&& builtins.elem "acl to_bridge dst ${hiveOtel.services.hyperhive.network.networkCidr}" lines
&&
lib.sort lib.lessThan ranges == lib.sort lib.lessThan [
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
"100.64.0.0/10"
"0.0.0.0/8"
"224.0.0.0/4"
"240.0.0.0/4"
"fc00::/7"
"ff00::/8"
];
}
{
# An unrestricted user is let through before the unlisted-name deny and
# everyone else is held to the listed names after it. Control: both are

87
nix/vm-tests/hive-squid.nix
View file

@ -66,6 +66,38 @@ let
config.services.hyperhive.network.exposeHostPorts;
};
# The uplink is a public range. The private ranges squid must refuse are
# extra subnets on the same link, so the proxy can reach a listener in each:
# a name that resolves there is refused only by squid's own rules.
privateNets = [
{
name = "priv10.t.local";
net = "10.99.0";
}
{
name = "priv172.t.local";
net = "172.16.99";
}
{
name = "priv192.t.local";
net = "192.168.99";
}
{
name = "cgnat.t.local";
net = "100.64.99";
}
];
privateAddrs =
host:
map (p: {
address = "${p.net}.${toString host}";
prefixLength = 24;
}) privateNets;
ulaAddr = host: {
address = "fd00:99::${toString host}";
prefixLength = 64;
};
# Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
# test reads.
listen443 = addrs: {
@ -104,23 +136,34 @@ pkgs.testers.runNixOSTest {
prefixLength = 24;
}
];
networking.interfaces.eth2.ipv4.addresses = lib.mkForce [
networking.interfaces.eth2.ipv4.addresses = lib.mkForce (
[
{
address = "192.168.2.1";
address = "203.0.113.1";
prefixLength = 24;
}
];
]
++ privateAddrs 1
);
networking.interfaces.eth2.ipv6.addresses = [ (ulaAddr 1) ];
# Names the proxy resolves, each a different destination class.
networking.hosts = {
# `other.t.local` resolves, is reachable and is on no allow-list.
"192.168.2.2" = [
"203.0.113.2" = [
"forge.t.local"
"other.t.local"
];
"127.0.0.1" = [ "loop.t.local" ];
${bridgeIp} = [ "bridge.t.local" ];
"169.254.169.254" = [ "meta.t.local" ];
};
"fd00:99::2" = [ "ula.t.local" ];
}
// lib.listToAttrs (
map (p: {
name = "${p.net}.2";
value = [ p.name ];
}) privateNets
);
# Renders the users file the way hive-c0re does.
environment.systemPackages = [ pkgs.mkpasswd ];
services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local";
@ -129,20 +172,31 @@ pkgs.testers.runNixOSTest {
"loop.t.local"
"bridge.t.local"
"meta.t.local"
];
"ula.t.local"
]
++ map (p: p.name) privateNets;
};
web =
{ lib, ... }:
{
imports = [ (listen443 [ "0.0.0.0" ]) ];
imports = [
(listen443 [
"0.0.0.0"
"[::]"
])
];
virtualisation.vlans = [ 2 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
networking.interfaces.eth1.ipv4.addresses = lib.mkForce (
[
{
address = "192.168.2.2";
address = "203.0.113.2";
prefixLength = 24;
}
];
]
++ privateAddrs 2
);
networking.interfaces.eth1.ipv6.addresses = [ (ulaAddr 2) ];
networking.firewall.allowedTCPPorts = [ 443 ];
};
@ -188,6 +242,8 @@ pkgs.testers.runNixOSTest {
web.wait_for_unit("nginx.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local"]
def connect(host, cred=None):
# The status squid answered the CONNECT with; 000 when none came back.
auth = f"-U '{cred}' " if cred else ""
@ -228,6 +284,11 @@ pkgs.testers.runNixOSTest {
time.sleep(1)
raise AssertionError(f"{host} as {cred.split(':')[0]}: wanted {want}, last got {got}")
with subtest("the private listeners answer the proxy itself, so a refusal below is squid's"):
for addr in ["10.99.0.2", "172.16.99.2", "192.168.99.2", "100.64.99.2", "fd00:99::2"]:
proxy.wait_until_succeeds(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")
assert connect("forge.t.local", None) == "407", "control: the public name is challenged, not refused"
with subtest("squid starts on the empty users file and list and refuses every login"):
assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c '%U:%G %a' {LIST}").strip() == "hive-core:squid 640"
@ -270,14 +331,16 @@ pkgs.testers.runNixOSTest {
assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, a name that does not resolve"
with subtest("neither kind of user reaches a host-local destination or another port"):
with subtest("neither kind of user reaches a host-local or private destination, or another port"):
for cred in (ATLAS, UMA):
who = cred.split(":")[0]
assert connect("loop.t.local", cred) == "403", f"{who}: loopback"
assert connect("bridge.t.local", cred) == "403", f"{who}: bridge address"
assert connect("meta.t.local", cred) == "403", f"{who}: link-local"
assert connect("127.0.0.1", cred) == "403", f"{who}: loopback literal"
assert connect("192.168.2.2", cred) == "403", f"{who}: IP literal"
assert connect("203.0.113.2", cred) == "403", f"{who}: IP literal"
for name in PRIVATE:
assert connect(name, cred) == "403", f"{who}: {name}, a listed name that resolves to a private address"
assert connect("other.t.local:22", cred) == "403", f"{who}: port 22"
assert connect("other.t.local:80", cred) == "403", f"{who}: CONNECT to port 80"