diff --git a/docs/networking/network.md b/docs/networking/network.md index 816155da..75a723f3 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -238,13 +238,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. (`/var/lib/hive-squid/unrestricted`, `hive-core:squid`, `0640`): the names of the users whose agent holds the `unrestricted_internet` flag in the permissions the hive last read from the swarm, one per line, rewritten when - it changes. An agent with no record, or a record that never set the flag, - holds it, the same default as the - `hyperhive.agent.unrestricted_internet` metric. Squid reads the list when it + it changes. The default is permissive: an agent with no record, or a record + that never set the flag, holds it, the same default as the + `hyperhive.agent.unrestricted_internet` metric. While the hive's read of the + permissions is stale or failing, a new agent that already holds a + credential counts as unrestricted. Squid reads the list when it parses its configuration, so `squid-unrestricted-reload.path` reloads squid when the file changes. A missing or empty file lists no one, and squid starts either way. A flag change reaches squid after the hive's next read of the permissions and its next 30-second pass. + An older `hive-c0re` writes no list, so every credential on that hive is + restricted. Agents must not start sending credentials until every hive runs + the `hive-c0re` that writes it, or a legacy agent loses general internet. - **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each @@ -253,11 +258,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. - **Unlisted names.** For a user who isn't on the unrestricted list, refused after the destination checks, which look the name up on the host. - **Denied before any allow.** Destinations that resolve to loopback, - link-local or the bridge subnet, and IP-literal hosts, for every user - including the unrestricted ones. Squid resolves names on the host, so - without these a name pointing at `127.0.0.1` would walk around the - bridge→loopback DROP rule. Only `CONNECT` to port 443 and - plain HTTP to ports 80 and 443 pass. + link-local, the bridge subnet, a private range (`10.0.0.0/8`, + `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`), shared address space + (`100.64.0.0/10`), `0.0.0.0/8`, multicast or reserved space, and IP-literal + hosts, for every user including the unrestricted ones and for listed names. + Squid resolves names on the host and connects from it, so without these a + name pointing at `127.0.0.1`, at the host's own LAN address or at a private + neighbour would walk around the bridge→loopback DROP rule and the swarm's + network. Squid checks the address it connects to, so a name that changes its + answer between a check and the connect gains nothing. A listed name must + therefore resolve to a public address on the host's resolver: a split-horizon + name that answers a LAN address gets a 403, and so does the self-check + canary. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass. - **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through `services.hyperhive.otel.scrapeTargets`. `squid_up` and diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index a0c38dc0..fa53d1bb 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -31,11 +31,30 @@ let selfcheckUnrestrictedUser = "selfcheck-unrestricted.squid"; usersDir = builtins.dirOf cfg.usersFile; - # Evaluated top to bottom, first match wins. The three destination denies - # sit before every allow, `allow unrestricted` included: squid resolves - # names on the host, so a name that answers 127.0.0.1, 169.254.169.254 or a - # bridge address would otherwise reach services the bridge firewall keeps - # agents out of, and an unrestricted agent is still an agent. The IP-literal + # Destinations that are not the public internet: loopback, link-local, the + # bridge, and the private, shared (CGNAT), unspecified, multicast and + # reserved ranges. Squid checks the address it connects to, so a name an + # agent controls cannot point past them. + privateRanges = [ + "10.0.0.0/8" + "172.16.0.0/12" + "192.168.0.0/16" + "100.64.0.0/10" + "0.0.0.0/8" + "224.0.0.0/4" + "240.0.0.0/4" + "fc00::/7" + "ff00::/8" + ]; + + # Evaluated top to bottom, first match wins. The destination denies sit + # before every allow, `allow unrestricted` included: squid resolves names on + # the host and connects from it, so a name that answers 127.0.0.1, + # 169.254.169.254, the host's own LAN address or a private neighbour would + # otherwise reach services the bridge firewall keeps agents out of, and an + # unrestricted agent is still an agent. The same denies bind a listed name, + # so a listed name must resolve to a public address on the host's resolver. + # The IP-literal # denies come first because `dstdomain` on an IP literal does a reverse # lookup. The credential check sits right before the first allow, so only a # request that would otherwise be let through gets the 407 challenge. An @@ -75,6 +94,7 @@ let acl selfcheck src ${netCfg.bridgeIp}/32 acl to_bridge dst ${netCfg.networkCidr} + acl to_private dst ${lib.concatStringsSep " " privateRanges} # `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record # an agent controls could match an allowed name. acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$) @@ -95,6 +115,7 @@ let http_access deny to_localhost http_access deny to_linklocal http_access deny to_bridge + http_access deny to_private http_access deny !authed http_access allow unrestricted ${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"} @@ -181,8 +202,12 @@ in ]; description = '' Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each - entry matches that exact name. Everything else gets a 403. The - self-check canary is allowed on top of this list. + entry matches that exact name. Everything else gets a 403, except + for an unrestricted user. The self-check canary is allowed on top of + this list. A name must resolve, on this host's resolver, to a public + address: one in a loopback, link-local, private, shared (100.64.0.0/10), + multicast or reserved range, or in the bridge subnet, gets a 403 for + every user. ''; }; @@ -246,9 +271,10 @@ in description = '' Hostname the self-check CONNECTs to on port 443 and expects squid to let through. Squid resolves it with this host's resolver, not the - hive's: a host whose resolver answers a loopback or bridge address - for it (a single-host swarm with the gateway's `localHostsEntry`) gets - a 403, and the self-check reports `ok=false`. + hive's: a host whose resolver answers a loopback, bridge or other + private address for it (a single-host swarm with the gateway's + `localHostsEntry`, or a split-horizon LAN) gets a 403, and the + self-check reports `ok=false`. ''; }; }; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2b03ba67..0e94ae03 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -491,7 +491,7 @@ let # reaches loopback, link-local and the bridge through the proxy. The two # IP-literal denies stay ahead of every allow: `dstdomain` on an IP # literal does a reverse lookup. - name = "squid refuses loopback, link-local and bridge destinations and IP literals, then a missing credential, before any allow"; + name = "squid refuses loopback, link-local, bridge and private destinations and IP literals, then a missing credential, before any allow"; ok = let lines = lib.splitString "\n" hiveOtel.services.squid.configText; @@ -507,22 +507,51 @@ let && at "http_access deny to_localhost" != null && at "http_access deny to_linklocal" != null && at "http_access deny to_bridge" != null + && at "http_access deny to_private" != null && at "http_access deny ip_v4_host" != null && at "http_access deny ip_v6_host" != null && lib.all (l: at l < firstAllow) [ "http_access deny to_localhost" "http_access deny to_linklocal" "http_access deny to_bridge" + "http_access deny to_private" "http_access deny ip_v4_host" "http_access deny ip_v6_host" ] && at "http_access deny to_localhost" < authed && at "http_access deny to_linklocal" < authed && at "http_access deny to_bridge" < authed + && at "http_access deny to_private" < authed && authed < firstAllow && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } + { + # The ranges are what keeps an unrestricted user on the public internet. + # Control: the bridge deny and the ACL line are found, so a range missing + # below is not a lookup of nothing. + name = "squid's private-destination ACL names every private, shared, multicast and reserved range"; + ok = + let + lines = lib.splitString "\n" hiveOtel.services.squid.configText; + acl = lib.findFirst (l: lib.hasPrefix "acl to_private dst " l) null lines; + ranges = lib.splitString " " (lib.removePrefix "acl to_private dst " acl); + in + acl != null + && builtins.elem "acl to_bridge dst ${hiveOtel.services.hyperhive.network.networkCidr}" lines + && + lib.sort lib.lessThan ranges == lib.sort lib.lessThan [ + "10.0.0.0/8" + "172.16.0.0/12" + "192.168.0.0/16" + "100.64.0.0/10" + "0.0.0.0/8" + "224.0.0.0/4" + "240.0.0.0/4" + "fc00::/7" + "ff00::/8" + ]; + } { # An unrestricted user is let through before the unlisted-name deny and # everyone else is held to the listed names after it. Control: both are diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index d08a86bc..5e1fcb03 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -66,6 +66,38 @@ let config.services.hyperhive.network.exposeHostPorts; }; + # The uplink is a public range. The private ranges squid must refuse are + # extra subnets on the same link, so the proxy can reach a listener in each: + # a name that resolves there is refused only by squid's own rules. + privateNets = [ + { + name = "priv10.t.local"; + net = "10.99.0"; + } + { + name = "priv172.t.local"; + net = "172.16.99"; + } + { + name = "priv192.t.local"; + net = "192.168.99"; + } + { + name = "cgnat.t.local"; + net = "100.64.99"; + } + ]; + privateAddrs = + host: + map (p: { + address = "${p.net}.${toString host}"; + prefixLength = 24; + }) privateNets; + ulaAddr = host: { + address = "fd00:99::${toString host}"; + prefixLength = 64; + }; + # Plain TCP on 443: a CONNECT is answered before any TLS, which is all the # test reads. listen443 = addrs: { @@ -104,23 +136,34 @@ pkgs.testers.runNixOSTest { prefixLength = 24; } ]; - networking.interfaces.eth2.ipv4.addresses = lib.mkForce [ - { - address = "192.168.2.1"; - prefixLength = 24; - } - ]; + networking.interfaces.eth2.ipv4.addresses = lib.mkForce ( + [ + { + address = "203.0.113.1"; + prefixLength = 24; + } + ] + ++ privateAddrs 1 + ); + networking.interfaces.eth2.ipv6.addresses = [ (ulaAddr 1) ]; # Names the proxy resolves, each a different destination class. networking.hosts = { # `other.t.local` resolves, is reachable and is on no allow-list. - "192.168.2.2" = [ + "203.0.113.2" = [ "forge.t.local" "other.t.local" ]; "127.0.0.1" = [ "loop.t.local" ]; ${bridgeIp} = [ "bridge.t.local" ]; "169.254.169.254" = [ "meta.t.local" ]; - }; + "fd00:99::2" = [ "ula.t.local" ]; + } + // lib.listToAttrs ( + map (p: { + name = "${p.net}.2"; + value = [ p.name ]; + }) privateNets + ); # Renders the users file the way hive-c0re does. environment.systemPackages = [ pkgs.mkpasswd ]; services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local"; @@ -129,20 +172,31 @@ pkgs.testers.runNixOSTest { "loop.t.local" "bridge.t.local" "meta.t.local" - ]; + "ula.t.local" + ] + ++ map (p: p.name) privateNets; }; web = { lib, ... }: { - imports = [ (listen443 [ "0.0.0.0" ]) ]; - virtualisation.vlans = [ 2 ]; - networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ - { - address = "192.168.2.2"; - prefixLength = 24; - } + imports = [ + (listen443 [ + "0.0.0.0" + "[::]" + ]) ]; + virtualisation.vlans = [ 2 ]; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce ( + [ + { + address = "203.0.113.2"; + prefixLength = 24; + } + ] + ++ privateAddrs 2 + ); + networking.interfaces.eth1.ipv6.addresses = [ (ulaAddr 2) ]; networking.firewall.allowedTCPPorts = [ 443 ]; }; @@ -188,6 +242,8 @@ pkgs.testers.runNixOSTest { web.wait_for_unit("nginx.service") proxy.wait_for_open_port(3128, "${bridgeIp}") + PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local"] + def connect(host, cred=None): # The status squid answered the CONNECT with; 000 when none came back. auth = f"-U '{cred}' " if cred else "" @@ -228,6 +284,11 @@ pkgs.testers.runNixOSTest { time.sleep(1) raise AssertionError(f"{host} as {cred.split(':')[0]}: wanted {want}, last got {got}") + with subtest("the private listeners answer the proxy itself, so a refusal below is squid's"): + for addr in ["10.99.0.2", "172.16.99.2", "192.168.99.2", "100.64.99.2", "fd00:99::2"]: + proxy.wait_until_succeeds(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'") + assert connect("forge.t.local", None) == "407", "control: the public name is challenged, not refused" + with subtest("squid starts on the empty users file and list and refuses every login"): assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640" assert proxy.succeed(f"stat -c '%U:%G %a' {LIST}").strip() == "hive-core:squid 640" @@ -270,14 +331,16 @@ pkgs.testers.runNixOSTest { assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password" assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, a name that does not resolve" - with subtest("neither kind of user reaches a host-local destination or another port"): + with subtest("neither kind of user reaches a host-local or private destination, or another port"): for cred in (ATLAS, UMA): who = cred.split(":")[0] assert connect("loop.t.local", cred) == "403", f"{who}: loopback" assert connect("bridge.t.local", cred) == "403", f"{who}: bridge address" assert connect("meta.t.local", cred) == "403", f"{who}: link-local" assert connect("127.0.0.1", cred) == "403", f"{who}: loopback literal" - assert connect("192.168.2.2", cred) == "403", f"{who}: IP literal" + assert connect("203.0.113.2", cred) == "403", f"{who}: IP literal" + for name in PRIVATE: + assert connect(name, cred) == "403", f"{who}: {name}, a listed name that resolves to a private address" assert connect("other.t.local:22", cred) == "403", f"{who}: port 22" assert connect("other.t.local:80", cred) == "403", f"{who}: CONNECT to port 80"