deny private and reserved destinations for every squid user

docs/networking/network.md

@@ -238,13 +238,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
   (`/var/lib/hive-squid/unrestricted`, `hive-core:squid`, `0640`): the names of
   the users whose agent holds the `unrestricted_internet` flag in the
   permissions the hive last read from the swarm, one per line, rewritten when
-  it changes. An agent with no record, or a record that never set the flag,
+  it changes. The default is permissive: an agent with no record, or a record
-  holds it, the same default as the
+  that never set the flag, holds it, the same default as the
-  `hyperhive.agent.unrestricted_internet` metric. Squid reads the list when it
+  `hyperhive.agent.unrestricted_internet` metric. While the hive's read of the
+  permissions is stale or failing, a new agent that already holds a
+  credential counts as unrestricted. Squid reads the list when it
   parses its configuration, so `squid-unrestricted-reload.path` reloads squid
   when the file changes. A missing or empty file lists no one, and squid
   starts either way. A flag change reaches squid after the hive's next read of
   the permissions and its next 30-second pass.
+  An older `hive-c0re` writes no list, so every credential on that hive is
+  restricted. Agents must not start sending credentials until every hive runs
+  the `hive-c0re` that writes it, or a legacy agent loses general internet.
 
 - **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the
   swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each
@@ -253,11 +258,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
 - **Unlisted names.** For a user who isn't on the unrestricted list, refused
   after the destination checks, which look the name up on the host.
 - **Denied before any allow.** Destinations that resolve to loopback,
-  link-local or the bridge subnet, and IP-literal hosts, for every user
+  link-local, the bridge subnet, a private range (`10.0.0.0/8`,
-  including the unrestricted ones. Squid resolves names on the host, so
+  `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`), shared address space
-  without these a name pointing at `127.0.0.1` would walk around the
+  (`100.64.0.0/10`), `0.0.0.0/8`, multicast or reserved space, and IP-literal
-  bridge→loopback DROP rule. Only `CONNECT` to port 443 and
+  hosts, for every user including the unrestricted ones and for listed names.
-  plain HTTP to ports 80 and 443 pass.
+  Squid resolves names on the host and connects from it, so without these a
+  name pointing at `127.0.0.1`, at the host's own LAN address or at a private
+  neighbour would walk around the bridge→loopback DROP rule and the swarm's
+  network. Squid checks the address it connects to, so a name that changes its
+  answer between a check and the connect gains nothing. A listed name must
+  therefore resolve to a public address on the host's resolver: a split-horizon
+  name that answers a LAN address gets a 403, and so does the self-check
+  canary. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass.
 - **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on
   `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through
   `services.hyperhive.otel.scrapeTargets`. `squid_up` and

nix/host-modules/hive-squid.nix

@@ -31,11 +31,30 @@ let
   selfcheckUnrestrictedUser = "selfcheck-unrestricted.squid";
   usersDir = builtins.dirOf cfg.usersFile;
 
-  # Evaluated top to bottom, first match wins. The three destination denies
+  # Destinations that are not the public internet: loopback, link-local, the
-  # sit before every allow, `allow unrestricted` included: squid resolves
+  # bridge, and the private, shared (CGNAT), unspecified, multicast and
-  # names on the host, so a name that answers 127.0.0.1, 169.254.169.254 or a
+  # reserved ranges. Squid checks the address it connects to, so a name an
-  # bridge address would otherwise reach services the bridge firewall keeps
+  # agent controls cannot point past them.
-  # agents out of, and an unrestricted agent is still an agent. The IP-literal
+  privateRanges = [
+    "10.0.0.0/8"
+    "172.16.0.0/12"
+    "192.168.0.0/16"
+    "100.64.0.0/10"
+    "0.0.0.0/8"
+    "224.0.0.0/4"
+    "240.0.0.0/4"
+    "fc00::/7"
+    "ff00::/8"
+  ];
+
+  # Evaluated top to bottom, first match wins. The destination denies sit
+  # before every allow, `allow unrestricted` included: squid resolves names on
+  # the host and connects from it, so a name that answers 127.0.0.1,
+  # 169.254.169.254, the host's own LAN address or a private neighbour would
+  # otherwise reach services the bridge firewall keeps agents out of, and an
+  # unrestricted agent is still an agent. The same denies bind a listed name,
+  # so a listed name must resolve to a public address on the host's resolver.
+  # The IP-literal
   # denies come first because `dstdomain` on an IP literal does a reverse
   # lookup. The credential check sits right before the first allow, so only a
   # request that would otherwise be let through gets the 407 challenge. An
@@ -75,6 +94,7 @@ let
     acl selfcheck src ${netCfg.bridgeIp}/32
 
     acl to_bridge dst ${netCfg.networkCidr}
+    acl to_private dst ${lib.concatStringsSep " " privateRanges}
     # `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record
     # an agent controls could match an allowed name.
     acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
@@ -95,6 +115,7 @@ let
     http_access deny to_localhost
     http_access deny to_linklocal
     http_access deny to_bridge
+    http_access deny to_private
     http_access deny !authed
     http_access allow unrestricted
     ${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"}
@@ -181,8 +202,12 @@ in
       ];
       description = ''
         Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
-        entry matches that exact name. Everything else gets a 403. The
+        entry matches that exact name. Everything else gets a 403, except
-        self-check canary is allowed on top of this list.
+        for an unrestricted user. The self-check canary is allowed on top of
+        this list. A name must resolve, on this host's resolver, to a public
+        address: one in a loopback, link-local, private, shared (100.64.0.0/10),
+        multicast or reserved range, or in the bridge subnet, gets a 403 for
+        every user.
       '';
     };
 
@@ -246,9 +271,10 @@ in
       description = ''
         Hostname the self-check CONNECTs to on port 443 and expects squid to
         let through. Squid resolves it with this host's resolver, not the
-        hive's: a host whose resolver answers a loopback or bridge address
+        hive's: a host whose resolver answers a loopback, bridge or other
-        for it (a single-host swarm with the gateway's `localHostsEntry`) gets
+        private address for it (a single-host swarm with the gateway's
-        a 403, and the self-check reports `ok=false`.
+        `localHostsEntry`, or a split-horizon LAN) gets a 403, and the
+        self-check reports `ok=false`.
       '';
     };
   };

nix/module-eval/hive-otel.nix

@@ -491,7 +491,7 @@ let
       # reaches loopback, link-local and the bridge through the proxy. The two
       # IP-literal denies stay ahead of every allow: `dstdomain` on an IP
       # literal does a reverse lookup.
-      name = "squid refuses loopback, link-local and bridge destinations and IP literals, then a missing credential, before any allow";
+      name = "squid refuses loopback, link-local, bridge and private destinations and IP literals, then a missing credential, before any allow";
       ok =
         let
           lines = lib.splitString "\n" hiveOtel.services.squid.configText;
@@ -507,22 +507,51 @@ let
         && at "http_access deny to_localhost" != null
         && at "http_access deny to_linklocal" != null
         && at "http_access deny to_bridge" != null
+        && at "http_access deny to_private" != null
         && at "http_access deny ip_v4_host" != null
         && at "http_access deny ip_v6_host" != null
         && lib.all (l: at l < firstAllow) [
           "http_access deny to_localhost"
           "http_access deny to_linklocal"
           "http_access deny to_bridge"
+          "http_access deny to_private"
           "http_access deny ip_v4_host"
           "http_access deny ip_v6_host"
         ]
         && at "http_access deny to_localhost" < authed
         && at "http_access deny to_linklocal" < authed
         && at "http_access deny to_bridge" < authed
+        && at "http_access deny to_private" < authed
         && authed < firstAllow
         && builtins.elem "cache deny all" lines
         && lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
     }
+    {
+      # The ranges are what keeps an unrestricted user on the public internet.
+      # Control: the bridge deny and the ACL line are found, so a range missing
+      # below is not a lookup of nothing.
+      name = "squid's private-destination ACL names every private, shared, multicast and reserved range";
+      ok =
+        let
+          lines = lib.splitString "\n" hiveOtel.services.squid.configText;
+          acl = lib.findFirst (l: lib.hasPrefix "acl to_private dst " l) null lines;
+          ranges = lib.splitString " " (lib.removePrefix "acl to_private dst " acl);
+        in
+        acl != null
+        && builtins.elem "acl to_bridge dst ${hiveOtel.services.hyperhive.network.networkCidr}" lines
+        &&
+          lib.sort lib.lessThan ranges == lib.sort lib.lessThan [
+            "10.0.0.0/8"
+            "172.16.0.0/12"
+            "192.168.0.0/16"
+            "100.64.0.0/10"
+            "0.0.0.0/8"
+            "224.0.0.0/4"
+            "240.0.0.0/4"
+            "fc00::/7"
+            "ff00::/8"
+          ];
+    }
     {
       # An unrestricted user is let through before the unlisted-name deny and
       # everyone else is held to the listed names after it. Control: both are

nix/vm-tests/hive-squid.nix

@@ -66,6 +66,38 @@ let
         config.services.hyperhive.network.exposeHostPorts;
     };
 
+  # The uplink is a public range. The private ranges squid must refuse are
+  # extra subnets on the same link, so the proxy can reach a listener in each:
+  # a name that resolves there is refused only by squid's own rules.
+  privateNets = [
+    {
+      name = "priv10.t.local";
+      net = "10.99.0";
+    }
+    {
+      name = "priv172.t.local";
+      net = "172.16.99";
+    }
+    {
+      name = "priv192.t.local";
+      net = "192.168.99";
+    }
+    {
+      name = "cgnat.t.local";
+      net = "100.64.99";
+    }
+  ];
+  privateAddrs =
+    host:
+    map (p: {
+      address = "${p.net}.${toString host}";
+      prefixLength = 24;
+    }) privateNets;
+  ulaAddr = host: {
+    address = "fd00:99::${toString host}";
+    prefixLength = 64;
+  };
+
   # Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
   # test reads.
   listen443 = addrs: {
@@ -104,23 +136,34 @@ pkgs.testers.runNixOSTest {
             prefixLength = 24;
           }
         ];
-        networking.interfaces.eth2.ipv4.addresses = lib.mkForce [
+        networking.interfaces.eth2.ipv4.addresses = lib.mkForce (
+          [
             {
-            address = "192.168.2.1";
+              address = "203.0.113.1";
               prefixLength = 24;
             }
-        ];
+          ]
+          ++ privateAddrs 1
+        );
+        networking.interfaces.eth2.ipv6.addresses = [ (ulaAddr 1) ];
         # Names the proxy resolves, each a different destination class.
         networking.hosts = {
           # `other.t.local` resolves, is reachable and is on no allow-list.
-          "192.168.2.2" = [
+          "203.0.113.2" = [
             "forge.t.local"
             "other.t.local"
           ];
           "127.0.0.1" = [ "loop.t.local" ];
           ${bridgeIp} = [ "bridge.t.local" ];
           "169.254.169.254" = [ "meta.t.local" ];
-        };
+          "fd00:99::2" = [ "ula.t.local" ];
+        }
+        // lib.listToAttrs (
+          map (p: {
+            name = "${p.net}.2";
+            value = [ p.name ];
+          }) privateNets
+        );
         # Renders the users file the way hive-c0re does.
         environment.systemPackages = [ pkgs.mkpasswd ];
         services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local";
@@ -129,20 +172,31 @@ pkgs.testers.runNixOSTest {
           "loop.t.local"
           "bridge.t.local"
           "meta.t.local"
-        ];
+          "ula.t.local"
+        ]
+        ++ map (p: p.name) privateNets;
       };
 
     web =
       { lib, ... }:
       {
-        imports = [ (listen443 [ "0.0.0.0" ]) ];
+        imports = [
+          (listen443 [
+            "0.0.0.0"
+            "[::]"
+          ])
+        ];
         virtualisation.vlans = [ 2 ];
-        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
+        networking.interfaces.eth1.ipv4.addresses = lib.mkForce (
+          [
             {
-            address = "192.168.2.2";
+              address = "203.0.113.2";
               prefixLength = 24;
             }
-        ];
+          ]
+          ++ privateAddrs 2
+        );
+        networking.interfaces.eth1.ipv6.addresses = [ (ulaAddr 2) ];
         networking.firewall.allowedTCPPorts = [ 443 ];
       };
 
@@ -188,6 +242,8 @@ pkgs.testers.runNixOSTest {
     web.wait_for_unit("nginx.service")
     proxy.wait_for_open_port(3128, "${bridgeIp}")
 
+    PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local"]
+
     def connect(host, cred=None):
         # The status squid answered the CONNECT with; 000 when none came back.
         auth = f"-U '{cred}' " if cred else ""
@@ -228,6 +284,11 @@ pkgs.testers.runNixOSTest {
             time.sleep(1)
         raise AssertionError(f"{host} as {cred.split(':')[0]}: wanted {want}, last got {got}")
 
+    with subtest("the private listeners answer the proxy itself, so a refusal below is squid's"):
+        for addr in ["10.99.0.2", "172.16.99.2", "192.168.99.2", "100.64.99.2", "fd00:99::2"]:
+            proxy.wait_until_succeeds(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")
+        assert connect("forge.t.local", None) == "407", "control: the public name is challenged, not refused"
+
     with subtest("squid starts on the empty users file and list and refuses every login"):
         assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
         assert proxy.succeed(f"stat -c '%U:%G %a' {LIST}").strip() == "hive-core:squid 640"
@@ -270,14 +331,16 @@ pkgs.testers.runNixOSTest {
         assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password"
         assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, a name that does not resolve"
 
-    with subtest("neither kind of user reaches a host-local destination or another port"):
+    with subtest("neither kind of user reaches a host-local or private destination, or another port"):
         for cred in (ATLAS, UMA):
             who = cred.split(":")[0]
             assert connect("loop.t.local", cred) == "403", f"{who}: loopback"
             assert connect("bridge.t.local", cred) == "403", f"{who}: bridge address"
             assert connect("meta.t.local", cred) == "403", f"{who}: link-local"
             assert connect("127.0.0.1", cred) == "403", f"{who}: loopback literal"
-            assert connect("192.168.2.2", cred) == "403", f"{who}: IP literal"
+            assert connect("203.0.113.2", cred) == "403", f"{who}: IP literal"
+            for name in PRIVATE:
+                assert connect(name, cred) == "403", f"{who}: {name}, a listed name that resolves to a private address"
             assert connect("other.t.local:22", cred) == "403", f"{who}: port 22"
             assert connect("other.t.local:80", cred) == "403", f"{who}: CONNECT to port 80"