squid self-check: use a public canary, drop the swarm service names from the allow list

docs/networking/network.md

@@ -289,9 +289,12 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
   restricted. Agents must not start sending credentials until every hive runs
   the `hive-c0re` that writes it, or a legacy agent loses general internet.
 
-- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the
-  swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each
-  entry matches that exact name. A user on the unrestricted list reaches any
+- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to empty,
+  plus the self-check canary (`services.hyperhive.squid.selfcheck.canary`), so
+  the canary widens the restricted list by that one name. The swarm's own
+  service names aren't on it: they resolve to the bridge or a private address,
+  which the rules below refuse, and agents reach them directly. Each entry
+  matches that exact name. A user on the unrestricted list reaches any
   other name too, on ports 80 and 443. Everyone else gets a 403 for it.
 - **Unlisted names.** For a user who isn't on the unrestricted list, refused
   after the destination checks, which look the name up on the host.
@@ -307,7 +310,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
   answer between a check and the connect gains nothing. A listed name must
   therefore resolve to a public address on the host's resolver: a split-horizon
   name that answers a LAN address gets a 403, and so does the self-check
-  canary. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass.
+  canary, which is why it isn't a swarm service name. Only `CONNECT` to port
+  443 and plain HTTP to ports 80 and 443 pass.
 - **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on
   `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through
   `services.hyperhive.otel.scrapeTargets`. `squid_up` and
@@ -318,7 +322,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
   client's source address.
 - **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service`
   every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT`
-  to `services.hyperhive.squid.selfcheck.canary` (default: the forge's name)
+  to `services.hyperhive.squid.selfcheck.canary` (default: `cache.nixos.org`,
+  a public name that's also on the allow-list)
   must answer 200, to `deny-canary.invalid` 403, and to the canary without a
   credential 407. It then logs in as `selfcheck-unrestricted.squid`, which has
   the same secret and is always on the unrestricted list, and a `CONNECT` to
@@ -333,8 +338,7 @@ credential=present|absent unrestricted_canary=… unrestricted_status=…` line
   written the users file, the line reads `credential=absent` or
   `allow_status=407` and `ok=false`, and nothing else breaks. The
   host's resolver looks up the canary, so a host whose resolver maps
-  that name to a loopback or bridge address (a single-host swarm with
-  `gateway.localHostsEntry`) fails the check.
+  that name to a loopback, bridge or private address fails the check.
 
 ## Container isolation
 

nix/host-modules/hive-squid.nix

@@ -213,17 +213,19 @@ in
 
     allowedDomains = lib.mkOption {
       type = lib.types.listOf lib.types.str;
-      default = hyperhiveCfg.swarm.serviceDomains;
-      defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains";
+      default = [ ];
       example = [
-        "forge.example.com"
         "github.com"
+        "crates.io"
       ];
       description = ''
         Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
         entry matches that exact name. Everything else gets a 403, except
-        for an unrestricted user. The self-check canary is allowed on top of
-        this list. A name must resolve, on this host's resolver, to a public
+        for an unrestricted user. The swarm's own service names are not
+        listed: they resolve to the bridge or a private address, which the
+        destination rules refuse, and agents reach them directly. The
+        self-check canary is allowed on top of this list, so it widens the
+        restricted allow list by that one name. A name must resolve, on this host's resolver, to a public
         address: one in a loopback, link-local, private, shared (100.64.0.0/10),
         multicast or reserved range (or is `::`), or in the bridge subnet, gets a 403 for
         every user.
@@ -285,15 +287,15 @@ in
 
     selfcheck.canary = lib.mkOption {
       type = lib.types.str;
-      default = hyperhiveCfg.swarm.forge.domain;
-      defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain";
+      default = "cache.nixos.org";
       description = ''
         Hostname the self-check CONNECTs to on port 443 and expects squid to
-        let through. Squid resolves it with this host's resolver, not the
-        hive's: a host whose resolver answers a loopback, bridge or other
-        private address for it (a single-host swarm with the gateway's
-        `localHostsEntry`, or a split-horizon LAN) gets a 403, and the
-        self-check reports `ok=false`.
+        let through. It is added to the allowed names, so every restricted
+        agent can reach it too. It must resolve to a public address: squid
+        resolves it with this host's resolver, not the hive's, and a name
+        that answers a loopback, bridge or other private address (a swarm
+        service name on a hive, a split-horizon LAN) gets a 403 before
+        authentication, and the self-check reports `ok=false`.
       '';
     };
   };

nix/module-eval/hive-otel.nix

@@ -526,6 +526,19 @@ let
         && builtins.elem "cache deny all" lines
         && lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
     }
+    {
+      # Control: the allow ACL line is found and holds the canary, so the
+      # absence of the swarm's service names is not a lookup of nothing.
+      name = "squid's default allow-list is the self-check canary alone, with no swarm service name";
+      ok =
+        let
+          lines = lib.splitString "\n" hiveOtel.services.squid.configText;
+          acl = lib.findFirst (l: lib.hasPrefix "acl allowed_dst dstdomain " l) "" (map lib.trim lines);
+        in
+        acl == "acl allowed_dst dstdomain ${hiveOtel.services.hyperhive.squid.selfcheck.canary}"
+        && hiveOtel.services.hyperhive.squid.selfcheck.canary == "cache.nixos.org"
+        && !(builtins.elem hiveOtel.services.hyperhive.swarm.forge.domain (lib.splitString " " acl));
+    }
     {
       # The ranges are what keeps an unrestricted user on the public internet.
       # Control: the bridge deny and the ACL line are found, so a range missing

nix/vm-tests/hive-squid.nix

@@ -20,14 +20,6 @@ let
           type = lib.types.bool;
           default = true;
         };
-        swarm.serviceDomains = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          default = [ "forge.t.local" ];
-        };
-        swarm.forge.domain = lib.mkOption {
-          type = lib.types.str;
-          default = "forge.t.local";
-        };
         network = {
           bridgeIp = lib.mkOption {
             type = lib.types.str;
@@ -157,6 +149,7 @@ pkgs.testers.runNixOSTest {
           # has no resolver to answer for an unknown one before the client gives up.
           "203.0.113.2" = [
             "forge.t.local"
+            "cache.t.local"
             "other.t.local"
             "deny-canary.invalid"
           ];
@@ -177,6 +170,7 @@ pkgs.testers.runNixOSTest {
         # Renders the users file the way hive-c0re does.
         environment.systemPackages = [ pkgs.mkpasswd ];
         services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local";
+        services.hyperhive.squid.selfcheck.canary = "cache.t.local";
         services.hyperhive.squid.allowedDomains = [
           "forge.t.local"
           "loop.t.local"
@@ -323,6 +317,7 @@ pkgs.testers.runNixOSTest {
 
     with subtest("an allowed name is tunnelled with a credential and challenged without one"):
         assert connect("forge.t.local", ATLAS) == "200", "allow canary"
+        assert connect("cache.t.local", ATLAS) == "200", "the self-check canary, allowed on top of the list"
         assert connect("forge.t.local") == "407", "no credential"
         assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
         assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user"
@@ -409,7 +404,7 @@ pkgs.testers.runNixOSTest {
         proxy.succeed("systemctl start squid-selfcheck.service")
         proxy.succeed(
             "journalctl -u squid-selfcheck.service -o cat | "
-            "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local "
+            "grep -F 'squid-selfcheck ok=true allow_canary=cache.t.local "
             "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
             "anon_status=407 credential=present "
             "unrestricted_canary=other.t.local unrestricted_status=200'"
@@ -421,7 +416,7 @@ pkgs.testers.runNixOSTest {
         proxy.fail("systemctl start squid-selfcheck.service")
         proxy.succeed(
             "journalctl -u squid-selfcheck.service -o cat | "
-            "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local "
+            "grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local "
             "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
             "anon_status=407 credential=present "
             "unrestricted_canary=other.t.local unrestricted_status=403'"
@@ -434,7 +429,7 @@ pkgs.testers.runNixOSTest {
         proxy.fail("systemctl start squid-selfcheck.service")
         proxy.succeed(
             "journalctl -u squid-selfcheck.service -o cat | "
-            "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local "
+            "grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local "
             "allow_status=407 deny_canary=deny-canary.invalid deny_status=407 "
             "anon_status=407 credential=absent "
             "unrestricted_canary=other.t.local unrestricted_status=407'"