diff --git a/docs/networking/network.md b/docs/networking/network.md index 9bfff2bf..7d136ece 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -289,9 +289,12 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. restricted. Agents must not start sending credentials until every hive runs the `hive-c0re` that writes it, or a legacy agent loses general internet. -- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the - swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each - entry matches that exact name. A user on the unrestricted list reaches any +- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to empty, + plus the self-check canary (`services.hyperhive.squid.selfcheck.canary`), so + the canary widens the restricted list by that one name. The swarm's own + service names aren't on it: they resolve to the bridge or a private address, + which the rules below refuse, and agents reach them directly. Each entry + matches that exact name. A user on the unrestricted list reaches any other name too, on ports 80 and 443. Everyone else gets a 403 for it. - **Unlisted names.** For a user who isn't on the unrestricted list, refused after the destination checks, which look the name up on the host. @@ -307,7 +310,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. answer between a check and the connect gains nothing. A listed name must therefore resolve to a public address on the host's resolver: a split-horizon name that answers a LAN address gets a 403, and so does the self-check - canary. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass. + canary, which is why it isn't a swarm service name. Only `CONNECT` to port + 443 and plain HTTP to ports 80 and 443 pass. - **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through `services.hyperhive.otel.scrapeTargets`. `squid_up` and @@ -318,7 +322,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. client's source address. - **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service` every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT` - to `services.hyperhive.squid.selfcheck.canary` (default: the forge's name) + to `services.hyperhive.squid.selfcheck.canary` (default: `cache.nixos.org`, + a public name that's also on the allow-list) must answer 200, to `deny-canary.invalid` 403, and to the canary without a credential 407. It then logs in as `selfcheck-unrestricted.squid`, which has the same secret and is always on the unrestricted list, and a `CONNECT` to @@ -333,8 +338,7 @@ credential=present|absent unrestricted_canary=… unrestricted_status=…` line written the users file, the line reads `credential=absent` or `allow_status=407` and `ok=false`, and nothing else breaks. The host's resolver looks up the canary, so a host whose resolver maps - that name to a loopback or bridge address (a single-host swarm with - `gateway.localHostsEntry`) fails the check. + that name to a loopback, bridge or private address fails the check. ## Container isolation diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 0875d9ee..07e1f177 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -213,17 +213,19 @@ in allowedDomains = lib.mkOption { type = lib.types.listOf lib.types.str; - default = hyperhiveCfg.swarm.serviceDomains; - defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains"; + default = [ ]; example = [ - "forge.example.com" "github.com" + "crates.io" ]; description = '' Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each entry matches that exact name. Everything else gets a 403, except - for an unrestricted user. The self-check canary is allowed on top of - this list. A name must resolve, on this host's resolver, to a public + for an unrestricted user. The swarm's own service names are not + listed: they resolve to the bridge or a private address, which the + destination rules refuse, and agents reach them directly. The + self-check canary is allowed on top of this list, so it widens the + restricted allow list by that one name. A name must resolve, on this host's resolver, to a public address: one in a loopback, link-local, private, shared (100.64.0.0/10), multicast or reserved range (or is `::`), or in the bridge subnet, gets a 403 for every user. @@ -285,15 +287,15 @@ in selfcheck.canary = lib.mkOption { type = lib.types.str; - default = hyperhiveCfg.swarm.forge.domain; - defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain"; + default = "cache.nixos.org"; description = '' Hostname the self-check CONNECTs to on port 443 and expects squid to - let through. Squid resolves it with this host's resolver, not the - hive's: a host whose resolver answers a loopback, bridge or other - private address for it (a single-host swarm with the gateway's - `localHostsEntry`, or a split-horizon LAN) gets a 403, and the - self-check reports `ok=false`. + let through. It is added to the allowed names, so every restricted + agent can reach it too. It must resolve to a public address: squid + resolves it with this host's resolver, not the hive's, and a name + that answers a loopback, bridge or other private address (a swarm + service name on a hive, a split-horizon LAN) gets a 403 before + authentication, and the self-check reports `ok=false`. ''; }; }; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 6bf1e61f..7d7441f5 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -526,6 +526,19 @@ let && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } + { + # Control: the allow ACL line is found and holds the canary, so the + # absence of the swarm's service names is not a lookup of nothing. + name = "squid's default allow-list is the self-check canary alone, with no swarm service name"; + ok = + let + lines = lib.splitString "\n" hiveOtel.services.squid.configText; + acl = lib.findFirst (l: lib.hasPrefix "acl allowed_dst dstdomain " l) "" (map lib.trim lines); + in + acl == "acl allowed_dst dstdomain ${hiveOtel.services.hyperhive.squid.selfcheck.canary}" + && hiveOtel.services.hyperhive.squid.selfcheck.canary == "cache.nixos.org" + && !(builtins.elem hiveOtel.services.hyperhive.swarm.forge.domain (lib.splitString " " acl)); + } { # The ranges are what keeps an unrestricted user on the public internet. # Control: the bridge deny and the ACL line are found, so a range missing diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 0d624c86..ce7a953a 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -20,14 +20,6 @@ let type = lib.types.bool; default = true; }; - swarm.serviceDomains = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ "forge.t.local" ]; - }; - swarm.forge.domain = lib.mkOption { - type = lib.types.str; - default = "forge.t.local"; - }; network = { bridgeIp = lib.mkOption { type = lib.types.str; @@ -157,6 +149,7 @@ pkgs.testers.runNixOSTest { # has no resolver to answer for an unknown one before the client gives up. "203.0.113.2" = [ "forge.t.local" + "cache.t.local" "other.t.local" "deny-canary.invalid" ]; @@ -177,6 +170,7 @@ pkgs.testers.runNixOSTest { # Renders the users file the way hive-c0re does. environment.systemPackages = [ pkgs.mkpasswd ]; services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local"; + services.hyperhive.squid.selfcheck.canary = "cache.t.local"; services.hyperhive.squid.allowedDomains = [ "forge.t.local" "loop.t.local" @@ -323,6 +317,7 @@ pkgs.testers.runNixOSTest { with subtest("an allowed name is tunnelled with a credential and challenged without one"): assert connect("forge.t.local", ATLAS) == "200", "allow canary" + assert connect("cache.t.local", ATLAS) == "200", "the self-check canary, allowed on top of the list" assert connect("forge.t.local") == "407", "no credential" assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password" assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user" @@ -409,7 +404,7 @@ pkgs.testers.runNixOSTest { proxy.succeed("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " - "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local " + "grep -F 'squid-selfcheck ok=true allow_canary=cache.t.local " "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " "anon_status=407 credential=present " "unrestricted_canary=other.t.local unrestricted_status=200'" @@ -421,7 +416,7 @@ pkgs.testers.runNixOSTest { proxy.fail("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " - "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " + "grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local " "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " "anon_status=407 credential=present " "unrestricted_canary=other.t.local unrestricted_status=403'" @@ -434,7 +429,7 @@ pkgs.testers.runNixOSTest { proxy.fail("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " - "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " + "grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local " "allow_status=407 deny_canary=deny-canary.invalid deny_status=407 " "anon_status=407 credential=absent " "unrestricted_canary=other.t.local unrestricted_status=407'"