From 10fdf06334161926d54d34f50f7bd3bc04e2556b Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 18:17:53 +0100 Subject: [PATCH 1/2] squid self-check: use a public canary, drop the swarm service names from the allow list --- docs/networking/network.md | 18 +++++++++++------- nix/host-modules/hive-squid.nix | 26 ++++++++++++++------------ nix/module-eval/hive-otel.nix | 13 +++++++++++++ nix/vm-tests/hive-squid.nix | 17 ++++++----------- 4 files changed, 44 insertions(+), 30 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index 9bfff2bfd..7d136ece5 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -289,9 +289,12 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. restricted. Agents must not start sending credentials until every hive runs the `hive-c0re` that writes it, or a legacy agent loses general internet. -- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the - swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each - entry matches that exact name. A user on the unrestricted list reaches any +- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to empty, + plus the self-check canary (`services.hyperhive.squid.selfcheck.canary`), so + the canary widens the restricted list by that one name. The swarm's own + service names aren't on it: they resolve to the bridge or a private address, + which the rules below refuse, and agents reach them directly. Each entry + matches that exact name. A user on the unrestricted list reaches any other name too, on ports 80 and 443. Everyone else gets a 403 for it. - **Unlisted names.** For a user who isn't on the unrestricted list, refused after the destination checks, which look the name up on the host. @@ -307,7 +310,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. answer between a check and the connect gains nothing. A listed name must therefore resolve to a public address on the host's resolver: a split-horizon name that answers a LAN address gets a 403, and so does the self-check - canary. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass. + canary, which is why it isn't a swarm service name. Only `CONNECT` to port + 443 and plain HTTP to ports 80 and 443 pass. - **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through `services.hyperhive.otel.scrapeTargets`. `squid_up` and @@ -318,7 +322,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. client's source address. - **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service` every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT` - to `services.hyperhive.squid.selfcheck.canary` (default: the forge's name) + to `services.hyperhive.squid.selfcheck.canary` (default: `cache.nixos.org`, + a public name that's also on the allow-list) must answer 200, to `deny-canary.invalid` 403, and to the canary without a credential 407. It then logs in as `selfcheck-unrestricted.squid`, which has the same secret and is always on the unrestricted list, and a `CONNECT` to @@ -333,8 +338,7 @@ credential=present|absent unrestricted_canary=… unrestricted_status=…` line written the users file, the line reads `credential=absent` or `allow_status=407` and `ok=false`, and nothing else breaks. The host's resolver looks up the canary, so a host whose resolver maps - that name to a loopback or bridge address (a single-host swarm with - `gateway.localHostsEntry`) fails the check. + that name to a loopback, bridge or private address fails the check. ## Container isolation diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 0875d9ee4..07e1f1779 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -213,17 +213,19 @@ in allowedDomains = lib.mkOption { type = lib.types.listOf lib.types.str; - default = hyperhiveCfg.swarm.serviceDomains; - defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains"; + default = [ ]; example = [ - "forge.example.com" "github.com" + "crates.io" ]; description = '' Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each entry matches that exact name. Everything else gets a 403, except - for an unrestricted user. The self-check canary is allowed on top of - this list. A name must resolve, on this host's resolver, to a public + for an unrestricted user. The swarm's own service names are not + listed: they resolve to the bridge or a private address, which the + destination rules refuse, and agents reach them directly. The + self-check canary is allowed on top of this list, so it widens the + restricted allow list by that one name. A name must resolve, on this host's resolver, to a public address: one in a loopback, link-local, private, shared (100.64.0.0/10), multicast or reserved range (or is `::`), or in the bridge subnet, gets a 403 for every user. @@ -285,15 +287,15 @@ in selfcheck.canary = lib.mkOption { type = lib.types.str; - default = hyperhiveCfg.swarm.forge.domain; - defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain"; + default = "cache.nixos.org"; description = '' Hostname the self-check CONNECTs to on port 443 and expects squid to - let through. Squid resolves it with this host's resolver, not the - hive's: a host whose resolver answers a loopback, bridge or other - private address for it (a single-host swarm with the gateway's - `localHostsEntry`, or a split-horizon LAN) gets a 403, and the - self-check reports `ok=false`. + let through. It is added to the allowed names, so every restricted + agent can reach it too. It must resolve to a public address: squid + resolves it with this host's resolver, not the hive's, and a name + that answers a loopback, bridge or other private address (a swarm + service name on a hive, a split-horizon LAN) gets a 403 before + authentication, and the self-check reports `ok=false`. ''; }; }; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 6bf1e61f6..7d7441f5e 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -526,6 +526,19 @@ let && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } + { + # Control: the allow ACL line is found and holds the canary, so the + # absence of the swarm's service names is not a lookup of nothing. + name = "squid's default allow-list is the self-check canary alone, with no swarm service name"; + ok = + let + lines = lib.splitString "\n" hiveOtel.services.squid.configText; + acl = lib.findFirst (l: lib.hasPrefix "acl allowed_dst dstdomain " l) "" (map lib.trim lines); + in + acl == "acl allowed_dst dstdomain ${hiveOtel.services.hyperhive.squid.selfcheck.canary}" + && hiveOtel.services.hyperhive.squid.selfcheck.canary == "cache.nixos.org" + && !(builtins.elem hiveOtel.services.hyperhive.swarm.forge.domain (lib.splitString " " acl)); + } { # The ranges are what keeps an unrestricted user on the public internet. # Control: the bridge deny and the ACL line are found, so a range missing diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 0d624c866..ce7a953a8 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -20,14 +20,6 @@ let type = lib.types.bool; default = true; }; - swarm.serviceDomains = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ "forge.t.local" ]; - }; - swarm.forge.domain = lib.mkOption { - type = lib.types.str; - default = "forge.t.local"; - }; network = { bridgeIp = lib.mkOption { type = lib.types.str; @@ -157,6 +149,7 @@ pkgs.testers.runNixOSTest { # has no resolver to answer for an unknown one before the client gives up. "203.0.113.2" = [ "forge.t.local" + "cache.t.local" "other.t.local" "deny-canary.invalid" ]; @@ -177,6 +170,7 @@ pkgs.testers.runNixOSTest { # Renders the users file the way hive-c0re does. environment.systemPackages = [ pkgs.mkpasswd ]; services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local"; + services.hyperhive.squid.selfcheck.canary = "cache.t.local"; services.hyperhive.squid.allowedDomains = [ "forge.t.local" "loop.t.local" @@ -323,6 +317,7 @@ pkgs.testers.runNixOSTest { with subtest("an allowed name is tunnelled with a credential and challenged without one"): assert connect("forge.t.local", ATLAS) == "200", "allow canary" + assert connect("cache.t.local", ATLAS) == "200", "the self-check canary, allowed on top of the list" assert connect("forge.t.local") == "407", "no credential" assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password" assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user" @@ -409,7 +404,7 @@ pkgs.testers.runNixOSTest { proxy.succeed("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " - "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local " + "grep -F 'squid-selfcheck ok=true allow_canary=cache.t.local " "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " "anon_status=407 credential=present " "unrestricted_canary=other.t.local unrestricted_status=200'" @@ -421,7 +416,7 @@ pkgs.testers.runNixOSTest { proxy.fail("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " - "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " + "grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local " "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " "anon_status=407 credential=present " "unrestricted_canary=other.t.local unrestricted_status=403'" @@ -434,7 +429,7 @@ pkgs.testers.runNixOSTest { proxy.fail("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " - "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " + "grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local " "allow_status=407 deny_canary=deny-canary.invalid deny_status=407 " "anon_status=407 credential=absent " "unrestricted_canary=other.t.local unrestricted_status=407'" From e78a529ec215e39dce4795a2a3ac68ee869f90e7 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 18:37:58 +0100 Subject: [PATCH 2/2] squid: base allow list for every authed user, and no start limit on the list reload --- docs/networking/network.md | 12 ++++++++ docs/process/conventions.md | 2 +- nix/host-modules/hive-squid.nix | 52 ++++++++++++++++++++++++++++----- nix/module-eval/hive-otel.nix | 11 +++++-- nix/vm-tests/hive-squid.nix | 17 ++++++++++- 5 files changed, 82 insertions(+), 12 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index 7d136ece5..641e7df14 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -289,6 +289,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. restricted. Agents must not start sending credentials until every hive runs the `hive-c0re` that writes it, or a legacy agent loses general internet. +- **Base list.** `services.hyperhive.squid.baseAllowedDomains` is open to every + authenticated user, whatever its flag: what the agent harness needs off the + swarm. The default is the hosts Claude Code requires + (): `api.anthropic.com` for + the model API and the usage probe (`hive-agent/src/claude_usage_watch.rs`), + `claude.ai` for the account login, and `platform.claude.com` for the OAuth + token exchange and refresh. An agent whose `ANTHROPIC_BASE_URL` or ACP + provider points elsewhere needs that host added to the option: the value + sits in the agent's own environment file, which the host doesn't read. The + list omits the plugin marketplace clone from `github.com`, since a failed add + isn't fatal. The rules below still refuse an entry that resolves to a + private address. - **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to empty, plus the self-check canary (`services.hyperhive.squid.selfcheck.canary`), so the canary widens the restricted list by that one name. The swarm's own diff --git a/docs/process/conventions.md b/docs/process/conventions.md index c876f2869..37271b960 100644 --- a/docs/process/conventions.md +++ b/docs/process/conventions.md @@ -323,7 +323,7 @@ that value. Read it with `swarm_queue_client::agent_permissions::unrestricted`. The hive reads it for the agent's squid policy: [the egress proxy](../networking/network.md#egress-proxy-squid) lets an unrestricted agent's credential reach any destination, and holds the -others to the allow-list. An agent sends its credential once it has fetched it, from its next start. +others to the base list and the allow-list. An agent sends its credential once it has fetched it, from its next start. **How a change applies** — - The agent's harness reads its own key every 10 seconds and builds every diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 07e1f1779..ca727f267 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -25,6 +25,7 @@ let denyCanary = "deny-canary.invalid"; allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]); + base = lib.unique cfg.baseAllowedDomains; # The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`. selfcheckUser = "selfcheck.squid"; @@ -63,9 +64,9 @@ let # The IP-literal # denies come first because `dstdomain` on an IP literal does a reverse # lookup. The credential check sits right before the first allow, so only a - # request that would otherwise be let through gets the 407 challenge. An - # unrestricted user is then let through to any other destination on a safe - # port, and every other user to the listed names. The `dst` denies cost an + # request that would otherwise be let through gets the 407 challenge. Every + # user is then let through to the base names, an unrestricted user to any other + # destination on a safe port, and every other user to the listed names. The `dst` denies cost an # unlisted name a lookup on the host's resolver before it is refused. squidConf = '' http_port ${listen} @@ -105,6 +106,7 @@ let # an agent controls could match an allowed name. acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$) acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$) + ${lib.optionalString (base != [ ]) "acl base_dst dstdomain ${lib.concatStringsSep " " base}"} ${lib.optionalString ( allowed != [ ] ) "acl allowed_dst dstdomain ${lib.concatStringsSep " " allowed}"} @@ -123,6 +125,7 @@ let http_access deny to_bridge http_access deny to_private http_access deny !authed + ${lib.optionalString (base != [ ]) "http_access allow base_dst"} http_access allow unrestricted ${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"} ${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"} @@ -225,13 +228,43 @@ in listed: they resolve to the bridge or a private address, which the destination rules refuse, and agents reach them directly. The self-check canary is allowed on top of this list, so it widens the - restricted allow list by that one name. A name must resolve, on this host's resolver, to a public - address: one in a loopback, link-local, private, shared (100.64.0.0/10), - multicast or reserved range (or is `::`), or in the bridge subnet, gets a 403 for + restricted allow list by that one name. A name must + resolve, on this host's resolver, to a public address: one in a + loopback, link-local, private, shared (100.64.0.0/10), multicast or + reserved range (or is `::`), or in the bridge subnet, gets a 403 for every user. ''; }; + baseAllowedDomains = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "api.anthropic.com" + "claude.ai" + "platform.claude.com" + ]; + example = [ + "api.anthropic.com" + "claude.ai" + "platform.claude.com" + "openrouter.ai" + ]; + description = '' + Hostnames every authenticated user may reach, whether or not its agent + holds the `unrestricted_internet` flag: what the agent harness needs + off the swarm to work at all. The defaults are the hosts Claude Code + requires. `api.anthropic.com` serves the model API and the usage + probe, `claude.ai` the account login, and `platform.claude.com` the + OAuth token exchange and refresh. An agent whose `claude` or ACP provider + points elsewhere (`ANTHROPIC_BASE_URL`, an `opencode` provider) needs + that host listed here: the value lives in the agent's own backend + environment, which this host does not read. Swarm-internal names are + not listed, since agents reach them directly. The destination denies + still apply to every entry, and the list is separate from + `allowedDomains`. + ''; + }; + usersFile = lib.mkOption { type = lib.types.path; default = "/var/lib/hive-squid/users"; @@ -340,6 +373,9 @@ in systemd.services.squid-unrestricted-reload = { description = "Reload squid to read its unrestricted-users list"; + # Several list writes in a few seconds must not trip the start limit: a + # tripped limit also fails the path unit, and later writes then reload nothing. + startLimitIntervalSec = 0; serviceConfig = { Type = "oneshot"; ExecStart = "${config.systemd.package}/bin/systemctl try-reload-or-restart squid.service"; @@ -348,10 +384,10 @@ in assertions = [ { - assertion = !(builtins.elem cfg.selfcheck.unrestrictedCanary allowed); + assertion = !(builtins.elem cfg.selfcheck.unrestrictedCanary (allowed ++ base)); message = '' services.hyperhive.squid.selfcheck.unrestrictedCanary must be a name outside - allowedDomains, or the self-check cannot tell an unrestricted user from a + allowedDomains and baseAllowedDomains, or the self-check cannot tell an unrestricted user from a restricted one. ''; } diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 7d7441f5e..b408268f9 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -497,11 +497,13 @@ let lines = lib.splitString "\n" hiveOtel.services.squid.configText; at = l: lib.lists.findFirstIndex (x: x == l) null lines; allow = at "http_access allow allowed_dst"; + base = at "http_access allow base_dst"; unrestricted = at "http_access allow unrestricted"; authed = at "http_access deny !authed"; - firstAllow = if unrestricted < allow then unrestricted else allow; + firstAllow = lib.min base (lib.min unrestricted allow); in allow != null + && base != null && unrestricted != null && authed != null && at "http_access deny to_localhost" != null @@ -529,13 +531,15 @@ let { # Control: the allow ACL line is found and holds the canary, so the # absence of the swarm's service names is not a lookup of nothing. - name = "squid's default allow-list is the self-check canary alone, with no swarm service name"; + name = "squid's default allow-list is the self-check canary alone and its base list is the Claude Code hosts, with no swarm service name"; ok = let lines = lib.splitString "\n" hiveOtel.services.squid.configText; acl = lib.findFirst (l: lib.hasPrefix "acl allowed_dst dstdomain " l) "" (map lib.trim lines); + baseAcl = lib.findFirst (l: lib.hasPrefix "acl base_dst dstdomain " l) "" (map lib.trim lines); in acl == "acl allowed_dst dstdomain ${hiveOtel.services.hyperhive.squid.selfcheck.canary}" + && baseAcl == "acl base_dst dstdomain api.anthropic.com claude.ai platform.claude.com" && hiveOtel.services.hyperhive.squid.selfcheck.canary == "cache.nixos.org" && !(builtins.elem hiveOtel.services.hyperhive.swarm.forge.domain (lib.splitString " " acl)); } @@ -580,6 +584,8 @@ let in at "http_access allow unrestricted" != null && at "http_access deny !allowed_dst" != null + && at "http_access deny !authed" < at "http_access allow base_dst" + && at "http_access allow base_dst" < at "http_access allow unrestricted" && at "http_access allow unrestricted" < at "http_access deny !allowed_dst" && at "http_access deny !allowed_dst" < at "http_access allow allowed_dst" && builtins.elem ''acl unrestricted proxy_auth "${list}"'' lines @@ -597,6 +603,7 @@ let svc = hiveOtel.systemd.services.squid-unrestricted-reload.serviceConfig; in path.pathConfig.PathChanged == list + && hiveOtel.systemd.services.squid-unrestricted-reload.startLimitIntervalSec == 0 && builtins.elem "paths.target" path.wantedBy && lib.hasSuffix "systemctl try-reload-or-restart squid.service" svc.ExecStart && builtins.elem "f ${list} 0640 hive-core squid - -" rules diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index ce7a953a8..87acdd4c9 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -150,11 +150,15 @@ pkgs.testers.runNixOSTest { "203.0.113.2" = [ "forge.t.local" "cache.t.local" + "api.t.local" "other.t.local" "deny-canary.invalid" ]; "127.0.0.1" = [ "loop.t.local" ]; - ${bridgeIp} = [ "bridge.t.local" ]; + ${bridgeIp} = [ + "bridge.t.local" + "basebridge.t.local" + ]; "169.254.169.254" = [ "meta.t.local" ]; "fd00:99::2" = [ "ula.t.local" ]; "::" = [ "any6.t.local" ]; @@ -171,6 +175,10 @@ pkgs.testers.runNixOSTest { environment.systemPackages = [ pkgs.mkpasswd ]; services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local"; services.hyperhive.squid.selfcheck.canary = "cache.t.local"; + services.hyperhive.squid.baseAllowedDomains = [ + "api.t.local" + "basebridge.t.local" + ]; services.hyperhive.squid.allowedDomains = [ "forge.t.local" "loop.t.local" @@ -324,6 +332,13 @@ pkgs.testers.runNixOSTest { # DES would accept the first eight characters; the hash must not. assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password" + with subtest("a base-list name is open to a restricted user, with a credential, and nothing else is"): + assert connect("api.t.local", ATLAS) == "200", "restricted user, base name" + assert connect("api.t.local") == "407", "no credential" + assert connect("api.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password" + assert connect("other.t.local", ATLAS) == "403", "control: an unlisted name is still refused" + assert connect("basebridge.t.local", ATLAS) == "403", "a base name that resolves to the bridge" + with subtest("with an empty list every user is restricted"): assert connect("other.t.local", UMA) == "403", "uma, not listed yet" assert connect("other.t.local", ATLAS) == "403"