From b7adb90f537852b79b7a46a5e9592b87e6703b2d Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 13:07:15 +0100 Subject: [PATCH] require a proxy credential at squid and prove it in the self-check --- nix/host-modules/hive-squid.nix | 85 +++++++++++++++++---- nix/module-eval/hive-otel.nix | 35 +++++++-- nix/vm-tests/hive-squid.nix | 130 +++++++++++++++++++++++++++----- 3 files changed, 207 insertions(+), 43 deletions(-) diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index c1b92645..46515662 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -1,8 +1,9 @@ # Egress proxy for this hive's agents: squid on the host, bound to the bridge # IP, default-deny on the destination hostname. It filters CONNECT targets and -# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is -# pointed at it yet; the exporter, journal forwarding and the self-check below -# are how a deploy shows it works. +# plain-HTTP hosts and never decrypts, so it holds no signing key. A request +# for a listed name must carry a per-agent proxy credential, checked against the +# users file hive-c0re renders. No agent sends one yet; the exporter, journal +# forwarding and the self-check below are how a deploy shows it works. { pkgs, lib, @@ -21,6 +22,8 @@ let allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]); + # The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`. + selfcheckUser = "selfcheck.squid"; usersDir = builtins.dirOf cfg.usersFile; # Evaluated top to bottom, first match wins. The three destination denies @@ -29,7 +32,10 @@ let # otherwise reach services the bridge firewall keeps agents out of. The # unlisted-name deny comes first because `dstdomain` needs no lookup while # the `dst` ACLs do: an unlisted name is refused at once instead of waiting - # on the host resolver, and costs a client no DNS query. + # on the host resolver, and costs a client no DNS query. The credential + # check sits right before the allow, so a refused destination answers 403 + # to a client with or without a credential, and only a request that would + # otherwise be let through gets the 407 challenge. squidConf = '' http_port ${netCfg.bridgeIp}:${toString port} pid_filename /run/squid/squid.pid @@ -42,8 +48,17 @@ let # Cachemgr scrapes are left out. access_log syslog:daemon.info squid !manager + # The helper reads the users file on every request and reloads it when its + # mtime moves. It exits when the file is missing, which is why the file is + # created before squid starts. A positive answer is cached for a minute, so + # that is also how long a removed user keeps working. + auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile} + auth_param basic realm hyperhive egress + auth_param basic credentialsttl 1 minute + acl SSL_ports port 443 acl Safe_ports port 80 443 + acl authed proxy_auth REQUIRED # Host-originated requests: the self-check and the exporter. A local # connect to the bridge IP takes the bridge IP as its source. @@ -71,6 +86,7 @@ let http_access deny to_localhost http_access deny to_linklocal http_access deny to_bridge + http_access deny !authed ${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"} http_access deny all ''; @@ -103,20 +119,39 @@ let name = "squid-selfcheck"; runtimeInputs = [ pkgs.curl ]; text = '' - # The status squid answers a CONNECT with; 000 when the proxy is down. + # The credential comes from the unit's imported credential, and goes to + # curl on its stdin so it never appears in a command line. Without one + # the requests are anonymous and the self-check reports it. + secret="" + credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}" + if [ -s "$credential_file" ]; then + secret=$(< "$credential_file") + fi + + # : the status squid answers a CONNECT with; 000 when + # the proxy is down. connect_status() { - curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \ + { + if [ "$2" = auth ] && [ -n "$secret" ]; then + printf 'proxy-user = "%s:%s"\n' ${selfcheckUser} "$secret" + fi + } | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \ -o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true } - allow_status=$(connect_status ${cfg.selfcheck.canary}) - deny_status=$(connect_status ${denyCanary}) + allow_status=$(connect_status ${cfg.selfcheck.canary} auth) + deny_status=$(connect_status ${denyCanary} auth) + anon_status=$(connect_status ${cfg.selfcheck.canary} anon) ok=false - if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then + if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ]; then ok=true fi - echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status" + credential=absent + if [ -n "$secret" ]; then + credential=present + fi + echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential" [ "$ok" = true ] ''; }; @@ -142,10 +177,24 @@ in type = lib.types.path; default = "/var/lib/hive-squid/users"; description = '' - The `:` file hive-c0re renders from the credentials in the - swarm secret store: one line per agent of this hive and one for the - self-check, as SHA-512 crypt hashes. Created empty, owned by `hive-core` - and readable by the `squid` group. Squid doesn't read it yet. + The `:` file squid checks proxy credentials against. hive-c0re + renders it from the credentials in the swarm secret store, one line per + agent of this hive and one for the self-check, as SHA-512 crypt hashes. + An empty file, which is what exists until hive-c0re writes one, makes + every login fail, and every request to a listed name answers 407. + Owned by `hive-core` and readable by the `squid` group. + ''; + }; + + selfcheck.credentialName = lib.mkOption { + type = lib.types.str; + default = "hive-squid.selfcheck"; + readOnly = true; + description = '' + Name of the systemd credential the self-check imports its squid + password from. `squid-selfcheck-credential` writes it to + `/run/credstore/`. Without one the self-check sends no + credential and reports `ok=false`. ''; }; @@ -181,7 +230,8 @@ in }; # The directory is setgid so the file hive-c0re renames into it is the - # squid group's. + # squid group's. The file exists before squid starts, empty: the helper + # exits on a missing file and squid then rejects every login. systemd.tmpfiles.rules = [ "d ${usersDir} 2750 hive-core squid - -" "f ${cfg.usersFile} 0640 hive-core squid - -" @@ -224,10 +274,13 @@ in }; systemd.services.squid-selfcheck = { - description = "Squid allow and deny canaries"; + description = "Squid allow, deny and credential canaries"; after = [ "squid.service" ]; serviceConfig = sandbox // { Type = "oneshot"; + # A glob that matches nothing is not an error: a host with no + # credential yet still runs the self-check, and it reports that. + ImportCredential = cfg.selfcheck.credentialName; ExecStart = lib.getExe selfcheck; DynamicUser = true; IPAddressAllow = netCfg.bridgeIp; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 3db59108..2cf4e19e 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -485,39 +485,60 @@ let && !(lib.hasInfix "localnet" conf); } { - # First match wins, so a destination deny after the allow is dead. - name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, before any allow"; + # First match wins, so a destination deny after the allow is dead, and + # a credential check after it never challenges anyone. The two IP-literal + # denies stay ahead of the unlisted-name deny: `dstdomain` on an IP + # literal does a reverse lookup. + name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, then a missing credential, before any allow"; ok = let lines = lib.splitString "\n" hiveOtel.services.squid.configText; at = l: lib.lists.findFirstIndex (x: x == l) null lines; allow = at "http_access allow allowed_dst"; + authed = at "http_access deny !authed"; in allow != null + && authed != null && at "http_access deny to_localhost" != null && at "http_access deny to_localhost" < allow && at "http_access deny to_linklocal" != null && at "http_access deny to_linklocal" < allow && at "http_access deny to_bridge" != null && at "http_access deny to_bridge" < allow + && at "http_access deny to_localhost" < authed + && at "http_access deny to_linklocal" < authed + && at "http_access deny to_bridge" < authed + && authed < allow + && at "http_access deny ip_v4_host" != null + && at "http_access deny ip_v4_host" < at "http_access deny !allowed_dst" + && at "http_access deny ip_v6_host" < at "http_access deny !allowed_dst" && at "http_access deny !allowed_dst" != null && at "http_access deny !allowed_dst" < at "http_access deny to_localhost" && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } { - # The file hive-c0re renders exists, owned by it and readable by squid's - # group, from boot; hive-c0re gets its path only on a host with a store - # identity. - name = "squid's users file exists for hive-c0re to write, and hive-c0re is told where it is"; + # The helper is the NCSA one, on the file hive-c0re renders. The file + # exists, empty, before squid starts (the helper exits on a missing one), + # and the self-check imports its credential without failing when there is + # none. + name = "squid authenticates with the NCSA helper on a users file that exists before it starts"; ok = let + conf = hiveOtel.services.squid.configText; + lines = lib.splitString "\n" conf; users = hiveOtel.services.hyperhive.squid.usersFile; rules = hiveOtel.systemd.tmpfiles.rules; + sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig; in - users == "/var/lib/hive-squid/users" + lib.any ( + l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l + ) lines + && builtins.elem "acl authed proxy_auth REQUIRED" lines + && users == "/var/lib/hive-squid/users" && builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules && builtins.elem "f ${users} 0640 hive-core squid - -" rules + && sc.ImportCredential == hiveOtel.services.hyperhive.squid.selfcheck.credentialName && hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users; } { diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 382fb971..801baa47 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -55,6 +55,12 @@ let }; }; }; + # The user hive-c0re runs as, which owns the users file squid reads. + config.users.users.hive-core = { + isSystemUser = true; + group = "hive-core"; + }; + config.users.groups.hive-core = { }; # The firewall hole hive-network.nix opens for `exposeHostPorts`. config.networking.firewall.interfaces.eth1.allowedTCPPorts = config.services.hyperhive.network.exposeHostPorts; @@ -77,7 +83,7 @@ pkgs.testers.runNixOSTest { nodes = { proxy = - { lib, ... }: + { lib, pkgs, ... }: { imports = [ hiveStubs @@ -111,6 +117,8 @@ pkgs.testers.runNixOSTest { ${bridgeIp} = [ "bridge.t.local" ]; "169.254.169.254" = [ "meta.t.local" ]; }; + # Renders the users file the way hive-c0re does. + environment.systemPackages = [ pkgs.mkpasswd ]; services.hyperhive.squid.allowedDomains = [ "forge.t.local" "loop.t.local" @@ -148,6 +156,15 @@ pkgs.testers.runNixOSTest { testScript = '' import re + import secrets + + # Drawn fresh each run. The users file holds only their hashes, as + # hive-c0re renders it. + PW = {u: secrets.token_urlsafe(24) for u in ("atlas", "selfcheck.squid", "ruth")} + ATLAS = "atlas:" + PW["atlas"] + SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"] + RUTH = "ruth:" + PW["ruth"] + USERS = "/var/lib/hive-squid/users" start_all() proxy.wait_for_unit("squid.service") @@ -156,29 +173,64 @@ pkgs.testers.runNixOSTest { web.wait_for_unit("nginx.service") proxy.wait_for_open_port(3128, "${bridgeIp}") - def connect(host): + def connect(host, cred=None): # The status squid answered the CONNECT with; 000 when none came back. + auth = f"-U '{cred}' " if cred else "" return agent.succeed( - "curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null " + "curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null " f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true" ).strip() - with subtest("an allowed name is tunnelled and an unlisted one is refused"): - assert connect("forge.t.local") == "200", "allow canary" - assert connect("deny-canary.invalid") == "403", "deny canary" + def write_users(*creds): + # What hive-c0re does: hashes only, written aside and renamed into + # place, in squid's group. The sleep keeps two writes in different + # seconds, which is how the helper tells the file changed. + lines = " ".join( + f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";" + for u, p in (c.split(":", 1) for c in creds) + ) + proxy.succeed( + f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new " + f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1" + ) + + with subtest("squid starts on the empty users file and refuses every login"): + assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640" + assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750" + assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0" + assert connect("forge.t.local", ATLAS) == "407", "no such user" + assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name needs no login" + + with subtest("the users file holds hashes, never a password"): + write_users(ATLAS, SELFCHECK) + users = proxy.succeed(f"cat {USERS}") + assert re.fullmatch(r"(atlas|selfcheck\.squid):\$6\$\S+\n(atlas|selfcheck\.squid):\$6\$\S+\n", users), users + assert not any(p in users for p in PW.values()), users + + with subtest("an allowed name is tunnelled with a credential and challenged without one"): + assert connect("forge.t.local", ATLAS) == "200", "allow canary" + assert connect("forge.t.local") == "407", "no credential" + assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password" + assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user" + # DES would accept the first eight characters; the hash must not. + assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password" + + with subtest("an unlisted name is refused with or without a credential"): + assert connect("deny-canary.invalid", ATLAS) == "403", "deny canary" + assert connect("deny-canary.invalid") == "403", "deny canary, anonymous" with subtest("a name resolving to a host-local destination is refused"): - assert connect("loop.t.local") == "403", "loopback" - assert connect("bridge.t.local") == "403", "bridge address" - assert connect("meta.t.local") == "403", "link-local" + assert connect("loop.t.local", ATLAS) == "403", "loopback" + assert connect("bridge.t.local", ATLAS) == "403", "bridge address" + assert connect("meta.t.local", ATLAS) == "403", "link-local" with subtest("IP literals are refused even when the address is reachable"): - assert connect("127.0.0.1") == "403" - assert connect("192.168.2.2") == "403" + assert connect("127.0.0.1", ATLAS) == "403" + assert connect("192.168.2.2", ATLAS) == "403" with subtest("only a port-443 CONNECT is allowed"): - assert connect("forge.t.local:22") == "403" - assert connect("forge.t.local:80") == "403" + assert connect("forge.t.local:22", ATLAS) == "403" + assert connect("forge.t.local:80", ATLAS) == "403" with subtest("an agent cannot read cachemgr"): agent.succeed( @@ -186,19 +238,42 @@ pkgs.testers.runNixOSTest { "http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403" ) - with subtest("the self-check passes against the same squid"): + with subtest("the self-check passes with its credential"): + proxy.succeed("install -d -m 0700 /run/credstore") + proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck") + proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck") proxy.succeed("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local " - "allow_status=200 deny_canary=deny-canary.invalid deny_status=403'" + "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " + "anon_status=407 credential=present'" ) - with subtest("squid's access log reaches the journal under squid.service"): + with subtest("the self-check fails, and says why, without a credential or with the wrong one"): + proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept") + proxy.fail("systemctl start squid-selfcheck.service") + proxy.succeed( + "journalctl -u squid-selfcheck.service -o cat | " + "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " + "allow_status=407 deny_canary=deny-canary.invalid deny_status=403 " + "anon_status=407 credential=absent'" + ) + proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck") + proxy.fail("systemctl start squid-selfcheck.service") + proxy.succeed( + "journalctl -u squid-selfcheck.service -o cat | " + "grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=403 " + "anon_status=407 credential=present'" + ) + proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck") + + with subtest("squid's access log reaches the journal under squid.service, with the user"): journal = proxy.succeed("journalctl -u squid.service -o cat") - assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", journal), journal - assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", journal), journal - assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443 ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal # Exporter scrapes are cachemgr requests and stay out of the log. assert "squid-internal-mgr" not in journal, journal @@ -215,6 +290,21 @@ pkgs.testers.runNixOSTest { with subtest("squid survives a reload"): proxy.succeed("systemctl reload squid.service") proxy.wait_for_unit("squid.service") - assert connect("forge.t.local") == "200" + assert connect("forge.t.local", ATLAS) == "200" + + with subtest("a user added while squid runs can log in, with no reload"): + write_users(ATLAS, SELFCHECK, RUTH) + assert connect("forge.t.local", RUTH) == "200" + assert connect("forge.t.local", ATLAS) == "200" + + with subtest("squid starts on a missing users file, refuses logins, and recovers when it appears"): + proxy.succeed(f"rm {USERS}") + proxy.succeed("systemctl restart squid.service") + proxy.wait_for_unit("squid.service") + proxy.wait_for_open_port(3128, "${bridgeIp}") + assert connect("forge.t.local", ATLAS) == "407", "no file, no login" + assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name still answers" + write_users(ATLAS, SELFCHECK) + assert connect("forge.t.local", ATLAS) == "200", "the file appeared" ''; }