require a proxy credential at squid and prove it in the self-check

nix/host-modules/hive-squid.nix

@@ -1,8 +1,9 @@
 # Egress proxy for this hive's agents: squid on the host, bound to the bridge
 # IP, default-deny on the destination hostname. It filters CONNECT targets and
-# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is
+# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
-# pointed at it yet; the exporter, journal forwarding and the self-check below
+# for a listed name must carry a per-agent proxy credential, checked against the
-# are how a deploy shows it works.
+# users file hive-c0re renders. No agent sends one yet; the exporter, journal
+# forwarding and the self-check below are how a deploy shows it works.
 {
   pkgs,
   lib,
@@ -21,6 +22,8 @@ let
 
   allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
 
+  # The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`.
+  selfcheckUser = "selfcheck.squid";
   usersDir = builtins.dirOf cfg.usersFile;
 
   # Evaluated top to bottom, first match wins. The three destination denies
@@ -29,7 +32,10 @@ let
   # otherwise reach services the bridge firewall keeps agents out of. The
   # unlisted-name deny comes first because `dstdomain` needs no lookup while
   # the `dst` ACLs do: an unlisted name is refused at once instead of waiting
-  # on the host resolver, and costs a client no DNS query.
+  # on the host resolver, and costs a client no DNS query. The credential
+  # check sits right before the allow, so a refused destination answers 403
+  # to a client with or without a credential, and only a request that would
+  # otherwise be let through gets the 407 challenge.
   squidConf = ''
     http_port ${netCfg.bridgeIp}:${toString port}
     pid_filename /run/squid/squid.pid
@@ -42,8 +48,17 @@ let
     # Cachemgr scrapes are left out.
     access_log syslog:daemon.info squid !manager
 
+    # The helper reads the users file on every request and reloads it when its
+    # mtime moves. It exits when the file is missing, which is why the file is
+    # created before squid starts. A positive answer is cached for a minute, so
+    # that is also how long a removed user keeps working.
+    auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile}
+    auth_param basic realm hyperhive egress
+    auth_param basic credentialsttl 1 minute
+
     acl SSL_ports port 443
     acl Safe_ports port 80 443
+    acl authed proxy_auth REQUIRED
 
     # Host-originated requests: the self-check and the exporter. A local
     # connect to the bridge IP takes the bridge IP as its source.
@@ -71,6 +86,7 @@ let
     http_access deny to_localhost
     http_access deny to_linklocal
     http_access deny to_bridge
+    http_access deny !authed
     ${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
     http_access deny all
   '';
@@ -103,20 +119,39 @@ let
     name = "squid-selfcheck";
     runtimeInputs = [ pkgs.curl ];
     text = ''
-      # The status squid answers a CONNECT with; 000 when the proxy is down.
+      # The credential comes from the unit's imported credential, and goes to
+      # curl on its stdin so it never appears in a command line. Without one
+      # the requests are anonymous and the self-check reports it.
+      secret=""
+      credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}"
+      if [ -s "$credential_file" ]; then
+        secret=$(< "$credential_file")
+      fi
+
+      # <host> <auth|anon>: the status squid answers a CONNECT with; 000 when
+      # the proxy is down.
       connect_status() {
-        curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
+        {
+          if [ "$2" = auth ] && [ -n "$secret" ]; then
+            printf 'proxy-user = "%s:%s"\n' ${selfcheckUser} "$secret"
+          fi
+        } | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
           -o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
       }
 
-      allow_status=$(connect_status ${cfg.selfcheck.canary})
+      allow_status=$(connect_status ${cfg.selfcheck.canary} auth)
-      deny_status=$(connect_status ${denyCanary})
+      deny_status=$(connect_status ${denyCanary} auth)
+      anon_status=$(connect_status ${cfg.selfcheck.canary} anon)
 
       ok=false
-      if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then
+      if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ]; then
         ok=true
       fi
-      echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status"
+      credential=absent
+      if [ -n "$secret" ]; then
+        credential=present
+      fi
+      echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential"
       [ "$ok" = true ]
     '';
   };
@@ -142,10 +177,24 @@ in
       type = lib.types.path;
       default = "/var/lib/hive-squid/users";
       description = ''
-        The `<user>:<hash>` file hive-c0re renders from the credentials in the
+        The `<user>:<hash>` file squid checks proxy credentials against. hive-c0re
-        swarm secret store: one line per agent of this hive and one for the
+        renders it from the credentials in the swarm secret store, one line per
-        self-check, as SHA-512 crypt hashes. Created empty, owned by `hive-core`
+        agent of this hive and one for the self-check, as SHA-512 crypt hashes.
-        and readable by the `squid` group. Squid doesn't read it yet.
+        An empty file, which is what exists until hive-c0re writes one, makes
+        every login fail, and every request to a listed name answers 407.
+        Owned by `hive-core` and readable by the `squid` group.
+      '';
+    };
+
+    selfcheck.credentialName = lib.mkOption {
+      type = lib.types.str;
+      default = "hive-squid.selfcheck";
+      readOnly = true;
+      description = ''
+        Name of the systemd credential the self-check imports its squid
+        password from. `squid-selfcheck-credential` writes it to
+        `/run/credstore/<name>`. Without one the self-check sends no
+        credential and reports `ok=false`.
       '';
     };
 
@@ -181,7 +230,8 @@ in
     };
 
     # The directory is setgid so the file hive-c0re renames into it is the
-    # squid group's.
+    # squid group's. The file exists before squid starts, empty: the helper
+    # exits on a missing file and squid then rejects every login.
     systemd.tmpfiles.rules = [
       "d ${usersDir} 2750 hive-core squid - -"
       "f ${cfg.usersFile} 0640 hive-core squid - -"
@@ -224,10 +274,13 @@ in
     };
 
     systemd.services.squid-selfcheck = {
-      description = "Squid allow and deny canaries";
+      description = "Squid allow, deny and credential canaries";
       after = [ "squid.service" ];
       serviceConfig = sandbox // {
         Type = "oneshot";
+        # A glob that matches nothing is not an error: a host with no
+        # credential yet still runs the self-check, and it reports that.
+        ImportCredential = cfg.selfcheck.credentialName;
         ExecStart = lib.getExe selfcheck;
         DynamicUser = true;
         IPAddressAllow = netCfg.bridgeIp;

nix/module-eval/hive-otel.nix

@@ -485,39 +485,60 @@ let
         && !(lib.hasInfix "localnet" conf);
     }
     {
-      # First match wins, so a destination deny after the allow is dead.
+      # First match wins, so a destination deny after the allow is dead, and
-      name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, before any allow";
+      # a credential check after it never challenges anyone. The two IP-literal
+      # denies stay ahead of the unlisted-name deny: `dstdomain` on an IP
+      # literal does a reverse lookup.
+      name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, then a missing credential, before any allow";
       ok =
         let
           lines = lib.splitString "\n" hiveOtel.services.squid.configText;
           at = l: lib.lists.findFirstIndex (x: x == l) null lines;
           allow = at "http_access allow allowed_dst";
+          authed = at "http_access deny !authed";
         in
         allow != null
+        && authed != null
         && at "http_access deny to_localhost" != null
         && at "http_access deny to_localhost" < allow
         && at "http_access deny to_linklocal" != null
         && at "http_access deny to_linklocal" < allow
         && at "http_access deny to_bridge" != null
         && at "http_access deny to_bridge" < allow
+        && at "http_access deny to_localhost" < authed
+        && at "http_access deny to_linklocal" < authed
+        && at "http_access deny to_bridge" < authed
+        && authed < allow
+        && at "http_access deny ip_v4_host" != null
+        && at "http_access deny ip_v4_host" < at "http_access deny !allowed_dst"
+        && at "http_access deny ip_v6_host" < at "http_access deny !allowed_dst"
         && at "http_access deny !allowed_dst" != null
         && at "http_access deny !allowed_dst" < at "http_access deny to_localhost"
         && builtins.elem "cache deny all" lines
         && lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
     }
     {
-      # The file hive-c0re renders exists, owned by it and readable by squid's
+      # The helper is the NCSA one, on the file hive-c0re renders. The file
-      # group, from boot; hive-c0re gets its path only on a host with a store
+      # exists, empty, before squid starts (the helper exits on a missing one),
-      # identity.
+      # and the self-check imports its credential without failing when there is
-      name = "squid's users file exists for hive-c0re to write, and hive-c0re is told where it is";
+      # none.
+      name = "squid authenticates with the NCSA helper on a users file that exists before it starts";
       ok =
         let
+          conf = hiveOtel.services.squid.configText;
+          lines = lib.splitString "\n" conf;
           users = hiveOtel.services.hyperhive.squid.usersFile;
           rules = hiveOtel.systemd.tmpfiles.rules;
+          sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig;
         in
-        users == "/var/lib/hive-squid/users"
+        lib.any (
+          l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l
+        ) lines
+        && builtins.elem "acl authed proxy_auth REQUIRED" lines
+        && users == "/var/lib/hive-squid/users"
         && builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules
         && builtins.elem "f ${users} 0640 hive-core squid - -" rules
+        && sc.ImportCredential == hiveOtel.services.hyperhive.squid.selfcheck.credentialName
         && hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users;
     }
     {

nix/vm-tests/hive-squid.nix

@@ -55,6 +55,12 @@ let
           };
         };
       };
+      # The user hive-c0re runs as, which owns the users file squid reads.
+      config.users.users.hive-core = {
+        isSystemUser = true;
+        group = "hive-core";
+      };
+      config.users.groups.hive-core = { };
       # The firewall hole hive-network.nix opens for `exposeHostPorts`.
       config.networking.firewall.interfaces.eth1.allowedTCPPorts =
         config.services.hyperhive.network.exposeHostPorts;
@@ -77,7 +83,7 @@ pkgs.testers.runNixOSTest {
 
   nodes = {
     proxy =
-      { lib, ... }:
+      { lib, pkgs, ... }:
       {
         imports = [
           hiveStubs
@@ -111,6 +117,8 @@ pkgs.testers.runNixOSTest {
           ${bridgeIp} = [ "bridge.t.local" ];
           "169.254.169.254" = [ "meta.t.local" ];
         };
+        # Renders the users file the way hive-c0re does.
+        environment.systemPackages = [ pkgs.mkpasswd ];
         services.hyperhive.squid.allowedDomains = [
           "forge.t.local"
           "loop.t.local"
@@ -148,6 +156,15 @@ pkgs.testers.runNixOSTest {
 
   testScript = ''
     import re
+    import secrets
+
+    # Drawn fresh each run. The users file holds only their hashes, as
+    # hive-c0re renders it.
+    PW = {u: secrets.token_urlsafe(24) for u in ("atlas", "selfcheck.squid", "ruth")}
+    ATLAS = "atlas:" + PW["atlas"]
+    SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"]
+    RUTH = "ruth:" + PW["ruth"]
+    USERS = "/var/lib/hive-squid/users"
 
     start_all()
     proxy.wait_for_unit("squid.service")
@@ -156,29 +173,64 @@ pkgs.testers.runNixOSTest {
     web.wait_for_unit("nginx.service")
     proxy.wait_for_open_port(3128, "${bridgeIp}")
 
-    def connect(host):
+    def connect(host, cred=None):
         # The status squid answered the CONNECT with; 000 when none came back.
+        auth = f"-U '{cred}' " if cred else ""
         return agent.succeed(
-            "curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null "
+            "curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null "
             f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
         ).strip()
 
-    with subtest("an allowed name is tunnelled and an unlisted one is refused"):
+    def write_users(*creds):
-        assert connect("forge.t.local") == "200", "allow canary"
+        # What hive-c0re does: hashes only, written aside and renamed into
-        assert connect("deny-canary.invalid") == "403", "deny canary"
+        # place, in squid's group. The sleep keeps two writes in different
+        # seconds, which is how the helper tells the file changed.
+        lines = " ".join(
+            f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";"
+            for u, p in (c.split(":", 1) for c in creds)
+        )
+        proxy.succeed(
+            f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new "
+            f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1"
+        )
+
+    with subtest("squid starts on the empty users file and refuses every login"):
+        assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
+        assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750"
+        assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0"
+        assert connect("forge.t.local", ATLAS) == "407", "no such user"
+        assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name needs no login"
+
+    with subtest("the users file holds hashes, never a password"):
+        write_users(ATLAS, SELFCHECK)
+        users = proxy.succeed(f"cat {USERS}")
+        assert re.fullmatch(r"(atlas|selfcheck\.squid):\$6\$\S+\n(atlas|selfcheck\.squid):\$6\$\S+\n", users), users
+        assert not any(p in users for p in PW.values()), users
+
+    with subtest("an allowed name is tunnelled with a credential and challenged without one"):
+        assert connect("forge.t.local", ATLAS) == "200", "allow canary"
+        assert connect("forge.t.local") == "407", "no credential"
+        assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
+        assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user"
+        # DES would accept the first eight characters; the hash must not.
+        assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password"
+
+    with subtest("an unlisted name is refused with or without a credential"):
+        assert connect("deny-canary.invalid", ATLAS) == "403", "deny canary"
+        assert connect("deny-canary.invalid") == "403", "deny canary, anonymous"
 
     with subtest("a name resolving to a host-local destination is refused"):
-        assert connect("loop.t.local") == "403", "loopback"
+        assert connect("loop.t.local", ATLAS) == "403", "loopback"
-        assert connect("bridge.t.local") == "403", "bridge address"
+        assert connect("bridge.t.local", ATLAS) == "403", "bridge address"
-        assert connect("meta.t.local") == "403", "link-local"
+        assert connect("meta.t.local", ATLAS) == "403", "link-local"
 
     with subtest("IP literals are refused even when the address is reachable"):
-        assert connect("127.0.0.1") == "403"
+        assert connect("127.0.0.1", ATLAS) == "403"
-        assert connect("192.168.2.2") == "403"
+        assert connect("192.168.2.2", ATLAS) == "403"
 
     with subtest("only a port-443 CONNECT is allowed"):
-        assert connect("forge.t.local:22") == "403"
+        assert connect("forge.t.local:22", ATLAS) == "403"
-        assert connect("forge.t.local:80") == "403"
+        assert connect("forge.t.local:80", ATLAS) == "403"
 
     with subtest("an agent cannot read cachemgr"):
         agent.succeed(
@@ -186,17 +238,40 @@ pkgs.testers.runNixOSTest {
             "http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
         )
 
-    with subtest("the self-check passes against the same squid"):
+    with subtest("the self-check passes with its credential"):
+        proxy.succeed("install -d -m 0700 /run/credstore")
+        proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck")
+        proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck")
         proxy.succeed("systemctl start squid-selfcheck.service")
         proxy.succeed(
             "journalctl -u squid-selfcheck.service -o cat | "
             "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local "
-            "allow_status=200 deny_canary=deny-canary.invalid deny_status=403'"
+            "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
+            "anon_status=407 credential=present'"
         )
 
-    with subtest("squid's access log reaches the journal under squid.service"):
+    with subtest("the self-check fails, and says why, without a credential or with the wrong one"):
+        proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept")
+        proxy.fail("systemctl start squid-selfcheck.service")
+        proxy.succeed(
+            "journalctl -u squid-selfcheck.service -o cat | "
+            "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local "
+            "allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
+            "anon_status=407 credential=absent'"
+        )
+        proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck")
+        proxy.fail("systemctl start squid-selfcheck.service")
+        proxy.succeed(
+            "journalctl -u squid-selfcheck.service -o cat | "
+            "grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
+            "anon_status=407 credential=present'"
+        )
+        proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck")
+
+    with subtest("squid's access log reaches the journal under squid.service, with the user"):
         journal = proxy.succeed("journalctl -u squid.service -o cat")
-        assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", journal), journal
+        assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal
+        assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal
         assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443 ", journal), journal
         assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal
         # Exporter scrapes are cachemgr requests and stay out of the log.
@@ -215,6 +290,21 @@ pkgs.testers.runNixOSTest {
     with subtest("squid survives a reload"):
         proxy.succeed("systemctl reload squid.service")
         proxy.wait_for_unit("squid.service")
-        assert connect("forge.t.local") == "200"
+        assert connect("forge.t.local", ATLAS) == "200"
+
+    with subtest("a user added while squid runs can log in, with no reload"):
+        write_users(ATLAS, SELFCHECK, RUTH)
+        assert connect("forge.t.local", RUTH) == "200"
+        assert connect("forge.t.local", ATLAS) == "200"
+
+    with subtest("squid starts on a missing users file, refuses logins, and recovers when it appears"):
+        proxy.succeed(f"rm {USERS}")
+        proxy.succeed("systemctl restart squid.service")
+        proxy.wait_for_unit("squid.service")
+        proxy.wait_for_open_port(3128, "${bridgeIp}")
+        assert connect("forge.t.local", ATLAS) == "407", "no file, no login"
+        assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name still answers"
+        write_users(ATLAS, SELFCHECK)
+        assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
   '';
 }