hyperhive/nix/host-modules/hive-squid.nix

# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is
# pointed at it yet; the exporter, journal forwarding and the self-check below
# are how a deploy shows it works.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  cfg = hyperhiveCfg.squid;
  netCfg = hyperhiveCfg.network;

  port = 3128;
  proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
  exporterListen = "127.0.0.1:9301";
  denyCanary = "deny-canary.invalid";

  allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);

  usersDir = builtins.dirOf cfg.usersFile;

  # Evaluated top to bottom, first match wins. The three destination denies
  # sit before the only allow: squid resolves names on the host, so a name
  # that answers 127.0.0.1, 169.254.169.254 or a bridge address would
  # otherwise reach services the bridge firewall keeps agents out of. The
  # unlisted-name deny comes first because `dstdomain` needs no lookup while
  # the `dst` ACLs do: an unlisted name is refused at once instead of waiting
  # on the host resolver, and costs a client no DNS query.
  squidConf = ''
    http_port ${netCfg.bridgeIp}:${toString port}
    pid_filename /run/squid/squid.pid
    cache_log stdio:/var/log/squid/cache.log
    coredump_dir /var/cache/squid

    cache deny all

    # One syslog line per request reaches the journal as squid.service.
    # Cachemgr scrapes are left out.
    access_log syslog:daemon.info squid !manager

    acl SSL_ports port 443
    acl Safe_ports port 80 443

    # Host-originated requests: the self-check and the exporter. A local
    # connect to the bridge IP takes the bridge IP as its source.
    acl selfcheck src ${netCfg.bridgeIp}/32

    acl to_bridge dst ${netCfg.networkCidr}
    # `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record
    # an agent controls could match an allowed name.
    acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
    acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$)
    ${lib.optionalString (
      allowed != [ ]
    ) "acl allowed_dst dstdomain ${lib.concatStringsSep " " allowed}"}

    # Answered by squid itself, never forwarded, and addressed to the bridge IP
    # on a port outside Safe_ports, so it precedes the destination rules.
    http_access allow manager selfcheck
    http_access deny manager

    http_access deny !Safe_ports
    http_access deny CONNECT !SSL_ports
    http_access deny ip_v4_host
    http_access deny ip_v6_host
    ${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"}
    http_access deny to_localhost
    http_access deny to_linklocal
    http_access deny to_bridge
    ${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
    http_access deny all
  '';

  sandbox = {
    NoNewPrivileges = true;
    ProtectSystem = "strict";
    ProtectHome = true;
    PrivateTmp = true;
    PrivateDevices = true;
    ProtectKernelTunables = true;
    ProtectKernelModules = true;
    ProtectKernelLogs = true;
    ProtectControlGroups = true;
    ProtectClock = true;
    RestrictAddressFamilies = [
      "AF_INET"
      "AF_INET6"
      "AF_UNIX"
    ];
    RestrictNamespaces = true;
    RestrictRealtime = true;
    RestrictSUIDSGID = true;
    LockPersonality = true;
    SystemCallArchitectures = "native";
    CapabilityBoundingSet = "";
  };

  selfcheck = pkgs.writeShellApplication {
    name = "squid-selfcheck";
    runtimeInputs = [ pkgs.curl ];
    text = ''
      # The status squid answers a CONNECT with; 000 when the proxy is down.
      connect_status() {
        curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
          -o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
      }

      allow_status=$(connect_status ${cfg.selfcheck.canary})
      deny_status=$(connect_status ${denyCanary})

      ok=false
      if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then
        ok=true
      fi
      echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status"
      [ "$ok" = true ]
    '';
  };
in
{
  options.services.hyperhive.squid = {
    allowedDomains = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = hyperhiveCfg.swarm.serviceDomains;
      defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains";
      example = [
        "forge.example.com"
        "github.com"
      ];
      description = ''
        Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
        entry matches that exact name. Everything else gets a 403. The
        self-check canary is allowed on top of this list.
      '';
    };

    usersFile = lib.mkOption {
      type = lib.types.path;
      default = "/var/lib/hive-squid/users";
      description = ''
        The `<user>:<hash>` file hive-c0re renders from the credentials in the
        swarm secret store: one line per agent of this hive and one for the
        self-check, as SHA-512 crypt hashes. Created empty, owned by `hive-core`
        and readable by the `squid` group. Squid doesn't read it yet.
      '';
    };

    selfcheck.canary = lib.mkOption {
      type = lib.types.str;
      default = hyperhiveCfg.swarm.forge.domain;
      defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain";
      description = ''
        Hostname the self-check CONNECTs to on port 443 and expects squid to
        let through. Squid resolves it with this host's resolver, not the
        hive's: a host whose resolver answers a loopback or bridge address
        for it (a single-host swarm with the gateway's `localHostsEntry`) gets
        a 403, and the self-check reports `ok=false`.
      '';
    };
  };

  config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable {
    # Reachable from agent containers and nowhere else: this opens the port on
    # the bridge interface only.
    services.hyperhive.network.exposeHostPorts = [ port ];
    services.hyperhive.network.enable = lib.mkDefault true;

    services.hyperhive.otel.scrapeTargets.squid = exporterListen;
    services.hyperhive.otel.journalUnits = [
      "squid.service"
      "squid-selfcheck.service"
    ];

    services.squid = {
      enable = true;
      configText = squidConf;
    };

    # The directory is setgid so the file hive-c0re renames into it is the
    # squid group's.
    systemd.tmpfiles.rules = [
      "d ${usersDir} 2750 hive-core squid - -"
      "f ${cfg.usersFile} 0640 hive-core squid - -"
    ];

    systemd.services.squid = {
      serviceConfig = sandbox // {
        User = "squid";
        Group = "squid";
        PIDFile = lib.mkForce "/run/squid/squid.pid";
        RuntimeDirectory = "squid";
        CacheDirectory = "squid";
        LogsDirectory = "squid";
        # The bridge address may not be assigned yet at first start.
        Restart = "on-failure";
        RestartSec = 5;
      };
    };

    systemd.services.squid-exporter = {
      description = "Prometheus exporter for squid";
      wantedBy = [ "multi-user.target" ];
      after = [ "squid.service" ];
      serviceConfig = sandbox // {
        ExecStart = lib.concatStringsSep " " [
          "${pkgs.prometheus-squid-exporter}/bin/squid-exporter"
          "-squid-hostname ${netCfg.bridgeIp}"
          "-squid-port ${toString port}"
          "-listen ${exporterListen}"
        ];
        DynamicUser = true;
        IPAddressAllow = [
          "127.0.0.0/8"
          netCfg.bridgeIp
        ];
        IPAddressDeny = "any";
        Restart = "on-failure";
        RestartSec = 5;
      };
    };

    systemd.services.squid-selfcheck = {
      description = "Squid allow and deny canaries";
      after = [ "squid.service" ];
      serviceConfig = sandbox // {
        Type = "oneshot";
        ExecStart = lib.getExe selfcheck;
        DynamicUser = true;
        IPAddressAllow = netCfg.bridgeIp;
        IPAddressDeny = "any";
      };
    };

    systemd.timers.squid-selfcheck = {
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnBootSec = "1min";
        OnUnitActiveSec = "1min";
      };
    };
  };
}