From a2cf6fe8bb0e9b1b44a8c80a094b4bba33fe89f6 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 27 Oct 2026 15:09:00 +0100 Subject: [PATCH] deploy squid on every hive host, with exporter, journal forwarding and self-check --- docs/networking/network.md | 38 +++++ docs/scheduler/observability.md | 12 +- nix/checks.nix | 2 + nix/host-modules/default.nix | 1 + nix/host-modules/hive-network.nix | 11 ++ nix/host-modules/hive-squid.nix | 222 ++++++++++++++++++++++++++++++ nix/host-modules/otel.nix | 26 +++- nix/module-eval/hive-otel.nix | 66 +++++++++ nix/vm-tests/hive-squid.nix | 220 +++++++++++++++++++++++++++++ 9 files changed, 589 insertions(+), 9 deletions(-) create mode 100644 nix/host-modules/hive-squid.nix create mode 100644 nix/vm-tests/hive-squid.nix diff --git a/docs/networking/network.md b/docs/networking/network.md index cf0087c2..3e183a4f 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -26,6 +26,7 @@ that touches it. │ │ │ nginx :80/:443 [hive-gateway] │ │ dnsmasq 10.42.0.1:53 (DNS) + :67 (DHCP) [hive-gateway] │ +│ squid 10.42.0.1:3128 (egress proxy) [hive-squid] │ │ forgejo :3000 http, :2222 git-ssh [hive-forge] │ │ hive-c0re dashboard 127.0.0.1:7000 (host service) │ │ wg-hive :51820/udp — swarm mesh, when enabled (host iface) │ @@ -44,6 +45,7 @@ that touches it. | container | netns | IPv4 | listens / reached via | | ------------------------- | ----------------------- | -------------- | ----------------------------------------------------------------------------------------------- | | gateway (host) | host | host addresses | nginx `:80`/`:443` (every vhost); dnsmasq `bridgeIp:53` + DHCP `:67` on the bridge | +| egress proxy (host) | host | `bridgeIp` | squid `bridgeIp:3128`; no agent uses it yet | | `hive-forge` | host (shared) | host addresses | forgejo `:3000` http, `:2222` git-ssh; fronted by the `forge.` vhost | | `hive-matrix` | private, veth on bridge | DHCP pool | tuwunel on UDS `/run/hive-matrix/tuwunel.sock` → the matrix vhost; out to store, SSO, peers | | `swarm-ci-runner` | private, veth on bridge | DHCP pool | outbound only (runner → forge); no inbound surface | @@ -206,6 +208,42 @@ service bound to `127.0.0.1` only stays unreachable; rebind it to The port is reachable by **every** agent on the bridge subnet (like DNS/gateway), so only expose services safe for any agent to reach. +## Egress proxy (squid) + +Every hive host runs squid in the host netns, listening on `:3128` +(`nix/host-modules/hive-squid.nix`). The port is open on the bridge interface +only. **No agent uses it yet**: agent traffic still goes straight out through +the NAT. The proxy goes first so you can check its metrics and logs before +anything reroutes. + +Squid filters on the destination hostname: a `CONNECT` target or a plain-HTTP +host. It never decrypts TLS, so it holds no signing key. It caches nothing. + +- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the + swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each + entry matches that exact name. Everything else gets a 403. +- **Denied before any allow.** Destinations that resolve to loopback, + link-local or the bridge subnet, and IP-literal hosts. Squid resolves names + on the host, so without these a listed name pointing at `127.0.0.1` would + walk around the bridge→loopback DROP rule. Only `CONNECT` to port 443 and + plain HTTP to ports 80 and 443 pass. +- **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on + `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through + `services.hyperhive.otel.scrapeTargets`. `squid_up` and + `squid_client_http_requests_total` are the two to watch. +- **Logs.** One syslog line per request reaches the journal as + `squid.service`, and `services.hyperhive.otel.journalUnits` forwards that + unit and the self-check's to the swarm's log store. A refusal shows as `TCP_DENIED/403` with the + client's source address. +- **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service` + every minute from the bridge IP: `CONNECT` to `services.hyperhive.squid.selfcheck.canary` + (default: the forge's name) must answer 200, and to `deny-canary.invalid` + must answer 403. It writes one `squid-selfcheck ok=… allow_status=… +deny_status=…` line per run and fails the unit when either is wrong. The + host's resolver looks up the canary, so a host whose resolver maps + that name to a loopback or bridge address (a single-host swarm with + `gateway.localHostsEntry`) fails the check. + ## Container isolation Each agent container runs in a private network namespace with a dedicated diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 66c7d6c0..c9e5b4a1 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -373,11 +373,13 @@ a crash and `tracing::warn!`s `"container crash detected"` — nothing else: no broker event, no per-hive dashboard banner. Both of those used to exist; this change removed them in favour of the log line. -The hive tier's own collector runs a `journald` -receiver scoped to `hive-c0re.service` (no bind mount needed — unlike an -agent's journal, this collector already shares a filesystem with the unit it -reads, being a host systemd service rather than a container) and forwards -that one log line alongside the rest of the hive's telemetry. A +The hive tier's own collector runs a `journald` receiver scoped to the units +in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, plus +`squid.service` and `squid-selfcheck.service` (see [Egress proxy](../networking/network.md#egress-proxy-squid)). +No bind mount is needed. Unlike an agent's journal, this collector already +shares a filesystem with the units it reads, because it's a host systemd +service rather than a container. It forwards the crash log line alongside the +rest of the hive's telemetry. A swarm-level grafana alert, `hh-container-crashed` in [`nix/host-modules/swarm-grafana.nix`](../../nix/host-modules/swarm-grafana.nix), matches it directly and pages the operator — no agent, and no per-hive diff --git a/nix/checks.nix b/nix/checks.nix index a5866ee5..49a36ce7 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -140,6 +140,8 @@ in inherit (pkgs) lib; shared = moduleEvalShared; }; + # Runs the squid module in a VM; needs KVM, unlike every check above. + vm-hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; }; module-eval-hive-otel = import ./module-eval/hive-otel.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index c76477c3..0a73c600 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -23,6 +23,7 @@ ./hive-matrix.nix ./hive-network.nix ./hive-priv.nix + ./hive-squid.nix ./hive-tls.nix ./otel.nix ./glue-bao-readers-policy-order.nix diff --git a/nix/host-modules/hive-network.nix b/nix/host-modules/hive-network.nix index a94c31c6..9a15bc2f 100644 --- a/nix/host-modules/hive-network.nix +++ b/nix/host-modules/hive-network.nix @@ -162,6 +162,17 @@ in ''; }; + networkCidr = lib.mkOption { + type = lib.types.str; + internal = true; + readOnly = true; + default = "${intToIp networkBase}/${toString cfg.bridgePrefixLength}"; + defaultText = lib.literalMD "bridge subnet in CIDR form, host bits masked off"; + description = '' + Read-only computed bridge subnet, e.g. `10.42.0.0/24`. + ''; + }; + }; config = lib.mkMerge [ diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix new file mode 100644 index 00000000..e6026294 --- /dev/null +++ b/nix/host-modules/hive-squid.nix @@ -0,0 +1,222 @@ +# Egress proxy for this hive's agents: squid on the host, bound to the bridge +# IP, default-deny on the destination hostname. It filters CONNECT targets and +# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is +# pointed at it yet; the exporter, journal forwarding and the self-check below +# are how a deploy shows it works. +{ + pkgs, + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + cfg = hyperhiveCfg.squid; + netCfg = hyperhiveCfg.network; + + port = 3128; + proxyUrl = "http://${netCfg.bridgeIp}:${toString port}"; + exporterListen = "127.0.0.1:9301"; + denyCanary = "deny-canary.invalid"; + + allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]); + + # Evaluated top to bottom, first match wins. The three destination denies + # sit before the only allow: squid resolves names on the host, so a name + # that answers 127.0.0.1, 169.254.169.254 or a bridge address would + # otherwise reach services the bridge firewall keeps agents out of. + squidConf = '' + http_port ${netCfg.bridgeIp}:${toString port} + pid_filename /run/squid/squid.pid + cache_log stdio:/var/log/squid/cache.log + coredump_dir /var/cache/squid + + cache deny all + + # One syslog line per request reaches the journal as squid.service. + # Cachemgr scrapes are left out. + access_log syslog:daemon.info squid !manager + + acl SSL_ports port 443 + acl Safe_ports port 80 443 + + # Host-originated requests: the self-check and the exporter. A local + # connect to the bridge IP takes the bridge IP as its source. + acl selfcheck src ${netCfg.bridgeIp}/32 + + acl to_bridge dst ${netCfg.networkCidr} + # `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record + # an agent controls could match an allowed name. + acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$) + acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$) + ${lib.optionalString ( + allowed != [ ] + ) "acl allowed_dst dstdomain ${lib.concatStringsSep " " allowed}"} + + # Answered by squid itself, never forwarded, and addressed to the bridge IP + # on a port outside Safe_ports, so it precedes the destination rules. + http_access allow manager selfcheck + http_access deny manager + + http_access deny !Safe_ports + http_access deny CONNECT !SSL_ports + http_access deny to_localhost + http_access deny to_linklocal + http_access deny to_bridge + http_access deny ip_v4_host + http_access deny ip_v6_host + ${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"} + http_access deny all + ''; + + sandbox = { + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectClock = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + LockPersonality = true; + SystemCallArchitectures = "native"; + CapabilityBoundingSet = ""; + }; + + selfcheck = pkgs.writeShellApplication { + name = "squid-selfcheck"; + runtimeInputs = [ pkgs.curl ]; + text = '' + # The status squid answers a CONNECT with; 000 when the proxy is down. + connect_status() { + curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \ + -o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true + } + + allow_status=$(connect_status ${cfg.selfcheck.canary}) + deny_status=$(connect_status ${denyCanary}) + + ok=false + if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then + ok=true + fi + echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status" + [ "$ok" = true ] + ''; + }; +in +{ + options.services.hyperhive.squid = { + allowedDomains = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = hyperhiveCfg.swarm.serviceDomains; + defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains"; + example = [ + "forge.example.com" + "github.com" + ]; + description = '' + Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each + entry matches that exact name. Everything else gets a 403. The + self-check canary is allowed on top of this list. + ''; + }; + + selfcheck.canary = lib.mkOption { + type = lib.types.str; + default = hyperhiveCfg.swarm.forge.domain; + defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain"; + description = '' + Hostname the self-check CONNECTs to on port 443 and expects squid to + let through. Squid resolves it with this host's resolver, not the + hive's: a host whose resolver answers a loopback or bridge address + for it (a single-host swarm with the gateway's `localHostsEntry`) gets + a 403, and the self-check reports `ok=false`. + ''; + }; + }; + + config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable { + # Reachable from agent containers and nowhere else: this opens the port on + # the bridge interface only. + services.hyperhive.network.exposeHostPorts = [ port ]; + services.hyperhive.network.enable = lib.mkDefault true; + + services.hyperhive.otel.scrapeTargets.squid = exporterListen; + services.hyperhive.otel.journalUnits = [ + "squid.service" + "squid-selfcheck.service" + ]; + + services.squid = { + enable = true; + configText = squidConf; + }; + + systemd.services.squid = { + serviceConfig = sandbox // { + User = "squid"; + Group = "squid"; + PIDFile = lib.mkForce "/run/squid/squid.pid"; + RuntimeDirectory = "squid"; + CacheDirectory = "squid"; + LogsDirectory = "squid"; + # The bridge address may not be assigned yet at first start. + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + systemd.services.squid-exporter = { + description = "Prometheus exporter for squid"; + wantedBy = [ "multi-user.target" ]; + after = [ "squid.service" ]; + serviceConfig = sandbox // { + ExecStart = lib.concatStringsSep " " [ + "${pkgs.prometheus-squid-exporter}/bin/squid-exporter" + "-squid-hostname ${netCfg.bridgeIp}" + "-squid-port ${toString port}" + "-listen ${exporterListen}" + ]; + DynamicUser = true; + IPAddressAllow = [ + "127.0.0.0/8" + netCfg.bridgeIp + ]; + IPAddressDeny = "any"; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + systemd.services.squid-selfcheck = { + description = "Squid allow and deny canaries"; + after = [ "squid.service" ]; + serviceConfig = sandbox // { + Type = "oneshot"; + ExecStart = lib.getExe selfcheck; + DynamicUser = true; + IPAddressAllow = netCfg.bridgeIp; + IPAddressDeny = "any"; + }; + }; + + systemd.timers.squid-selfcheck = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "1min"; + OnUnitActiveSec = "1min"; + }; + }; + }; +} diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 62081e81..6ac3ecdf 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -187,6 +187,21 @@ in ''; }; + journalUnits = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "squid.service" ]; + description = '' + Host systemd units whose journal this hive's collector forwards to + the swarm's log store, by full unit name. + + **A service adds its own unit, from its own module, under its own + `enable`**, like `scrapeTargets`. The list is an allow-list on + purpose: a hive host can be someone's workstation, so a unit that no + module named never leaves the machine. + ''; + }; + clientSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = @@ -326,6 +341,9 @@ in # not for agent containers. services.hyperhive.otel.scrapeTargets.collector = "127.0.0.1:${toString otel.telemetryPort}"; + # The unit whose crash-watch warning a swarm-level alert reads. + services.hyperhive.otel.journalUnits = [ "hive-c0re.service" ]; + services.opentelemetry-collector = { enable = true; # Contrib, matching the swarm tier (./swarm-otel.nix). The upstream @@ -449,11 +467,11 @@ in }; }; - # This tier's own host journal, scoped to the one unit whose - # crash-watch warning (`"container crash detected"`, + # This tier's own host journal, scoped to `journalUnits`: hive-c0re, + # whose crash-watch warning (`"container crash detected"`, # `crash_watch.rs`) a swarm-level grafana alert # (`hh-container-crashed` in `swarm-grafana.nix`) depends on - # reaching VictoriaLogs. Unlike the swarm tier's journald + # reaching VictoriaLogs, plus whatever other modules name. Unlike the swarm tier's journald # receiver (`swarm-otel.nix`) this needs no bind mount: this # collector is a HOST systemd unit, not a container, so # `/var/log/journal` is the same filesystem hive-c0re itself @@ -467,7 +485,7 @@ in journald = { directory = "/var/log/journal"; storage = "file_storage"; - units = [ "hive-c0re.service" ]; + units = otel.journalUnits; # Severity table shared with the agent and swarm tiers' # journald receivers — see ../journald-severity.nix for why # it is its own file. diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 08a6e7be..f2434b59 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -421,6 +421,72 @@ let && hiveOtelSettings.extensions ? file_storage && builtins.elem "file_storage" hiveOtelSettings.service.extensions; } + { + # Squid rides this fixture because its journal and metrics reach the + # swarm through this collector. The allow-list of journal units must stay + # exactly the three named: hive-c0re's (control: the list is not just + # squid's), squid's and its self-check's. + name = "the hive collector forwards the journal of squid, its self-check and hive-c0re and no other unit"; + ok = + lib.sort builtins.lessThan hiveOtelSettings.receivers.journald.units == [ + "hive-c0re.service" + "squid-selfcheck.service" + "squid.service" + ]; + } + { + name = "the hive collector scrapes the squid exporter on loopback"; + ok = + let + j = hiveScrapeJob "squid"; + in + j != null + && j.static_configs == [ { targets = [ "127.0.0.1:9301" ]; } ] + && lib.hasInfix "-listen 127.0.0.1:9301" hiveOtel.systemd.services.squid-exporter.serviceConfig.ExecStart; + } + { + # Control for the absence checks: the bridge listener is there, so the + # absences below are not a misspelled lookup. + name = "squid listens on the bridge IP only, without a second listener or ssl_bump"; + ok = + let + conf = hiveOtel.services.squid.configText; + lines = lib.splitString "\n" conf; + in + builtins.elem "http_port ${hiveOtel.services.hyperhive.network.bridgeIp}:3128" lines + && lib.length (lib.filter (l: lib.hasPrefix "http_port " l) lines) == 1 + && !(lib.hasInfix "ssl_bump" conf) + && !(lib.hasInfix "ssl-bump" conf) + && !(lib.hasInfix "localnet" conf); + } + { + # First match wins, so a destination deny after the allow is dead. + name = "squid denies loopback, link-local and bridge destinations before any destination allow"; + ok = + let + lines = lib.splitString "\n" hiveOtel.services.squid.configText; + at = l: lib.lists.findFirstIndex (x: x == l) null lines; + allow = at "http_access allow allowed_dst"; + in + allow != null + && at "http_access deny to_localhost" != null + && at "http_access deny to_localhost" < allow + && at "http_access deny to_linklocal" != null + && at "http_access deny to_linklocal" < allow + && at "http_access deny to_bridge" != null + && at "http_access deny to_bridge" < allow + && builtins.elem "cache deny all" lines + && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; + } + { + name = "squid's port is open on the bridge interface and on no other"; + ok = + let + fw = hiveOtel.networking.firewall; + in + builtins.elem 3128 fw.interfaces.${hiveOtel.services.hyperhive.network.bridgeName}.allowedTCPPorts + && !(builtins.elem 3128 fw.allowedTCPPorts); + } ]; in runGroup "hive-otel" cases diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix new file mode 100644 index 00000000..50fc3d91 --- /dev/null +++ b/nix/vm-tests/hive-squid.nix @@ -0,0 +1,220 @@ +# `checks.vm-hive-squid`: runs ../host-modules/hive-squid.nix in a VM, which +# the module-eval suite can only read. Three nodes on two networks: +# +# agent (vlan 1) --> proxy (vlan 1 = the stand-in bridge, vlan 2 = uplink) --> web (vlan 2) +# +# `proxy` imports the squid module against stub options for the few hive +# options it reads, so no hive-c0re build is pulled in; the module-eval suite +# covers the real option wiring. +{ pkgs }: +let + bridgeIp = "192.168.1.1"; + + hiveStubs = + { lib, config, ... }: + { + options.services.hyperhive = { + deploy.hive-controller.enable = lib.mkOption { + type = lib.types.bool; + default = true; + }; + swarm.serviceDomains = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "forge.t.local" ]; + }; + swarm.forge.domain = lib.mkOption { + type = lib.types.str; + default = "forge.t.local"; + }; + network = { + bridgeIp = lib.mkOption { + type = lib.types.str; + default = bridgeIp; + }; + networkCidr = lib.mkOption { + type = lib.types.str; + default = "192.168.1.0/24"; + }; + exposeHostPorts = lib.mkOption { + type = lib.types.listOf lib.types.port; + default = [ ]; + }; + enable = lib.mkOption { + type = lib.types.bool; + default = true; + }; + }; + otel = { + scrapeTargets = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + }; + journalUnits = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + }; + }; + # The firewall hole hive-network.nix opens for `exposeHostPorts`. + config.networking.firewall.interfaces.eth1.allowedTCPPorts = + config.services.hyperhive.network.exposeHostPorts; + }; + + # Plain TCP on 443: a CONNECT is answered before any TLS, which is all the + # test reads. + listen443 = addrs: { + services.nginx = { + enable = true; + virtualHosts.default.listen = map (addr: { + inherit addr; + port = 443; + }) addrs; + }; + }; +in +pkgs.testers.runNixOSTest { + name = "hive-squid"; + + nodes = { + proxy = + { lib, ... }: + { + imports = [ + hiveStubs + ../host-modules/hive-squid.nix + # Listeners a leaked destination would reach: loopback and the bridge. + (listen443 [ + "127.0.0.1" + bridgeIp + ]) + ]; + virtualisation.vlans = [ + 1 + 2 + ]; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ + { + address = bridgeIp; + prefixLength = 24; + } + ]; + networking.interfaces.eth2.ipv4.addresses = lib.mkForce [ + { + address = "192.168.2.1"; + prefixLength = 24; + } + ]; + # Names the proxy resolves, each a different destination class. + networking.hosts = { + "192.168.2.2" = [ "forge.t.local" ]; + "127.0.0.1" = [ "loop.t.local" ]; + ${bridgeIp} = [ "bridge.t.local" ]; + "169.254.169.254" = [ "meta.t.local" ]; + }; + services.hyperhive.squid.allowedDomains = [ + "forge.t.local" + "loop.t.local" + "bridge.t.local" + "meta.t.local" + ]; + }; + + web = + { lib, ... }: + { + imports = [ (listen443 [ "0.0.0.0" ]) ]; + virtualisation.vlans = [ 2 ]; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ + { + address = "192.168.2.2"; + prefixLength = 24; + } + ]; + networking.firewall.allowedTCPPorts = [ 443 ]; + }; + + agent = + { lib, ... }: + { + virtualisation.vlans = [ 1 ]; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ + { + address = "192.168.1.3"; + prefixLength = 24; + } + ]; + }; + }; + + testScript = '' + import re + + start_all() + proxy.wait_for_unit("squid.service") + proxy.wait_for_unit("squid-exporter.service") + proxy.wait_for_unit("nginx.service") + web.wait_for_unit("nginx.service") + proxy.wait_for_open_port(3128, "${bridgeIp}") + + def connect(host): + # The status squid answered the CONNECT with; 000 when none came back. + return agent.succeed( + "curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null " + f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true" + ).strip() + + with subtest("an allowed name is tunnelled and an unlisted one is refused"): + assert connect("forge.t.local") == "200", "allow canary" + assert connect("deny-canary.invalid") == "403", "deny canary" + + with subtest("a name resolving to a host-local destination is refused"): + assert connect("loop.t.local") == "403", "loopback" + assert connect("bridge.t.local") == "403", "bridge address" + assert connect("meta.t.local") == "403", "link-local" + + with subtest("IP literals are refused even when the address is reachable"): + assert connect("127.0.0.1") == "403" + assert connect("192.168.2.2") == "403" + + with subtest("only a port-443 CONNECT is allowed"): + assert connect("forge.t.local:22") == "403" + assert connect("forge.t.local:80") == "403" + + with subtest("an agent cannot read cachemgr"): + agent.succeed( + "test $(curl -s -m 15 -o /dev/null -w '%{http_code}' " + "http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403" + ) + + with subtest("the self-check passes against the same squid"): + proxy.succeed("systemctl start squid-selfcheck.service") + proxy.succeed( + "journalctl -u squid-selfcheck.service -o cat | " + "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local " + "allow_status=200 deny_canary=deny-canary.invalid deny_status=403'" + ) + + with subtest("squid's access log reaches the journal under squid.service"): + log = proxy.succeed("journalctl -u squid.service -o cat") + assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", log), log + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", log), log + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", log), log + # Exporter scrapes are cachemgr requests and stay out of the log. + assert "squid-internal-mgr" not in log, log + + with subtest("the exporter serves squid's counters on loopback"): + metrics = proxy.succeed("curl -sf http://127.0.0.1:9301/metrics") + print("\n".join(l for l in metrics.splitlines() if l.startswith(("squid_up", "squid_client_http")))) + assert 'squid_up{host="${bridgeIp}"} 1' in metrics + m = re.search(r"^squid_client_http_requests_total (\S+)$", metrics, re.M) + assert m and float(m.group(1)) >= 9, metrics + m = re.search(r"^squid_client_http_errors_total (\S+)$", metrics, re.M) + assert m and float(m.group(1)) >= 1, metrics + proxy.fail("curl -sf -m 5 http://${bridgeIp}:9301/metrics") + + with subtest("squid survives a reload"): + proxy.succeed("systemctl reload squid.service") + proxy.wait_for_unit("squid.service") + assert connect("forge.t.local") == "200" + ''; +}