vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

deploy squid on every hive host, with exporter, journal forwarding and self-check

Browse source
This commit is contained in:
atlas 2026-10-27 15:09:00 +01:00 • committed by mara
commit a2cf6fe8bb
1 changed files with 589 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

220
nix/vm-tests/hive-squid.nix Normal file
View file

@ -0,0 +1,220 @@
# `checks.vm-hive-squid`: runs ../host-modules/hive-squid.nix in a VM, which
# the module-eval suite can only read. Three nodes on two networks:
#
# agent (vlan 1) --> proxy (vlan 1 = the stand-in bridge, vlan 2 = uplink) --> web (vlan 2)
#
# `proxy` imports the squid module against stub options for the few hive
# options it reads, so no hive-c0re build is pulled in; the module-eval suite
# covers the real option wiring.
{ pkgs }:
let
bridgeIp = "192.168.1.1";
hiveStubs =
{ lib, config, ... }:
{
options.services.hyperhive = {
deploy.hive-controller.enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
swarm.serviceDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "forge.t.local" ];
};
swarm.forge.domain = lib.mkOption {
type = lib.types.str;
default = "forge.t.local";
};
network = {
bridgeIp = lib.mkOption {
type = lib.types.str;
default = bridgeIp;
};
networkCidr = lib.mkOption {
type = lib.types.str;
default = "192.168.1.0/24";
};
exposeHostPorts = lib.mkOption {
type = lib.types.listOf lib.types.port;
default = [ ];
};
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
};
otel = {
scrapeTargets = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
};
journalUnits = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
};
};
};
# The firewall hole hive-network.nix opens for `exposeHostPorts`.
config.networking.firewall.interfaces.eth1.allowedTCPPorts =
config.services.hyperhive.network.exposeHostPorts;
};
# Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
# test reads.
listen443 = addrs: {
services.nginx = {
enable = true;
virtualHosts.default.listen = map (addr: {
inherit addr;
port = 443;
}) addrs;
};
};
in
pkgs.testers.runNixOSTest {
name = "hive-squid";
nodes = {
proxy =
{ lib, ... }:
{
imports = [
hiveStubs
../host-modules/hive-squid.nix
# Listeners a leaked destination would reach: loopback and the bridge.
(listen443 [
"127.0.0.1"
bridgeIp
])
];
virtualisation.vlans = [
1
2
];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = bridgeIp;
prefixLength = 24;
}
];
networking.interfaces.eth2.ipv4.addresses = lib.mkForce [
{
address = "192.168.2.1";
prefixLength = 24;
}
];
# Names the proxy resolves, each a different destination class.
networking.hosts = {
"192.168.2.2" = [ "forge.t.local" ];
"127.0.0.1" = [ "loop.t.local" ];
${bridgeIp} = [ "bridge.t.local" ];
"169.254.169.254" = [ "meta.t.local" ];
};
services.hyperhive.squid.allowedDomains = [
"forge.t.local"
"loop.t.local"
"bridge.t.local"
"meta.t.local"
];
};
web =
{ lib, ... }:
{
imports = [ (listen443 [ "0.0.0.0" ]) ];
virtualisation.vlans = [ 2 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.2.2";
prefixLength = 24;
}
];
networking.firewall.allowedTCPPorts = [ 443 ];
};
agent =
{ lib, ... }:
{
virtualisation.vlans = [ 1 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.3";
prefixLength = 24;
}
];
};
};
testScript = ''
import re
start_all()
proxy.wait_for_unit("squid.service")
proxy.wait_for_unit("squid-exporter.service")
proxy.wait_for_unit("nginx.service")
web.wait_for_unit("nginx.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
def connect(host):
# The status squid answered the CONNECT with; 000 when none came back.
return agent.succeed(
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null "
f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
).strip()
with subtest("an allowed name is tunnelled and an unlisted one is refused"):
assert connect("forge.t.local") == "200", "allow canary"
assert connect("deny-canary.invalid") == "403", "deny canary"
with subtest("a name resolving to a host-local destination is refused"):
assert connect("loop.t.local") == "403", "loopback"
assert connect("bridge.t.local") == "403", "bridge address"
assert connect("meta.t.local") == "403", "link-local"
with subtest("IP literals are refused even when the address is reachable"):
assert connect("127.0.0.1") == "403"
assert connect("192.168.2.2") == "403"
with subtest("only a port-443 CONNECT is allowed"):
assert connect("forge.t.local:22") == "403"
assert connect("forge.t.local:80") == "403"
with subtest("an agent cannot read cachemgr"):
agent.succeed(
"test $(curl -s -m 15 -o /dev/null -w '%{http_code}' "
"http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
)
with subtest("the self-check passes against the same squid"):
proxy.succeed("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local "
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403'"
)
with subtest("squid's access log reaches the journal under squid.service"):
log = proxy.succeed("journalctl -u squid.service -o cat")
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", log), log
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", log), log
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", log), log
# Exporter scrapes are cachemgr requests and stay out of the log.
assert "squid-internal-mgr" not in log, log
with subtest("the exporter serves squid's counters on loopback"):
metrics = proxy.succeed("curl -sf http://127.0.0.1:9301/metrics")
print("\n".join(l for l in metrics.splitlines() if l.startswith(("squid_up", "squid_client_http"))))
assert 'squid_up{host="${bridgeIp}"} 1' in metrics
m = re.search(r"^squid_client_http_requests_total (\S+)$", metrics, re.M)
assert m and float(m.group(1)) >= 9, metrics
m = re.search(r"^squid_client_http_errors_total (\S+)$", metrics, re.M)
assert m and float(m.group(1)) >= 1, metrics
proxy.fail("curl -sf -m 5 http://${bridgeIp}:9301/metrics")
with subtest("squid survives a reload"):
proxy.succeed("systemctl reload squid.service")
proxy.wait_for_unit("squid.service")
assert connect("forge.t.local") == "200"
'';
}