vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

deploy squid on every hive host, with exporter, journal forwarding and self-check

Browse source
This commit is contained in:
atlas 2026-10-27 15:09:00 +01:00 • committed by mara
commit a2cf6fe8bb
7 changed files with 589 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/checks.nix
View file

@ -140,6 +140,8 @@ in
inherit (pkgs) lib;
shared = moduleEvalShared;
};
# Runs the squid module in a VM; needs KVM, unlike every check above.
vm-hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
module-eval-hive-otel = import ./module-eval/hive-otel.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;

1
nix/host-modules/default.nix
View file

@ -23,6 +23,7 @@
./hive-matrix.nix
./hive-network.nix
./hive-priv.nix
./hive-squid.nix
./hive-tls.nix
./otel.nix
./glue-bao-readers-policy-order.nix

11
nix/host-modules/hive-network.nix
View file

@ -162,6 +162,17 @@ in
'';
};
networkCidr = lib.mkOption {
type = lib.types.str;
internal = true;
readOnly = true;
default = "${intToIp networkBase}/${toString cfg.bridgePrefixLength}";
defaultText = lib.literalMD "bridge subnet in CIDR form, host bits masked off";
description = ''
Read-only computed bridge subnet, e.g. `10.42.0.0/24`.
'';
};
};
config = lib.mkMerge [

222
nix/host-modules/hive-squid.nix Normal file
View file

@ -0,0 +1,222 @@
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is
# pointed at it yet; the exporter, journal forwarding and the self-check below
# are how a deploy shows it works.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.squid;
netCfg = hyperhiveCfg.network;
port = 3128;
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
exporterListen = "127.0.0.1:9301";
denyCanary = "deny-canary.invalid";
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
# Evaluated top to bottom, first match wins. The three destination denies
# sit before the only allow: squid resolves names on the host, so a name
# that answers 127.0.0.1, 169.254.169.254 or a bridge address would
# otherwise reach services the bridge firewall keeps agents out of.
squidConf = ''
http_port ${netCfg.bridgeIp}:${toString port}
pid_filename /run/squid/squid.pid
cache_log stdio:/var/log/squid/cache.log
coredump_dir /var/cache/squid
cache deny all
# One syslog line per request reaches the journal as squid.service.
# Cachemgr scrapes are left out.
access_log syslog:daemon.info squid !manager
acl SSL_ports port 443
acl Safe_ports port 80 443
# Host-originated requests: the self-check and the exporter. A local
# connect to the bridge IP takes the bridge IP as its source.
acl selfcheck src ${netCfg.bridgeIp}/32
acl to_bridge dst ${netCfg.networkCidr}
# `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record
# an agent controls could match an allowed name.
acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$)
${lib.optionalString (
allowed != [ ]
) "acl allowed_dst dstdomain ${lib.concatStringsSep " " allowed}"}
# Answered by squid itself, never forwarded, and addressed to the bridge IP
# on a port outside Safe_ports, so it precedes the destination rules.
http_access allow manager selfcheck
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny to_localhost
http_access deny to_linklocal
http_access deny to_bridge
http_access deny ip_v4_host
http_access deny ip_v6_host
${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
http_access deny all
'';
sandbox = {
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectControlGroups = true;
ProtectClock = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
"AF_UNIX"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
LockPersonality = true;
SystemCallArchitectures = "native";
CapabilityBoundingSet = "";
};
selfcheck = pkgs.writeShellApplication {
name = "squid-selfcheck";
runtimeInputs = [ pkgs.curl ];
text = ''
# The status squid answers a CONNECT with; 000 when the proxy is down.
connect_status() {
curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
-o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
}
allow_status=$(connect_status ${cfg.selfcheck.canary})
deny_status=$(connect_status ${denyCanary})
ok=false
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then
ok=true
fi
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status"
[ "$ok" = true ]
'';
};
in
{
options.services.hyperhive.squid = {
allowedDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = hyperhiveCfg.swarm.serviceDomains;
defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains";
example = [
"forge.example.com"
"github.com"
];
description = ''
Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
entry matches that exact name. Everything else gets a 403. The
self-check canary is allowed on top of this list.
'';
};
selfcheck.canary = lib.mkOption {
type = lib.types.str;
default = hyperhiveCfg.swarm.forge.domain;
defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain";
description = ''
Hostname the self-check CONNECTs to on port 443 and expects squid to
let through. Squid resolves it with this host's resolver, not the
hive's: a host whose resolver answers a loopback or bridge address
for it (a single-host swarm with the gateway's `localHostsEntry`) gets
a 403, and the self-check reports `ok=false`.
'';
};
};
config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable {
# Reachable from agent containers and nowhere else: this opens the port on
# the bridge interface only.
services.hyperhive.network.exposeHostPorts = [ port ];
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
services.hyperhive.otel.journalUnits = [
"squid.service"
"squid-selfcheck.service"
];
services.squid = {
enable = true;
configText = squidConf;
};
systemd.services.squid = {
serviceConfig = sandbox // {
User = "squid";
Group = "squid";
PIDFile = lib.mkForce "/run/squid/squid.pid";
RuntimeDirectory = "squid";
CacheDirectory = "squid";
LogsDirectory = "squid";
# The bridge address may not be assigned yet at first start.
Restart = "on-failure";
RestartSec = 5;
};
};
systemd.services.squid-exporter = {
description = "Prometheus exporter for squid";
wantedBy = [ "multi-user.target" ];
after = [ "squid.service" ];
serviceConfig = sandbox // {
ExecStart = lib.concatStringsSep " " [
"${pkgs.prometheus-squid-exporter}/bin/squid-exporter"
"-squid-hostname ${netCfg.bridgeIp}"
"-squid-port ${toString port}"
"-listen ${exporterListen}"
];
DynamicUser = true;
IPAddressAllow = [
"127.0.0.0/8"
netCfg.bridgeIp
];
IPAddressDeny = "any";
Restart = "on-failure";
RestartSec = 5;
};
};
systemd.services.squid-selfcheck = {
description = "Squid allow and deny canaries";
after = [ "squid.service" ];
serviceConfig = sandbox // {
Type = "oneshot";
ExecStart = lib.getExe selfcheck;
DynamicUser = true;
IPAddressAllow = netCfg.bridgeIp;
IPAddressDeny = "any";
};
};
systemd.timers.squid-selfcheck = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "1min";
OnUnitActiveSec = "1min";
};
};
};
}

26
nix/host-modules/otel.nix
View file

@ -187,6 +187,21 @@ in
'';
};
journalUnits = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "squid.service" ];
description = ''
Host systemd units whose journal this hive's collector forwards to
the swarm's log store, by full unit name.
**A service adds its own unit, from its own module, under its own
`enable`**, like `scrapeTargets`. The list is an allow-list on
purpose: a hive host can be someone's workstation, so a unit that no
module named never leaves the machine.
'';
};
clientSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default =
@ -326,6 +341,9 @@ in
# not for agent containers.
services.hyperhive.otel.scrapeTargets.collector = "127.0.0.1:${toString otel.telemetryPort}";
# The unit whose crash-watch warning a swarm-level alert reads.
services.hyperhive.otel.journalUnits = [ "hive-c0re.service" ];
services.opentelemetry-collector = {
enable = true;
# Contrib, matching the swarm tier (./swarm-otel.nix). The upstream
@ -449,11 +467,11 @@ in
};
};
# This tier's own host journal, scoped to the one unit whose
# crash-watch warning (`"container crash detected"`,
# This tier's own host journal, scoped to `journalUnits`: hive-c0re,
# whose crash-watch warning (`"container crash detected"`,
# `crash_watch.rs`) a swarm-level grafana alert
# (`hh-container-crashed` in `swarm-grafana.nix`) depends on
# reaching VictoriaLogs. Unlike the swarm tier's journald
# reaching VictoriaLogs, plus whatever other modules name. Unlike the swarm tier's journald
# receiver (`swarm-otel.nix`) this needs no bind mount: this
# collector is a HOST systemd unit, not a container, so
# `/var/log/journal` is the same filesystem hive-c0re itself
@ -467,7 +485,7 @@ in
journald = {
directory = "/var/log/journal";
storage = "file_storage";
units = [ "hive-c0re.service" ];
units = otel.journalUnits;
# Severity table shared with the agent and swarm tiers'
# journald receivers — see ../journald-severity.nix for why
# it is its own file.

66
nix/module-eval/hive-otel.nix
View file

@ -421,6 +421,72 @@ let
&& hiveOtelSettings.extensions ? file_storage
&& builtins.elem "file_storage" hiveOtelSettings.service.extensions;
}
{
# Squid rides this fixture because its journal and metrics reach the
# swarm through this collector. The allow-list of journal units must stay
# exactly the three named: hive-c0re's (control: the list is not just
# squid's), squid's and its self-check's.
name = "the hive collector forwards the journal of squid, its self-check and hive-c0re and no other unit";
ok =
lib.sort builtins.lessThan hiveOtelSettings.receivers.journald.units == [
"hive-c0re.service"
"squid-selfcheck.service"
"squid.service"
];
}
{
name = "the hive collector scrapes the squid exporter on loopback";
ok =
let
j = hiveScrapeJob "squid";
in
j != null
&& j.static_configs == [ { targets = [ "127.0.0.1:9301" ]; } ]
&& lib.hasInfix "-listen 127.0.0.1:9301" hiveOtel.systemd.services.squid-exporter.serviceConfig.ExecStart;
}
{
# Control for the absence checks: the bridge listener is there, so the
# absences below are not a misspelled lookup.
name = "squid listens on the bridge IP only, without a second listener or ssl_bump";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
in
builtins.elem "http_port ${hiveOtel.services.hyperhive.network.bridgeIp}:3128" lines
&& lib.length (lib.filter (l: lib.hasPrefix "http_port " l) lines) == 1
&& !(lib.hasInfix "ssl_bump" conf)
&& !(lib.hasInfix "ssl-bump" conf)
&& !(lib.hasInfix "localnet" conf);
}
{
# First match wins, so a destination deny after the allow is dead.
name = "squid denies loopback, link-local and bridge destinations before any destination allow";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
allow = at "http_access allow allowed_dst";
in
allow != null
&& at "http_access deny to_localhost" != null
&& at "http_access deny to_localhost" < allow
&& at "http_access deny to_linklocal" != null
&& at "http_access deny to_linklocal" < allow
&& at "http_access deny to_bridge" != null
&& at "http_access deny to_bridge" < allow
&& builtins.elem "cache deny all" lines
&& lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
}
{
name = "squid's port is open on the bridge interface and on no other";
ok =
let
fw = hiveOtel.networking.firewall;
in
builtins.elem 3128 fw.interfaces.${hiveOtel.services.hyperhive.network.bridgeName}.allowedTCPPorts
&& !(builtins.elem 3128 fw.allowedTCPPorts);
}
];
in
runGroup "hive-otel" cases

220
nix/vm-tests/hive-squid.nix Normal file
View file

@ -0,0 +1,220 @@
# `checks.vm-hive-squid`: runs ../host-modules/hive-squid.nix in a VM, which
# the module-eval suite can only read. Three nodes on two networks:
#
# agent (vlan 1) --> proxy (vlan 1 = the stand-in bridge, vlan 2 = uplink) --> web (vlan 2)
#
# `proxy` imports the squid module against stub options for the few hive
# options it reads, so no hive-c0re build is pulled in; the module-eval suite
# covers the real option wiring.
{ pkgs }:
let
bridgeIp = "192.168.1.1";
hiveStubs =
{ lib, config, ... }:
{
options.services.hyperhive = {
deploy.hive-controller.enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
swarm.serviceDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "forge.t.local" ];
};
swarm.forge.domain = lib.mkOption {
type = lib.types.str;
default = "forge.t.local";
};
network = {
bridgeIp = lib.mkOption {
type = lib.types.str;
default = bridgeIp;
};
networkCidr = lib.mkOption {
type = lib.types.str;
default = "192.168.1.0/24";
};
exposeHostPorts = lib.mkOption {
type = lib.types.listOf lib.types.port;
default = [ ];
};
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
};
otel = {
scrapeTargets = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
};
journalUnits = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
};
};
};
# The firewall hole hive-network.nix opens for `exposeHostPorts`.
config.networking.firewall.interfaces.eth1.allowedTCPPorts =
config.services.hyperhive.network.exposeHostPorts;
};
# Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
# test reads.
listen443 = addrs: {
services.nginx = {
enable = true;
virtualHosts.default.listen = map (addr: {
inherit addr;
port = 443;
}) addrs;
};
};
in
pkgs.testers.runNixOSTest {
name = "hive-squid";
nodes = {
proxy =
{ lib, ... }:
{
imports = [
hiveStubs
../host-modules/hive-squid.nix
# Listeners a leaked destination would reach: loopback and the bridge.
(listen443 [
"127.0.0.1"
bridgeIp
])
];
virtualisation.vlans = [
1
2
];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = bridgeIp;
prefixLength = 24;
}
];
networking.interfaces.eth2.ipv4.addresses = lib.mkForce [
{
address = "192.168.2.1";
prefixLength = 24;
}
];
# Names the proxy resolves, each a different destination class.
networking.hosts = {
"192.168.2.2" = [ "forge.t.local" ];
"127.0.0.1" = [ "loop.t.local" ];
${bridgeIp} = [ "bridge.t.local" ];
"169.254.169.254" = [ "meta.t.local" ];
};
services.hyperhive.squid.allowedDomains = [
"forge.t.local"
"loop.t.local"
"bridge.t.local"
"meta.t.local"
];
};
web =
{ lib, ... }:
{
imports = [ (listen443 [ "0.0.0.0" ]) ];
virtualisation.vlans = [ 2 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.2.2";
prefixLength = 24;
}
];
networking.firewall.allowedTCPPorts = [ 443 ];
};
agent =
{ lib, ... }:
{
virtualisation.vlans = [ 1 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.3";
prefixLength = 24;
}
];
};
};
testScript = ''
import re
start_all()
proxy.wait_for_unit("squid.service")
proxy.wait_for_unit("squid-exporter.service")
proxy.wait_for_unit("nginx.service")
web.wait_for_unit("nginx.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
def connect(host):
# The status squid answered the CONNECT with; 000 when none came back.
return agent.succeed(
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null "
f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
).strip()
with subtest("an allowed name is tunnelled and an unlisted one is refused"):
assert connect("forge.t.local") == "200", "allow canary"
assert connect("deny-canary.invalid") == "403", "deny canary"
with subtest("a name resolving to a host-local destination is refused"):
assert connect("loop.t.local") == "403", "loopback"
assert connect("bridge.t.local") == "403", "bridge address"
assert connect("meta.t.local") == "403", "link-local"
with subtest("IP literals are refused even when the address is reachable"):
assert connect("127.0.0.1") == "403"
assert connect("192.168.2.2") == "403"
with subtest("only a port-443 CONNECT is allowed"):
assert connect("forge.t.local:22") == "403"
assert connect("forge.t.local:80") == "403"
with subtest("an agent cannot read cachemgr"):
agent.succeed(
"test $(curl -s -m 15 -o /dev/null -w '%{http_code}' "
"http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
)
with subtest("the self-check passes against the same squid"):
proxy.succeed("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local "
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403'"
)
with subtest("squid's access log reaches the journal under squid.service"):
log = proxy.succeed("journalctl -u squid.service -o cat")
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", log), log
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", log), log
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", log), log
# Exporter scrapes are cachemgr requests and stay out of the log.
assert "squid-internal-mgr" not in log, log
with subtest("the exporter serves squid's counters on loopback"):
metrics = proxy.succeed("curl -sf http://127.0.0.1:9301/metrics")
print("\n".join(l for l in metrics.splitlines() if l.startswith(("squid_up", "squid_client_http"))))
assert 'squid_up{host="${bridgeIp}"} 1' in metrics
m = re.search(r"^squid_client_http_requests_total (\S+)$", metrics, re.M)
assert m and float(m.group(1)) >= 9, metrics
m = re.search(r"^squid_client_http_errors_total (\S+)$", metrics, re.M)
assert m and float(m.group(1)) >= 1, metrics
proxy.fail("curl -sf -m 5 http://${bridgeIp}:9301/metrics")
with subtest("squid survives a reload"):
proxy.succeed("systemctl reload squid.service")
proxy.wait_for_unit("squid.service")
assert connect("forge.t.local") == "200"
'';
}