vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity
hyperhive/nix/host-modules/hive-squid.nix
atlas e46ad07226 add a per-agent egress log mode
2026-10-30 08:52:22 +01:00

578 lines
24 KiB
Nix
Raw Blame History

# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
# must carry a per-agent proxy credential, checked against the users file
# hive-c0re renders. A credential's agent either has the swarm's
# `unrestricted_internet` flag, and may then reach any destination the deny
# rules leave, or reaches only the listed names: the base list, the allow-list
# the swarm keeps plus this host's `allowedDomains` (rendered by hive-c0re
# beside the users file) and the names
# the swarm lists for that agent alone. A third kind, set per agent
# (`internet_log`), reaches the same names but is also let through to any other
# destination the deny rules leave; the request is logged as `would_deny`.
# An agent sends its credential
# once it has fetched it (../agent-modules/egress-proxy.nix); the exporter,
# journal forwarding and the self-check below show the proxy works.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.deploy.hive-controller.squid;
netCfg = hyperhiveCfg.network;
port = 3128;
listen = "${netCfg.bridgeIp}:${toString port}";
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
exporterListen = "127.0.0.1:9301";
denyCanary = "deny-canary.invalid";
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
base = lib.unique cfg.baseAllowedDomains;
# The names squid allows every authenticated user whatever the swarm lists.
# hive-c0re skips an allow-list entry that one of them already covers.
alwaysAllowed = base ++ [ cfg.selfcheck.canary ];
# What `allowedFile` starts as and always includes: this hive's own
# `allowedDomains`, one per line. The canary is in the configuration.
seedFile = pkgs.writeText "squid-allowed-seed" (
lib.concatMapStrings (d: d + "\n") (
lib.unique (lib.subtractLists [ cfg.selfcheck.canary ] cfg.allowedDomains)
)
);
# The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`.
selfcheckUser = "selfcheck.squid";
# The same string in `swarm_secret_client::proxy::SELFCHECK_UNRESTRICTED_USER`.
selfcheckUnrestrictedUser = "selfcheck-unrestricted.squid";
usersDir = builtins.dirOf cfg.usersFile;
# Destinations that are not the public internet: loopback, link-local, the
# bridge, and the private, shared (CGNAT), unspecified, multicast and
# reserved ranges. Squid checks the address it connects to, so a name an
# agent controls cannot point past them.
# `::` is listed because a connect to it lands on loopback; squid's
# built-in `to_localhost` has `0.0.0.0/32` and `::1` but not `::`. NAT64
# (`64:ff9b::/96`) and 6to4 (`2002::/16`) embed IPv4 addresses and are left
# out: they matter only on a host with such a route.
privateRanges = [
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
"100.64.0.0/10"
"0.0.0.0/8"
"224.0.0.0/4"
"240.0.0.0/4"
"fc00::/7"
"ff00::/8"
"::/128"
];
# Evaluated top to bottom, first match wins. The destination denies sit
# before every allow, `allow unrestricted` included: squid resolves names on
# the host and connects from it, so a name that answers 127.0.0.1,
# 169.254.169.254, the host's own LAN address or a private neighbour would
# otherwise reach services the bridge firewall keeps agents out of, and an
# unrestricted agent is still an agent. The same denies bind a listed name,
# so a listed name must resolve to a public address on the host's resolver.
# The IP-literal
# denies come first because `dstdomain` on an IP literal does a reverse
# lookup. The credential check sits right before the first allow, so only a
# request that would otherwise be let through gets the 407 challenge. Every
# user is then let through to the base names, a user the swarm lists names
# for to those names (the include, whose rules hive-c0re generates and whose
# file may be empty), an unrestricted user to any other destination on a
# safe port, and every other user to the listed names. A `log`-mode agent's
# rule sits in the include after the agent's own names and before
# `deny !allowed_dst`, so it sees the same denies and only turns that last
# refusal into an allow. The `dst` denies cost an unlisted name a lookup on
# the host's resolver before it is refused.
squidConf = ''
http_port ${listen}
pid_filename /run/squid/squid.pid
cache_log stdio:/var/log/squid/cache.log
coredump_dir /var/cache/squid
cache deny all
# One syslog line per request reaches the journal as squid.service.
# Cachemgr scrapes are left out.
access_log syslog:daemon.info squid !manager
# A request that a `log`-mode agent's rule (hive-c0re's include, below)
# let through carries the annotation `would_deny=1`. It gets a second line
# in its own format, so the first line's format stays what the
# dashboards and lints parse, and `swarm-logs query '"would_deny"'` finds
# exactly these requests.
logformat would_deny would_deny user=%[un method=%rm host=%>rd port=%>rP result=%Ss/%03>Hs
acl would_denied note would_deny 1
access_log syslog:daemon.info logformat=would_deny would_denied
# The helper reads the users file on every request and reloads it when its
# mtime moves. It exits when the file is missing, which is why the file is
# created before squid starts. A positive answer is cached for a minute, so
# that is also how long a removed user keeps working.
auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile}
auth_param basic realm hyperhive egress
auth_param basic credentialsttl 1 minute
acl SSL_ports port 443
acl Safe_ports port 80 443
acl authed proxy_auth REQUIRED
# The users whose agent holds the flag. Squid reads the file when it parses
# its configuration, so `squid-unrestricted-reload` reloads squid when
# hive-c0re rewrites it. An empty or missing file lists no one.
acl unrestricted proxy_auth "${cfg.unrestrictedFile}"
# Host-originated requests: the self-check and the exporter. A local
# connect to the bridge IP takes the bridge IP as its source.
acl selfcheck src ${netCfg.bridgeIp}/32
acl to_bridge dst ${netCfg.networkCidr}
acl to_private dst ${lib.concatStringsSep " " privateRanges}
# `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record
# an agent controls could match an allowed name.
acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$)
${lib.optionalString (base != [ ]) "acl base_dst dstdomain ${lib.concatStringsSep " " base}"}
acl allowed_dst dstdomain ${cfg.selfcheck.canary}
acl allowed_dst dstdomain "${cfg.allowedFile}"
# Answered by squid itself, never forwarded, and addressed to the bridge IP
# on a port outside Safe_ports, so it precedes the destination rules.
http_access allow manager selfcheck
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny ip_v4_host
http_access deny ip_v6_host
http_access deny to_localhost
http_access deny to_linklocal
http_access deny to_bridge
http_access deny to_private
http_access deny !authed
${lib.optionalString (base != [ ]) "http_access allow base_dst"}
include ${cfg.agentAllowFile}
http_access allow unrestricted
http_access deny !allowed_dst
http_access allow allowed_dst
http_access deny all
'';
# `squid -k parse` on the configuration with the include pointing at an empty
# file. The nixpkgs module runs the same parse on the real text, which
# cannot open the include in the build sandbox.
squidConfCheck =
pkgs.runCommand "squid-conf-check"
{
nativeBuildInputs = [ config.services.squid.package ];
}
''
squid -k parse -f ${
pkgs.writeText "squid.conf" (
builtins.replaceStrings [ cfg.agentAllowFile ] [ "/dev/null" ] squidConf
)
}
touch $out
'';
sandbox = {
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectControlGroups = true;
ProtectClock = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
"AF_UNIX"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
LockPersonality = true;
SystemCallArchitectures = "native";
CapabilityBoundingSet = "";
};
selfcheck = pkgs.writeShellApplication {
name = "squid-selfcheck";
runtimeInputs = [ pkgs.curl ];
text = ''
# The credential comes from the unit's imported credential, and goes to
# curl on its stdin so it never appears in a command line. Without one
# the requests are anonymous and the self-check reports it.
secret=""
credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}"
if [ -s "$credential_file" ]; then
secret=$(< "$credential_file")
fi
# <host> <user|anon>: the status squid answers a CONNECT with; 000 when
# the proxy is down. Both users log in with the one secret.
connect_status() {
{
if [ "$2" != anon ] && [ -n "$secret" ]; then
printf 'proxy-user = "%s:%s"\n' "$2" "$secret"
fi
} | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
-o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
}
allow_status=$(connect_status ${cfg.selfcheck.canary} ${selfcheckUser})
deny_status=$(connect_status ${denyCanary} ${selfcheckUser})
anon_status=$(connect_status ${cfg.selfcheck.canary} anon)
unrestricted_status=$(connect_status ${cfg.selfcheck.unrestrictedCanary} ${selfcheckUnrestrictedUser})
ok=false
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ] \
&& [ "$unrestricted_status" = 200 ]; then
ok=true
fi
credential=absent
if [ -n "$secret" ]; then
credential=present
fi
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential unrestricted_canary=${cfg.selfcheck.unrestrictedCanary} unrestricted_status=$unrestricted_status"
[ "$ok" = true ]
'';
};
in
{
options.services.hyperhive.deploy.hive-controller.squid = {
listen = lib.mkOption {
type = lib.types.str;
default = listen;
readOnly = true;
defaultText = lib.literalMD "`<network.bridgeIp>:3128`";
description = ''
The address squid listens on, as `<bridge IP>:<port>`. Read-only.
hive-c0re hands it to every container as the `HIVE_SQUID_PROXY` fact,
which is what tells an agent that this host's squid checks its proxy
credential and applies its `unrestricted_internet` flag.
'';
};
allowedDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [
"github.com"
"crates.io"
];
description = ''
Hostnames squid always lets a client reach, by CONNECT or plain HTTP, on
this hive. The entries the swarm's controller keeps
(`/api/egress/allowed-domains`) add to this list and never replace it:
hive-c0re writes these names and the swarm-wide entries to
`allowedFile`. A hive-c0re that predates this rule replaced this list
with the controller's once the controller held an entry, so every hive
must run the new one before the controller creates its first entry.
Each entry matches that exact name, or that name and its subdomains
with a leading dot. Everything else gets a 403, except
for an unrestricted user. The swarm's own service names are not
listed: they resolve to the bridge or a private address, which the
destination rules refuse, and agents reach them directly. The
self-check canary is allowed on top of this list, so it widens the
restricted allow list by that one name. A name must
resolve, on this host's resolver, to a public address: one in a
loopback, link-local, private, shared (100.64.0.0/10), multicast or
reserved range (or is `::`), or in the bridge subnet, gets a 403 for
every user.
'';
};
baseAllowedDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = import ./lib/squid-base-domains.nix;
example = [
"api.anthropic.com"
"claude.ai"
"platform.claude.com"
"openrouter.ai"
];
description = ''
Hostnames every authenticated user may reach, whether or not its agent
holds the `unrestricted_internet` flag: what the agent harness needs
off the swarm to work at all. The defaults are the hosts Claude Code
requires. `api.anthropic.com` serves the model API and the usage
probe, `claude.ai` the account login, and `platform.claude.com` the
OAuth token exchange and refresh. The swarm controller adds the host of an
agent's stored `ANTHROPIC_BASE_URL` to that agent's swarm entries; an
agent on the hand-placed environment file alone, or an `opencode`
provider, needs its host in the swarm UI or in `allowedDomains`.
Swarm-internal names are
not listed, since agents reach them directly. The destination denies
still apply to every entry, and the list is separate from
`allowedDomains`. A name the swarm's allow-list holds that this list
already covers is left out of the rendered files, and the controller
refuses it as an entry when it is among the defaults.
'';
};
usersFile = lib.mkOption {
type = lib.types.path;
default = "/var/lib/hive-squid/users";
description = ''
The `<user>:<hash>` file squid checks proxy credentials against. hive-c0re
renders it from the credentials in the swarm secret store, one line per
agent of this hive and one for the self-check, as SHA-512 crypt hashes.
An empty file, which is what exists until hive-c0re writes one, makes
every login fail, and every request to a listed name answers 407.
Owned by `hive-core` and readable by the `squid` group.
'';
};
unrestrictedFile = lib.mkOption {
type = lib.types.path;
default = "${builtins.dirOf cfg.usersFile}/unrestricted";
readOnly = true;
defaultText = lib.literalMD "`unrestricted`, in the directory of `usersFile`";
description = ''
The user names, one per line, that squid lets reach any destination the
deny rules leave open: the users of the agents that hold the swarm's
`unrestricted_internet` flag (set unless the swarm records it off), and
the self-check's unrestricted user. hive-c0re renders it beside
`usersFile`, and squid is reloaded when it changes. Every other user
reaches only the base list, the allow-list and its own names. An empty file, which is what exists
until hive-c0re writes one, makes every user a restricted one.
Owned by `hive-core` and readable by the `squid` group.
'';
};
allowedFile = lib.mkOption {
type = lib.types.path;
default = "${builtins.dirOf cfg.usersFile}/allowed";
readOnly = true;
defaultText = lib.literalMD "`allowed`, in the directory of `usersFile`";
description = ''
The names squid lets every restricted user reach, one per line, read as
part of its allow ACL beside the self-check canary. It starts as a copy
of `allowedDomains`. hive-c0re rewrites it every pass: with
`allowedDomains` and the swarm-wide entries once the controller has a
list, otherwise with `allowedDomains` alone. Squid is reloaded when it changes. Owned by
`hive-core` and readable by the `squid` group.
'';
};
agentAllowFile = lib.mkOption {
type = lib.types.path;
default = "${builtins.dirOf cfg.usersFile}/agent-allow.conf";
readOnly = true;
defaultText = lib.literalMD "`agent-allow.conf`, in the directory of `usersFile`";
description = ''
Squid configuration that hive-c0re generates and the main configuration
includes after every destination deny and `deny !authed`, where the
base list is allowed: for each agent of this hive the swarm lists
names for, an ACL for its user, an ACL for its names and the
`http_access allow` joining the two. After those, for each agent in
`log` mode, an `http_access allow` for names `allowed_dst` lacks that
annotates the request `would_deny=1`. Empty until the controller has a
list or an agent is in `log` mode. Squid is reloaded when it changes. Owned by `hive-core` and
readable by the `squid` group.
'';
};
allowedSeedFile = lib.mkOption {
type = lib.types.path;
default = seedFile;
readOnly = true;
defaultText = lib.literalMD "`allowedDomains`, one per line, minus the self-check canary";
description = ''
The names `allowedFile` starts as and hive-c0re writes into it on every
pass, with or without a swarm allow-list. Read-only.
'';
};
alwaysAllowed = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = alwaysAllowed;
readOnly = true;
defaultText = lib.literalMD "`baseAllowedDomains` and the self-check canary";
description = ''
The names squid allows every authenticated user whatever the swarm
lists. hive-c0re leaves a swarm allow-list entry out of the rendered
files when one of these covers it. Read-only.
'';
};
selfcheck.credentialName = lib.mkOption {
type = lib.types.str;
default = "hive-squid.selfcheck";
readOnly = true;
description = ''
Name of the systemd credential the self-check imports its squid
password from. `squid-selfcheck-credential` writes it to
`/run/credstore/<name>`. Without one the self-check sends no
credential and reports `ok=false`.
'';
};
selfcheck.unrestrictedCanary = lib.mkOption {
type = lib.types.str;
default = "example.com";
description = ''
Hostname outside `allowedDomains` that the self-check CONNECTs to on
port 443, as its unrestricted user, and expects squid to let through.
Squid resolves it with this host's resolver and connects to it, so a
host with no route to it reports `ok=false`.
'';
};
selfcheck.canary = lib.mkOption {
type = lib.types.str;
default = "cache.nixos.org";
description = ''
Hostname the self-check CONNECTs to on port 443 and expects squid to
let through. It is added to the allowed names, so every restricted
agent can reach it too. It must resolve to a public address: squid
resolves it with this host's resolver, not the hive's, and a name
that answers a loopback, bridge or other private address (a swarm
service name on a hive, a split-horizon LAN) gets a 403 before
authentication, and the self-check reports `ok=false`.
'';
};
};
config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable {
# Reachable from agent containers and nowhere else: this opens the port on
# the bridge interface only.
services.hyperhive.network.exposeHostPorts = [ port ];
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
services.hyperhive.otel.journalUnits = [
"squid.service"
"squid-selfcheck.service"
];
services.squid = {
enable = true;
configText = squidConf;
# The module's build-time parse fails on the `include` of a file only the
# running host has; `squidConfCheck` parses the same text without it.
validateConfig = false;
};
system.extraDependencies = [ squidConfCheck ];
# The directory is setgid so the file hive-c0re renames into it is the
# squid group's. The users file exists before squid starts, empty: the
# helper exits on a missing file and squid then rejects every login. The
# unrestricted list is created the same way, though squid treats a missing
# one as empty. The allow-list file starts as a copy of `allowedDomains`
# and an existing one is left alone, so the list hive-c0re wrote survives a
# switch. The include is empty until hive-c0re has a list: squid does not
# start on a missing one.
systemd.tmpfiles.rules = [
"d ${usersDir} 2750 hive-core squid - -"
"f ${cfg.usersFile} 0640 hive-core squid - -"
"f ${cfg.unrestrictedFile} 0640 hive-core squid - -"
"C ${cfg.allowedFile} 0640 hive-core squid - ${seedFile}"
"f ${cfg.agentAllowFile} 0640 hive-core squid - -"
];
# Squid reads the unrestricted list, the allow-list and the per-agent include
# when it parses its configuration, not when they change. hive-c0re writes
# each only when its contents change, so each change is one reload. Nothing
# happens to a squid that is not running; it reads them when it starts.
systemd.paths.squid-unrestricted-reload = {
description = "Watch the lists hive-c0re renders for squid";
wantedBy = [ "paths.target" ];
pathConfig.PathChanged = [
cfg.unrestrictedFile
cfg.allowedFile
cfg.agentAllowFile
];
};
systemd.services.squid-unrestricted-reload = {
description = "Reload squid to read the lists hive-c0re renders";
# Several list writes in a few seconds must not trip the start limit: a
# tripped limit also fails the path unit, and later writes then reload nothing.
startLimitIntervalSec = 0;
serviceConfig = {
Type = "oneshot";
ExecStart = "${config.systemd.package}/bin/systemctl try-reload-or-restart squid.service";
};
};
assertions = [
{
assertion = !(builtins.elem cfg.selfcheck.unrestrictedCanary (allowed ++ base));
message = ''
services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary must be a name outside
allowedDomains and baseAllowedDomains, or the self-check cannot tell an unrestricted user from a
restricted one.
'';
}
];
systemd.services.squid = {
serviceConfig = sandbox // {
User = "squid";
Group = "squid";
PIDFile = lib.mkForce "/run/squid/squid.pid";
RuntimeDirectory = "squid";
CacheDirectory = "squid";
LogsDirectory = "squid";
# The bridge address may not be assigned yet at first start.
Restart = "on-failure";
RestartSec = 5;
};
};
systemd.services.squid-exporter = {
description = "Prometheus exporter for squid";
wantedBy = [ "multi-user.target" ];
after = [ "squid.service" ];
serviceConfig = sandbox // {
ExecStart = lib.concatStringsSep " " [
"${pkgs.prometheus-squid-exporter}/bin/squid-exporter"
"-squid-hostname ${netCfg.bridgeIp}"
"-squid-port ${toString port}"
"-listen ${exporterListen}"
];
DynamicUser = true;
IPAddressAllow = [
"127.0.0.0/8"
netCfg.bridgeIp
];
IPAddressDeny = "any";
Restart = "on-failure";
RestartSec = 5;
};
};
systemd.services.squid-selfcheck = {
description = "Squid allow, deny and credential canaries";
after = [ "squid.service" ];
serviceConfig = sandbox // {
Type = "oneshot";
# A glob that matches nothing is not an error: a host with no
# credential yet still runs the self-check, and it reports that.
ImportCredential = cfg.selfcheck.credentialName;
ExecStart = lib.getExe selfcheck;
DynamicUser = true;
IPAddressAllow = netCfg.bridgeIp;
IPAddressDeny = "any";
};
};
systemd.timers.squid-selfcheck = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "1min";
OnUnitActiveSec = "1min";
};
};
};
}
View git blame Copy permalink