vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

make squid's allowed domains editable at runtime in the swarm ui

Browse source
This commit is contained in:
atlas 2026-10-28 20:39:22 +01:00 • committed by mara
commit c9abc2a42b
7 changed files with 2307 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

7
nix/host-modules/hive-c0re/environment.nix
View file

@ -301,6 +301,13 @@ in
# store under the identity above.
HIVE_SQUID_USERS_FILE = config.services.hyperhive.squid.usersFile;
HIVE_SQUID_UNRESTRICTED_FILE = config.services.hyperhive.squid.unrestrictedFile;
# The swarm's egress allow-list, which `workers::squid_allow` renders into
# the two files; the seed is the list the files hold until the swarm has one,
# and the always-allowed names are those it never repeats.
HIVE_SQUID_ALLOWED_FILE = config.services.hyperhive.squid.allowedFile;
HIVE_SQUID_AGENT_ALLOW_FILE = config.services.hyperhive.squid.agentAllowFile;
HIVE_SQUID_ALLOWED_SEED_FILE = config.services.hyperhive.squid.allowedSeedFile;
HIVE_SQUID_ALWAYS_ALLOWED = lib.concatStringsSep " " config.services.hyperhive.squid.alwaysAllowed;
# The egress proxy facts every container is handed (`HiveFacts` in
# `hive-priv-sock`). They sit with the users file above because that is what
# makes squid's credential check and per-agent policy real on this host: an

154
nix/host-modules/hive-squid.nix
View file

@ -4,7 +4,10 @@
# must carry a per-agent proxy credential, checked against the users file
# hive-c0re renders. A credential's agent either has the swarm's
# `unrestricted_internet` flag, and may then reach any destination the deny
# rules leave, or reaches only the listed names. An agent sends its credential
# rules leave, or reaches only the listed names: the base list, the allow-list
# the swarm keeps (rendered by hive-c0re beside the users file, with this
# host's `allowedDomains` as the list until the swarm has one) and the names
# the swarm lists for that agent alone. An agent sends its credential
# once it has fetched it (../agent-modules/egress-proxy.nix); the exporter,
# journal forwarding and the self-check below show the proxy works.
{
@ -26,6 +29,16 @@ let
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
base = lib.unique cfg.baseAllowedDomains;
# The names squid allows every authenticated user whatever the swarm lists.
# hive-c0re skips an allow-list entry that one of them already covers.
alwaysAllowed = base ++ [ cfg.selfcheck.canary ];
# What `allowedFile` holds until the swarm has an allow-list: this hive's
# own `allowedDomains`, one per line. The canary is in the configuration.
seedFile = pkgs.writeText "squid-allowed-seed" (
lib.concatMapStrings (d: d + "\n") (
lib.unique (lib.subtractLists [ cfg.selfcheck.canary ] cfg.allowedDomains)
)
);
# The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`.
selfcheckUser = "selfcheck.squid";
@ -65,8 +78,10 @@ let
# denies come first because `dstdomain` on an IP literal does a reverse
# lookup. The credential check sits right before the first allow, so only a
# request that would otherwise be let through gets the 407 challenge. Every
# user is then let through to the base names, an unrestricted user to any other
# destination on a safe port, and every other user to the listed names. The `dst` denies cost an
# user is then let through to the base names, a user the swarm lists names
# for to those names (the include, whose rules hive-c0re generates and whose
# file may be empty), an unrestricted user to any other destination on a
# safe port, and every other user to the listed names. The `dst` denies cost an
# unlisted name a lookup on the host's resolver before it is refused.
squidConf = ''
http_port ${listen}
@ -107,9 +122,8 @@ let
acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$)
${lib.optionalString (base != [ ]) "acl base_dst dstdomain ${lib.concatStringsSep " " base}"}
${lib.optionalString (
allowed != [ ]
) "acl allowed_dst dstdomain ${lib.concatStringsSep " " allowed}"}
acl allowed_dst dstdomain ${cfg.selfcheck.canary}
acl allowed_dst dstdomain "${cfg.allowedFile}"
# Answered by squid itself, never forwarded, and addressed to the bridge IP
# on a port outside Safe_ports, so it precedes the destination rules.
@ -126,12 +140,30 @@ let
http_access deny to_private
http_access deny !authed
${lib.optionalString (base != [ ]) "http_access allow base_dst"}
include ${cfg.agentAllowFile}
http_access allow unrestricted
${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"}
${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
http_access deny !allowed_dst
http_access allow allowed_dst
http_access deny all
'';
# `squid -k parse` on the configuration with the include pointing at an empty
# file. The nixpkgs module runs the same parse on the real text, which
# cannot open the include in the build sandbox.
squidConfCheck =
pkgs.runCommand "squid-conf-check"
{
nativeBuildInputs = [ config.services.squid.package ];
}
''
squid -k parse -f ${
pkgs.writeText "squid.conf" (
builtins.replaceStrings [ cfg.agentAllowFile ] [ "/dev/null" ] squidConf
)
}
touch $out
'';
sandbox = {
NoNewPrivileges = true;
ProtectSystem = "strict";
@ -222,8 +254,14 @@ in
"crates.io"
];
description = ''
Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
entry matches that exact name. Everything else gets a 403, except
Hostnames squid lets a client reach, by CONNECT or plain HTTP, until the
swarm has an allow-list of its own: the list the swarm's controller
keeps (`/api/egress/allowed-domains`) replaces this one as soon as it
holds an entry or has had one, and this option is the list again only
while the controller has none, for instance because it predates the
feature. hive-c0re writes whichever list applies to `allowedFile`.
Each entry matches that exact name, or that name and its subdomains
with a leading dot. Everything else gets a 403, except
for an unrestricted user. The swarm's own service names are not
listed: they resolve to the bridge or a private address, which the
destination rules refuse, and agents reach them directly. The
@ -238,11 +276,7 @@ in
baseAllowedDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"api.anthropic.com"
"claude.ai"
"platform.claude.com"
];
default = import ./lib/squid-base-domains.nix;
example = [
"api.anthropic.com"
"claude.ai"
@ -261,7 +295,9 @@ in
environment, which this host does not read. Swarm-internal names are
not listed, since agents reach them directly. The destination denies
still apply to every entry, and the list is separate from
`allowedDomains`.
`allowedDomains`. A name the swarm's allow-list holds that this list
already covers is left out of the rendered files, and the controller
refuses it as an entry when it is among the defaults.
'';
};
@ -289,12 +325,66 @@ in
`unrestricted_internet` flag (set unless the swarm records it off), and
the self-check's unrestricted user. hive-c0re renders it beside
`usersFile`, and squid is reloaded when it changes. Every other user
reaches only `allowedDomains`. An empty file, which is what exists
reaches only the base list, the allow-list and its own names. An empty file, which is what exists
until hive-c0re writes one, makes every user a restricted one.
Owned by `hive-core` and readable by the `squid` group.
'';
};
allowedFile = lib.mkOption {
type = lib.types.path;
default = "${builtins.dirOf cfg.usersFile}/allowed";
readOnly = true;
defaultText = lib.literalMD "`allowed`, in the directory of `usersFile`";
description = ''
The names squid lets every restricted user reach, one per line, read as
part of its allow ACL beside the self-check canary. It starts as a copy
of `allowedDomains`. hive-c0re rewrites it every pass: with the swarm's
swarm-wide entries once the controller has a list, otherwise with
`allowedDomains`. Squid is reloaded when it changes. Owned by
`hive-core` and readable by the `squid` group.
'';
};
agentAllowFile = lib.mkOption {
type = lib.types.path;
default = "${builtins.dirOf cfg.usersFile}/agent-allow.conf";
readOnly = true;
defaultText = lib.literalMD "`agent-allow.conf`, in the directory of `usersFile`";
description = ''
Squid configuration that hive-c0re generates and the main configuration
includes after every destination deny and `deny !authed`, where the
base list is allowed: for each agent of this hive the swarm lists
names for, an ACL for its user, an ACL for its names and the
`http_access allow` joining the two. Empty until the controller has a
list. Squid is reloaded when it changes. Owned by `hive-core` and
readable by the `squid` group.
'';
};
allowedSeedFile = lib.mkOption {
type = lib.types.path;
default = seedFile;
readOnly = true;
defaultText = lib.literalMD "`allowedDomains`, one per line, minus the self-check canary";
description = ''
The names `allowedFile` starts as and hive-c0re writes to it again while
the swarm has no allow-list. Read-only.
'';
};
alwaysAllowed = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = alwaysAllowed;
readOnly = true;
defaultText = lib.literalMD "`baseAllowedDomains` and the self-check canary";
description = ''
The names squid allows every authenticated user whatever the swarm
lists. hive-c0re leaves a swarm allow-list entry out of the rendered
files when one of these covers it. Read-only.
'';
};
selfcheck.credentialName = lib.mkOption {
type = lib.types.str;
default = "hive-squid.selfcheck";
@ -348,31 +438,45 @@ in
services.squid = {
enable = true;
configText = squidConf;
# The module's build-time parse fails on the `include` of a file only the
# running host has; `squidConfCheck` parses the same text without it.
validateConfig = false;
};
system.extraDependencies = [ squidConfCheck ];
# The directory is setgid so the file hive-c0re renames into it is the
# squid group's. The users file exists before squid starts, empty: the
# helper exits on a missing file and squid then rejects every login. The
# unrestricted list is created the same way, though squid treats a missing
# one as empty.
# one as empty. The allow-list file starts as a copy of `allowedDomains`
# and an existing one is left alone, so the list hive-c0re wrote survives a
# switch. The include is empty until hive-c0re has a list: squid does not
# start on a missing one.
systemd.tmpfiles.rules = [
"d ${usersDir} 2750 hive-core squid - -"
"f ${cfg.usersFile} 0640 hive-core squid - -"
"f ${cfg.unrestrictedFile} 0640 hive-core squid - -"
"C ${cfg.allowedFile} 0640 hive-core squid - ${seedFile}"
"f ${cfg.agentAllowFile} 0640 hive-core squid - -"
];
# Squid reads the unrestricted list when it parses its configuration, not
# when it changes. hive-c0re writes the list only when its contents change,
# so each change is one reload. Nothing happens to a squid that is not
# running; it reads the list when it starts.
# Squid reads the unrestricted list, the allow-list and the per-agent include
# when it parses its configuration, not when they change. hive-c0re writes
# each only when its contents change, so each change is one reload. Nothing
# happens to a squid that is not running; it reads them when it starts.
systemd.paths.squid-unrestricted-reload = {
description = "Watch squid's unrestricted-users list";
description = "Watch the lists hive-c0re renders for squid";
wantedBy = [ "paths.target" ];
pathConfig.PathChanged = cfg.unrestrictedFile;
pathConfig.PathChanged = [
cfg.unrestrictedFile
cfg.allowedFile
cfg.agentAllowFile
];
};
systemd.services.squid-unrestricted-reload = {
description = "Reload squid to read its unrestricted-users list";
description = "Reload squid to read the lists hive-c0re renders";
# Several list writes in a few seconds must not trip the start limit: a
# tripped limit also fails the path unit, and later writes then reload nothing.
startLimitIntervalSec = 0;

9
nix/host-modules/lib/squid-base-domains.nix Normal file
View file

@ -0,0 +1,9 @@
# The default of `services.hyperhive.squid.baseAllowedDomains`, and the names
# swarm-controller refuses as allow-list entries because squid already allows
# them to every authenticated user: the hosts Claude Code requires
# (<https://code.claude.com/docs/en/network-config>).
[
"api.anthropic.com"
"claude.ai"
"platform.claude.com"
]

8
nix/host-modules/swarm-controller.nix
View file

@ -864,6 +864,14 @@ in
# token is in the store, written by `swarm-bao-grafana-controller-token`
# on Grafana's host.
SWARM_CONTROLLER_GRAFANA_URL = "https://${swarmCfg.grafana.domain}";
# The names squid allows every authenticated agent by default. The
# allow-list route refuses them as entries
# (swarm-controller/src/egress_allow.rs). A hive that overrides
# `squid.baseAllowedDomains` is not reflected here; its hive-c0re
# skips the entries its own list covers.
SWARM_CONTROLLER_SQUID_BASE_DOMAINS = lib.concatStringsSep " " (
import ./lib/squid-base-domains.nix
);
}
// queueEnv
// forgeEnv

14
nix/module-eval/bao-controller.nix
View file

@ -249,6 +249,20 @@ let
controllerNoStore
];
}
{
# The allow-list route refuses the names squid already allows everyone.
# The list the controller is handed is the hive module's default, from
# one file; the second clause is the control that the default is not empty.
name = "the controller is handed the base list squid defaults to";
ok =
let
base = baoControllerHere.services.hyperhive.squid.baseAllowedDomains;
in
base != [ ]
&&
baoControllerHere.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_SQUID_BASE_DOMAINS
== lib.concatStringsSep " " base;
}
{
# A hive's cert-auth role carries the authority by value, so the daemon
# has to be handed the file rather than a path into the store's own

58
nix/module-eval/hive-otel.nix
View file

@ -47,6 +47,12 @@ let
hiveOtelAllLocal = hiveWithController {
deploy.bao.enable = true;
deploy.authelia.enable = true;
# The seed `allowedFile` starts as; the canary is listed too, to show it
# is not written twice.
squid.allowedDomains = [
"github.com"
"cache.nixos.org"
];
};
# Neither a leaf nor a secret file: the one shape the collector has no
@ -500,12 +506,18 @@ let
base = at "http_access allow base_dst";
unrestricted = at "http_access allow unrestricted";
authed = at "http_access deny !authed";
firstAllow = lib.min base (lib.min unrestricted allow);
# The generated per-agent `http_access allow` rules sit in this file.
agentRules = at "include ${hiveOtel.services.hyperhive.squid.agentAllowFile}";
firstAllow = lib.min agentRules (lib.min base (lib.min unrestricted allow));
in
allow != null
&& base != null
&& unrestricted != null
&& authed != null
&& agentRules != null
&& authed < agentRules
&& base < agentRules
&& agentRules < unrestricted
&& at "http_access deny to_localhost" != null
&& at "http_access deny to_linklocal" != null
&& at "http_access deny to_bridge" != null
@ -602,13 +614,55 @@ let
path = hiveOtel.systemd.paths.squid-unrestricted-reload;
svc = hiveOtel.systemd.services.squid-unrestricted-reload.serviceConfig;
in
path.pathConfig.PathChanged == list
path.pathConfig.PathChanged == [
list
hiveOtel.services.hyperhive.squid.allowedFile
hiveOtel.services.hyperhive.squid.agentAllowFile
]
&& hiveOtel.systemd.services.squid-unrestricted-reload.startLimitIntervalSec == 0
&& builtins.elem "paths.target" path.wantedBy
&& lib.hasSuffix "systemctl try-reload-or-restart squid.service" svc.ExecStart
&& builtins.elem "f ${list} 0640 hive-core squid - -" rules
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_UNRESTRICTED_FILE == list;
}
{
# The swarm's allow-list reaches squid as a file of names, which starts
# as this hive's own `allowedDomains` (the canary left out, since the
# configuration names it), and as an include that holds the per-agent
# rules. Control: the seed holds the listed name, and the configuration
# no longer names it, so the absence of the canary and of the list from
# the seed and the configuration is not a lookup of nothing.
name = "squid reads the swarm's allow-list from a file seeded with the hive's own names";
ok =
let
sq = hiveOtelAllLocal.services.hyperhive.squid;
lines = lib.splitString "\n" hiveOtelAllLocal.services.squid.configText;
rules = hiveOtelAllLocal.systemd.tmpfiles.rules;
env = hiveOtelAllLocal.systemd.services.hive-c0re.environment;
in
builtins.elem ''acl allowed_dst dstdomain "${sq.allowedFile}"'' lines
&& builtins.elem "acl allowed_dst dstdomain ${sq.selfcheck.canary}" lines
&& !(lib.hasInfix "github.com" hiveOtelAllLocal.services.squid.configText)
&& sq.allowedFile == "/var/lib/hive-squid/allowed"
&& sq.agentAllowFile == "/var/lib/hive-squid/agent-allow.conf"
&& sq.allowedSeedFile.text == "github.com\n"
&& builtins.elem "C ${sq.allowedFile} 0640 hive-core squid - ${sq.allowedSeedFile}" rules
&& builtins.elem "f ${sq.agentAllowFile} 0640 hive-core squid - -" rules
&& env.HIVE_SQUID_ALLOWED_FILE == sq.allowedFile
&& env.HIVE_SQUID_AGENT_ALLOW_FILE == sq.agentAllowFile
&& env.HIVE_SQUID_ALLOWED_SEED_FILE == toString sq.allowedSeedFile
&&
env.HIVE_SQUID_ALWAYS_ALLOWED == "api.anthropic.com claude.ai platform.claude.com cache.nixos.org";
}
{
# The nixpkgs module parses the configuration at build time, which cannot
# open the include; the parse moves to a check on the same text, so a
# broken configuration still fails the host build.
name = "squid's configuration is still parsed at build time, without the include";
ok =
!hiveOtel.services.squid.validateConfig
&& lib.any (d: (d.name or "") == "squid-conf-check") hiveOtel.system.extraDependencies;
}
{
# The helper is the NCSA one, on the file hive-c0re renders. The file
# exists, empty, before squid starts (the helper exits on a missing one),

45
nix/vm-tests/hive-squid.nix
View file

@ -152,6 +152,8 @@ pkgs.testers.runNixOSTest {
"cache.t.local"
"api.t.local"
"other.t.local"
"swarm.t.local"
"agentonly.t.local"
"deny-canary.invalid"
];
"127.0.0.1" = [ "loop.t.local" ];
@ -231,6 +233,7 @@ pkgs.testers.runNixOSTest {
testScript = ''
import re
import secrets
import shlex
import time
# Drawn fresh each run. The users file holds only their hashes, as
@ -248,6 +251,8 @@ pkgs.testers.runNixOSTest {
RUTH = "ruth:" + PW["ruth"]
USERS = "/var/lib/hive-squid/users"
LIST = "/var/lib/hive-squid/unrestricted"
ALLOWED = "/var/lib/hive-squid/allowed"
AGENT_ALLOW = "/var/lib/hive-squid/agent-allow.conf"
UNRESTRICTED = ["uma", "selfcheck-unrestricted.squid"]
start_all()
@ -289,6 +294,16 @@ pkgs.testers.runNixOSTest {
f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}"
)
def write_allow(allowed, agent_conf):
# What hive-c0re does for its two files: written aside and renamed into
# place, in squid's group. Squid reads them on a reload, which
# squid-unrestricted-reload does when either changes.
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf)):
proxy.succeed(
f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new "
f"&& chmod 640 {path}.new && mv {path}.new {path}"
)
def wait_status(host, cred, want):
# The reload that follows a list write is asynchronous.
got = None
@ -310,6 +325,11 @@ pkgs.testers.runNixOSTest {
assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750"
assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0"
assert proxy.succeed(f"stat -c %s {LIST}").strip() == "0"
assert proxy.succeed(f"stat -c '%U:%G %a' {ALLOWED}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c '%U:%G %a' {AGENT_ALLOW}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c %s {AGENT_ALLOW}").strip() == "0"
# The allow-list file starts as the hive's own allowedDomains.
proxy.succeed(f"grep -Fx forge.t.local {ALLOWED}")
assert connect("forge.t.local", ATLAS) == "407", "no such user"
assert connect("other.t.local", UMA) == "407", "no such user, unlisted name"
@ -500,6 +520,31 @@ pkgs.testers.runNixOSTest {
wait_status("other.t.local", RUTH, "403")
assert connect("other.t.local", UMA) == "200", "control: uma is still listed"
with subtest("the swarm's allow-list replaces the hive's names, opens one to every restricted user, and a per-agent rule opens one to that agent alone"):
seeded = proxy.succeed(f"cat {ALLOWED}")
assert connect("swarm.t.local", RUTH) == "403", "control: not allowed yet"
assert connect("agentonly.t.local", ATLAS) == "403", "control: not allowed yet"
assert connect("forge.t.local", RUTH) == "200", "control: the hive's own name"
write_allow(
"# source: swarm\nswarm.t.local\n",
"# source: swarm\n"
"acl agent_allow_atlas proxy_auth atlas\n"
"acl agent_allow_atlas_dst dstdomain agentonly.t.local priv10.t.local\n"
"http_access allow agent_allow_atlas agent_allow_atlas_dst\n",
)
wait_status("swarm.t.local", RUTH, "200")
wait_status("agentonly.t.local", ATLAS, "200")
assert connect("swarm.t.local", ATLAS) == "200", "swarm-wide, another user"
assert connect("agentonly.t.local", RUTH) == "403", "the per-agent name, another restricted user"
assert connect("agentonly.t.local") == "407", "no credential"
assert connect("forge.t.local", RUTH) == "403", "the hive's own name is replaced"
assert connect("priv10.t.local", ATLAS) == "403", "a per-agent name that resolves privately"
assert connect("agentonly.t.local", UMA) == "200", "control: unrestricted reaches it anyway"
write_allow(seeded, "")
wait_status("swarm.t.local", RUTH, "403")
wait_status("agentonly.t.local", ATLAS, "403")
assert connect("forge.t.local", RUTH) == "200", "the hive's own name is back"
with subtest("squid starts on a missing list, treats every user as restricted, and recovers when it appears"):
proxy.succeed(f"rm {LIST}")
proxy.succeed("systemctl restart squid.service")