mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

squid blocks a runtime deny list from the controller in every egress mode, ahead of the allow list

Browse source
This commit is contained in:
atlas 2026-11-04 22:00:42 +01:00 • committed by mara
commit dd5fde848f
4 changed files with 1814 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

3
nix/host-modules/hive-c0re/environment.nix
View file

@ -299,6 +299,9 @@ in
HIVE_SQUID_ALLOWED_SEED_FILE =
config.services.hyperhive.deploy.hive-controller.squid.allowedSeedFile;
HIVE_SQUID_ALWAYS_ALLOWED = lib.concatStringsSep " " config.services.hyperhive.deploy.hive-controller.squid.alwaysAllowed;
# The swarm's egress deny list, which `workers::squid_deny` renders into the
# include squid reads before every allow.
HIVE_SQUID_DENY_FILE = config.services.hyperhive.deploy.hive-controller.squid.denyFile;
# The egress proxy facts every container is handed (`HiveFacts` in
# `hive-priv-sock`). They sit with the users file above because that is what
# makes squid's credential check and per-agent policy real on this host: an

41
nix/host-modules/hive-squid.nix
View file

@ -10,6 +10,8 @@
# the swarm lists for that agent alone. A third kind, set per agent
# (`internet_log`), reaches the same names but is also let through to any other
# destination the deny rules leave; the request is logged as `would_deny`.
# The swarm's deny list (rendered by hive-c0re into `denyFile`) is refused for
# all three before any allow, and logged as `egress_denied`.
# An agent sends its credential
# once it has fetched it (../agent-modules/egress-proxy.nix); the exporter,
# journal forwarding and the self-check below show the proxy works.
@ -89,6 +91,10 @@ let
# `deny !allowed_dst`, so it sees the same denies and only turns that last
# refusal into an allow. The `dst` denies cost an unlisted name a lookup on
# the host's resolver before it is refused.
# The swarm's deny list (`denyFile`) sits right after those destination
# denies and before `deny !authed`, so a denied name is refused before any
# credential check and before every allow, in every mode: its rules name the
# destination first, and a swarm-wide one names no user at all.
squidConf = ''
http_port ${listen}
pid_filename /run/squid/squid.pid
@ -110,6 +116,14 @@ let
acl would_denied note would_deny 1
access_log syslog:daemon.info logformat=would_deny would_denied
# A request one of the deny list's rules (hive-c0re's `denyFile`, below)
# refused carries the annotation `egress_deny=1` and gets a line of its own,
# distinct from `would_deny` and from the 403 line every refusal has, in
# every mode.
logformat egress_denied egress_denied user=%[un method=%rm host=%>rd port=%>rP result=%Ss/%03>Hs
acl egress_denied_hit note egress_deny 1
access_log syslog:daemon.info logformat=egress_denied egress_denied_hit
# The helper reads the users file on every request and reloads it when its
# mtime moves. It exits when the file is missing, which is why the file is
# created before squid starts. A positive answer is cached for a minute, so
@ -153,6 +167,7 @@ let
http_access deny to_linklocal
http_access deny to_bridge
http_access deny to_private
include ${cfg.denyFile}
http_access deny !authed
${lib.optionalString (base != [ ]) "http_access allow base_dst"}
include ${cfg.agentAllowFile}
@ -162,9 +177,9 @@ let
http_access deny all
'';
# `squid -k parse` on the configuration with the include pointing at an empty
# `squid -k parse` on the configuration with the includes pointing at an empty
# file. The nixpkgs module runs the same parse on the real text, which
# cannot open the include in the build sandbox.
# cannot open the includes in the build sandbox.
squidConfCheck =
pkgs.runCommand "squid-conf-check"
{
@ -173,7 +188,7 @@ let
''
squid -k parse -f ${
pkgs.writeText "squid.conf" (
builtins.replaceStrings [ cfg.agentAllowFile ] [ "/dev/null" ] squidConf
builtins.replaceStrings [ cfg.agentAllowFile cfg.denyFile ] [ "/dev/null" "/dev/null" ] squidConf
)
}
touch $out
@ -381,6 +396,24 @@ in
'';
};
denyFile = lib.mkOption {
type = lib.types.path;
default = "${builtins.dirOf cfg.usersFile}/deny.conf";
readOnly = true;
defaultText = lib.literalMD "`deny.conf`, in the directory of `usersFile`";
description = ''
Squid configuration that hive-c0re generates from the swarm's deny list
(`/api/egress/denied-domains`) and the main configuration includes after
the destination denies and before `deny !authed` and every allow: a
`http_access deny` for the swarm-wide names, and one per agent of this
hive the swarm lists names for. A denied name is refused for every user
in every egress mode, whether or not an allow rule covers it. The file
is empty, and squid configured as without it, until the swarm lists a
name. Squid is reloaded when it changes. Owned by `hive-core` and readable
by the `squid` group.
'';
};
allowedSeedFile = lib.mkOption {
type = lib.types.path;
default = seedFile;
@ -481,6 +514,7 @@ in
"f ${cfg.unrestrictedFile} 0640 hive-core squid - -"
"C ${cfg.allowedFile} 0640 hive-core squid - ${seedFile}"
"f ${cfg.agentAllowFile} 0640 hive-core squid - -"
"f ${cfg.denyFile} 0640 hive-core squid - -"
];
# Squid reads the unrestricted list, the allow-list and the per-agent include
@ -494,6 +528,7 @@ in
cfg.unrestrictedFile
cfg.allowedFile
cfg.agentAllowFile
cfg.denyFile
];
};

53
nix/module-eval/hive-otel.nix
View file

@ -636,6 +636,58 @@ let
&& builtins.elem ''acl unrestricted proxy_auth "${list}"'' lines
&& list == "/var/lib/hive-squid/unrestricted";
}
{
# The swarm's deny list is an include that precedes every allow and the
# credential check, so a denied name is refused in every mode and for a
# client that sent no credential. Control: the include and the allows are
# found, so a position compared below is not a lookup of nothing.
name = "squid includes the swarm's deny list after the destination denies and before the credential check and every allow";
ok =
let
sq = hiveOtel.services.hyperhive.deploy.hive-controller.squid;
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
deny = at "include ${sq.denyFile}";
allows = map at [
"http_access allow base_dst"
"include ${sq.agentAllowFile}"
"http_access allow unrestricted"
"http_access allow allowed_dst"
];
in
deny != null
&& sq.denyFile == "/var/lib/hive-squid/deny.conf"
&& !(lib.elem null allows)
&& lib.all (a: deny < a) allows
&& at "http_access deny to_private" < deny
&& at "http_access deny ip_v4_host" < deny
&& deny < at "http_access deny !authed"
# Two includes: this one and the per-agent allow rules.
&& lib.length (lib.filter (l: lib.hasPrefix "include " l) lines) == 2;
}
{
# What the deny list adds to the configuration is the include and the
# lines that log a hit; with the file empty, which tmpfiles creates, squid
# has no rule it did not have. Control: the lines are found, so the
# equality below is not against an empty filter.
name = "squid's deny list adds one include and the log lines, and its file exists empty before squid starts";
ok =
let
sq = hiveOtel.services.hyperhive.deploy.hive-controller.squid;
conf = hiveOtel.services.squid.configText;
mentions = lib.filter (
l: !(lib.hasPrefix "#" l) && (lib.hasInfix "egress_den" l || lib.hasInfix sq.denyFile l)
) (lib.splitString "\n" conf);
in
mentions == [
"logformat egress_denied egress_denied user=%[un method=%rm host=%>rd port=%>rP result=%Ss/%03>Hs"
"acl egress_denied_hit note egress_deny 1"
"access_log syslog:daemon.info logformat=egress_denied egress_denied_hit"
"include ${sq.denyFile}"
]
&& builtins.elem "f ${sq.denyFile} 0640 hive-core squid - -" hiveOtel.systemd.tmpfiles.rules
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_DENY_FILE == sq.denyFile;
}
{
# The list is read when squid parses its configuration, so a change to
# the file reloads squid. Control: the watched path is the list's own.
@ -651,6 +703,7 @@ let
list
hiveOtel.services.hyperhive.deploy.hive-controller.squid.allowedFile
hiveOtel.services.hyperhive.deploy.hive-controller.squid.agentAllowFile
hiveOtel.services.hyperhive.deploy.hive-controller.squid.denyFile
]
&& hiveOtel.systemd.services.squid-unrestricted-reload.startLimitIntervalSec == 0
&& builtins.elem "paths.target" path.wantedBy

58
nix/vm-tests/hive-squid.nix
View file

@ -253,6 +253,7 @@ pkgs.testers.runNixOSTest {
LIST = "/var/lib/hive-squid/unrestricted"
ALLOWED = "/var/lib/hive-squid/allowed"
AGENT_ALLOW = "/var/lib/hive-squid/agent-allow.conf"
DENY = "/var/lib/hive-squid/deny.conf"
UNRESTRICTED = ["uma", "selfcheck-unrestricted.squid"]
start_all()
@ -304,6 +305,14 @@ pkgs.testers.runNixOSTest {
f"&& chmod 640 {path}.new && mv {path}.new {path}"
)
def write_deny(body):
# What hive-c0re does for the deny list: written aside and renamed into
# place, in squid's group; the reload is squid-unrestricted-reload's.
proxy.succeed(
f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new "
f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}"
)
def wait_status(host, cred, want):
# The reload that follows a list write is asynchronous.
got = None
@ -328,6 +337,8 @@ pkgs.testers.runNixOSTest {
assert proxy.succeed(f"stat -c '%U:%G %a' {ALLOWED}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c '%U:%G %a' {AGENT_ALLOW}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c %s {AGENT_ALLOW}").strip() == "0"
assert proxy.succeed(f"stat -c '%U:%G %a' {DENY}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c %s {DENY}").strip() == "0", "no deny list, no rule"
# The allow-list file starts as the hive's own allowedDomains.
proxy.succeed(f"grep -Fx forge.t.local {ALLOWED}")
assert connect("forge.t.local", ATLAS) == "407", "no such user"
@ -573,6 +584,53 @@ pkgs.testers.runNixOSTest {
write_allow(seeded, "")
wait_status("other.t.local", RUTH, "403")
with subtest("the swarm's deny list is refused before any allow, in every mode, with or without a credential, and logged as egress_denied"):
# What hive-c0re renders: `forge.t.local` and `other.t.local` for every
# agent, `api.t.local` (a base-list name) for atlas alone. `ruth` is in
# log mode, so her rule follows. The names are allowed or reachable
# without the list, which the controls establish first.
assert connect("forge.t.local", ATLAS) == "200", "control: allowed by the hive"
assert connect("forge.t.local", UMA) == "200", "control: allowed, unrestricted"
assert connect("other.t.local", UMA) == "200", "control: unrestricted reaches it"
assert connect("api.t.local", ATLAS) == "200", "control: base list"
write_allow(
seeded,
"# source: swarm\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_ruth proxy_auth ruth\n"
"http_access allow agent_log_ruth !allowed_dst log_mark\n",
)
wait_status("other.t.local", RUTH, "200")
write_deny(
"acl deny_hit_mark annotate_transaction egress_deny=1\n"
"acl deny_swarm_dst dstdomain forge.t.local other.t.local\n"
"http_access deny deny_swarm_dst deny_hit_mark\n"
"acl deny_agent_atlas_dst dstdomain api.t.local\n"
"acl deny_agent_atlas proxy_auth atlas\n"
"http_access deny deny_agent_atlas_dst deny_agent_atlas deny_hit_mark\n"
)
wait_status("forge.t.local", ATLAS, "403")
assert connect("forge.t.local", RUTH) == "403", "a listed name, log mode"
assert connect("forge.t.local", UMA) == "403", "a listed name, unrestricted"
assert connect("other.t.local", UMA) == "403", "unrestricted"
assert connect("other.t.local", RUTH) == "403", "log mode"
assert connect("forge.t.local") == "403", "no credential: refused, not challenged"
assert connect("api.t.local", ATLAS) == "403", "a base-list name, denied for atlas"
assert connect("api.t.local", RUTH) == "200", "control: another agent keeps the base name"
assert connect("api.t.local") == "407", "control: a per-agent deny does not answer a client it cannot name"
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
)
assert squid_saw(r"egress_denied user=- method=CONNECT host=forge\.t\.local port=443 "), "the anonymous client's hit"
assert not squid_saw(r"egress_denied .* host=loop\.t\.local"), "another refusal is not a deny-list hit"
assert not squid_saw(r"egress_denied user=ruth .* host=api\.t\.local"), "another agent's name is not a hit"
write_deny("")
write_allow(seeded, "")
wait_status("forge.t.local", ATLAS, "200")
assert connect("api.t.local", ATLAS) == "200", "the list emptied"
assert connect("other.t.local", UMA) == "200", "the list emptied, unrestricted"
with subtest("squid starts on a missing list, treats every user as restricted, and recovers when it appears"):
proxy.succeed(f"rm {LIST}")
proxy.succeed("systemctl restart squid.service")