vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

require a proxy credential at squid and prove it in the self-check

Browse source
This commit is contained in:
atlas 2026-10-28 13:07:15 +01:00 • committed by mara
commit b7adb90f53
3 changed files with 207 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

85
nix/host-modules/hive-squid.nix
View file

@ -1,8 +1,9 @@
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is
# pointed at it yet; the exporter, journal forwarding and the self-check below
# are how a deploy shows it works.
# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
# for a listed name must carry a per-agent proxy credential, checked against the
# users file hive-c0re renders. No agent sends one yet; the exporter, journal
# forwarding and the self-check below are how a deploy shows it works.
{
pkgs,
lib,
@ -21,6 +22,8 @@ let
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
# The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`.
selfcheckUser = "selfcheck.squid";
usersDir = builtins.dirOf cfg.usersFile;
# Evaluated top to bottom, first match wins. The three destination denies
@ -29,7 +32,10 @@ let
# otherwise reach services the bridge firewall keeps agents out of. The
# unlisted-name deny comes first because `dstdomain` needs no lookup while
# the `dst` ACLs do: an unlisted name is refused at once instead of waiting
# on the host resolver, and costs a client no DNS query.
# on the host resolver, and costs a client no DNS query. The credential
# check sits right before the allow, so a refused destination answers 403
# to a client with or without a credential, and only a request that would
# otherwise be let through gets the 407 challenge.
squidConf = ''
http_port ${netCfg.bridgeIp}:${toString port}
pid_filename /run/squid/squid.pid
@ -42,8 +48,17 @@ let
# Cachemgr scrapes are left out.
access_log syslog:daemon.info squid !manager
# The helper reads the users file on every request and reloads it when its
# mtime moves. It exits when the file is missing, which is why the file is
# created before squid starts. A positive answer is cached for a minute, so
# that is also how long a removed user keeps working.
auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile}
auth_param basic realm hyperhive egress
auth_param basic credentialsttl 1 minute
acl SSL_ports port 443
acl Safe_ports port 80 443
acl authed proxy_auth REQUIRED
# Host-originated requests: the self-check and the exporter. A local
# connect to the bridge IP takes the bridge IP as its source.
@ -71,6 +86,7 @@ let
http_access deny to_localhost
http_access deny to_linklocal
http_access deny to_bridge
http_access deny !authed
${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
http_access deny all
'';
@ -103,20 +119,39 @@ let
name = "squid-selfcheck";
runtimeInputs = [ pkgs.curl ];
text = ''
# The status squid answers a CONNECT with; 000 when the proxy is down.
# The credential comes from the unit's imported credential, and goes to
# curl on its stdin so it never appears in a command line. Without one
# the requests are anonymous and the self-check reports it.
secret=""
credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}"
if [ -s "$credential_file" ]; then
secret=$(< "$credential_file")
fi
# <host> <auth|anon>: the status squid answers a CONNECT with; 000 when
# the proxy is down.
connect_status() {
curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
{
if [ "$2" = auth ] && [ -n "$secret" ]; then
printf 'proxy-user = "%s:%s"\n' ${selfcheckUser} "$secret"
fi
} | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
-o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
}
allow_status=$(connect_status ${cfg.selfcheck.canary})
deny_status=$(connect_status ${denyCanary})
allow_status=$(connect_status ${cfg.selfcheck.canary} auth)
deny_status=$(connect_status ${denyCanary} auth)
anon_status=$(connect_status ${cfg.selfcheck.canary} anon)
ok=false
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ]; then
ok=true
fi
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status"
credential=absent
if [ -n "$secret" ]; then
credential=present
fi
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential"
[ "$ok" = true ]
'';
};
@ -142,10 +177,24 @@ in
type = lib.types.path;
default = "/var/lib/hive-squid/users";
description = ''
The `<user>:<hash>` file hive-c0re renders from the credentials in the
swarm secret store: one line per agent of this hive and one for the
self-check, as SHA-512 crypt hashes. Created empty, owned by `hive-core`
and readable by the `squid` group. Squid doesn't read it yet.
The `<user>:<hash>` file squid checks proxy credentials against. hive-c0re
renders it from the credentials in the swarm secret store, one line per
agent of this hive and one for the self-check, as SHA-512 crypt hashes.
An empty file, which is what exists until hive-c0re writes one, makes
every login fail, and every request to a listed name answers 407.
Owned by `hive-core` and readable by the `squid` group.
'';
};
selfcheck.credentialName = lib.mkOption {
type = lib.types.str;
default = "hive-squid.selfcheck";
readOnly = true;
description = ''
Name of the systemd credential the self-check imports its squid
password from. `squid-selfcheck-credential` writes it to
`/run/credstore/<name>`. Without one the self-check sends no
credential and reports `ok=false`.
'';
};
@ -181,7 +230,8 @@ in
};
# The directory is setgid so the file hive-c0re renames into it is the
# squid group's.
# squid group's. The file exists before squid starts, empty: the helper
# exits on a missing file and squid then rejects every login.
systemd.tmpfiles.rules = [
"d ${usersDir} 2750 hive-core squid - -"
"f ${cfg.usersFile} 0640 hive-core squid - -"
@ -224,10 +274,13 @@ in
};
systemd.services.squid-selfcheck = {
description = "Squid allow and deny canaries";
description = "Squid allow, deny and credential canaries";
after = [ "squid.service" ];
serviceConfig = sandbox // {
Type = "oneshot";
# A glob that matches nothing is not an error: a host with no
# credential yet still runs the self-check, and it reports that.
ImportCredential = cfg.selfcheck.credentialName;
ExecStart = lib.getExe selfcheck;
DynamicUser = true;
IPAddressAllow = netCfg.bridgeIp;

35
nix/module-eval/hive-otel.nix
View file

@ -485,39 +485,60 @@ let
&& !(lib.hasInfix "localnet" conf);
}
{
# First match wins, so a destination deny after the allow is dead.
name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, before any allow";
# First match wins, so a destination deny after the allow is dead, and
# a credential check after it never challenges anyone. The two IP-literal
# denies stay ahead of the unlisted-name deny: `dstdomain` on an IP
# literal does a reverse lookup.
name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, then a missing credential, before any allow";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
allow = at "http_access allow allowed_dst";
authed = at "http_access deny !authed";
in
allow != null
&& authed != null
&& at "http_access deny to_localhost" != null
&& at "http_access deny to_localhost" < allow
&& at "http_access deny to_linklocal" != null
&& at "http_access deny to_linklocal" < allow
&& at "http_access deny to_bridge" != null
&& at "http_access deny to_bridge" < allow
&& at "http_access deny to_localhost" < authed
&& at "http_access deny to_linklocal" < authed
&& at "http_access deny to_bridge" < authed
&& authed < allow
&& at "http_access deny ip_v4_host" != null
&& at "http_access deny ip_v4_host" < at "http_access deny !allowed_dst"
&& at "http_access deny ip_v6_host" < at "http_access deny !allowed_dst"
&& at "http_access deny !allowed_dst" != null
&& at "http_access deny !allowed_dst" < at "http_access deny to_localhost"
&& builtins.elem "cache deny all" lines
&& lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
}
{
# The file hive-c0re renders exists, owned by it and readable by squid's
# group, from boot; hive-c0re gets its path only on a host with a store
# identity.
name = "squid's users file exists for hive-c0re to write, and hive-c0re is told where it is";
# The helper is the NCSA one, on the file hive-c0re renders. The file
# exists, empty, before squid starts (the helper exits on a missing one),
# and the self-check imports its credential without failing when there is
# none.
name = "squid authenticates with the NCSA helper on a users file that exists before it starts";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
users = hiveOtel.services.hyperhive.squid.usersFile;
rules = hiveOtel.systemd.tmpfiles.rules;
sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig;
in
users == "/var/lib/hive-squid/users"
lib.any (
l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l
) lines
&& builtins.elem "acl authed proxy_auth REQUIRED" lines
&& users == "/var/lib/hive-squid/users"
&& builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules
&& builtins.elem "f ${users} 0640 hive-core squid - -" rules
&& sc.ImportCredential == hiveOtel.services.hyperhive.squid.selfcheck.credentialName
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users;
}
{

130
nix/vm-tests/hive-squid.nix
View file

@ -55,6 +55,12 @@ let
};
};
};
# The user hive-c0re runs as, which owns the users file squid reads.
config.users.users.hive-core = {
isSystemUser = true;
group = "hive-core";
};
config.users.groups.hive-core = { };
# The firewall hole hive-network.nix opens for `exposeHostPorts`.
config.networking.firewall.interfaces.eth1.allowedTCPPorts =
config.services.hyperhive.network.exposeHostPorts;
@ -77,7 +83,7 @@ pkgs.testers.runNixOSTest {
nodes = {
proxy =
{ lib, ... }:
{ lib, pkgs, ... }:
{
imports = [
hiveStubs
@ -111,6 +117,8 @@ pkgs.testers.runNixOSTest {
${bridgeIp} = [ "bridge.t.local" ];
"169.254.169.254" = [ "meta.t.local" ];
};
# Renders the users file the way hive-c0re does.
environment.systemPackages = [ pkgs.mkpasswd ];
services.hyperhive.squid.allowedDomains = [
"forge.t.local"
"loop.t.local"
@ -148,6 +156,15 @@ pkgs.testers.runNixOSTest {
testScript = ''
import re
import secrets
# Drawn fresh each run. The users file holds only their hashes, as
# hive-c0re renders it.
PW = {u: secrets.token_urlsafe(24) for u in ("atlas", "selfcheck.squid", "ruth")}
ATLAS = "atlas:" + PW["atlas"]
SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"]
RUTH = "ruth:" + PW["ruth"]
USERS = "/var/lib/hive-squid/users"
start_all()
proxy.wait_for_unit("squid.service")
@ -156,29 +173,64 @@ pkgs.testers.runNixOSTest {
web.wait_for_unit("nginx.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
def connect(host):
def connect(host, cred=None):
# The status squid answered the CONNECT with; 000 when none came back.
auth = f"-U '{cred}' " if cred else ""
return agent.succeed(
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null "
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null "
f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
).strip()
with subtest("an allowed name is tunnelled and an unlisted one is refused"):
assert connect("forge.t.local") == "200", "allow canary"
assert connect("deny-canary.invalid") == "403", "deny canary"
def write_users(*creds):
# What hive-c0re does: hashes only, written aside and renamed into
# place, in squid's group. The sleep keeps two writes in different
# seconds, which is how the helper tells the file changed.
lines = " ".join(
f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";"
for u, p in (c.split(":", 1) for c in creds)
)
proxy.succeed(
f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new "
f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1"
)
with subtest("squid starts on the empty users file and refuses every login"):
assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750"
assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0"
assert connect("forge.t.local", ATLAS) == "407", "no such user"
assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name needs no login"
with subtest("the users file holds hashes, never a password"):
write_users(ATLAS, SELFCHECK)
users = proxy.succeed(f"cat {USERS}")
assert re.fullmatch(r"(atlas|selfcheck\.squid):\$6\$\S+\n(atlas|selfcheck\.squid):\$6\$\S+\n", users), users
assert not any(p in users for p in PW.values()), users
with subtest("an allowed name is tunnelled with a credential and challenged without one"):
assert connect("forge.t.local", ATLAS) == "200", "allow canary"
assert connect("forge.t.local") == "407", "no credential"
assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user"
# DES would accept the first eight characters; the hash must not.
assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password"
with subtest("an unlisted name is refused with or without a credential"):
assert connect("deny-canary.invalid", ATLAS) == "403", "deny canary"
assert connect("deny-canary.invalid") == "403", "deny canary, anonymous"
with subtest("a name resolving to a host-local destination is refused"):
assert connect("loop.t.local") == "403", "loopback"
assert connect("bridge.t.local") == "403", "bridge address"
assert connect("meta.t.local") == "403", "link-local"
assert connect("loop.t.local", ATLAS) == "403", "loopback"
assert connect("bridge.t.local", ATLAS) == "403", "bridge address"
assert connect("meta.t.local", ATLAS) == "403", "link-local"
with subtest("IP literals are refused even when the address is reachable"):
assert connect("127.0.0.1") == "403"
assert connect("192.168.2.2") == "403"
assert connect("127.0.0.1", ATLAS) == "403"
assert connect("192.168.2.2", ATLAS) == "403"
with subtest("only a port-443 CONNECT is allowed"):
assert connect("forge.t.local:22") == "403"
assert connect("forge.t.local:80") == "403"
assert connect("forge.t.local:22", ATLAS) == "403"
assert connect("forge.t.local:80", ATLAS) == "403"
with subtest("an agent cannot read cachemgr"):
agent.succeed(
@ -186,19 +238,42 @@ pkgs.testers.runNixOSTest {
"http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
)
with subtest("the self-check passes against the same squid"):
with subtest("the self-check passes with its credential"):
proxy.succeed("install -d -m 0700 /run/credstore")
proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck")
proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck")
proxy.succeed("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local "
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403'"
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present'"
)
with subtest("squid's access log reaches the journal under squid.service"):
with subtest("the self-check fails, and says why, without a credential or with the wrong one"):
proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local "
"allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=absent'"
)
proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present'"
)
proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck")
with subtest("squid's access log reaches the journal under squid.service, with the user"):
journal = proxy.succeed("journalctl -u squid.service -o cat")
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443 ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal
# Exporter scrapes are cachemgr requests and stay out of the log.
assert "squid-internal-mgr" not in journal, journal
@ -215,6 +290,21 @@ pkgs.testers.runNixOSTest {
with subtest("squid survives a reload"):
proxy.succeed("systemctl reload squid.service")
proxy.wait_for_unit("squid.service")
assert connect("forge.t.local") == "200"
assert connect("forge.t.local", ATLAS) == "200"
with subtest("a user added while squid runs can log in, with no reload"):
write_users(ATLAS, SELFCHECK, RUTH)
assert connect("forge.t.local", RUTH) == "200"
assert connect("forge.t.local", ATLAS) == "200"
with subtest("squid starts on a missing users file, refuses logins, and recovers when it appears"):
proxy.succeed(f"rm {USERS}")
proxy.succeed("systemctl restart squid.service")
proxy.wait_for_unit("squid.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
assert connect("forge.t.local", ATLAS) == "407", "no file, no login"
assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name still answers"
write_users(ATLAS, SELFCHECK)
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
'';
}