403 lines
16 KiB
Nix
403 lines
16 KiB
Nix
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
|
|
# IP, default-deny on the destination hostname. It filters CONNECT targets and
|
|
# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
|
|
# must carry a per-agent proxy credential, checked against the users file
|
|
# hive-c0re renders. A credential's agent either has the swarm's
|
|
# `unrestricted_internet` flag, and may then reach any destination the deny
|
|
# rules leave, or reaches only the listed names. No agent sends a credential
|
|
# yet; the exporter, journal forwarding and the self-check below are how a
|
|
# deploy shows it works.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
cfg = hyperhiveCfg.squid;
|
|
netCfg = hyperhiveCfg.network;
|
|
|
|
port = 3128;
|
|
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
|
|
exporterListen = "127.0.0.1:9301";
|
|
denyCanary = "deny-canary.invalid";
|
|
|
|
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
|
|
|
|
# The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`.
|
|
selfcheckUser = "selfcheck.squid";
|
|
# The same string in `swarm_secret_client::proxy::SELFCHECK_UNRESTRICTED_USER`.
|
|
selfcheckUnrestrictedUser = "selfcheck-unrestricted.squid";
|
|
usersDir = builtins.dirOf cfg.usersFile;
|
|
|
|
# Destinations that are not the public internet: loopback, link-local, the
|
|
# bridge, and the private, shared (CGNAT), unspecified, multicast and
|
|
# reserved ranges. Squid checks the address it connects to, so a name an
|
|
# agent controls cannot point past them.
|
|
# `::` is listed because a connect to it lands on loopback; squid's
|
|
# built-in `to_localhost` has `0.0.0.0/32` and `::1` but not `::`. NAT64
|
|
# (`64:ff9b::/96`) and 6to4 (`2002::/16`) embed IPv4 addresses and are left
|
|
# out: they matter only on a host with such a route.
|
|
privateRanges = [
|
|
"10.0.0.0/8"
|
|
"172.16.0.0/12"
|
|
"192.168.0.0/16"
|
|
"100.64.0.0/10"
|
|
"0.0.0.0/8"
|
|
"224.0.0.0/4"
|
|
"240.0.0.0/4"
|
|
"fc00::/7"
|
|
"ff00::/8"
|
|
"::/128"
|
|
];
|
|
|
|
# Evaluated top to bottom, first match wins. The destination denies sit
|
|
# before every allow, `allow unrestricted` included: squid resolves names on
|
|
# the host and connects from it, so a name that answers 127.0.0.1,
|
|
# 169.254.169.254, the host's own LAN address or a private neighbour would
|
|
# otherwise reach services the bridge firewall keeps agents out of, and an
|
|
# unrestricted agent is still an agent. The same denies bind a listed name,
|
|
# so a listed name must resolve to a public address on the host's resolver.
|
|
# The IP-literal
|
|
# denies come first because `dstdomain` on an IP literal does a reverse
|
|
# lookup. The credential check sits right before the first allow, so only a
|
|
# request that would otherwise be let through gets the 407 challenge. An
|
|
# unrestricted user is then let through to any other destination on a safe
|
|
# port, and every other user to the listed names. The `dst` denies cost an
|
|
# unlisted name a lookup on the host's resolver before it is refused.
|
|
squidConf = ''
|
|
http_port ${netCfg.bridgeIp}:${toString port}
|
|
pid_filename /run/squid/squid.pid
|
|
cache_log stdio:/var/log/squid/cache.log
|
|
coredump_dir /var/cache/squid
|
|
|
|
cache deny all
|
|
|
|
# One syslog line per request reaches the journal as squid.service.
|
|
# Cachemgr scrapes are left out.
|
|
access_log syslog:daemon.info squid !manager
|
|
|
|
# The helper reads the users file on every request and reloads it when its
|
|
# mtime moves. It exits when the file is missing, which is why the file is
|
|
# created before squid starts. A positive answer is cached for a minute, so
|
|
# that is also how long a removed user keeps working.
|
|
auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile}
|
|
auth_param basic realm hyperhive egress
|
|
auth_param basic credentialsttl 1 minute
|
|
|
|
acl SSL_ports port 443
|
|
acl Safe_ports port 80 443
|
|
acl authed proxy_auth REQUIRED
|
|
# The users whose agent holds the flag. Squid reads the file when it parses
|
|
# its configuration, so `squid-unrestricted-reload` reloads squid when
|
|
# hive-c0re rewrites it. An empty or missing file lists no one.
|
|
acl unrestricted proxy_auth "${cfg.unrestrictedFile}"
|
|
|
|
# Host-originated requests: the self-check and the exporter. A local
|
|
# connect to the bridge IP takes the bridge IP as its source.
|
|
acl selfcheck src ${netCfg.bridgeIp}/32
|
|
|
|
acl to_bridge dst ${netCfg.networkCidr}
|
|
acl to_private dst ${lib.concatStringsSep " " privateRanges}
|
|
# `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record
|
|
# an agent controls could match an allowed name.
|
|
acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
|
|
acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$)
|
|
${lib.optionalString (
|
|
allowed != [ ]
|
|
) "acl allowed_dst dstdomain ${lib.concatStringsSep " " allowed}"}
|
|
|
|
# Answered by squid itself, never forwarded, and addressed to the bridge IP
|
|
# on a port outside Safe_ports, so it precedes the destination rules.
|
|
http_access allow manager selfcheck
|
|
http_access deny manager
|
|
|
|
http_access deny !Safe_ports
|
|
http_access deny CONNECT !SSL_ports
|
|
http_access deny ip_v4_host
|
|
http_access deny ip_v6_host
|
|
http_access deny to_localhost
|
|
http_access deny to_linklocal
|
|
http_access deny to_bridge
|
|
http_access deny to_private
|
|
http_access deny !authed
|
|
http_access allow unrestricted
|
|
${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"}
|
|
${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
|
|
http_access deny all
|
|
'';
|
|
|
|
sandbox = {
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
PrivateDevices = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectKernelModules = true;
|
|
ProtectKernelLogs = true;
|
|
ProtectControlGroups = true;
|
|
ProtectClock = true;
|
|
RestrictAddressFamilies = [
|
|
"AF_INET"
|
|
"AF_INET6"
|
|
"AF_UNIX"
|
|
];
|
|
RestrictNamespaces = true;
|
|
RestrictRealtime = true;
|
|
RestrictSUIDSGID = true;
|
|
LockPersonality = true;
|
|
SystemCallArchitectures = "native";
|
|
CapabilityBoundingSet = "";
|
|
};
|
|
|
|
selfcheck = pkgs.writeShellApplication {
|
|
name = "squid-selfcheck";
|
|
runtimeInputs = [ pkgs.curl ];
|
|
text = ''
|
|
# The credential comes from the unit's imported credential, and goes to
|
|
# curl on its stdin so it never appears in a command line. Without one
|
|
# the requests are anonymous and the self-check reports it.
|
|
secret=""
|
|
credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}"
|
|
if [ -s "$credential_file" ]; then
|
|
secret=$(< "$credential_file")
|
|
fi
|
|
|
|
# <host> <user|anon>: the status squid answers a CONNECT with; 000 when
|
|
# the proxy is down. Both users log in with the one secret.
|
|
connect_status() {
|
|
{
|
|
if [ "$2" != anon ] && [ -n "$secret" ]; then
|
|
printf 'proxy-user = "%s:%s"\n' "$2" "$secret"
|
|
fi
|
|
} | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
|
|
-o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
|
|
}
|
|
|
|
allow_status=$(connect_status ${cfg.selfcheck.canary} ${selfcheckUser})
|
|
deny_status=$(connect_status ${denyCanary} ${selfcheckUser})
|
|
anon_status=$(connect_status ${cfg.selfcheck.canary} anon)
|
|
unrestricted_status=$(connect_status ${cfg.selfcheck.unrestrictedCanary} ${selfcheckUnrestrictedUser})
|
|
|
|
ok=false
|
|
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ] \
|
|
&& [ "$unrestricted_status" = 200 ]; then
|
|
ok=true
|
|
fi
|
|
credential=absent
|
|
if [ -n "$secret" ]; then
|
|
credential=present
|
|
fi
|
|
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential unrestricted_canary=${cfg.selfcheck.unrestrictedCanary} unrestricted_status=$unrestricted_status"
|
|
[ "$ok" = true ]
|
|
'';
|
|
};
|
|
in
|
|
{
|
|
options.services.hyperhive.squid = {
|
|
allowedDomains = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = hyperhiveCfg.swarm.serviceDomains;
|
|
defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains";
|
|
example = [
|
|
"forge.example.com"
|
|
"github.com"
|
|
];
|
|
description = ''
|
|
Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
|
|
entry matches that exact name. Everything else gets a 403, except
|
|
for an unrestricted user. The self-check canary is allowed on top of
|
|
this list. A name must resolve, on this host's resolver, to a public
|
|
address: one in a loopback, link-local, private, shared (100.64.0.0/10),
|
|
multicast or reserved range (or is `::`), or in the bridge subnet, gets a 403 for
|
|
every user.
|
|
'';
|
|
};
|
|
|
|
usersFile = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "/var/lib/hive-squid/users";
|
|
description = ''
|
|
The `<user>:<hash>` file squid checks proxy credentials against. hive-c0re
|
|
renders it from the credentials in the swarm secret store, one line per
|
|
agent of this hive and one for the self-check, as SHA-512 crypt hashes.
|
|
An empty file, which is what exists until hive-c0re writes one, makes
|
|
every login fail, and every request to a listed name answers 407.
|
|
Owned by `hive-core` and readable by the `squid` group.
|
|
'';
|
|
};
|
|
|
|
unrestrictedFile = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "${builtins.dirOf cfg.usersFile}/unrestricted";
|
|
readOnly = true;
|
|
defaultText = lib.literalMD "`unrestricted`, in the directory of `usersFile`";
|
|
description = ''
|
|
The user names, one per line, that squid lets reach any destination the
|
|
deny rules leave open: the users of the agents that hold the swarm's
|
|
`unrestricted_internet` flag (set unless the swarm records it off), and
|
|
the self-check's unrestricted user. hive-c0re renders it beside
|
|
`usersFile`, and squid is reloaded when it changes. Every other user
|
|
reaches only `allowedDomains`. An empty file, which is what exists
|
|
until hive-c0re writes one, makes every user a restricted one.
|
|
Owned by `hive-core` and readable by the `squid` group.
|
|
'';
|
|
};
|
|
|
|
selfcheck.credentialName = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-squid.selfcheck";
|
|
readOnly = true;
|
|
description = ''
|
|
Name of the systemd credential the self-check imports its squid
|
|
password from. `squid-selfcheck-credential` writes it to
|
|
`/run/credstore/<name>`. Without one the self-check sends no
|
|
credential and reports `ok=false`.
|
|
'';
|
|
};
|
|
|
|
selfcheck.unrestrictedCanary = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "example.com";
|
|
description = ''
|
|
Hostname outside `allowedDomains` that the self-check CONNECTs to on
|
|
port 443, as its unrestricted user, and expects squid to let through.
|
|
Squid resolves it with this host's resolver and connects to it, so a
|
|
host with no route to it reports `ok=false`.
|
|
'';
|
|
};
|
|
|
|
selfcheck.canary = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = hyperhiveCfg.swarm.forge.domain;
|
|
defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain";
|
|
description = ''
|
|
Hostname the self-check CONNECTs to on port 443 and expects squid to
|
|
let through. Squid resolves it with this host's resolver, not the
|
|
hive's: a host whose resolver answers a loopback, bridge or other
|
|
private address for it (a single-host swarm with the gateway's
|
|
`localHostsEntry`, or a split-horizon LAN) gets a 403, and the
|
|
self-check reports `ok=false`.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable {
|
|
# Reachable from agent containers and nowhere else: this opens the port on
|
|
# the bridge interface only.
|
|
services.hyperhive.network.exposeHostPorts = [ port ];
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
|
|
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
|
|
services.hyperhive.otel.journalUnits = [
|
|
"squid.service"
|
|
"squid-selfcheck.service"
|
|
];
|
|
|
|
services.squid = {
|
|
enable = true;
|
|
configText = squidConf;
|
|
};
|
|
|
|
# The directory is setgid so the file hive-c0re renames into it is the
|
|
# squid group's. The users file exists before squid starts, empty: the
|
|
# helper exits on a missing file and squid then rejects every login. The
|
|
# unrestricted list is created the same way, though squid treats a missing
|
|
# one as empty.
|
|
systemd.tmpfiles.rules = [
|
|
"d ${usersDir} 2750 hive-core squid - -"
|
|
"f ${cfg.usersFile} 0640 hive-core squid - -"
|
|
"f ${cfg.unrestrictedFile} 0640 hive-core squid - -"
|
|
];
|
|
|
|
# Squid reads the unrestricted list when it parses its configuration, not
|
|
# when it changes. hive-c0re writes the list only when its contents change,
|
|
# so each change is one reload. Nothing happens to a squid that is not
|
|
# running; it reads the list when it starts.
|
|
systemd.paths.squid-unrestricted-reload = {
|
|
description = "Watch squid's unrestricted-users list";
|
|
wantedBy = [ "paths.target" ];
|
|
pathConfig.PathChanged = cfg.unrestrictedFile;
|
|
};
|
|
|
|
systemd.services.squid-unrestricted-reload = {
|
|
description = "Reload squid to read its unrestricted-users list";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = "${config.systemd.package}/bin/systemctl try-reload-or-restart squid.service";
|
|
};
|
|
};
|
|
|
|
assertions = [
|
|
{
|
|
assertion = !(builtins.elem cfg.selfcheck.unrestrictedCanary allowed);
|
|
message = ''
|
|
services.hyperhive.squid.selfcheck.unrestrictedCanary must be a name outside
|
|
allowedDomains, or the self-check cannot tell an unrestricted user from a
|
|
restricted one.
|
|
'';
|
|
}
|
|
];
|
|
|
|
systemd.services.squid = {
|
|
serviceConfig = sandbox // {
|
|
User = "squid";
|
|
Group = "squid";
|
|
PIDFile = lib.mkForce "/run/squid/squid.pid";
|
|
RuntimeDirectory = "squid";
|
|
CacheDirectory = "squid";
|
|
LogsDirectory = "squid";
|
|
# The bridge address may not be assigned yet at first start.
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
};
|
|
};
|
|
|
|
systemd.services.squid-exporter = {
|
|
description = "Prometheus exporter for squid";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "squid.service" ];
|
|
serviceConfig = sandbox // {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${pkgs.prometheus-squid-exporter}/bin/squid-exporter"
|
|
"-squid-hostname ${netCfg.bridgeIp}"
|
|
"-squid-port ${toString port}"
|
|
"-listen ${exporterListen}"
|
|
];
|
|
DynamicUser = true;
|
|
IPAddressAllow = [
|
|
"127.0.0.0/8"
|
|
netCfg.bridgeIp
|
|
];
|
|
IPAddressDeny = "any";
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
};
|
|
};
|
|
|
|
systemd.services.squid-selfcheck = {
|
|
description = "Squid allow, deny and credential canaries";
|
|
after = [ "squid.service" ];
|
|
serviceConfig = sandbox // {
|
|
Type = "oneshot";
|
|
# A glob that matches nothing is not an error: a host with no
|
|
# credential yet still runs the self-check, and it reports that.
|
|
ImportCredential = cfg.selfcheck.credentialName;
|
|
ExecStart = lib.getExe selfcheck;
|
|
DynamicUser = true;
|
|
IPAddressAllow = netCfg.bridgeIp;
|
|
IPAddressDeny = "any";
|
|
};
|
|
};
|
|
|
|
systemd.timers.squid-selfcheck = {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnBootSec = "1min";
|
|
OnUnitActiveSec = "1min";
|
|
};
|
|
};
|
|
};
|
|
}
|