deploy squid on every hive host, with exporter, journal forwarding and self-check
This commit is contained in:
atlas
mara
parent
f66b169158
commit
a2cf6fe8bb
4 changed files with 589 additions and 9 deletions
|
|
@ -23,6 +23,7 @@
|
|||
./hive-matrix.nix
|
||||
./hive-network.nix
|
||||
./hive-priv.nix
|
||||
./hive-squid.nix
|
||||
./hive-tls.nix
|
||||
./otel.nix
|
||||
./glue-bao-readers-policy-order.nix
|
||||
|
|
|
|||
|
|
@ -162,6 +162,17 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
networkCidr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
default = "${intToIp networkBase}/${toString cfg.bridgePrefixLength}";
|
||||
defaultText = lib.literalMD "bridge subnet in CIDR form, host bits masked off";
|
||||
description = ''
|
||||
Read-only computed bridge subnet, e.g. `10.42.0.0/24`.
|
||||
'';
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
config = lib.mkMerge [
|
||||
|
|
|
|||
222
nix/host-modules/hive-squid.nix
Normal file
222
nix/host-modules/hive-squid.nix
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
|
||||
# IP, default-deny on the destination hostname. It filters CONNECT targets and
|
||||
# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is
|
||||
# pointed at it yet; the exporter, journal forwarding and the self-check below
|
||||
# are how a deploy shows it works.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
cfg = hyperhiveCfg.squid;
|
||||
netCfg = hyperhiveCfg.network;
|
||||
|
||||
port = 3128;
|
||||
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
|
||||
exporterListen = "127.0.0.1:9301";
|
||||
denyCanary = "deny-canary.invalid";
|
||||
|
||||
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
|
||||
|
||||
# Evaluated top to bottom, first match wins. The three destination denies
|
||||
# sit before the only allow: squid resolves names on the host, so a name
|
||||
# that answers 127.0.0.1, 169.254.169.254 or a bridge address would
|
||||
# otherwise reach services the bridge firewall keeps agents out of.
|
||||
squidConf = ''
|
||||
http_port ${netCfg.bridgeIp}:${toString port}
|
||||
pid_filename /run/squid/squid.pid
|
||||
cache_log stdio:/var/log/squid/cache.log
|
||||
coredump_dir /var/cache/squid
|
||||
|
||||
cache deny all
|
||||
|
||||
# One syslog line per request reaches the journal as squid.service.
|
||||
# Cachemgr scrapes are left out.
|
||||
access_log syslog:daemon.info squid !manager
|
||||
|
||||
acl SSL_ports port 443
|
||||
acl Safe_ports port 80 443
|
||||
|
||||
# Host-originated requests: the self-check and the exporter. A local
|
||||
# connect to the bridge IP takes the bridge IP as its source.
|
||||
acl selfcheck src ${netCfg.bridgeIp}/32
|
||||
|
||||
acl to_bridge dst ${netCfg.networkCidr}
|
||||
# `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record
|
||||
# an agent controls could match an allowed name.
|
||||
acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$)
|
||||
acl ip_v6_host url_regex ^([a-z]+://)?\[[0-9a-fA-F:.]+\](:[0-9]+)?(/|$)
|
||||
${lib.optionalString (
|
||||
allowed != [ ]
|
||||
) "acl allowed_dst dstdomain ${lib.concatStringsSep " " allowed}"}
|
||||
|
||||
# Answered by squid itself, never forwarded, and addressed to the bridge IP
|
||||
# on a port outside Safe_ports, so it precedes the destination rules.
|
||||
http_access allow manager selfcheck
|
||||
http_access deny manager
|
||||
|
||||
http_access deny !Safe_ports
|
||||
http_access deny CONNECT !SSL_ports
|
||||
http_access deny to_localhost
|
||||
http_access deny to_linklocal
|
||||
http_access deny to_bridge
|
||||
http_access deny ip_v4_host
|
||||
http_access deny ip_v6_host
|
||||
${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
|
||||
http_access deny all
|
||||
'';
|
||||
|
||||
sandbox = {
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectClock = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
"AF_UNIX"
|
||||
];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
CapabilityBoundingSet = "";
|
||||
};
|
||||
|
||||
selfcheck = pkgs.writeShellApplication {
|
||||
name = "squid-selfcheck";
|
||||
runtimeInputs = [ pkgs.curl ];
|
||||
text = ''
|
||||
# The status squid answers a CONNECT with; 000 when the proxy is down.
|
||||
connect_status() {
|
||||
curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
|
||||
-o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
|
||||
}
|
||||
|
||||
allow_status=$(connect_status ${cfg.selfcheck.canary})
|
||||
deny_status=$(connect_status ${denyCanary})
|
||||
|
||||
ok=false
|
||||
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then
|
||||
ok=true
|
||||
fi
|
||||
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status"
|
||||
[ "$ok" = true ]
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.squid = {
|
||||
allowedDomains = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = hyperhiveCfg.swarm.serviceDomains;
|
||||
defaultText = lib.literalExpression "config.services.hyperhive.swarm.serviceDomains";
|
||||
example = [
|
||||
"forge.example.com"
|
||||
"github.com"
|
||||
];
|
||||
description = ''
|
||||
Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each
|
||||
entry matches that exact name. Everything else gets a 403. The
|
||||
self-check canary is allowed on top of this list.
|
||||
'';
|
||||
};
|
||||
|
||||
selfcheck.canary = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = hyperhiveCfg.swarm.forge.domain;
|
||||
defaultText = lib.literalExpression "config.services.hyperhive.swarm.forge.domain";
|
||||
description = ''
|
||||
Hostname the self-check CONNECTs to on port 443 and expects squid to
|
||||
let through. Squid resolves it with this host's resolver, not the
|
||||
hive's: a host whose resolver answers a loopback or bridge address
|
||||
for it (a single-host swarm with the gateway's `localHostsEntry`) gets
|
||||
a 403, and the self-check reports `ok=false`.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable {
|
||||
# Reachable from agent containers and nowhere else: this opens the port on
|
||||
# the bridge interface only.
|
||||
services.hyperhive.network.exposeHostPorts = [ port ];
|
||||
services.hyperhive.network.enable = lib.mkDefault true;
|
||||
|
||||
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
|
||||
services.hyperhive.otel.journalUnits = [
|
||||
"squid.service"
|
||||
"squid-selfcheck.service"
|
||||
];
|
||||
|
||||
services.squid = {
|
||||
enable = true;
|
||||
configText = squidConf;
|
||||
};
|
||||
|
||||
systemd.services.squid = {
|
||||
serviceConfig = sandbox // {
|
||||
User = "squid";
|
||||
Group = "squid";
|
||||
PIDFile = lib.mkForce "/run/squid/squid.pid";
|
||||
RuntimeDirectory = "squid";
|
||||
CacheDirectory = "squid";
|
||||
LogsDirectory = "squid";
|
||||
# The bridge address may not be assigned yet at first start.
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.squid-exporter = {
|
||||
description = "Prometheus exporter for squid";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "squid.service" ];
|
||||
serviceConfig = sandbox // {
|
||||
ExecStart = lib.concatStringsSep " " [
|
||||
"${pkgs.prometheus-squid-exporter}/bin/squid-exporter"
|
||||
"-squid-hostname ${netCfg.bridgeIp}"
|
||||
"-squid-port ${toString port}"
|
||||
"-listen ${exporterListen}"
|
||||
];
|
||||
DynamicUser = true;
|
||||
IPAddressAllow = [
|
||||
"127.0.0.0/8"
|
||||
netCfg.bridgeIp
|
||||
];
|
||||
IPAddressDeny = "any";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.squid-selfcheck = {
|
||||
description = "Squid allow and deny canaries";
|
||||
after = [ "squid.service" ];
|
||||
serviceConfig = sandbox // {
|
||||
Type = "oneshot";
|
||||
ExecStart = lib.getExe selfcheck;
|
||||
DynamicUser = true;
|
||||
IPAddressAllow = netCfg.bridgeIp;
|
||||
IPAddressDeny = "any";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.squid-selfcheck = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "1min";
|
||||
OnUnitActiveSec = "1min";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -187,6 +187,21 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
journalUnits = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "squid.service" ];
|
||||
description = ''
|
||||
Host systemd units whose journal this hive's collector forwards to
|
||||
the swarm's log store, by full unit name.
|
||||
|
||||
**A service adds its own unit, from its own module, under its own
|
||||
`enable`**, like `scrapeTargets`. The list is an allow-list on
|
||||
purpose: a hive host can be someone's workstation, so a unit that no
|
||||
module named never leaves the machine.
|
||||
'';
|
||||
};
|
||||
|
||||
clientSecretFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default =
|
||||
|
|
@ -326,6 +341,9 @@ in
|
|||
# not for agent containers.
|
||||
services.hyperhive.otel.scrapeTargets.collector = "127.0.0.1:${toString otel.telemetryPort}";
|
||||
|
||||
# The unit whose crash-watch warning a swarm-level alert reads.
|
||||
services.hyperhive.otel.journalUnits = [ "hive-c0re.service" ];
|
||||
|
||||
services.opentelemetry-collector = {
|
||||
enable = true;
|
||||
# Contrib, matching the swarm tier (./swarm-otel.nix). The upstream
|
||||
|
|
@ -449,11 +467,11 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# This tier's own host journal, scoped to the one unit whose
|
||||
# crash-watch warning (`"container crash detected"`,
|
||||
# This tier's own host journal, scoped to `journalUnits`: hive-c0re,
|
||||
# whose crash-watch warning (`"container crash detected"`,
|
||||
# `crash_watch.rs`) a swarm-level grafana alert
|
||||
# (`hh-container-crashed` in `swarm-grafana.nix`) depends on
|
||||
# reaching VictoriaLogs. Unlike the swarm tier's journald
|
||||
# reaching VictoriaLogs, plus whatever other modules name. Unlike the swarm tier's journald
|
||||
# receiver (`swarm-otel.nix`) this needs no bind mount: this
|
||||
# collector is a HOST systemd unit, not a container, so
|
||||
# `/var/log/journal` is the same filesystem hive-c0re itself
|
||||
|
|
@ -467,7 +485,7 @@ in
|
|||
journald = {
|
||||
directory = "/var/log/journal";
|
||||
storage = "file_storage";
|
||||
units = [ "hive-c0re.service" ];
|
||||
units = otel.journalUnits;
|
||||
# Severity table shared with the agent and swarm tiers'
|
||||
# journald receivers — see ../journald-severity.nix for why
|
||||
# it is its own file.
|
||||
|
|
|
|||
Loading…
Reference in a new issue