Watch
0
0
Fork
You've already forked hyperhive
0

egress: the forward chain allows each agent's declared ssh connections

This commit is contained in:
atlas 2026-11-20 03:07:14 +01:00
commit 8ece0b7e0b
4 changed files with 1110 additions and 20 deletions

View file

@ -298,6 +298,14 @@ in
HIVE_SQUID_PROXY = config.services.hyperhive.deploy.hive-controller.squid.listen;
HIVE_EGRESS_NO_PROXY = lib.concatStringsSep "," egressDirectHosts;
}
//
lib.optionalAttrs (config.services.hyperhive.deploy.hive-controller.egress.connectionsFile != null)
{
# Where `workers::egress_connections` renders the SSH connections each agent
# may open, for the egress forward chain.
HIVE_EGRESS_CONNECTIONS_FILE =
config.services.hyperhive.deploy.hive-controller.egress.connectionsFile;
}
// lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) {
# Absent means the system trust store — right for a deployment with a real
# CA, wrong for the self-signed one ../glue-bao-tls.nix mints, which is why

View file

@ -74,6 +74,9 @@ let
}
// lib.optionalAttrs (egressCfg.intercept.unrestrictedFile != null) {
HIVE_EGRESS_UNRESTRICTED = toString egressCfg.intercept.unrestrictedFile;
}
// lib.optionalAttrs (egressCfg.connectionsFile != null) {
HIVE_EGRESS_CONNECTIONS = toString egressCfg.connectionsFile;
};
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
@ -186,6 +189,7 @@ let
chain forward {
type filter hook forward priority filter - 10; policy accept;
${fromBridge} ether saddr @direct_mac accept
${fromBridge} meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept
${fromBridge} counter
${lib.concatStringsSep "\n " (
logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules
@ -309,6 +313,7 @@ let
${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''}
}
set direct_mac { type ether_addr; }
set agent_allow { type ether_addr . ipv4_addr . inet_service; }
${lib.optionalString egressCfg.intercept.enable interceptChain}
${lib.optionalString (egressCfg.forward != "accept") forwardChain}
}
@ -342,7 +347,9 @@ in
What the `inet hive_egress` table does with traffic the host forwards
from the agent bridge to anywhere but the bridge. In `log` and
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
agents and the service containers) is accepted and never logged.
agents and the service containers) is accepted and never logged, and so
is TCP from an agent to an address and port in the `agent_allow` set
(`connectionsFile`).
- `accept`: nothing; the table has no forward chain.
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
@ -359,6 +366,22 @@ in
'';
};
options.services.hyperhive.deploy.hive-controller.egress.connectionsFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The file hive-c0re renders with the SSH connections each agent may open,
one `<agent> <ipv4> <port>` per line: the addresses behind the `host` of
each connection in the permissions record of an agent whose `ssh` switch
is on. `hive-priv egress-pin` turns each line into elements of the
`agent_allow` set of the `inet hive_egress` table, keyed by the MAC of
the agent's port, and the forward chain accepts TCP to an address and
port in it before it logs or refuses. The file is read when a port is
bound, on each pass of the 30 second timer and when the file changes.
`null` leaves the set alone, empty: no agent has an allowance.
'';
};
options.services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkOption {
type = lib.types.enum [
"drop"
@ -664,6 +687,17 @@ in
};
};
# The same pass refills `agent_allow`, so a connection an operator declares
# is allowed at once.
systemd.paths.hive-egress-connections = lib.mkIf (egressCfg.connectionsFile != null) {
description = "Watch the agents' allowed connections list for the egress forward chain";
wantedBy = [ "paths.target" ];
pathConfig = {
PathChanged = toString egressCfg.connectionsFile;
Unit = "hive-egress-pin.service";
};
};
# nspawn names the host end of a container's veth `vb-<machine>` when the
# container joins a bridge.
services.udev.extraRules = ''

View file

@ -43,6 +43,7 @@ let
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
exporterListen = "127.0.0.1:9301";
interceptCfg = hyperhiveCfg.deploy.hive-controller.egress.intercept;
egressCfg = hyperhiveCfg.deploy.hive-controller.egress;
# The path every platform container's unit runs hive-priv by (hive-priv.nix
# points it at the current build). Squid runs `egress-whois` through it.
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
@ -555,6 +556,11 @@ in
enable = lib.mkDefault true;
inherit (cfg) unrestrictedFile agentAllowFile;
};
# The file `hive-c0re` renders with the SSH connections each agent may open,
# which the egress forward chain reads (`workers::egress_connections`). It
# sits in the directory hive-c0re may write.
services.hyperhive.deploy.hive-controller.egress.connectionsFile =
lib.mkDefault "${usersDir}/ssh-connections";
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
@ -591,7 +597,10 @@ in
"C ${cfg.allowedFile} 0640 hive-core squid - ${seedFile}"
"f ${cfg.agentAllowFile} 0640 hive-core squid - -"
"f ${cfg.denyFile} 0640 hive-core squid - -"
];
]
++ lib.optional (
egressCfg.connectionsFile != null
) "f ${egressCfg.connectionsFile} 0640 hive-core squid - -";
# Squid reads the unrestricted list, the allow-list and the per-agent include
# when it parses its configuration, not when they change. hive-c0re writes

View file

@ -1645,6 +1645,57 @@ let
&& !(lib.any (l: lib.hasSuffix " drop" l || lib.hasSuffix " reject" l) rules)
&& lib.hasInfix "chain input" text;
}
{
# An agent's declared SSH connections are accepted before the chain
# counts, logs or refuses, in `log` and in `reject`. The set takes MAC,
# address and port. The host without a controller has the set and no
# chain (the presence control) and no wiring of the connections file.
# hive-c0re, the units that fill the sets and the path unit that runs a
# pass are told the same file.
name = "the egress forward chain accepts the agents' declared ssh connections before it logs or refuses";
ok =
let
rulesetOf = m: m.services.hyperhive.network.egress.ruleset;
inetOf = m: lib.head (lib.splitString "table bridge hive_egress {" (rulesetOf m));
rulesOf =
m:
lib.filter (l: l != "" && l != "}") (
map lib.trim (lib.splitString "\n" (lib.last (lib.splitString "chain forward" (inetOf m))))
);
idxIn = rules: pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
allow = ''iifname "hive-br0" oifname != "hive-br0" meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept'';
set = "set agent_allow { type ether_addr . ipv4_addr . inet_service; }";
inOrder =
m:
let
rules = rulesOf m;
idx = idxIn rules;
in
lib.elem allow rules
&& lib.length (lib.filter (r: r == allow) rules) == 1
&& idx "ether saddr @direct_mac accept" < idx "@agent_allow accept"
&& idx "@agent_allow accept" < idx "counter"
&& idx "@agent_allow accept" < idx "update @log_v4"
&& lib.hasInfix set (inetOf m);
file = "/var/lib/hive-squid/ssh-connections";
envOf = m: name: m.systemd.services.${name}.environment;
in
inOrder bare
&& inOrder egressForwardReject
&& lib.hasInfix set (inetOf egressForwardAccept)
&& !(lib.hasInfix "@agent_allow accept" (rulesetOf egressForwardAccept))
&& bare.services.hyperhive.deploy.hive-controller.egress.connectionsFile == file
&& bare.systemd.services.hive-c0re.environment.HIVE_EGRESS_CONNECTIONS_FILE == file
&& lib.all (name: ((envOf bare name).HIVE_EGRESS_CONNECTIONS or null) == file) [
"hive-egress"
"hive-egress-pin@"
"hive-egress-pin"
]
&& bare.systemd.paths.hive-egress-connections.pathConfig.PathChanged == file
&& bare.systemd.paths.hive-egress-connections.pathConfig.Unit == "hive-egress-pin.service"
&& !((envOf egressForwardAccept "hive-egress") ? HIVE_EGRESS_CONNECTIONS)
&& !(egressForwardAccept.systemd.paths ? hive-egress-connections);
}
{
# The bridge table drops what a port sends from an address that is not
# its own, and logs it. The ruleset is read as text: each `spoof_<reason>`
@ -1807,7 +1858,7 @@ let
chain = lib.last (lib.splitString "chain forward" text);
rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain));
idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
tcp = lib.filter (lib.hasInfix " meta l4proto tcp ") rules;
tcp = lib.filter (l: lib.hasInfix " meta l4proto tcp " l && !(lib.hasInfix "@agent_allow" l)) rules;
other = lib.filter (lib.hasInfix " meta l4proto != tcp") rules;
logged = lib.filter (lib.hasInfix "log prefix") rules;
bareText = bare.services.hyperhive.network.egress.ruleset;
@ -1820,7 +1871,7 @@ let
&& !(lib.hasInfix "wg-hive" text)
&& idx "ether saddr @direct_mac accept" < idx "counter"
&& idx "counter" < idx "reject"
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 1
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 2
&& lib.length tcp == 4
&& lib.length other == 4
&& lib.all (lib.hasSuffix " reject with tcp reset") tcp