egress: the forward chain allows each agent's declared ssh connections
This commit is contained in:
parent
934da6f31f
commit
8ece0b7e0b
4 changed files with 1110 additions and 20 deletions
|
|
@ -298,6 +298,14 @@ in
|
|||
HIVE_SQUID_PROXY = config.services.hyperhive.deploy.hive-controller.squid.listen;
|
||||
HIVE_EGRESS_NO_PROXY = lib.concatStringsSep "," egressDirectHosts;
|
||||
}
|
||||
//
|
||||
lib.optionalAttrs (config.services.hyperhive.deploy.hive-controller.egress.connectionsFile != null)
|
||||
{
|
||||
# Where `workers::egress_connections` renders the SSH connections each agent
|
||||
# may open, for the egress forward chain.
|
||||
HIVE_EGRESS_CONNECTIONS_FILE =
|
||||
config.services.hyperhive.deploy.hive-controller.egress.connectionsFile;
|
||||
}
|
||||
// lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) {
|
||||
# Absent means the system trust store — right for a deployment with a real
|
||||
# CA, wrong for the self-signed one ../glue-bao-tls.nix mints, which is why
|
||||
|
|
|
|||
|
|
@ -74,6 +74,9 @@ let
|
|||
}
|
||||
// lib.optionalAttrs (egressCfg.intercept.unrestrictedFile != null) {
|
||||
HIVE_EGRESS_UNRESTRICTED = toString egressCfg.intercept.unrestrictedFile;
|
||||
}
|
||||
// lib.optionalAttrs (egressCfg.connectionsFile != null) {
|
||||
HIVE_EGRESS_CONNECTIONS = toString egressCfg.connectionsFile;
|
||||
};
|
||||
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
|
||||
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
|
||||
|
|
@ -186,6 +189,7 @@ let
|
|||
chain forward {
|
||||
type filter hook forward priority filter - 10; policy accept;
|
||||
${fromBridge} ether saddr @direct_mac accept
|
||||
${fromBridge} meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept
|
||||
${fromBridge} counter
|
||||
${lib.concatStringsSep "\n " (
|
||||
logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules
|
||||
|
|
@ -309,6 +313,7 @@ let
|
|||
${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''}
|
||||
}
|
||||
set direct_mac { type ether_addr; }
|
||||
set agent_allow { type ether_addr . ipv4_addr . inet_service; }
|
||||
${lib.optionalString egressCfg.intercept.enable interceptChain}
|
||||
${lib.optionalString (egressCfg.forward != "accept") forwardChain}
|
||||
}
|
||||
|
|
@ -342,7 +347,9 @@ in
|
|||
What the `inet hive_egress` table does with traffic the host forwards
|
||||
from the agent bridge to anywhere but the bridge. In `log` and
|
||||
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
|
||||
agents and the service containers) is accepted and never logged.
|
||||
agents and the service containers) is accepted and never logged, and so
|
||||
is TCP from an agent to an address and port in the `agent_allow` set
|
||||
(`connectionsFile`).
|
||||
|
||||
- `accept`: nothing; the table has no forward chain.
|
||||
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
|
||||
|
|
@ -359,6 +366,22 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.hive-controller.egress.connectionsFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
default = null;
|
||||
description = ''
|
||||
The file hive-c0re renders with the SSH connections each agent may open,
|
||||
one `<agent> <ipv4> <port>` per line: the addresses behind the `host` of
|
||||
each connection in the permissions record of an agent whose `ssh` switch
|
||||
is on. `hive-priv egress-pin` turns each line into elements of the
|
||||
`agent_allow` set of the `inet hive_egress` table, keyed by the MAC of
|
||||
the agent's port, and the forward chain accepts TCP to an address and
|
||||
port in it before it logs or refuses. The file is read when a port is
|
||||
bound, on each pass of the 30 second timer and when the file changes.
|
||||
`null` leaves the set alone, empty: no agent has an allowance.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"drop"
|
||||
|
|
@ -664,6 +687,17 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# The same pass refills `agent_allow`, so a connection an operator declares
|
||||
# is allowed at once.
|
||||
systemd.paths.hive-egress-connections = lib.mkIf (egressCfg.connectionsFile != null) {
|
||||
description = "Watch the agents' allowed connections list for the egress forward chain";
|
||||
wantedBy = [ "paths.target" ];
|
||||
pathConfig = {
|
||||
PathChanged = toString egressCfg.connectionsFile;
|
||||
Unit = "hive-egress-pin.service";
|
||||
};
|
||||
};
|
||||
|
||||
# nspawn names the host end of a container's veth `vb-<machine>` when the
|
||||
# container joins a bridge.
|
||||
services.udev.extraRules = ''
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ let
|
|||
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
|
||||
exporterListen = "127.0.0.1:9301";
|
||||
interceptCfg = hyperhiveCfg.deploy.hive-controller.egress.intercept;
|
||||
egressCfg = hyperhiveCfg.deploy.hive-controller.egress;
|
||||
# The path every platform container's unit runs hive-priv by (hive-priv.nix
|
||||
# points it at the current build). Squid runs `egress-whois` through it.
|
||||
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
|
||||
|
|
@ -555,6 +556,11 @@ in
|
|||
enable = lib.mkDefault true;
|
||||
inherit (cfg) unrestrictedFile agentAllowFile;
|
||||
};
|
||||
# The file `hive-c0re` renders with the SSH connections each agent may open,
|
||||
# which the egress forward chain reads (`workers::egress_connections`). It
|
||||
# sits in the directory hive-c0re may write.
|
||||
services.hyperhive.deploy.hive-controller.egress.connectionsFile =
|
||||
lib.mkDefault "${usersDir}/ssh-connections";
|
||||
services.hyperhive.network.enable = lib.mkDefault true;
|
||||
|
||||
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
|
||||
|
|
@ -591,7 +597,10 @@ in
|
|||
"C ${cfg.allowedFile} 0640 hive-core squid - ${seedFile}"
|
||||
"f ${cfg.agentAllowFile} 0640 hive-core squid - -"
|
||||
"f ${cfg.denyFile} 0640 hive-core squid - -"
|
||||
];
|
||||
]
|
||||
++ lib.optional (
|
||||
egressCfg.connectionsFile != null
|
||||
) "f ${egressCfg.connectionsFile} 0640 hive-core squid - -";
|
||||
|
||||
# Squid reads the unrestricted list, the allow-list and the per-agent include
|
||||
# when it parses its configuration, not when they change. hive-c0re writes
|
||||
|
|
|
|||
|
|
@ -1645,6 +1645,57 @@ let
|
|||
&& !(lib.any (l: lib.hasSuffix " drop" l || lib.hasSuffix " reject" l) rules)
|
||||
&& lib.hasInfix "chain input" text;
|
||||
}
|
||||
{
|
||||
# An agent's declared SSH connections are accepted before the chain
|
||||
# counts, logs or refuses, in `log` and in `reject`. The set takes MAC,
|
||||
# address and port. The host without a controller has the set and no
|
||||
# chain (the presence control) and no wiring of the connections file.
|
||||
# hive-c0re, the units that fill the sets and the path unit that runs a
|
||||
# pass are told the same file.
|
||||
name = "the egress forward chain accepts the agents' declared ssh connections before it logs or refuses";
|
||||
ok =
|
||||
let
|
||||
rulesetOf = m: m.services.hyperhive.network.egress.ruleset;
|
||||
inetOf = m: lib.head (lib.splitString "table bridge hive_egress {" (rulesetOf m));
|
||||
rulesOf =
|
||||
m:
|
||||
lib.filter (l: l != "" && l != "}") (
|
||||
map lib.trim (lib.splitString "\n" (lib.last (lib.splitString "chain forward" (inetOf m))))
|
||||
);
|
||||
idxIn = rules: pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
|
||||
allow = ''iifname "hive-br0" oifname != "hive-br0" meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept'';
|
||||
set = "set agent_allow { type ether_addr . ipv4_addr . inet_service; }";
|
||||
inOrder =
|
||||
m:
|
||||
let
|
||||
rules = rulesOf m;
|
||||
idx = idxIn rules;
|
||||
in
|
||||
lib.elem allow rules
|
||||
&& lib.length (lib.filter (r: r == allow) rules) == 1
|
||||
&& idx "ether saddr @direct_mac accept" < idx "@agent_allow accept"
|
||||
&& idx "@agent_allow accept" < idx "counter"
|
||||
&& idx "@agent_allow accept" < idx "update @log_v4"
|
||||
&& lib.hasInfix set (inetOf m);
|
||||
file = "/var/lib/hive-squid/ssh-connections";
|
||||
envOf = m: name: m.systemd.services.${name}.environment;
|
||||
in
|
||||
inOrder bare
|
||||
&& inOrder egressForwardReject
|
||||
&& lib.hasInfix set (inetOf egressForwardAccept)
|
||||
&& !(lib.hasInfix "@agent_allow accept" (rulesetOf egressForwardAccept))
|
||||
&& bare.services.hyperhive.deploy.hive-controller.egress.connectionsFile == file
|
||||
&& bare.systemd.services.hive-c0re.environment.HIVE_EGRESS_CONNECTIONS_FILE == file
|
||||
&& lib.all (name: ((envOf bare name).HIVE_EGRESS_CONNECTIONS or null) == file) [
|
||||
"hive-egress"
|
||||
"hive-egress-pin@"
|
||||
"hive-egress-pin"
|
||||
]
|
||||
&& bare.systemd.paths.hive-egress-connections.pathConfig.PathChanged == file
|
||||
&& bare.systemd.paths.hive-egress-connections.pathConfig.Unit == "hive-egress-pin.service"
|
||||
&& !((envOf egressForwardAccept "hive-egress") ? HIVE_EGRESS_CONNECTIONS)
|
||||
&& !(egressForwardAccept.systemd.paths ? hive-egress-connections);
|
||||
}
|
||||
{
|
||||
# The bridge table drops what a port sends from an address that is not
|
||||
# its own, and logs it. The ruleset is read as text: each `spoof_<reason>`
|
||||
|
|
@ -1807,7 +1858,7 @@ let
|
|||
chain = lib.last (lib.splitString "chain forward" text);
|
||||
rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain));
|
||||
idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
|
||||
tcp = lib.filter (lib.hasInfix " meta l4proto tcp ") rules;
|
||||
tcp = lib.filter (l: lib.hasInfix " meta l4proto tcp " l && !(lib.hasInfix "@agent_allow" l)) rules;
|
||||
other = lib.filter (lib.hasInfix " meta l4proto != tcp") rules;
|
||||
logged = lib.filter (lib.hasInfix "log prefix") rules;
|
||||
bareText = bare.services.hyperhive.network.egress.ruleset;
|
||||
|
|
@ -1820,7 +1871,7 @@ let
|
|||
&& !(lib.hasInfix "wg-hive" text)
|
||||
&& idx "ether saddr @direct_mac accept" < idx "counter"
|
||||
&& idx "counter" < idx "reject"
|
||||
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 1
|
||||
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 2
|
||||
&& lib.length tcp == 4
|
||||
&& lib.length other == 4
|
||||
&& lib.all (lib.hasSuffix " reject with tcp reset") tcp
|
||||
|
|
|
|||
Loading…
Reference in a new issue