diff --git a/docs/2-handbook/external-accounts/ssh.md b/docs/2-handbook/external-accounts/ssh.md index 805a1db7c..ad2581f40 100644 --- a/docs/2-handbook/external-accounts/ssh.md +++ b/docs/2-handbook/external-accounts/ssh.md @@ -61,10 +61,10 @@ key changed and you didn't change the machine, don't clear it: something else answers at that address. The SSH switch and the CA don't open the network. The default `egress.forward` -of `log` lets the agent's direct connection to the machine's port through. If -you set it to `reject`, the hive refuses the connection unless you gave the -agent a named connection to that host and port (see -[Named connections](#named-connections)). See +is `reject`, so the hive refuses the agent's direct connection to the machine's +port unless you gave the agent a named connection to that host and port (see +[Named connections](#named-connections)). If you set `egress.forward` to `log`, +the connection goes through. See [the network page](../../4-implementation/networking/network.md). ## 2. Make a machine trust the CA diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 4c477ef2a..c329d2389 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -233,9 +233,10 @@ when it speaks HTTP on port 80 or TLS on port 443 and its agent is `restricted` or in `log` mode: the host redirects that traffic into squid's intercept ports (see _Intercepted web traffic_ below). Everything else goes straight out through the NAT, as before: the `unrestricted` agents, the service containers and every -other port and protocol. By default nothing blocks that. The host logs the direct -connections of the `restricted` and `log` agents, and `egress.forward = -"reject"` refuses them (see _Agent bridge egress log_ below). +other port and protocol. A hive controller host refuses the direct connections +of the `restricted` and `log` agents (`egress.forward = "reject"`, the default +while the intercept is on); `egress.forward = "log"` only logs them (see _Agent +bridge egress log_ below). Squid filters on the destination hostname: a `CONNECT` target or a plain-HTTP host. It never decrypts TLS. The one TLS key it loads is a throwaway that the @@ -727,7 +728,8 @@ loopback — the hive-c0re admin socket is a UDS, not TCP. `services.hyperhive.deploy.hive-controller.egress.forward` sets what the `forward` chain of the `inet hive_egress` table does with traffic the host routes from the agent bridge to anywhere but the bridge. It takes -`accept`, `log` (the default) or `reject`. In `log` mode the chain blocks +`accept`, `log` or `reject`; the default is `reject` where the intercept is on +(see _The default_ below) and `log` elsewhere. In `log` mode the chain blocks nothing. It logs each new flow and counts every packet. In `reject` mode it also refuses the flow. @@ -756,12 +758,15 @@ also refuses the flow. (and `hive-egress-reject-unmetered `); the rate limit and the counter work as in `log` mode. Still open to every agent are the host's own addresses: the proxy ports, DNS and DHCP from the bridge address, and the gateway. -- **Switching on.** Leave `log` running for about a week, then read the - `hive-egress-would-drop` lines. Each is a flow that `reject` would refuse; a - flow that belongs to a legitimate use needs a fix first (a service container - missing from `direct_mac`, an agent that should be `unrestricted`). Then set - `services.hyperhive.deploy.hive-controller.egress.forward = "reject"`. Making it the default - is a one-line change of the option's default. +- **The default.** `reject` while `egress.intercept.enable` is on, which + `hive-squid.nix` turns on for a hive controller host, and `log` on a host + without the intercept, because `reject` there would refuse all web traffic + too. The soak ran with `log` first: the `hive-egress-would-drop` lines are + each a flow that `reject` would refuse, and a flow that belongs to a + legitimate use needed a fix before the switch (a service container missing + from `direct_mac`, an agent that should be `unrestricted`). To go back to + logging, set + `services.hyperhive.deploy.hive-controller.egress.forward = "log"`. - **What `reject` can break.** - A service container with no pin yet isn't in `direct_mac`, so the chain refuses it until `hive-egress-pin` binds its port. The bridge table diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index bcda6af19..bfb980233 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -344,10 +344,16 @@ in "log" "reject" ]; - default = "log"; + default = if egressCfg.intercept.enable then "reject" else "log"; + defaultText = lib.literalMD '' + `"reject"` while `egress.intercept.enable` is on, `"log"` otherwise + ''; description = '' What the `inet hive_egress` table does with traffic the host forwards - from the agent bridge to anywhere but the bridge. In `log` and + from the agent bridge to anywhere but the bridge. The default is `reject` + on a host with the intercept on, which a hive controller host has, and + `log` on a host without it, because `reject` without the intercept would + refuse every agent's web traffic. In `log` and `reject` the traffic of the MACs in the `direct_mac` set (the unrestricted agents and the service containers) is accepted and never logged, and so is TCP from an agent to an address and port in the `agent_allow` set diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 1f240a92b..cfefc4273 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -114,6 +114,14 @@ let network.enable = true; deploy.hive-controller.egress.forward = "reject"; }; + # The default of `egress.forward`: the controller host has the intercept on, + # the host with the network and no controller does not, and a controller host + # can still ask for `log`. + egressForwardDefault = hiveWithController { }; + egressForwardDefaultNoIntercept = hive { network.enable = true; }; + egressForwardLogged = hiveWithController { + deploy.hive-controller.egress.forward = "log"; + }; # Every hive defaults the agents' queue address to the queue's name, so the # only hive without one is a hive told to have none. Also the host still @@ -1628,15 +1636,15 @@ let && !(centralToggleOff.systemd.services ? hive-egress); } { - # The forward chain is in the ordinary host's table and logs the bridge's - # forwarded flows without a verdict that blocks. The ruleset is read as - # text; the last rule of the chain is the accept that keeps the log from - # changing any outcome. - name = "the default egress table logs the bridge's forwarded flows and blocks none"; + # The forward chain of a controller host that asks for `log` is in its + # table and logs the bridge's forwarded flows without a verdict that + # blocks. The ruleset is read as text; the last rule of the chain is the + # accept that keeps the log from changing any outcome. + name = "the log-mode egress table logs the bridge's forwarded flows and blocks none"; ok = let text = lib.head ( - lib.splitString "table bridge hive_egress {" bare.services.hyperhive.network.egress.ruleset + lib.splitString "table bridge hive_egress {" egressForwardLogged.services.hyperhive.network.egress.ruleset ); chain = lib.last (lib.splitString "chain forward" text); rules = lib.filter (l: l != "") (map lib.trim (lib.splitString "\n" chain)); @@ -1700,7 +1708,7 @@ let file = "/var/lib/hive-squid/ssh-connections"; envOf = m: name: m.systemd.services.${name}.environment; in - inOrder bare + inOrder egressForwardLogged && inOrder egressForwardReject && lib.hasInfix set (inetOf egressForwardAccept) && !(lib.hasInfix "@agent_allow accept" (rulesetOf egressForwardAccept)) @@ -1866,9 +1874,9 @@ let # `reject` logs and refuses in one rule, as a TCP rule (reset) and a # non-TCP rule (ICMP admin-prohibited) for each of the four log/verdict # positions, after the exempt MACs. The ruleset is read as - # text. The `accept` count (the exempt MACs only) and the - # `bare` host, whose chain has no refusal, are the controls that the - # lookups find what is there. + # text. The `accept` count (the exempt MACs only) and the controller + # host that asks for `log`, whose chain has no refusal, are the controls + # that the lookups find what is there. name = "egress.forward reject refuses the bridge's forwarded flows after the exempt MACs"; ok = let @@ -1881,7 +1889,7 @@ let tcp = lib.filter (l: lib.hasInfix " meta l4proto tcp " l && !(lib.hasInfix "@agent_allow" l)) rules; other = lib.filter (lib.hasInfix " meta l4proto != tcp") rules; logged = lib.filter (lib.hasInfix "log prefix") rules; - bareText = bare.services.hyperhive.network.egress.ruleset; + loggedText = egressForwardLogged.services.hyperhive.network.egress.ruleset; in !(lib.any ( a: !a.assertion && lib.hasInfix "deploy.hive-controller.egress" a.message @@ -1905,14 +1913,14 @@ let && lib.last rules == ''iifname "hive-br0" oifname != "hive-br0" meta l4proto != tcp reject with icmpx type admin-prohibited'' - && !(lib.hasInfix "reject with" bareText) - && lib.hasInfix "would-drop" bareText; + && !(lib.hasInfix "reject with" loggedText) + && lib.hasInfix "would-drop" loggedText; } { # `reject` without the intercept is refused by an assertion that names # the option and both ways out; the controller host with `reject` (the - # intercept on) and `bare` (`log`) are the controls that the lookup - # finds nothing for them. + # intercept on) and the host without the intercept on its default + # (`log`) are the controls that the lookup finds nothing for them. name = "egress.forward reject without the intercept is refused"; ok = let @@ -1928,7 +1936,36 @@ let && lib.hasInfix "intercept.enable = true" (lib.head (refused egressRejectNoIntercept)).message && lib.hasInfix "keep forward = \"log\"" (lib.head (refused egressRejectNoIntercept)).message && refused egressForwardReject == [ ] - && refused bare == [ ]; + && refused egressForwardDefaultNoIntercept == [ ]; + } + { + # The default of `egress.forward` follows the intercept: `reject` on the + # controller host, whose ruleset then refuses and carries no would-drop + # line, and `log` on the host without the intercept, which a `reject` + # default would fail to evaluate. A controller host can still ask for + # `log`, and its ruleset then only logs. + name = "egress.forward defaults to reject with the intercept on and to log without it"; + ok = + let + forwardOf = m: m.services.hyperhive.deploy.hive-controller.egress.forward; + interceptOf = m: m.services.hyperhive.deploy.hive-controller.egress.intercept.enable; + rulesetOf = m: m.services.hyperhive.network.egress.ruleset; + unmet = + m: + lib.filter ( + a: !a.assertion && lib.hasInfix "hive-controller.egress.forward" a.message + ) m.assertions; + in + forwardOf egressForwardDefault == "reject" + && interceptOf egressForwardDefault + && lib.hasInfix "hive-egress-reject" (rulesetOf egressForwardDefault) + && !(lib.hasInfix "hive-egress-would-drop" (rulesetOf egressForwardDefault)) + && forwardOf egressForwardDefaultNoIntercept == "log" + && !(interceptOf egressForwardDefaultNoIntercept) + && unmet egressForwardDefaultNoIntercept == [ ] + && forwardOf egressForwardLogged == "log" + && lib.hasInfix "hive-egress-would-drop" (rulesetOf egressForwardLogged) + && !(lib.hasInfix "hive-egress-reject" (rulesetOf egressForwardLogged)); } { # The redirect of the agents' web traffic: IPv4 TCP 80 and 443 from the diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 864f0e1d7..b6356fc6d 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -228,9 +228,12 @@ pkgs.testers.runNixOSTest { pkgs.mkpasswd pkgs.nftables ]; - # The reject-mode half of the test switches to this at its end. The - # forward mode is fixed when the node evaluates; the switch reloads - # `hive-egress` with the other ruleset. + # The log-mode half of the test runs on this node, and the reject-mode + # half switches to the specialisation at its end. The forward mode is + # fixed when the node evaluates; the switch reloads `hive-egress` with + # the other ruleset. The default is `reject` while the intercept is on, + # so the base pins `log`, at a priority the specialisation overrides. + services.hyperhive.deploy.hive-controller.egress.forward = lib.mkDefault "log"; specialisation.reject.configuration.services.hyperhive.deploy.hive-controller.egress.forward = "reject"; services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local";