From 892675628d0c2c919da00625be85d98e68f80ecb Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 9 Nov 2026 17:32:03 +0100 Subject: [PATCH] nats and bao can listen on a routable address, opt-in --- docs/2-handbook/swarm/README.md | 14 ++++++++ nix/host-modules/swarm-bao.nix | 47 +++++++++++++++++++++++++ nix/host-modules/swarm-nats-service.nix | 5 +-- nix/host-modules/swarm-nats.nix | 34 +++++++++++++++--- nix/module-eval/bao-grants.nix | 33 +++++++++++++++++ nix/module-eval/bao-otel-collector.nix | 27 ++++++++++++++ nix/module-eval/nats-tls.nix | 20 +++++++++++ 7 files changed, 174 insertions(+), 6 deletions(-) diff --git a/docs/2-handbook/swarm/README.md b/docs/2-handbook/swarm/README.md index a0897de44..fffbee3ab 100644 --- a/docs/2-handbook/swarm/README.md +++ b/docs/2-handbook/swarm/README.md @@ -602,6 +602,20 @@ mechanism. entry's `wireguardPublicKey`/`wireguardEndpoint`/`wireguardAddress` to configure `wg-hive`. See [below](#wireguard-inter-hive-mesh-optional). +### Reaching the queue and the store on a routable address + +The mesh isn't the only way to reach the queue and the secret store from +another host. Set `deploy.nats.openFirewall` on the queue's host to open the +queue's port on every interface, and `deploy.bao.openFirewall` together with +`deploy.bao.extraListenAddresses` on the store's host to open the store's port +on the addresses it listens on. Both default to off. Each exposes a service +beyond the tunnel, which is the operator's decision. The queue requires TLS and +its `auth_callout` refuses every client it can't identify. The store requires a +client certificate on every listener, and evaluation fails when you +set `deploy.bao.openFirewall` without `deploy.bao.clientCaFile`. Point +`nats.` and `bao.` in your upstream DNS at the +address the other hives reach. + ### WireGuard inter-hive mesh (optional) The peer config above uses public HTTPS for all inter-hive traffic. diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index d11e9c029..0f184d285 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -2891,6 +2891,30 @@ in misconfiguration rather than a topology. ''; }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Open {option}`services.hyperhive.swarm.bao.port` in the host firewall + on every interface, so hives on other hosts can reach the addresses in + {option}`services.hyperhive.deploy.bao.extraListenAddresses` over the + LAN or the internet. Off by default: the port is otherwise open only on + the agent bridge. + + ⚠️ This exposes the secret store beyond the tunnel. Every listener + requires a client certificate signed by + {option}`services.hyperhive.deploy.bao.clientCaFile`, and nothing + answers without one, which is why setting this without that file, or + without any extra address, fails evaluation. Whether the store may be + reachable at all from outside a private network is the operator's + decision. + + Point `bao.` in the operator's upstream DNS at the + address the other hives reach. + ''; + }; }; # ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than @@ -2936,6 +2960,24 @@ in deploy.bao.otel.telemetryPort. Pick free ports. ''; } + { + assertion = !baoDeploy.openFirewall || baoDeploy.clientCaFile != null; + message = '' + services.hyperhive.deploy.bao.openFirewall opens the store's port to + every interface, but services.hyperhive.deploy.bao.clientCaFile is + null, so its listeners would not require a client certificate. Set + clientCaFile, or leave openFirewall off. + ''; + } + { + assertion = !baoDeploy.openFirewall || baoDeploy.extraListenAddresses != [ ]; + message = '' + services.hyperhive.deploy.bao.openFirewall is set, but + services.hyperhive.deploy.bao.extraListenAddresses is empty: the + store listens on loopback only, so the open port reaches nothing. + Name the address other hives reach, or leave openFirewall off. + ''; + } { assertion = cfg.ui.domain != cfg.domain; message = '' @@ -3090,6 +3132,11 @@ in # certificate its CA signed. services.hyperhive.network.exposeHostPorts = [ cfg.port ]; + # Every listener, bridge-IP passthrough included, requires the client + # certificate (see `listenerTls`), so opening the port host-wide adds + # reach and no unauthenticated path. + networking.firewall.allowedTCPPorts = lib.mkIf baoDeploy.openFirewall [ cfg.port ]; + # The operators' door to the UI, shaped like ./swarm-victorialogs.nix's # vhost. It proxies to the container's nginx over loopback (shared # netns), so no port opens anywhere. `auth_request` only proves a diff --git a/nix/host-modules/swarm-nats-service.nix b/nix/host-modules/swarm-nats-service.nix index 614b76e85..17541dcc5 100644 --- a/nix/host-modules/swarm-nats-service.nix +++ b/nix/host-modules/swarm-nats-service.nix @@ -57,8 +57,9 @@ in 3000, every agent in 8100..8999 via FNV-1a hash). Opened on the bridge interface, for agent containers, and on - `wg-hive` when this host is on the mesh, for other hives. Never - host-wide. TLS only: a client that does not speak it is refused. + `wg-hive` when this host is on the mesh, for other hives. Host-wide + only when {option}`services.hyperhive.deploy.nats.openFirewall` is + set. TLS only: a client that does not speak it is refused. Unlike {option}`services.hyperhive.deploy.nats.monitorPort` and {option}`services.hyperhive.deploy.nats.metricsPort`, the diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 3947b4d45..89510cd3d 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -505,6 +505,28 @@ in A path, never a value. ''; }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Open {option}`services.hyperhive.swarm.nats.port` in the host firewall + on every interface, so hives on other hosts can reach the queue on + this host's LAN or public address. Off by default: the port is + otherwise open only on the agent bridge and, when this host is on the + mesh, on `wg-hive`. + + ⚠️ This exposes the queue beyond the tunnel. The server binds every + address, so the firewall is the only thing this option changes. What + protects the port is the queue itself: TLS is required, and + `auth_callout` refuses every client it cannot identify. Whether that + is enough to expose the queue is the operator's decision. + + Point `nats.` in the operator's upstream DNS at the + address the other hives reach. + ''; + }; }; config = lib.mkIf deployCfg.nats.enable { @@ -681,14 +703,18 @@ in # by the firewall and looks exactly like every other NATS failure, # a timeout. # - # ⚠️ Never the world. What makes it safe to open at all is that the - # queue is fail-closed: `auth_callout` admits nobody until the responder - # above answers for them, so an agent that reaches this port still has to - # present a token authelia vouches for. + # ⚠️ The bridge only, unless `openFirewall` opts in. What makes it safe to + # open at all is that the queue is fail-closed: `auth_callout` admits + # nobody until the responder above answers for them, so a client that + # reaches this port still has to present a token authelia vouches for. # # An agent reaches it by name, which dnsmasq answers with the bridge IP. services.hyperhive.network.exposeHostPorts = [ cfg.port ]; + # The server binds every address (the container shares the host netns and + # sets no listen host), so reachability is the firewall alone. + networking.firewall.allowedTCPPorts = lib.mkIf deployCfg.nats.openFirewall [ cfg.port ]; + # The bind source has to exist before the container starts, leaf or not: # nixos-container refuses to start over a missing one, which would take # the whole queue down rather than only its TLS. diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 76844985e..c1b7b2957 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -76,6 +76,9 @@ let deploy.bao.enable = true; deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; swarm.hives.h2.domain = "h2.example.org"; + # The firewall opened with no address to listen on: the combination + # `deploy.bao.openFirewall` refuses. + deploy.bao.openFirewall = true; }; # The store and the token, with no CA to trust. `mkForce` because the PKI @@ -86,6 +89,10 @@ let deploy.bao.enable = true; deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; deploy.bao.clientCaFile = lib.mkForce null; + # An off-host listener opened in the firewall with no client CA: the + # combination `deploy.bao.openFirewall` refuses. + deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; + deploy.bao.openFirewall = true; }; # The store plus every reader that used to log in as the hive. One fixture # rather than one per reader: the claim they are *separate* principals is @@ -1695,6 +1702,32 @@ let lib.any (a: !a.assertion && names a) baoGranterOddHiveLeafRole.assertions && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions); } + { + # Control: `baoGrantHere` leaves the option off and fails no assertion + # naming it. + name = "openFirewall without a client CA is refused, naming both options"; + ok = + let + names = + a: + lib.hasInfix "services.hyperhive.deploy.bao.openFirewall" a.message + && lib.hasInfix "services.hyperhive.deploy.bao.clientCaFile" a.message; + in + lib.any (a: !a.assertion && names a) baoGrantNoClientCa.assertions + && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions); + } + { + name = "openFirewall without an extra listen address is refused, naming both options"; + ok = + let + names = + a: + lib.hasInfix "services.hyperhive.deploy.bao.openFirewall" a.message + && lib.hasInfix "services.hyperhive.deploy.bao.extraListenAddresses" a.message; + in + lib.any (a: !a.assertion && names a) baoGrantTwoHives.assertions + && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions); + } { # The granter writes pki roles through `roles/swarm-*` only, so a role # named otherwise is refused at eval rather than 403'd at deploy. diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index ddae66801..14ce3ac67 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -21,6 +21,7 @@ let hive runGroup baoSettings + bridgePorts forceServiceConfigs selfScrapes ; @@ -28,6 +29,7 @@ let baoTwoAddresses = hive { deploy.bao.enable = true; deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; + deploy.bao.openFirewall = true; # Pinned, not incidental: the case counting these listeners is about the # declared API addresses. ⚠️ Leaving the swarm tier off does not keep the # METRICS listener out of the count — that one is a function of the @@ -106,6 +108,31 @@ let in l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200"; } + { + # On: the port is open host-wide and the extra listener demands a client + # certificate, like loopback. Evaluates with no assertion naming the + # option, so the two refusals in bao-grants are not firing here. + name = "openFirewall opens the store port host-wide, and the extra listener requires a client certificate"; + ok = + let + l = (baoSettings baoTwoAddresses).listener; + in + lib.elem 8200 baoTwoAddresses.networking.firewall.allowedTCPPorts + && l.extra-1.tls_require_and_verify_client_cert == true + && l.extra-1.tls_client_ca_file == l.loopback.tls_client_ca_file + && !(lib.any ( + a: !a.assertion && lib.hasInfix "deploy.bao.openFirewall" a.message + ) baoTwoAddresses.assertions); + } + { + # Off is today's config: the port is open on the bridge for agents and on + # no other interface. The bridge arm is the control that the lookup finds + # a port that is there. + name = "without openFirewall the store port is open on the bridge only"; + ok = + !(lib.elem 8200 baoOtelElsewhere.networking.firewall.allowedTCPPorts) + && lib.elem 8200 (bridgePorts baoOtelElsewhere); + } { # Control for the case above: these settings are rendered per deployment, # not constants a passing case could be indifferent to. The metrics and diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index d38240a47..ec3985ae8 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -42,6 +42,7 @@ let # The queue on a host whose store is elsewhere: no local policy unit. queueNoStore = hive { deploy.nats.enable = true; + deploy.nats.openFirewall = true; }; inherit (shared) natsOnStoreHostReaderMoved; @@ -256,6 +257,12 @@ let name = "the server requires TLS: no allow_non_tls"; ok = !(natsContainer.services.nats.settings ? allow_non_tls); } + { + # Reachability is the firewall alone because no listen address is set. + name = "the server sets no listen address, so it binds every address"; + ok = + !(natsContainer.services.nats.settings ? host) && !(natsContainer.services.nats.settings ? listen); + } { name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide"; ok = @@ -263,6 +270,19 @@ let && !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts) && lib.elem 4222 (bridgePorts allLocalMesh); } + { + # Off is today's config; the bridge arm is the control that the lookup + # finds a port that is there. On adds the host-wide port and keeps the + # bridge one. + name = "4222 is open host-wide only with openFirewall, and the bridge keeps it either way"; + ok = + queueNoStore.services.hyperhive.deploy.nats.openFirewall + && lib.elem 4222 queueNoStore.networking.firewall.allowedTCPPorts + && lib.elem 4222 (bridgePorts queueNoStore) + && !allLocal.services.hyperhive.deploy.nats.openFirewall + && !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts) + && lib.elem 4222 (bridgePorts allLocal); + } { name = "4222 is not opened on wg-hive when this host is not on the mesh"; ok =