nats and bao can listen on a routable address, opt-in

docs/2-handbook/swarm/README.md

@@ -602,6 +602,20 @@ mechanism.
    entry's `wireguardPublicKey`/`wireguardEndpoint`/`wireguardAddress`
    to configure `wg-hive`. See [below](#wireguard-inter-hive-mesh-optional).
 
+### Reaching the queue and the store on a routable address
+
+The mesh isn't the only way to reach the queue and the secret store from
+another host. Set `deploy.nats.openFirewall` on the queue's host to open the
+queue's port on every interface, and `deploy.bao.openFirewall` together with
+`deploy.bao.extraListenAddresses` on the store's host to open the store's port
+on the addresses it listens on. Both default to off. Each exposes a service
+beyond the tunnel, which is the operator's decision. The queue requires TLS and
+its `auth_callout` refuses every client it can't identify. The store requires a
+client certificate on every listener, and evaluation fails when you
+set `deploy.bao.openFirewall` without `deploy.bao.clientCaFile`. Point
+`nats.<swarm.domain>` and `bao.<swarm.domain>` in your upstream DNS at the
+address the other hives reach.
+
 ### WireGuard inter-hive mesh (optional)
 
 The peer config above uses public HTTPS for all inter-hive traffic.

nix/host-modules/swarm-bao.nix

@@ -2891,6 +2891,30 @@ in
         misconfiguration rather than a topology.
       '';
     };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = ''
+        Open {option}`services.hyperhive.swarm.bao.port` in the host firewall
+        on every interface, so hives on other hosts can reach the addresses in
+        {option}`services.hyperhive.deploy.bao.extraListenAddresses` over the
+        LAN or the internet. Off by default: the port is otherwise open only on
+        the agent bridge.
+
+        ⚠️ This exposes the secret store beyond the tunnel. Every listener
+        requires a client certificate signed by
+        {option}`services.hyperhive.deploy.bao.clientCaFile`, and nothing
+        answers without one, which is why setting this without that file, or
+        without any extra address, fails evaluation. Whether the store may be
+        reachable at all from outside a private network is the operator's
+        decision.
+
+        Point `bao.<swarm domain>` in the operator's upstream DNS at the
+        address the other hives reach.
+      '';
+    };
   };
 
   # ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
@@ -2936,6 +2960,24 @@ in
             deploy.bao.otel.telemetryPort. Pick free ports.
           '';
         }
+        {
+          assertion = !baoDeploy.openFirewall || baoDeploy.clientCaFile != null;
+          message = ''
+            services.hyperhive.deploy.bao.openFirewall opens the store's port to
+            every interface, but services.hyperhive.deploy.bao.clientCaFile is
+            null, so its listeners would not require a client certificate. Set
+            clientCaFile, or leave openFirewall off.
+          '';
+        }
+        {
+          assertion = !baoDeploy.openFirewall || baoDeploy.extraListenAddresses != [ ];
+          message = ''
+            services.hyperhive.deploy.bao.openFirewall is set, but
+            services.hyperhive.deploy.bao.extraListenAddresses is empty: the
+            store listens on loopback only, so the open port reaches nothing.
+            Name the address other hives reach, or leave openFirewall off.
+          '';
+        }
         {
           assertion = cfg.ui.domain != cfg.domain;
           message = ''
@@ -3090,6 +3132,11 @@ in
       # certificate its CA signed.
       services.hyperhive.network.exposeHostPorts = [ cfg.port ];
 
+      # Every listener, bridge-IP passthrough included, requires the client
+      # certificate (see `listenerTls`), so opening the port host-wide adds
+      # reach and no unauthenticated path.
+      networking.firewall.allowedTCPPorts = lib.mkIf baoDeploy.openFirewall [ cfg.port ];
+
       # The operators' door to the UI, shaped like ./swarm-victorialogs.nix's
       # vhost. It proxies to the container's nginx over loopback (shared
       # netns), so no port opens anywhere. `auth_request` only proves a

nix/host-modules/swarm-nats-service.nix

@@ -57,8 +57,9 @@ in
         3000, every agent in 8100..8999 via FNV-1a hash).
 
         Opened on the bridge interface, for agent containers, and on
-        `wg-hive` when this host is on the mesh, for other hives. Never
+        `wg-hive` when this host is on the mesh, for other hives. Host-wide
-        host-wide. TLS only: a client that does not speak it is refused.
+        only when {option}`services.hyperhive.deploy.nats.openFirewall` is
+        set. TLS only: a client that does not speak it is refused.
 
         Unlike {option}`services.hyperhive.deploy.nats.monitorPort` and
         {option}`services.hyperhive.deploy.nats.metricsPort`, the

nix/host-modules/swarm-nats.nix

@@ -505,6 +505,28 @@ in
         A path, never a value.
       '';
     };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = ''
+        Open {option}`services.hyperhive.swarm.nats.port` in the host firewall
+        on every interface, so hives on other hosts can reach the queue on
+        this host's LAN or public address. Off by default: the port is
+        otherwise open only on the agent bridge and, when this host is on the
+        mesh, on `wg-hive`.
+
+        ⚠️ This exposes the queue beyond the tunnel. The server binds every
+        address, so the firewall is the only thing this option changes. What
+        protects the port is the queue itself: TLS is required, and
+        `auth_callout` refuses every client it cannot identify. Whether that
+        is enough to expose the queue is the operator's decision.
+
+        Point `nats.<swarm domain>` in the operator's upstream DNS at the
+        address the other hives reach.
+      '';
+    };
   };
 
   config = lib.mkIf deployCfg.nats.enable {
@@ -681,14 +703,18 @@ in
     # by the firewall and looks exactly like every other NATS failure,
     # a timeout.
     #
-    # ⚠️ Never the world. What makes it safe to open at all is that the
+    # ⚠️ The bridge only, unless `openFirewall` opts in. What makes it safe to
-    # queue is fail-closed: `auth_callout` admits nobody until the responder
+    # open at all is that the queue is fail-closed: `auth_callout` admits
-    # above answers for them, so an agent that reaches this port still has to
+    # nobody until the responder above answers for them, so a client that
-    # present a token authelia vouches for.
+    # reaches this port still has to present a token authelia vouches for.
     #
     # An agent reaches it by name, which dnsmasq answers with the bridge IP.
     services.hyperhive.network.exposeHostPorts = [ cfg.port ];
 
+    # The server binds every address (the container shares the host netns and
+    # sets no listen host), so reachability is the firewall alone.
+    networking.firewall.allowedTCPPorts = lib.mkIf deployCfg.nats.openFirewall [ cfg.port ];
+
     # The bind source has to exist before the container starts, leaf or not:
     # nixos-container refuses to start over a missing one, which would take
     # the whole queue down rather than only its TLS.

nix/module-eval/bao-grants.nix

@@ -76,6 +76,9 @@ let
     deploy.bao.enable = true;
     deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
     swarm.hives.h2.domain = "h2.example.org";
+    # The firewall opened with no address to listen on: the combination
+    # `deploy.bao.openFirewall` refuses.
+    deploy.bao.openFirewall = true;
   };
 
   # The store and the token, with no CA to trust. `mkForce` because the PKI
@@ -86,6 +89,10 @@ let
     deploy.bao.enable = true;
     deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
     deploy.bao.clientCaFile = lib.mkForce null;
+    # An off-host listener opened in the firewall with no client CA: the
+    # combination `deploy.bao.openFirewall` refuses.
+    deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
+    deploy.bao.openFirewall = true;
   };
   # The store plus every reader that used to log in as the hive. One fixture
   # rather than one per reader: the claim they are *separate* principals is
@@ -1695,6 +1702,32 @@ let
         lib.any (a: !a.assertion && names a) baoGranterOddHiveLeafRole.assertions
         && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions);
     }
+    {
+      # Control: `baoGrantHere` leaves the option off and fails no assertion
+      # naming it.
+      name = "openFirewall without a client CA is refused, naming both options";
+      ok =
+        let
+          names =
+            a:
+            lib.hasInfix "services.hyperhive.deploy.bao.openFirewall" a.message
+            && lib.hasInfix "services.hyperhive.deploy.bao.clientCaFile" a.message;
+        in
+        lib.any (a: !a.assertion && names a) baoGrantNoClientCa.assertions
+        && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions);
+    }
+    {
+      name = "openFirewall without an extra listen address is refused, naming both options";
+      ok =
+        let
+          names =
+            a:
+            lib.hasInfix "services.hyperhive.deploy.bao.openFirewall" a.message
+            && lib.hasInfix "services.hyperhive.deploy.bao.extraListenAddresses" a.message;
+        in
+        lib.any (a: !a.assertion && names a) baoGrantTwoHives.assertions
+        && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions);
+    }
     {
       # The granter writes pki roles through `roles/swarm-*` only, so a role
       # named otherwise is refused at eval rather than 403'd at deploy.

nix/module-eval/bao-otel-collector.nix

@@ -21,6 +21,7 @@ let
     hive
     runGroup
     baoSettings
+    bridgePorts
     forceServiceConfigs
     selfScrapes
     ;
@@ -28,6 +29,7 @@ let
   baoTwoAddresses = hive {
     deploy.bao.enable = true;
     deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
+    deploy.bao.openFirewall = true;
     # Pinned, not incidental: the case counting these listeners is about the
     # declared API addresses. ⚠️ Leaving the swarm tier off does not keep the
     # METRICS listener out of the count — that one is a function of the
@@ -106,6 +108,31 @@ let
         in
         l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
     }
+    {
+      # On: the port is open host-wide and the extra listener demands a client
+      # certificate, like loopback. Evaluates with no assertion naming the
+      # option, so the two refusals in bao-grants are not firing here.
+      name = "openFirewall opens the store port host-wide, and the extra listener requires a client certificate";
+      ok =
+        let
+          l = (baoSettings baoTwoAddresses).listener;
+        in
+        lib.elem 8200 baoTwoAddresses.networking.firewall.allowedTCPPorts
+        && l.extra-1.tls_require_and_verify_client_cert == true
+        && l.extra-1.tls_client_ca_file == l.loopback.tls_client_ca_file
+        && !(lib.any (
+          a: !a.assertion && lib.hasInfix "deploy.bao.openFirewall" a.message
+        ) baoTwoAddresses.assertions);
+    }
+    {
+      # Off is today's config: the port is open on the bridge for agents and on
+      # no other interface. The bridge arm is the control that the lookup finds
+      # a port that is there.
+      name = "without openFirewall the store port is open on the bridge only";
+      ok =
+        !(lib.elem 8200 baoOtelElsewhere.networking.firewall.allowedTCPPorts)
+        && lib.elem 8200 (bridgePorts baoOtelElsewhere);
+    }
     {
       # Control for the case above: these settings are rendered per deployment,
       # not constants a passing case could be indifferent to. The metrics and

nix/module-eval/nats-tls.nix

@@ -42,6 +42,7 @@ let
   # The queue on a host whose store is elsewhere: no local policy unit.
   queueNoStore = hive {
     deploy.nats.enable = true;
+    deploy.nats.openFirewall = true;
   };
 
   inherit (shared) natsOnStoreHostReaderMoved;
@@ -256,6 +257,12 @@ let
       name = "the server requires TLS: no allow_non_tls";
       ok = !(natsContainer.services.nats.settings ? allow_non_tls);
     }
+    {
+      # Reachability is the firewall alone because no listen address is set.
+      name = "the server sets no listen address, so it binds every address";
+      ok =
+        !(natsContainer.services.nats.settings ? host) && !(natsContainer.services.nats.settings ? listen);
+    }
     {
       name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
       ok =
@@ -263,6 +270,19 @@ let
         && !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
         && lib.elem 4222 (bridgePorts allLocalMesh);
     }
+    {
+      # Off is today's config; the bridge arm is the control that the lookup
+      # finds a port that is there. On adds the host-wide port and keeps the
+      # bridge one.
+      name = "4222 is open host-wide only with openFirewall, and the bridge keeps it either way";
+      ok =
+        queueNoStore.services.hyperhive.deploy.nats.openFirewall
+        && lib.elem 4222 queueNoStore.networking.firewall.allowedTCPPorts
+        && lib.elem 4222 (bridgePorts queueNoStore)
+        && !allLocal.services.hyperhive.deploy.nats.openFirewall
+        && !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts)
+        && lib.elem 4222 (bridgePorts allLocal);
+    }
     {
       name = "4222 is not opened on wg-hive when this host is not on the mesh";
       ok =