mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

nats and bao can listen on a routable address, opt-in

Browse source
This commit is contained in:
atlas 2026-11-09 17:32:03 +01:00 • committed by mara
commit 892675628d
7 changed files with 174 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

14
docs/2-handbook/swarm/README.md
View file

@ -602,6 +602,20 @@ mechanism.
entry's `wireguardPublicKey`/`wireguardEndpoint`/`wireguardAddress`
to configure `wg-hive`. See [below](#wireguard-inter-hive-mesh-optional).
### Reaching the queue and the store on a routable address
The mesh isn't the only way to reach the queue and the secret store from
another host. Set `deploy.nats.openFirewall` on the queue's host to open the
queue's port on every interface, and `deploy.bao.openFirewall` together with
`deploy.bao.extraListenAddresses` on the store's host to open the store's port
on the addresses it listens on. Both default to off. Each exposes a service
beyond the tunnel, which is the operator's decision. The queue requires TLS and
its `auth_callout` refuses every client it can't identify. The store requires a
client certificate on every listener, and evaluation fails when you
set `deploy.bao.openFirewall` without `deploy.bao.clientCaFile`. Point
`nats.<swarm.domain>` and `bao.<swarm.domain>` in your upstream DNS at the
address the other hives reach.
### WireGuard inter-hive mesh (optional)
The peer config above uses public HTTPS for all inter-hive traffic.

47
nix/host-modules/swarm-bao.nix
View file

@ -2891,6 +2891,30 @@ in
misconfiguration rather than a topology.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Open {option}`services.hyperhive.swarm.bao.port` in the host firewall
on every interface, so hives on other hosts can reach the addresses in
{option}`services.hyperhive.deploy.bao.extraListenAddresses` over the
LAN or the internet. Off by default: the port is otherwise open only on
the agent bridge.
⚠️ This exposes the secret store beyond the tunnel. Every listener
requires a client certificate signed by
{option}`services.hyperhive.deploy.bao.clientCaFile`, and nothing
answers without one, which is why setting this without that file, or
without any extra address, fails evaluation. Whether the store may be
reachable at all from outside a private network is the operator's
decision.
Point `bao.<swarm domain>` in the operator's upstream DNS at the
address the other hives reach.
'';
};
};
# ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
@ -2936,6 +2960,24 @@ in
deploy.bao.otel.telemetryPort. Pick free ports.
'';
}
{
assertion = !baoDeploy.openFirewall || baoDeploy.clientCaFile != null;
message = ''
services.hyperhive.deploy.bao.openFirewall opens the store's port to
every interface, but services.hyperhive.deploy.bao.clientCaFile is
null, so its listeners would not require a client certificate. Set
clientCaFile, or leave openFirewall off.
'';
}
{
assertion = !baoDeploy.openFirewall || baoDeploy.extraListenAddresses != [ ];
message = ''
services.hyperhive.deploy.bao.openFirewall is set, but
services.hyperhive.deploy.bao.extraListenAddresses is empty: the
store listens on loopback only, so the open port reaches nothing.
Name the address other hives reach, or leave openFirewall off.
'';
}
{
assertion = cfg.ui.domain != cfg.domain;
message = ''
@ -3090,6 +3132,11 @@ in
# certificate its CA signed.
services.hyperhive.network.exposeHostPorts = [ cfg.port ];
# Every listener, bridge-IP passthrough included, requires the client
# certificate (see `listenerTls`), so opening the port host-wide adds
# reach and no unauthenticated path.
networking.firewall.allowedTCPPorts = lib.mkIf baoDeploy.openFirewall [ cfg.port ];
# The operators' door to the UI, shaped like ./swarm-victorialogs.nix's
# vhost. It proxies to the container's nginx over loopback (shared
# netns), so no port opens anywhere. `auth_request` only proves a

5
nix/host-modules/swarm-nats-service.nix
View file

@ -57,8 +57,9 @@ in
3000, every agent in 8100..8999 via FNV-1a hash).
Opened on the bridge interface, for agent containers, and on
`wg-hive` when this host is on the mesh, for other hives. Never
host-wide. TLS only: a client that does not speak it is refused.
`wg-hive` when this host is on the mesh, for other hives. Host-wide
only when {option}`services.hyperhive.deploy.nats.openFirewall` is
set. TLS only: a client that does not speak it is refused.
Unlike {option}`services.hyperhive.deploy.nats.monitorPort` and
{option}`services.hyperhive.deploy.nats.metricsPort`, the

34
nix/host-modules/swarm-nats.nix
View file

@ -505,6 +505,28 @@ in
A path, never a value.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Open {option}`services.hyperhive.swarm.nats.port` in the host firewall
on every interface, so hives on other hosts can reach the queue on
this host's LAN or public address. Off by default: the port is
otherwise open only on the agent bridge and, when this host is on the
mesh, on `wg-hive`.
⚠️ This exposes the queue beyond the tunnel. The server binds every
address, so the firewall is the only thing this option changes. What
protects the port is the queue itself: TLS is required, and
`auth_callout` refuses every client it cannot identify. Whether that
is enough to expose the queue is the operator's decision.
Point `nats.<swarm domain>` in the operator's upstream DNS at the
address the other hives reach.
'';
};
};
config = lib.mkIf deployCfg.nats.enable {
@ -681,14 +703,18 @@ in
# by the firewall and looks exactly like every other NATS failure,
# a timeout.
#
# ⚠️ Never the world. What makes it safe to open at all is that the
# queue is fail-closed: `auth_callout` admits nobody until the responder
# above answers for them, so an agent that reaches this port still has to
# present a token authelia vouches for.
# ⚠️ The bridge only, unless `openFirewall` opts in. What makes it safe to
# open at all is that the queue is fail-closed: `auth_callout` admits
# nobody until the responder above answers for them, so a client that
# reaches this port still has to present a token authelia vouches for.
#
# An agent reaches it by name, which dnsmasq answers with the bridge IP.
services.hyperhive.network.exposeHostPorts = [ cfg.port ];
# The server binds every address (the container shares the host netns and
# sets no listen host), so reachability is the firewall alone.
networking.firewall.allowedTCPPorts = lib.mkIf deployCfg.nats.openFirewall [ cfg.port ];
# The bind source has to exist before the container starts, leaf or not:
# nixos-container refuses to start over a missing one, which would take
# the whole queue down rather than only its TLS.

33
nix/module-eval/bao-grants.nix
View file

@ -76,6 +76,9 @@ let
deploy.bao.enable = true;
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
swarm.hives.h2.domain = "h2.example.org";
# The firewall opened with no address to listen on: the combination
# `deploy.bao.openFirewall` refuses.
deploy.bao.openFirewall = true;
};
# The store and the token, with no CA to trust. `mkForce` because the PKI
@ -86,6 +89,10 @@ let
deploy.bao.enable = true;
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
deploy.bao.clientCaFile = lib.mkForce null;
# An off-host listener opened in the firewall with no client CA: the
# combination `deploy.bao.openFirewall` refuses.
deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
deploy.bao.openFirewall = true;
};
# The store plus every reader that used to log in as the hive. One fixture
# rather than one per reader: the claim they are *separate* principals is
@ -1695,6 +1702,32 @@ let
lib.any (a: !a.assertion && names a) baoGranterOddHiveLeafRole.assertions
&& !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions);
}
{
# Control: `baoGrantHere` leaves the option off and fails no assertion
# naming it.
name = "openFirewall without a client CA is refused, naming both options";
ok =
let
names =
a:
lib.hasInfix "services.hyperhive.deploy.bao.openFirewall" a.message
&& lib.hasInfix "services.hyperhive.deploy.bao.clientCaFile" a.message;
in
lib.any (a: !a.assertion && names a) baoGrantNoClientCa.assertions
&& !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions);
}
{
name = "openFirewall without an extra listen address is refused, naming both options";
ok =
let
names =
a:
lib.hasInfix "services.hyperhive.deploy.bao.openFirewall" a.message
&& lib.hasInfix "services.hyperhive.deploy.bao.extraListenAddresses" a.message;
in
lib.any (a: !a.assertion && names a) baoGrantTwoHives.assertions
&& !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions);
}
{
# The granter writes pki roles through `roles/swarm-*` only, so a role
# named otherwise is refused at eval rather than 403'd at deploy.

27
nix/module-eval/bao-otel-collector.nix
View file

@ -21,6 +21,7 @@ let
hive
runGroup
baoSettings
bridgePorts
forceServiceConfigs
selfScrapes
;
@ -28,6 +29,7 @@ let
baoTwoAddresses = hive {
deploy.bao.enable = true;
deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
deploy.bao.openFirewall = true;
# Pinned, not incidental: the case counting these listeners is about the
# declared API addresses. ⚠️ Leaving the swarm tier off does not keep the
# METRICS listener out of the count — that one is a function of the
@ -106,6 +108,31 @@ let
in
l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
}
{
# On: the port is open host-wide and the extra listener demands a client
# certificate, like loopback. Evaluates with no assertion naming the
# option, so the two refusals in bao-grants are not firing here.
name = "openFirewall opens the store port host-wide, and the extra listener requires a client certificate";
ok =
let
l = (baoSettings baoTwoAddresses).listener;
in
lib.elem 8200 baoTwoAddresses.networking.firewall.allowedTCPPorts
&& l.extra-1.tls_require_and_verify_client_cert == true
&& l.extra-1.tls_client_ca_file == l.loopback.tls_client_ca_file
&& !(lib.any (
a: !a.assertion && lib.hasInfix "deploy.bao.openFirewall" a.message
) baoTwoAddresses.assertions);
}
{
# Off is today's config: the port is open on the bridge for agents and on
# no other interface. The bridge arm is the control that the lookup finds
# a port that is there.
name = "without openFirewall the store port is open on the bridge only";
ok =
!(lib.elem 8200 baoOtelElsewhere.networking.firewall.allowedTCPPorts)
&& lib.elem 8200 (bridgePorts baoOtelElsewhere);
}
{
# Control for the case above: these settings are rendered per deployment,
# not constants a passing case could be indifferent to. The metrics and

20
nix/module-eval/nats-tls.nix
View file

@ -42,6 +42,7 @@ let
# The queue on a host whose store is elsewhere: no local policy unit.
queueNoStore = hive {
deploy.nats.enable = true;
deploy.nats.openFirewall = true;
};
inherit (shared) natsOnStoreHostReaderMoved;
@ -256,6 +257,12 @@ let
name = "the server requires TLS: no allow_non_tls";
ok = !(natsContainer.services.nats.settings ? allow_non_tls);
}
{
# Reachability is the firewall alone because no listen address is set.
name = "the server sets no listen address, so it binds every address";
ok =
!(natsContainer.services.nats.settings ? host) && !(natsContainer.services.nats.settings ? listen);
}
{
name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
ok =
@ -263,6 +270,19 @@ let
&& !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
&& lib.elem 4222 (bridgePorts allLocalMesh);
}
{
# Off is today's config; the bridge arm is the control that the lookup
# finds a port that is there. On adds the host-wide port and keeps the
# bridge one.
name = "4222 is open host-wide only with openFirewall, and the bridge keeps it either way";
ok =
queueNoStore.services.hyperhive.deploy.nats.openFirewall
&& lib.elem 4222 queueNoStore.networking.firewall.allowedTCPPorts
&& lib.elem 4222 (bridgePorts queueNoStore)
&& !allLocal.services.hyperhive.deploy.nats.openFirewall
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts)
&& lib.elem 4222 (bridgePorts allLocal);
}
{
name = "4222 is not opened on wg-hive when this host is not on the mesh";
ok =