swarm-bao: reach the store through a TLS passthrough, not a vhost
An agent container cannot dial the store's loopback listener: the bridge to-loopback DROP rule is there precisely to stop that, and the store authenticates every reader by client certificate, so the usual answer — a gateway vhost — is the one shape that cannot work. A terminating proxy strips the certificate and bao sees nginx as the client for every hive. nginx's stream module does not terminate. `ssl_preread` reads the SNI off the ClientHello and splices the rest of the connection through byte for byte, so bao completes the handshake itself and authenticates the client it actually has. That is the no-vhost rule kept, not bent. The listener binds the bridge IP rather than every address, because bao already holds `127.0.0.1:<port>` in the same netns and a wildcard bind there is EADDRINUSE — nginx would fail to start, taking the gateway with it. Nothing moves as a result: the name already resolves two ways, so a host-side reader still goes straight to loopback and an agent goes through the passthrough, both on one `BAO_ADDR`. Renders only inside the store's own `deploy.bao.enable` region; a host that runs no store grows no listener and opens no port. Per-agent certificates and per-agent policy are separate work. Refs #4386
This commit is contained in:
parent
cab6910cbb
commit
ef2dfbfb31
3 changed files with 145 additions and 2 deletions
|
|
@ -439,6 +439,18 @@ let
|
|||
|
||||
baoNames = machine: machine.services.hyperhive.gateway.localNames;
|
||||
|
||||
# What nginx is handed for its `stream {}` block. Deliberately not
|
||||
# `virtualHosts`: a vhost is the terminating shape ./host-modules/
|
||||
# swarm-bao.nix's header refuses, and this is the passthrough that is not.
|
||||
baoStream = machine: machine.services.nginx.streamConfig;
|
||||
|
||||
# The bridge-interface firewall — the list `network.exposeHostPorts` merges
|
||||
# into, and the only place a port is opened for agents. The host's own
|
||||
# `allowedTCPPorts` is a different list and a different exposure.
|
||||
bridgePorts =
|
||||
machine:
|
||||
machine.networking.firewall.interfaces.${machine.services.hyperhive.network.bridgeName}.allowedTCPPorts;
|
||||
|
||||
baoTwoAddresses = hive {
|
||||
deploy.bao.enable = true;
|
||||
deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
|
||||
|
|
@ -2462,6 +2474,75 @@ let
|
|||
name = "a hive that does not run the store claims no name for it";
|
||||
ok = !(builtins.elem "bao.t.local" (baoNames bare));
|
||||
}
|
||||
{
|
||||
# What makes that name reachable from inside an agent container, and the
|
||||
# single reason it is a stream server rather than a vhost: `ssl_preread`
|
||||
# routes on the SNI without decrypting, so the handshake bao completes is
|
||||
# still the client's own and the certificate it authenticates by arrives
|
||||
# intact. Terminating here would hand the store one identity for the
|
||||
# whole swarm.
|
||||
name = "the store's host passes connections through without terminating TLS";
|
||||
ok =
|
||||
let
|
||||
s = baoStream baoPkcs11;
|
||||
in
|
||||
lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s;
|
||||
}
|
||||
{
|
||||
# The address half, and it is bridge-only for a reason a wildcard would
|
||||
# hide until deploy: the store already holds `127.0.0.1:8200` in this
|
||||
# same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start
|
||||
# — taking every hive domain behind the gateway down with it.
|
||||
name = "the passthrough listens on the bridge, not on every address";
|
||||
ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11);
|
||||
}
|
||||
{
|
||||
# The routing half: the SNI picks the backend and the only name that
|
||||
# resolves to one is the store's own. A `default` that pointed anywhere
|
||||
# would make this host a relay for whatever name a client invented.
|
||||
name = "the passthrough routes only the store's name, to its loopback listener";
|
||||
ok =
|
||||
let
|
||||
s = baoStream baoPkcs11;
|
||||
in
|
||||
lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s
|
||||
&& lib.hasInfix "bao.t.local 127.0.0.1:8200;" s
|
||||
&& lib.hasInfix ''default "";'' s;
|
||||
}
|
||||
{
|
||||
# ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a
|
||||
# host-wide option, so a block rendered outside the store's own `mkIf`
|
||||
# gives every hive in the swarm a listener — on the port the store
|
||||
# answers on, in front of no store at all.
|
||||
name = "a hive that does not run the store renders no stream passthrough";
|
||||
ok = baoStream bare == "";
|
||||
}
|
||||
{
|
||||
# The listener is only half of reachable: the bridge firewall drops
|
||||
# everything not named here, and a silent drop is the failure that reads
|
||||
# as "the store is down" from inside a container.
|
||||
name = "the store's port is open on the bridge where the store runs";
|
||||
ok = builtins.elem 8200 (bridgePorts baoPkcs11);
|
||||
}
|
||||
{
|
||||
# Absence arm for the case above — a hive with no store has no reason to
|
||||
# open the store's port, and opening it would point agents at a host that
|
||||
# answers nothing.
|
||||
name = "a hive that does not run the store opens no bridge port for it";
|
||||
ok = !(builtins.elem 8200 (bridgePorts bare));
|
||||
}
|
||||
{
|
||||
# The store stays behind the passthrough rather than beside it: loopback
|
||||
# plus whatever was declared, never the bridge. A store that also bound
|
||||
# the bridge itself would collide with the listener above, and the
|
||||
# colliding one is nginx — the whole gateway, not just this port.
|
||||
name = "the store binds loopback and its declared addresses, never the bridge";
|
||||
ok =
|
||||
let
|
||||
l = (baoSettings baoTwoAddresses).listener;
|
||||
in
|
||||
l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
|
||||
}
|
||||
{
|
||||
# Raft refuses to start without it, and says so in a message that names
|
||||
# neither the setting nor the stanza.
|
||||
|
|
|
|||
Loading…
Reference in a new issue