diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 3a0a91e8..ec69baea 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -238,6 +238,33 @@ A hive that reads from a store on **another** machine names the reader's half It's the one credential that can't come out of the store, being what opens it; everything else a hive needs does. +## How a reader reaches the store + +Every reader dials the same URL — `https://bao.:` — and on +the host that runs the store that name resolves twice. `/etc/hosts` answers a +host-side unit with loopback, where openbao binds; the hive's dnsmasq answers a +container with the bridge IP, where nginx holds the same port. One `BAO_ADDR`, +and the network namespace that asked decides which half of it answers. + +The container's half goes through nginx and **still doesn't terminate TLS**. It +is an nginx `stream` server with `ssl_preread on`: it reads the SNI off the +ClientHello and splices the rest of the connection through byte for byte, so +openbao completes the handshake itself. A vhost would decrypt here instead, and +the client certificate — the store's whole authentication — would stop at nginx, +leaving openbao seeing one client for every hive in the swarm. That's why the +store has no vhost and gets a passthrough: not an exception to the no-vhost +rule, the one shape that keeps it. + +The stream listener binds the bridge IP rather than every address, because +openbao already holds loopback on that port in the same network namespace and a +wildcard bind would fail with `EADDRINUSE` — taking nginx, and with it every +hive domain behind the gateway, down. `network.exposeHostPorts` opens the port +on the bridge firewall and nowhere else. + +Reaching the port grants nothing by itself: openbao answers nothing without a +client certificate signed by `deploy.bao.clientCaFile`. The passthrough carries +whichever certificate the reader presents, unchanged. + ## The constraint that decides where the root lives A hive CA carries `nameConstraints=permitted;DNS:`, and **a swarm diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index a1d2b5bf..19f36ca5 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -15,8 +15,10 @@ # a non-HTTP protocol. It speaks HTTPS, so nginx *could* front it: **the client # certificate IS the authentication**, and a terminating proxy strips it, # leaving bao seeing nginx as the client for every hive in the swarm — one -# identity where there must be many. Reach is loopback plus whatever -# `deploy.bao.extraListenAddresses` names. +# identity where there must be many. Reach is loopback, whatever +# `deploy.bao.extraListenAddresses` names, and — for readers inside an agent +# container — the nginx *stream* passthrough below, which routes on the SNI +# without decrypting and so leaves the client certificate intact. # # ⚠️ THIS MODULE HAS NO OPINION ABOUT WHERE THE STORE'S IDENTITY COMES FROM. # A store must not take its certificates from an authority it will itself @@ -795,6 +797,39 @@ in # dead exporter rather than a wrong address. bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus"; }; + + # ⚠️ The one nginx exception to this file's header, and it is one because + # it never terminates. `ssl_preread` reads the SNI off the ClientHello + # and splices the rest through byte for byte, so bao completes the + # handshake itself and sees the *client's* certificate — the exact + # property the no-vhost rule protects. A vhost would decrypt here and + # hand the store one identity for every hive in the swarm. + # + # Bridge IP, never `0.0.0.0`: the store shares this netns and already + # holds `127.0.0.1:`, so a wildcard bind on that port is + # `EADDRINUSE` and nginx would not start at all. Nothing is lost to the + # split, because the name already resolves two ways — `/etc/hosts` sends + # a host-side reader straight to loopback, dnsmasq answers an agent's + # query with the bridge IP. One name, one port, one `BAO_ADDR`; only + # which netns asked decides whether the passthrough is in the path. + services.nginx.streamConfig = '' + map $ssl_preread_server_name $swarm_bao_backend { + ${cfg.domain} 127.0.0.1:${toString cfg.port}; + default ""; + } + + server { + listen ${networkCfg.bridgeIp}:${toString cfg.port}; + ssl_preread on; + proxy_pass $swarm_bao_backend; + } + ''; + + # Firewall only: the listener above is already the bridge-IP bind this + # option's own documentation asks a host service for. Reaching the port + # still proves nothing — bao answers nothing without a client + # certificate its CA signed. + services.hyperhive.network.exposeHostPorts = [ cfg.port ]; }) (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) { diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7956957b..d893217d 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -439,6 +439,18 @@ let baoNames = machine: machine.services.hyperhive.gateway.localNames; + # What nginx is handed for its `stream {}` block. Deliberately not + # `virtualHosts`: a vhost is the terminating shape ./host-modules/ + # swarm-bao.nix's header refuses, and this is the passthrough that is not. + baoStream = machine: machine.services.nginx.streamConfig; + + # The bridge-interface firewall — the list `network.exposeHostPorts` merges + # into, and the only place a port is opened for agents. The host's own + # `allowedTCPPorts` is a different list and a different exposure. + bridgePorts = + machine: + machine.networking.firewall.interfaces.${machine.services.hyperhive.network.bridgeName}.allowedTCPPorts; + baoTwoAddresses = hive { deploy.bao.enable = true; deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; @@ -2462,6 +2474,75 @@ let name = "a hive that does not run the store claims no name for it"; ok = !(builtins.elem "bao.t.local" (baoNames bare)); } + { + # What makes that name reachable from inside an agent container, and the + # single reason it is a stream server rather than a vhost: `ssl_preread` + # routes on the SNI without decrypting, so the handshake bao completes is + # still the client's own and the certificate it authenticates by arrives + # intact. Terminating here would hand the store one identity for the + # whole swarm. + name = "the store's host passes connections through without terminating TLS"; + ok = + let + s = baoStream baoPkcs11; + in + lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s; + } + { + # The address half, and it is bridge-only for a reason a wildcard would + # hide until deploy: the store already holds `127.0.0.1:8200` in this + # same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start + # — taking every hive domain behind the gateway down with it. + name = "the passthrough listens on the bridge, not on every address"; + ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11); + } + { + # The routing half: the SNI picks the backend and the only name that + # resolves to one is the store's own. A `default` that pointed anywhere + # would make this host a relay for whatever name a client invented. + name = "the passthrough routes only the store's name, to its loopback listener"; + ok = + let + s = baoStream baoPkcs11; + in + lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s + && lib.hasInfix "bao.t.local 127.0.0.1:8200;" s + && lib.hasInfix ''default "";'' s; + } + { + # ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a + # host-wide option, so a block rendered outside the store's own `mkIf` + # gives every hive in the swarm a listener — on the port the store + # answers on, in front of no store at all. + name = "a hive that does not run the store renders no stream passthrough"; + ok = baoStream bare == ""; + } + { + # The listener is only half of reachable: the bridge firewall drops + # everything not named here, and a silent drop is the failure that reads + # as "the store is down" from inside a container. + name = "the store's port is open on the bridge where the store runs"; + ok = builtins.elem 8200 (bridgePorts baoPkcs11); + } + { + # Absence arm for the case above — a hive with no store has no reason to + # open the store's port, and opening it would point agents at a host that + # answers nothing. + name = "a hive that does not run the store opens no bridge port for it"; + ok = !(builtins.elem 8200 (bridgePorts bare)); + } + { + # The store stays behind the passthrough rather than beside it: loopback + # plus whatever was declared, never the bridge. A store that also bound + # the bridge itself would collide with the listener above, and the + # colliding one is nginx — the whole gateway, not just this port. + name = "the store binds loopback and its declared addresses, never the bridge"; + ok = + let + l = (baoSettings baoTwoAddresses).listener; + in + l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200"; + } { # Raft refuses to start without it, and says so in a message that names # neither the setting nor the stanza.