From 3c79c5eb5d5d862a4221167987489bb1db8ece47 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 6 Oct 2026 09:52:26 +0200 Subject: [PATCH] swarm-nats: the auth responder's store identity path is its own, not the hive's glue-nats-auth-bao-identity.nix defaulted deploy.nats.authBaoClient{Cert,Key}File to a sibling of deploy.bao.clientCertFile, which is the hive's own store reader. That tied a swarm service to a hive identity, pointed a store host whose reader was moved at a file nobody mints, and on a queue host without the store that held only a placed hive reader invented a nats-auth.pem path that passed the eval assertion and failed at runtime. The glue is now gated on the store running on this host (deploy.bao.enable, the condition glue-bao-tls.nix mints under) and names the directory the minting unit writes to, shared through nix/host-modules/lib/bao-pki-dir.nix. A queue host without the store gets no default and the eval assertion refuses it until the responder's own leaf is named; the assertion message now names deploy.bao.enable as where the default comes from. On a default store host the evaluated paths are unchanged: /var/lib/swarm-bao-pki/nats-auth{,-key}.pem. Refs #4921 --- docs/swarm/secrets.md | 7 +++ nix/host-modules/glue-bao-tls.nix | 2 +- .../glue-nats-auth-bao-identity.nix | 19 +++----- nix/host-modules/lib/bao-pki-dir.nix | 5 +++ nix/host-modules/swarm-nats.nix | 10 +++-- nix/module-eval/nats-authelia.nix | 45 +++++++++++++++++++ 6 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 nix/host-modules/lib/bao-pki-dir.nix diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index a04f08a9..4c55dfc1 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -190,6 +190,13 @@ start reads the seed the queue refuses every client, OIDC ones included. The retry that lands the file starts or restarts the server, and the responder comes up with it. +The responder's store leaf is `deploy.nats.authBaoClientCertFile` / +`authBaoClientKeyFile`. On the store's host both default to the leaf +`glue-bao-tls.nix` mints there. A queue host without the store gets no +default, even when it holds the hive's own reader `deploy.bao.clientCertFile`: +issue a leaf whose subject is `deploy.bao.natsAuthCommonName`, place it, and +name it in both options. Until you do, evaluation fails naming both. + Where the queue and the store run on different hosts, deploy the store's host first. `swarm-bao-nats-auth-policy` there writes the role's grants on the two paths. A queue host deployed before it can't read or create the key, so its diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 50ba2a27..cced87a9 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -39,7 +39,7 @@ let # Host-side, outside the container's tree, for the same reason the raft data # is: `nixos-container destroy` must not take it. Losing the CA key means # re-issuing every client certificate in the swarm. - pkiDir = "/var/lib/swarm-bao-pki"; + pkiDir = import ./lib/bao-pki-dir.nix; # What a reader calls itself to the store. The hive's name, because a bao # cert-auth role matches on the CN — this is an interface, not a label. diff --git a/nix/host-modules/glue-nats-auth-bao-identity.nix b/nix/host-modules/glue-nats-auth-bao-identity.nix index 855ad3ac..19858c6f 100644 --- a/nix/host-modules/glue-nats-auth-bao-identity.nix +++ b/nix/host-modules/glue-nats-auth-bao-identity.nix @@ -7,9 +7,10 @@ # ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the # leaf. What belongs here is the pairing: which paths the responder presents. # -# ⚠️ Gated on the leaf existing, not on the store being enabled, for the reason -# ./glue-nats-bao-identity.nix states: the hive hosting the queue need not be -# the hive hosting the store. +# ⚠️ Gated on the store running on this host, the only place the leaf is +# minted. Not on `deploy.bao.clientCertFile`, the hive's own reader: a queue +# host need not run a hive. A queue host without the store gets no default +# here; the operator places the responder's leaf and names it. # # Everything is `mkDefault`. An operator naming their own paths wins. { @@ -18,17 +19,11 @@ ... }: let - hyperhiveCfg = config.services.hyperhive; - deployCfg = hyperhiveCfg.deploy; - baoDeploy = deployCfg.bao; - - # Where ./glue-bao-tls.nix puts the leaves, derived from the reader's own path - # rather than repeating that file's directory literal. - haveMintedPki = baoDeploy.clientCertFile != null; - pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null; + deployCfg = config.services.hyperhive.deploy; + pkiDir = import ./lib/bao-pki-dir.nix; in { - config = lib.mkIf (deployCfg.nats.enable && haveMintedPki) { + config = lib.mkIf (deployCfg.nats.enable && deployCfg.bao.enable) { services.hyperhive.deploy.nats = { authBaoClientCertFile = lib.mkDefault "${pkiDir}/nats-auth.pem"; authBaoClientKeyFile = lib.mkDefault "${pkiDir}/nats-auth-key.pem"; diff --git a/nix/host-modules/lib/bao-pki-dir.nix b/nix/host-modules/lib/bao-pki-dir.nix new file mode 100644 index 00000000..186921a0 --- /dev/null +++ b/nix/host-modules/lib/bao-pki-dir.nix @@ -0,0 +1,5 @@ +# Where ./glue-bao-tls.nix keeps the store's own CA and the leaves it signs, +# on the host that runs the store. Shared so a glue that points a principal at +# its minted leaf names the directory the minting unit writes to, not a copy +# of it. +"/var/lib/swarm-bao-pki" diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index ea9ff8f0..4fe11650 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -523,8 +523,11 @@ in Required when `deploy.nats.enable` is set; unset fails at eval. ./glue-nats-auth-bao-identity.nix points it at the leaf - ./glue-bao-tls.nix mints, where this host mints one; on any other - queue host it is a file the operator places. Set to a file that does + ./glue-bao-tls.nix mints on a host that runs the store + ({option}`services.hyperhive.deploy.bao.enable`); on any other queue + host it is a file the operator places. Never derived from + {option}`services.hyperhive.deploy.bao.clientCertFile`, which is the + hive's own reader. Set to a file that does not exist, `swarm-nats-auth-secrets`, `swarm-bao-nats-auth-oidc` and `swarm-bao-nats-callout-xkey` fail, and the queue's server does not start without the xkey. @@ -633,7 +636,8 @@ in The queue's callout xkey and its responder's OIDC client secret are read from the swarm's secret store with this certificate, and neither the server nor the responder starts without them. They - default to the leaf the store mints when it runs on this host; + default to the leaf the store mints where + services.hyperhive.deploy.bao.enable is set on this host; elsewhere, issue a leaf whose CN is services.hyperhive.deploy.bao.natsAuthCommonName and name it in both options. diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index e417da1e..005df500 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -80,6 +80,28 @@ let deploy.nats.enable = true; }; + # The store's host with the hive's own reader pointed somewhere else. + natsOnStoreHostReaderMoved = hive { + deploy.bao.enable = true; + deploy.nats.enable = true; + deploy.bao.clientCertFile = "/etc/pki/h1.pem"; + deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; + }; + + # A queue host holding only the hive's own store reader, placed by hand: no + # store here, and no responder leaf named. + natsWithHiveReaderOnly = hive { + deploy.nats.enable = true; + deploy.nats.autoGenerateCallout = false; + deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; + deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; + deploy.nats.authPackage = pkgs.emptyDirectory; + deploy.bao.clientCertFile = "/etc/pki/h1.pem"; + deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; + }; + responderOf = h: h.containers.swarm-nats.config.systemd.services.swarm-nats-auth; refusedWithoutIdentity = @@ -108,6 +130,15 @@ let && !(refusedWithoutIdentity natsWithStoreIdentity) && !(refusedWithoutIdentity natsOnStoreHost); } + { + # The hive's own reader is not the responder's identity, and no path is + # derived from it: a queue host without the store that holds only the + # hive's reader is refused like one holding nothing. + name = "a queue host without the store is refused when it holds only the hive's reader"; + ok = + refusedWithoutIdentity natsWithHiveReaderOnly + && natsWithHiveReaderOnly.services.hyperhive.deploy.nats.authBaoClientCertFile == null; + } { # Reads the RENDERED settings, not the option: `calloutBlocks {…} // { # … }` is a shallow merge, and a future edit that dropped or shadowed @@ -350,6 +381,20 @@ let && n.authBaoClientCertFile != n.baoClientCertFile && n.authBaoClientCertFile != b.clientCertFile; } + { + # The default is the file the minting unit writes, wherever the hive's + # own reader is pointed. + name = "on the store's host the responder's leaf is the one minted there, not one beside the hive's reader"; + ok = + let + n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats; + mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script; + in + n.authBaoClientCertFile == "/var/lib/swarm-bao-pki/nats-auth.pem" + && n.authBaoClientKeyFile == "/var/lib/swarm-bao-pki/nats-auth-key.pem" + && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats-auth.pem ] ||" mint + && lib.hasInfix "/var/lib/swarm-bao-pki nats-auth " mint; + } { # Not a rename test. `hostClientSecretDir` is `readOnly`, so the fixture # cannot define it; what can break is a reader left pointing at