From 1b18f6dcbd9fcb1819e0db95d8e9c9cfe340a60b Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 6 Oct 2026 12:04:48 +0200 Subject: [PATCH] swarm-nats, matrix: the queue's and matrix-ctl's store identity paths are their own, not the hive's glue-nats-bao-identity.nix and glue-matrix-ctl-bao-identity.nix defaulted deploy.nats.baoClient{Cert,Key}File and deploy.matrix.ctlBaoClient{Cert,Key}File to siblings of deploy.bao.clientCertFile, the hive's own store reader. That pointed a store host whose reader was moved at files nobody mints, and on a queue or homeserver host without the store that held a placed hive reader invented a nats.pem / matrix-ctl.pem path beside it. Both glues are now gated on the store running on this host (deploy.bao.enable, the condition glue-bao-tls.nix mints under) and name the directory the minting unit writes to, through nix/host-modules/lib/bao-pki-dir.nix. Host shapes: - Store host, default reader: unchanged. The evaluated paths stay /var/lib/swarm-bao-pki/nats{,-key}.pem and /var/lib/swarm-bao-pki/matrix-ctl{,-key}.pem, and the rendered units are the same. - Store host whose hive reader was moved: now gets the leaf minted there. - Queue or homeserver host without the store, holding a hive reader and not naming these options: no default any more. It still evaluates (neither option has an eval assertion). swarm-bao-nats-tls exits 1 naming deploy.nats.baoClient{Cert,Key}File when the queue's certificate is missing or due for renewal; swarm-bao-matrix-oidc exits 1 naming ctlBaoClient{Cert,Key}File, and the swarm appservice render/publish units and the PKI bind mount no longer render. If that host had placed nats.pem or matrix-ctl.pem beside its hive reader, it worked before and now needs both options named. - Hosts without the store and without a hive reader, and hosts naming the options: unchanged. Both refusals and option descriptions now say the default comes from deploy.bao.enable. Refs #4921 --- docs/integrations/matrix.md | 7 +++ docs/swarm/README.md | 9 ++++ .../glue-matrix-ctl-bao-identity.nix | 21 +++------ nix/host-modules/glue-nats-bao-identity.nix | 20 +++----- nix/host-modules/hive-matrix.nix | 8 +++- nix/host-modules/swarm-nats.nix | 9 +++- nix/module-eval/bao-matrix-reader.nix | 38 +++++++++++++++ nix/module-eval/nats-tls.nix | 46 +++++++++++++++++++ 8 files changed, 127 insertions(+), 31 deletions(-) diff --git a/docs/integrations/matrix.md b/docs/integrations/matrix.md index 289d3f56..74032614 100644 --- a/docs/integrations/matrix.md +++ b/docs/integrations/matrix.md @@ -143,6 +143,13 @@ A homeserver whose `swarm-matrix-ctl` has no store identity (`services.hyperhive.deploy.matrix.ctlBaoClientCertFile`) loads no registration, and nothing creates accounts on it. +On the store's host `ctlBaoClientCertFile` / `ctlBaoClientKeyFile` default +to the leaf `glue-bao-tls.nix` mints there. A homeserver host without the +store gets no default, even when it holds the hive's own reader +`deploy.bao.clientCertFile`: issue a leaf whose subject is +`deploy.bao.matrixCtlCommonName`, place it, and name it in both options. +Until you do, `swarm-bao-matrix-oidc` fails naming both. + ### The swarm's appservice, and its admin sender The registration's id is `swarm`, its sender `@swarm`, and its user diff --git a/docs/swarm/README.md b/docs/swarm/README.md index 9fefa3ff..f891b997 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -114,6 +114,15 @@ upstream DNS record**, `nats.` pointing at the queue host's mesh address, the same contract as `bao.`. The queue's port is open on `wg-hive` when that host is on the mesh. +The queue's host asks the secret store for that certificate under a store +leaf of its own, `deploy.nats.baoClientCertFile` / `baoClientKeyFile`. On the +store's host both default to the leaf `glue-bao-tls.nix` mints there. A queue +host without the store gets no default, even when it holds the hive's own +reader `deploy.bao.clientCertFile`: issue a leaf whose subject is +`deploy.bao.natsCommonName`, place it, and name it in both options. Until you +do, `swarm-bao-nats-tls` fails naming both whenever the queue's certificate is +missing or due for renewal. + The identity isn't a choice — a hive authenticates as `hive-` and publishes under `hiveName`, the same name that keys `swarm.hives`. diff --git a/nix/host-modules/glue-matrix-ctl-bao-identity.nix b/nix/host-modules/glue-matrix-ctl-bao-identity.nix index ac7cfda8..1ffa6db9 100644 --- a/nix/host-modules/glue-matrix-ctl-bao-identity.nix +++ b/nix/host-modules/glue-matrix-ctl-bao-identity.nix @@ -8,10 +8,10 @@ # leaf, because the thing that owns a private key owns issuing from it. What # belongs here is the pairing: which paths matrix-ctl presents. # -# ⚠️ Gated on the leaf existing, not on the store being enabled — the same rule -# ./glue-secret-publisher-bao-identity.nix states, and it bites harder here: a -# swarm runs ONE homeserver, so the hive hosting it is the one least likely to -# also be the hive hosting the store. +# ⚠️ Gated on the store running on this host, the only place the leaf is +# minted. Not on `deploy.bao.clientCertFile`, the hive's own reader. A +# homeserver host without the store gets no default here; the operator places +# matrix-ctl's leaf and names it. # # Everything is `mkDefault`. An operator naming their own paths wins. { @@ -20,18 +20,11 @@ ... }: let - hyperhiveCfg = config.services.hyperhive; - deployCfg = hyperhiveCfg.deploy; - baoDeploy = deployCfg.bao; - - # Where ./glue-bao-tls.nix puts the leaves, derived from the reader's own path - # rather than repeating that file's directory literal: an operator who moves - # the PKI moves both, and the two cannot drift apart. - haveMintedPki = baoDeploy.clientCertFile != null; - pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null; + deployCfg = config.services.hyperhive.deploy; + pkiDir = import ./lib/bao-pki-dir.nix; in { - config = lib.mkIf (deployCfg.matrix.enable && haveMintedPki) { + config = lib.mkIf (deployCfg.matrix.enable && deployCfg.bao.enable) { services.hyperhive.deploy.matrix = { ctlBaoClientCertFile = lib.mkDefault "${pkiDir}/matrix-ctl.pem"; ctlBaoClientKeyFile = lib.mkDefault "${pkiDir}/matrix-ctl-key.pem"; diff --git a/nix/host-modules/glue-nats-bao-identity.nix b/nix/host-modules/glue-nats-bao-identity.nix index 6daaddde..3ee69dae 100644 --- a/nix/host-modules/glue-nats-bao-identity.nix +++ b/nix/host-modules/glue-nats-bao-identity.nix @@ -9,9 +9,10 @@ # leaf, because the thing that owns a private key owns issuing from it. What # belongs here is the pairing: which paths `swarm-bao-nats-tls` presents. # -# ⚠️ Gated on the leaf existing, not on the store being enabled — the same rule -# ./glue-matrix-ctl-bao-identity.nix states: a swarm runs ONE queue, and the -# hive hosting it need not be the hive hosting the store. +# ⚠️ Gated on the store running on this host, the only place the leaf is +# minted. Not on `deploy.bao.clientCertFile`, the hive's own reader: a queue +# host need not run a hive. A queue host without the store gets no default +# here; the operator places the queue's leaf and names it. # # Everything is `mkDefault`. An operator naming their own paths wins. { @@ -20,18 +21,11 @@ ... }: let - hyperhiveCfg = config.services.hyperhive; - deployCfg = hyperhiveCfg.deploy; - baoDeploy = deployCfg.bao; - - # Where ./glue-bao-tls.nix puts the leaves, derived from the reader's own path - # rather than repeating that file's directory literal: an operator who moves - # the PKI moves both, and the two cannot drift apart. - haveMintedPki = baoDeploy.clientCertFile != null; - pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null; + deployCfg = config.services.hyperhive.deploy; + pkiDir = import ./lib/bao-pki-dir.nix; in { - config = lib.mkIf (deployCfg.nats.enable && haveMintedPki) { + config = lib.mkIf (deployCfg.nats.enable && deployCfg.bao.enable) { services.hyperhive.deploy.nats = { baoClientCertFile = lib.mkDefault "${pkiDir}/nats.pem"; baoClientKeyFile = lib.mkDefault "${pkiDir}/nats-key.pem"; diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index e79d4659..e5a6e29f 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -419,7 +419,11 @@ in No default: a module that guessed would be holding the CA opinion ./swarm-bao.nix deliberately does not hold. ./glue-matrix-ctl-bao-identity.nix points it at the leaf - ./glue-bao-tls.nix mints, where this host mints one. + ./glue-bao-tls.nix mints on a host that runs the store + ({option}`services.hyperhive.deploy.bao.enable`); on any other + homeserver host it is a file the operator places. Never derived from + {option}`services.hyperhive.deploy.bao.clientCertFile`, which is the + hive's own reader. The file and its key are bind-mounted into the container read-only. Co-located in the hive's filesystem and therefore readable by hive @@ -631,7 +635,7 @@ in script = if !ctlActive then '' - echo "the homeserver's host has no matrix-ctl store identity, so it cannot read the homeserver's OIDC client secret: set services.hyperhive.deploy.matrix.ctlBaoClientCertFile and ctlBaoClientKeyFile" >&2 + echo "the homeserver's host has no matrix-ctl store identity, so it cannot read the homeserver's OIDC client secret: set services.hyperhive.deploy.matrix.ctlBaoClientCertFile and ctlBaoClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2 exit 1 '' else diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 4fe11650..11b2f39e 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -491,9 +491,12 @@ in `pki/issue/swarm-nats` and nothing else. No default. ./glue-nats-bao-identity.nix points it at the leaf - ./glue-bao-tls.nix mints, where this host mints one. On a queue host + ./glue-bao-tls.nix mints on a host that runs the store + ({option}`services.hyperhive.deploy.bao.enable`). On a queue host without the store, it is a file the operator copies from the store's - host. + host. Never derived from + {option}`services.hyperhive.deploy.bao.clientCertFile`, which is the + hive's own reader. A path, never a value. ''; @@ -1340,6 +1343,8 @@ in echo "cannot be requested from the store." >&2 echo "Set services.hyperhive.deploy.nats.baoClient{Cert,Key}File" >&2 echo "to a leaf the store's CA signed with CN=${baoDeploy.natsCommonName}." >&2 + echo "They default to the leaf the store mints only where" >&2 + echo "services.hyperhive.deploy.bao.enable is set on this host." >&2 exit 1'' else "" diff --git a/nix/module-eval/bao-matrix-reader.nix b/nix/module-eval/bao-matrix-reader.nix index 32f3885d..08c049ab 100644 --- a/nix/module-eval/bao-matrix-reader.nix +++ b/nix/module-eval/bao-matrix-reader.nix @@ -71,6 +71,14 @@ let deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; }; + # The store's host with the hive's own reader pointed somewhere else. + matrixOnStoreHostReaderMoved = hiveWithController { + deploy.bao.enable = true; + deploy.matrix.enable = true; + deploy.bao.clientCertFile = "/etc/pki/h1.pem"; + deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; + }; + # Did a module refuse this host, and over which option. An assertion is a # config VALUE until something forces it — `.config` never throws — so a # fixture in a state the module refuses stays evaluable and the refusal reads @@ -243,6 +251,36 @@ let && p.ctlBaoClientCertFile == "/var/lib/swarm-bao-pki/matrix-ctl.pem" && p.ctlBaoClientKeyFile == "/var/lib/swarm-bao-pki/matrix-ctl-key.pem"; } + { + # The hive's own reader is not matrix-ctl's identity, and no path is + # derived from it: a homeserver host without the store that holds only + # the hive's reader gets the secret fetch that fails naming the options, + # like one holding nothing. + name = "a homeserver host without the store is refused when it holds only the hive's reader"; + ok = + let + m = baoRemoteReaderMissingLeaves; + u = m.systemd.services.swarm-bao-matrix-oidc; + in + m.services.hyperhive.deploy.matrix.ctlBaoClientCertFile == null + && !(u.environment ? BAO_CLIENT_CERT) + && lib.hasInfix "deploy.matrix.ctlBaoClientCertFile" u.script + && !(lib.hasInfix "bao login" u.script); + } + { + # The default is the file the minting unit writes, wherever the hive's + # own reader is pointed. + name = "on the store's host matrix-ctl's leaf is the one minted there, not one beside the hive's reader"; + ok = + let + p = matrixOnStoreHostReaderMoved.services.hyperhive.deploy.matrix; + mint = matrixOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script; + in + p.ctlBaoClientCertFile == "/var/lib/swarm-bao-pki/matrix-ctl.pem" + && p.ctlBaoClientKeyFile == "/var/lib/swarm-bao-pki/matrix-ctl-key.pem" + && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/matrix-ctl.pem ] ||" mint + && lib.hasInfix "/var/lib/swarm-bao-pki matrix-ctl " mint; + } { # 🩸 The identity separation this whole arrangement buys, stated as the # one thing that would silently undo it. The container gets MATRIX-CTL's diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index 22bb5c6a..9e5ef09b 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -46,6 +46,24 @@ let deploy.nats.autoGenerateCallout = true; }; + # The store's host with the hive's own reader pointed somewhere else. + natsOnStoreHostReaderMoved = hive { + deploy.bao.enable = true; + deploy.nats.enable = true; + deploy.nats.autoGenerateCallout = true; + deploy.bao.clientCertFile = "/etc/pki/h1.pem"; + deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; + }; + + # A queue host holding only the hive's own store reader, placed by hand: no + # store here, and no queue leaf named. + queueWithHiveReaderOnly = hive { + deploy.nats.enable = true; + deploy.nats.autoGenerateCallout = true; + deploy.bao.clientCertFile = "/etc/pki/h1.pem"; + deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; + }; + # A hive that is not the queue's host, with nothing about the queue's # address set by hand: what every hive but one in a multi-host swarm looks # like. The controller is on too, since it may run away from the queue. @@ -215,6 +233,34 @@ let && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script && lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script; } + { + # The hive's own reader is not the queue's identity, and no path is + # derived from it: a queue host without the store that holds only the + # hive's reader gets the leaf unit that fails naming the options, like + # one holding nothing. + name = "a queue host without the store is refused when it holds only the hive's reader"; + ok = + let + u = queueWithHiveReaderOnly.systemd.services.swarm-bao-nats-tls; + in + queueWithHiveReaderOnly.services.hyperhive.deploy.nats.baoClientCertFile == null + && !(u.environment ? BAO_CLIENT_CERT) + && lib.hasInfix "services.hyperhive.deploy.nats.baoClient{Cert,Key}File" u.script; + } + { + # The default is the file the minting unit writes, wherever the hive's + # own reader is pointed. + name = "on the store's host the queue's leaf is the one minted there, not one beside the hive's reader"; + ok = + let + n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats; + mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script; + in + n.baoClientCertFile == "/var/lib/swarm-bao-pki/nats.pem" + && n.baoClientKeyFile == "/var/lib/swarm-bao-pki/nats-key.pem" + && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ] ||" mint + && lib.hasInfix "/var/lib/swarm-bao-pki nats " mint; + } { name = "the server requires TLS: no allow_non_tls"; ok = !(natsContainer.services.nats.settings ? allow_non_tls);