From a66670be8765d15338854bdda860216bed71bcba Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 6 Oct 2026 13:32:55 +0200 Subject: [PATCH] hive-tls: the root fetch presents the hive identity, as on main On a hive fronting no swarm name, swarm-services-cert's token-less GET pki/ca/pem presents baoHiveIdentity{Cert,Key}File again. The fetch never logs in and grants nothing; the client certificate only completes the store listener's mTLS. A hive host without the store therefore needs no copy of the services-issuer pair. The issue path, which logs in to pki/issue/swarm-services, still presents only deploy.bao.servicesIssuerClient{Cert,Key}File. module-eval-hive-tls: the root-fetch cases pin the hive identity and the absence of the services-issuer pair (a fixture holding both, and one holding only the hive identity); the issue-path case pins the pair on a hive host and on a host without a hive; a gateway without the pair evaluates and its issue path names both options. Root-fetch identity: option (a), mara on PR #5002 (comment 92591). Refs #4921 --- docs/swarm/ca.md | 29 +++--- nix/host-modules/hive-tls.nix | 39 ++++---- nix/host-modules/swarm-bao.nix | 20 +++-- nix/module-eval/hive-tls.nix | 158 +++++++++++++++++++-------------- 4 files changed, 143 insertions(+), 103 deletions(-) diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 5b5f6416..a052e042 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -39,10 +39,11 @@ from the store's own client CA, with the subject and set both options, before the host's next deploy. Without it, `hive-tls-resign.service` fails with a message naming both -options. The unit checks the identity before the leaf, so it fails even while a +options, and so does `swarm-services-cert.service` on a hive fronting no +swarm name. The unit checks the identity before the leaf, so it fails even while a valid leaf keeps the gateway up. The identity is the hive's alone: -it issues this hive's leaf and nothing else, and it's never the identity a -swarm service uses. +it issues this hive's leaf and fetches the store's root, and it's never the +identity a swarm service uses. **When it issues.** The unit runs at boot, on every deploy, and from `hive-tls-resign.timer` once a day. It asks the store for a new leaf when @@ -113,8 +114,10 @@ the services-issuer identity, calls copies from. The store's role, not a key on the host, decides which names a gateway can serve. -**Every gateway that serves the self-signed leaf needs the services-issuer -identity**, whether or not a hive runs on its host. It's a client certificate +**Every self-signed gateway that fronts a swarm name needs the +services-issuer identity**, whether or not a hive runs on its host. A hive +fronting none needs no copy of it: it fetches the store's root with its own +store identity, as below. It's a client certificate from the store's own client CA, with the subject `deploy.bao.servicesIssuerCommonName` (`swarm-services-issuer` by default), and it's named by `services.hyperhive.deploy.bao.servicesIssuerClientCertFile` @@ -125,11 +128,11 @@ every gateway host: - On the host running the store, `swarm-bao-pki.service` mints it at `/var/lib/swarm-bao-pki/services-issuer.pem` and the two options default to it. -- On every other gateway host, copy `services-issuer.pem` and - `services-issuer-key.pem` from the store's host and set both options. +- On every other gateway host fronting a swarm name, copy + `services-issuer.pem` and `services-issuer-key.pem` from the store's host + and set both options. -A host whose gateway serves the self-signed leaf without both options set -still evaluates and deploys, and `swarm-services-cert.service` fails, naming +A host fronting a swarm name without both options set still evaluates and deploys, and `swarm-services-cert.service` fails, naming them. One that names files it can't read fails the same unit, naming the paths. The old spellings, `deploy.hive-controller.tls.baoClientCertFile` and `baoClientKeyFile`, are @@ -146,10 +149,12 @@ gateway can't serve a swarm service that runs behind another. **The root's public certificate is a file, on every self-signed gateway host:** `/var/lib/hive-tls/swarm-services-root.pem` (0644), written beside the -leaf it signed and folded into `trust-bundle.pem`. A host fronting no +leaf it signed and folded into `trust-bundle.pem`. A hive fronting no swarm name asks for no services leaf and reads the root alone, from the store's -`pki/ca/pem`, presenting the services-issuer identity for the handshake -without logging in with it. With either file absent, +`pki/ca/pem`, presenting its own store identity +(`deploy.hive-controller.tls.baoHiveIdentityCertFile` / +`baoHiveIdentityKeyFile`) for the handshake without logging in with it. +Every hive must hold one: without it, or with either file absent, `swarm-services-cert.service` fails with a message naming both options. On the host running the store it's also at diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index b8d92d2d..1f27697e 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -116,7 +116,9 @@ let # The hive's own store identity: its subject is # `-`, and its policy may issue from - # `hiveLeafRole` alone. Every hive must hold one. + # `hiveLeafRole` alone. It also carries the root fetch's handshake. Every + # hive must hold one. + haveHiveIdentity = cfg.baoHiveIdentityCertFile != null && cfg.baoHiveIdentityKeyFile != null; requireHiveIdentity = requireIdentity { holder = "this hive"; what = "store identity"; @@ -126,13 +128,12 @@ let keyOption = "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile"; }; - # The swarm-wide services-issuer identity `swarm-services-cert` presents, - # whether or not a hive runs here. Its subject is - # `servicesIssuerCommonName`, and its policy may issue from + # The swarm-wide services-issuer identity `swarm-services-cert` logs in + # with to issue the services leaf, whether or not a hive runs here. Its + # subject is `servicesIssuerCommonName`, and its policy may issue from # `pki/issue/swarm-services` alone. servicesIssuerCert = baoDeploy.servicesIssuerClientCertFile; servicesIssuerKey = baoDeploy.servicesIssuerClientKeyFile; - haveServicesIssuer = servicesIssuerCert != null && servicesIssuerKey != null; requireServicesIssuer = requireIdentity { holder = "this host"; what = "services-issuer identity"; @@ -244,8 +245,9 @@ let BAO_CACERT = baoDeploy.serverCaFile; }; - # The root-only fetch: the same store, server CA and services-issuer pair - # as `servicesCertEnvironment`, handed to curl. + # The root-only fetch. Same store and server CA as `servicesCertEnvironment`, + # but the hive's own identity: a host fronting no swarm name holds no + # services-issuer pair. storeRootCurlTls = lib.escapeShellArgs ( lib.optionals (baoDeploy.serverCaFile != null) [ "--cacert" @@ -253,9 +255,9 @@ let ] ++ [ "--cert" - servicesIssuerCert + cfg.baoHiveIdentityCertFile "--key" - servicesIssuerKey + cfg.baoHiveIdentityKeyFile ] ); @@ -303,15 +305,17 @@ in example = "/var/lib/swarm-bao-pki/hive-leaf.pem"; description = '' This hive's own store identity. `hive-tls-resign` presents it to ask - the secret store for the gateway leaf. + the secret store for the gateway leaf, and `swarm-services-cert` on a + hive fronting no swarm name presents it for the handshake of the root + fetch. Its subject has to be `-`; that cert-auth role's policy may issue from this hive's gateway-leaf role and nothing else. - Every hive with a self-signed gateway needs it: without it that unit - fails, naming this option and + Every hive with a self-signed gateway needs it: without it both units + fail, naming this option and {option}`services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile`. On the host that runs the store a glue module supplies the leaf as a `mkDefault`; on every other host an operator places it. @@ -637,12 +641,13 @@ in # bundle: every other hive's swarm-service names chain to it. # `ca/pem` takes no token; the TLS client certificate is the # store's listener's demand, not bao's, so this presents the - # services-issuer pair and never logs in with it. Without a - # readable pair the unit fails here, naming the options. + # hive's own identity and never logs in with it. + # A hive without a readable identity fails here, naming the + # options: every hive must hold one. if [ "$want_svc" = 0 ]; then ${ - if haveServicesIssuer then - requireServicesIssuer "fetch the store's ${baoServicesPkiMount} root" + if haveHiveIdentity then + requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root" + '' echo "no swarm service domains configured — fetching only the store's pki root" err="$(mktemp)" @@ -676,7 +681,7 @@ in fi exit 0'' else - requireServicesIssuer "fetch the store's ${baoServicesPkiMount} root" + requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root" } fi diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index c84b220a..d73a2c3f 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1955,10 +1955,8 @@ in default = null; example = "/var/lib/swarm-bao-pki/services-issuer.pem"; description = '' - Certificate `swarm-services-cert` presents to the store: to log in and - ask for the swarm-services leaf the gateway serves, and for the - handshake of the root fetch on a host fronting no swarm name. Its - subject must be + Certificate `swarm-services-cert` logs in to the store with to ask for + the swarm-services leaf the gateway serves. Its subject must be {option}`services.hyperhive.deploy.bao.servicesIssuerCommonName`. ⚠️ Neither the reader's @@ -1967,13 +1965,17 @@ in `pki/issue/swarm-services` and nothing else, and it is the same on every gateway host, whether or not a hive runs there. - Required on every host whose gateway serves the self-signed leaf. - Unset, the host evaluates and `swarm-services-cert` fails, naming - both options. + Required on every self-signed gateway host that fronts a swarm-service + name ({option}`services.hyperhive.swarm.localServiceDomains` not + empty). Unset there, the host evaluates and `swarm-services-cert` + fails, naming both options. A host fronting none only fetches the + store's root, presenting + {option}`services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile`, + and needs no services-issuer pair. ./glue-bao-tls.nix points it at the leaf it mints on the store's host. - On a gateway host without the store, it is a file the operator copies - from the store's host. A path, never a value. + On any other gateway host that needs it, it is a file the operator + copies from the store's host. A path, never a value. ''; }; diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 22576502..686b7709 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -54,6 +54,17 @@ let }; noSvcFetch = noService.systemd.services.swarm-services-cert.script; + # The same hive holding its own identity and no services-issuer pair: the + # shape of every hive host without the store that fronts no swarm name. + noServiceNoPair = hiveWithController { + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem"; + deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem"; + deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem"; + }; + noPairFetch = noServiceNoPair.systemd.services.swarm-services-cert.script; + # The same hive with no store identity at all. noIdentity = hiveWithController { }; noIdFetch = noIdentity.systemd.services.swarm-services-cert.script; @@ -83,6 +94,15 @@ let lib.splitString "\nfi\n\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script)) ); + # Everything after that branch: the path that logs in and issues the leaf. + issuePath = + script: + lib.concatStringsSep "\nfi\n\n" ( + lib.tail ( + lib.splitString "\nfi\n\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script)) + ) + ); + units = allLocal.systemd.services; boot = units.swarm-services-cert; caUnit = units.hive-tls-ca; @@ -210,55 +230,59 @@ let && lib.hasInfix ''cat "$svcroot" > "$bundle"'' noService.systemd.services.hive-tls-ca.script; } { - # The services-issuer pair, not the reader identity, whose policy reads - # every secret the hive is entitled to, and not the hive's own. - name = "the root fetch reaches the store with the services-issuer pair and server CA"; + # The hive's own identity, not its reader identity, whose policy reads + # every secret the hive is entitled to, and not the services-issuer + # pair, which `noService` also holds. A hive holding no pair fetches + # the same way. + name = "the root fetch reaches the store with the hive's own identity and server CA, never the services-issuer pair"; ok = + let + hiveTls = + "--cert ${lib.escapeShellArg "/etc/pki/bao-hive.pem"}" + + " --key ${lib.escapeShellArg "/etc/pki/bao-hive-key.pem"}"; + in lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch - && lib.hasInfix ( - "--cert ${lib.escapeShellArg "/etc/pki/bao-services-issuer.pem"}" - + " --key ${lib.escapeShellArg "/etc/pki/bao-services-issuer-key.pem"}" - ) noSvcFetch + && lib.hasInfix hiveTls (noSvcBranch noSvcFetch) && !(lib.hasInfix "/etc/pki/bao-client.pem" (noSvcBranch noSvcFetch)) - && !(lib.hasInfix "/etc/pki/bao-hive" (noSvcBranch noSvcFetch)); + && !(lib.hasInfix "/etc/pki/bao-services-issuer" (noSvcBranch noSvcFetch)) + && noServiceNoPair.services.hyperhive.deploy.bao.servicesIssuerClientCertFile == null + && lib.hasInfix hiveTls (noSvcBranch noPairFetch) + && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noPairFetch) + && !(lib.hasInfix "servicesIssuerClient" (noSvcBranch noPairFetch)); } { # No identity to present: the unit fails, naming both options, rather # than leaving the bundle without the root behind a green unit. The - # control is the same hive with the pair, whose branch fetches. - name = "a host fronting no swarm name with no services-issuer pair fails the unit, naming both options"; + # control is the same hive with an identity, whose branch fetches. + name = "a hive fronting no swarm name with no store identity fails the unit, naming the hive identity's options"; ok = noIdentity.services.hyperhive.swarm.localServiceDomains == [ ] - && noIdentity.services.hyperhive.deploy.bao.servicesIssuerClientCertFile == null + && noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null && !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch) - && lib.hasInfix "servicesIssuerClientCertFile (unset)" (noSvcBranch noIdFetch) - && lib.hasInfix "servicesIssuerClientKeyFile (unset)" (noSvcBranch noIdFetch) + && lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch) + && lib.hasInfix "baoHiveIdentityKeyFile (unset)" (noSvcBranch noIdFetch) + && !(lib.hasInfix "servicesIssuerClient" (noSvcBranch noIdFetch)) && lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (noSvcBranch noIdFetch)) && !(lib.hasInfix "exit 0" (noSvcBranch noIdFetch)) && !(lib.hasInfix "not staged" noIdFetch) && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch); } { - # A pair named but not on disk fails before curl, naming the paths. - name = "a host fronting no swarm name whose services-issuer files are absent fails the unit, naming the paths"; + # An identity named but not on disk fails before curl, naming the paths. + name = "a hive fronting no swarm name whose identity files are absent fails the unit, naming the paths"; ok = - lib.hasInfix "for f in /etc/pki/bao-services-issuer.pem /etc/pki/bao-services-issuer-key.pem; do" ( - noSvcBranch noSvcFetch - ) + lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (noSvcBranch noSvcFetch) && lib.hasInfix ''[ -r "$f" ] || missing="$missing $f"'' (noSvcBranch noSvcFetch) && lib.hasInfix "cannot read$missing" (noSvcBranch noSvcFetch) - && lib.hasInfix "servicesIssuerClientCertFile (/etc/pki/bao-services-issuer.pem)" ( - noSvcBranch noSvcFetch - ) - && lib.hasInfix "servicesIssuerClientKeyFile (/etc/pki/bao-services-issuer-key.pem)" ( - noSvcBranch noSvcFetch - ); + && lib.hasInfix "baoHiveIdentityCertFile (/etc/pki/bao-hive.pem)" (noSvcBranch noSvcFetch) + && lib.hasInfix "baoHiveIdentityKeyFile (/etc/pki/bao-hive-key.pem)" (noSvcBranch noSvcFetch); } { - # A gateway host with no hive, fronting no swarm name, the pair placed - # by hand. The control is the hive host, whose gateway-leaf unit does - # present the hive identity, so the scan can find one. - name = "on a host without a hive the swarm-services units present no hive identity, and the root fetch runs"; + # Each path presents one identity: the issue path the services-issuer + # pair, on a hive host and on a host without a hive alike, and the root + # fetch the hive's own. The control for the scan is `hive-tls-resign`, + # which does present the hive identity. + name = "the issue path presents only the services-issuer pair, and the root fetch only the hive identity"; ok = let hiveRefs = [ @@ -266,33 +290,40 @@ let "hive-leaf" "/etc/pki/bao-hive" ]; - refsHive = - units: - lib.any ( - u: lib.any (r: lib.hasInfix r u.script || lib.hasInfix r (builtins.toJSON u.environment)) hiveRefs - ) units; - units = gatewayNoHive.systemd.services; - s = units.swarm-services-cert; + refsHive = s: lib.any (r: lib.hasInfix r s) hiveRefs; + issuesWithPair = + m: + let + u = m.systemd.services; + path = issuePath u.swarm-services-cert.script; + in + lib.hasInfix "for f in /etc/pki/bao-services-issuer.pem /etc/pki/bao-services-issuer-key.pem; do" path + && !(refsHive path) + && + lib.all + ( + e: + e.BAO_CLIENT_CERT == "/etc/pki/bao-services-issuer.pem" + && e.BAO_CLIENT_KEY == "/etc/pki/bao-services-issuer-key.pem" + ) + [ + u.swarm-services-cert.environment + u.swarm-services-cert-renew.environment + ]; in - !gatewayNoHive.services.hyperhive.deploy.hive-controller.enable - && gatewayNoHive.services.hyperhive.swarm.localServiceDomains == [ ] - && !(refsHive [ - s - units.swarm-services-cert-renew - ]) - && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch s.script) - && lib.hasInfix "--cert ${lib.escapeShellArg "/etc/pki/bao-services-issuer.pem"}" ( - noSvcBranch s.script - ) - && s.environment.BAO_CLIENT_CERT == "/etc/pki/bao-services-issuer.pem" - && refsHive [ noService.systemd.services.hive-tls-resign ]; + issuesWithPair noService + && issuesWithPair gatewayNoHive + && !gatewayNoHive.services.hyperhive.deploy.hive-controller.enable + && refsHive (noSvcBranch noSvcFetch) + && !(lib.hasInfix "/etc/pki/bao-services-issuer" (noSvcBranch noSvcFetch)) + && refsHive noService.systemd.services.hive-tls-resign.script; } { # A host with no store and no pair placed is a supported shape: it - # evaluates, and its unit goes red. The control is the same host with - # the pair placed, whose branch fetches, and the store's host, where - # the glue supplies the pair. - name = "a self-signed gateway with no services-issuer pair evaluates, and its unit fails naming both options"; + # evaluates, and its issue path goes red. A hive among them still + # fetches the root. The controls are the same host without a hive with + # the pair placed, and the store's host, where the glue supplies it. + name = "a self-signed gateway with no services-issuer pair evaluates, and its issue path fails naming both options"; ok = let refused = @@ -305,24 +336,21 @@ let || lib.hasInfix "servicesIssuerClientKeyFile" a.message ) ) m.assertions; - redUnit = + redIssue = m: - let - branch = noSvcBranch m.systemd.services.swarm-services-cert.script; - in - lib.hasInfix "services.hyperhive.deploy.bao.servicesIssuerClientCertFile (unset)" branch - && lib.hasInfix "services.hyperhive.deploy.bao.servicesIssuerClientKeyFile (unset)" branch - && lib.hasSuffix "exit 1" (lib.removeSuffix "\n" branch) - && !(lib.hasInfix "/v1/pki/ca/pem" branch); + lib.hasInfix ( + "set services.hyperhive.deploy.bao.servicesIssuerClientCertFile (unset)" + + " and services.hyperhive.deploy.bao.servicesIssuerClientKeyFile (unset)" + ) (issuePath m.systemd.services.swarm-services-cert.script); in !(refused gatewayNoHiveNoPair) && !(refused noIdentity) - && redUnit gatewayNoHiveNoPair - && redUnit noIdentity - && !(redUnit gatewayNoHive) - && lib.hasInfix "/v1/pki/ca/pem" ( - noSvcBranch gatewayNoHive.systemd.services.swarm-services-cert.script - ) + && !(refused noServiceNoPair) + && redIssue gatewayNoHiveNoPair + && redIssue noIdentity + && redIssue noServiceNoPair + && !(redIssue gatewayNoHive) + && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noPairFetch) && allLocal.services.hyperhive.deploy.bao.servicesIssuerClientCertFile == "/var/lib/swarm-bao-pki/services-issuer.pem";