From 56335b24c0cc9346373c62623a6d8dca56d33017 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 30 Sep 2026 21:34:07 +0200 Subject: [PATCH] hive-tls: skip the store root fetch on a hive with no store identity The `want_svc=0` root fetch presents `deploy.bao.clientCertFile`/ `clientKeyFile` to a listener that demands a client certificate. On a hive with neither set, the handshake is refused every time, and `swarm-services-cert` went from green to failed-and-retrying for 24h. The branch is now chosen at eval: with no reader identity, the unit logs that the store root was not staged and names the option, then exits 0, and no fetch is rendered. A hive with an identity fetches as before. `hivectl peer-config` already handles the absent file. Refs #4588 (review c88774, point 2). --- docs/swarm/ca.md | 3 +- docs/swarm/secrets.md | 2 +- nix/host-modules/hive-tls.nix | 75 ++++++++++++++++++++--------------- nix/module-eval/hive-tls.nix | 26 ++++++++++++ 4 files changed, 72 insertions(+), 34 deletions(-) diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 9bdfed91..d4420503 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -65,7 +65,8 @@ gateway can't serve a swarm service that runs behind another. `/var/lib/hive-tls/swarm-services-root.pem` (0644), written beside the leaf it signed and folded into `trust-bundle.pem`. A hive fronting no swarm name asks for no leaf and reads the root alone, from the store's -`pki/ca/pem`. On the host running +`pki/ca/pem`, when it holds a store identity (`deploy.bao.clientCertFile`). +On the host running the store it's also at `/var/lib/swarm-bao-services-pki/services-root.pem`. That's the file to hand a browser, and reading it needs no store login — which matters, diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 327cda36..ad0c727d 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -45,7 +45,7 @@ neither is a renaming of the other. | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | swarm root CA cert | `swarm-ca.nix` first-boot unit, when the operator sets `autoConfigure` | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | | swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | -| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name | +| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and holds a store identity | | authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | | authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | | authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index fdbf6d7f..5d599384 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -306,6 +306,7 @@ let // lib.optionalAttrs (baoDeploy.serverCaFile != null) { BAO_CACERT = baoDeploy.serverCaFile; }; + haveReaderIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null; # The root-only fetch. Same store and server CA as `servicesCertEnvironment`, # but the hive's reader identity: a host fronting no swarm name holds no # services-issuer pair. @@ -314,7 +315,7 @@ let "--cacert" baoDeploy.serverCaFile ] - ++ lib.optionals (baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null) [ + ++ [ "--cert" baoDeploy.clientCertFile "--key" @@ -795,38 +796,48 @@ in # `ca/pem` takes no token; the TLS client certificate is the # store's listener's demand, not bao's, so this presents the # hive's reader identity and never logs in with it. + # A hive with no reader identity has no certificate to present, so it + # skips the fetch and the bundle goes without this root. if [ "$want_svc" = 0 ]; then - echo "no swarm service domains configured — fetching only the store's pki root" - err="$(mktemp)" - raw="$(mktemp "$d/swarm-services-root.XXXXXX")" - trap 'rm -f "$err" "$raw"' EXIT - if ! curl --fail --silent --show-error \ - ${storeRootCurlTls} --output "$raw" \ - "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then - echo "could not read the store's ${baoServicesPkiMount} root." >&2 - cat "$err" >&2 - exit 1 - fi - # Re-encoded rather than copied: it fails on a body that does not - # parse as a certificate, and ends the file in the newline the - # bundle's `cat` relies on. - if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then - echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2 - cat "$err" >&2 - exit 1 - fi - if cmp -s "$svcroot.new" "$svcroot"; then - rm -f "$svcroot.new" - echo "the store's pki root is unchanged" - exit 0 - fi - mv -f "$svcroot.new" "$svcroot" - chmod 0644 "$svcroot" - if systemctl is-active --quiet hive-tls-ca.service; then - echo "the services root changed — rebuilding the trust bundle" - systemctl restart --no-block hive-tls-ca.service - fi - exit 0 + ${ + if haveReaderIdentity then + '' + echo "no swarm service domains configured — fetching only the store's pki root" + err="$(mktemp)" + raw="$(mktemp "$d/swarm-services-root.XXXXXX")" + trap 'rm -f "$err" "$raw"' EXIT + if ! curl --fail --silent --show-error \ + ${storeRootCurlTls} --output "$raw" \ + "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then + echo "could not read the store's ${baoServicesPkiMount} root." >&2 + cat "$err" >&2 + exit 1 + fi + # Re-encoded rather than copied: it fails on a body that does not + # parse as a certificate, and ends the file in the newline the + # bundle's `cat` relies on. + if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then + echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2 + cat "$err" >&2 + exit 1 + fi + if cmp -s "$svcroot.new" "$svcroot"; then + rm -f "$svcroot.new" + echo "the store's pki root is unchanged" + exit 0 + fi + mv -f "$svcroot.new" "$svcroot" + chmod 0644 "$svcroot" + if systemctl is-active --quiet hive-tls-ca.service; then + echo "the services root changed — rebuilding the trust bundle" + systemctl restart --no-block hive-tls-ca.service + fi + exit 0'' + else + '' + echo "store pki root not staged: this hive has no store identity (services.hyperhive.deploy.bao.clientCertFile/clientKeyFile)" + exit 0'' + } fi # Re-issue when the file is missing, past half the lifetime the diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 6e92a050..8f2a14dd 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -46,6 +46,17 @@ let }; noSvcFetch = noService.systemd.services.swarm-services-cert.script; + # The same hive with no reader identity at all. + noIdentity = hive { }; + noIdFetch = noIdentity.systemd.services.swarm-services-cert.script; + + # The `want_svc=0` branch of a rendered script, up to its closing `fi`. + noSvcBranch = + script: + builtins.head ( + lib.splitString "\nfi\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script)) + ); + units = allLocal.systemd.services; boot = units.swarm-services-cert; timer = allLocal.systemd.timers.swarm-services-cert-renew; @@ -147,6 +158,21 @@ let + " --key ${lib.escapeShellArg "/etc/pki/bao-client-key.pem"}" ) noSvcFetch; } + { + # Nothing to present means no fetch to fail: the unit stays green and + # says why the root is missing. The control is the same hive with an + # identity, whose branch does fetch and can fail. + name = "a hive fronting no swarm name with no store identity skips the fetch and cannot fail on it"; + ok = + noIdentity.services.hyperhive.swarm.localServiceDomains == [ ] + && noIdentity.services.hyperhive.deploy.bao.clientCertFile == null + && !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch) + && lib.hasInfix "store pki root not staged" (noSvcBranch noIdFetch) + && lib.hasInfix "deploy.bao.clientCertFile" (noSvcBranch noIdFetch) + && !(lib.hasInfix "exit 1" (noSvcBranch noIdFetch)) + && lib.hasInfix "exit 1" (noSvcBranch noSvcFetch) + && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch); + } { # The fetch retries against a sealed store; a gateway `Requires=` on it # would go down with each attempt. The control is a host fronting