hive-tls: skip the store root fetch on a hive with no store identity
The `want_svc=0` root fetch presents `deploy.bao.clientCertFile`/ `clientKeyFile` to a listener that demands a client certificate. On a hive with neither set, the handshake is refused every time, and `swarm-services-cert` went from green to failed-and-retrying for 24h. The branch is now chosen at eval: with no reader identity, the unit logs that the store root was not staged and names the option, then exits 0, and no fetch is rendered. A hive with an identity fetches as before. `hivectl peer-config` already handles the absent file. Refs #4588 (review c88774, point 2).
This commit is contained in:
atlas
mara
parent
72f5e39278
commit
56335b24c0
4 changed files with 72 additions and 34 deletions
|
|
@ -65,7 +65,8 @@ gateway can't serve a swarm service that runs behind another.
|
|||
`/var/lib/hive-tls/swarm-services-root.pem` (0644), written beside the
|
||||
leaf it signed and folded into `trust-bundle.pem`. A hive fronting no
|
||||
swarm name asks for no leaf and reads the root alone, from the store's
|
||||
`pki/ca/pem`. On the host running
|
||||
`pki/ca/pem`, when it holds a store identity (`deploy.bao.clientCertFile`).
|
||||
On the host running
|
||||
the store it's also at
|
||||
`/var/lib/swarm-bao-services-pki/services-root.pem`. That's the file to
|
||||
hand a browser, and reading it needs no store login — which matters,
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ neither is a renaming of the other.
|
|||
| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| swarm root CA cert | `swarm-ca.nix` first-boot unit, when the operator sets `autoConfigure` | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public |
|
||||
| swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below |
|
||||
| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name |
|
||||
| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and holds a store identity |
|
||||
| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them |
|
||||
| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same |
|
||||
| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` |
|
||||
|
|
|
|||
|
|
@ -306,6 +306,7 @@ let
|
|||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
haveReaderIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null;
|
||||
# The root-only fetch. Same store and server CA as `servicesCertEnvironment`,
|
||||
# but the hive's reader identity: a host fronting no swarm name holds no
|
||||
# services-issuer pair.
|
||||
|
|
@ -314,7 +315,7 @@ let
|
|||
"--cacert"
|
||||
baoDeploy.serverCaFile
|
||||
]
|
||||
++ lib.optionals (baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null) [
|
||||
++ [
|
||||
"--cert"
|
||||
baoDeploy.clientCertFile
|
||||
"--key"
|
||||
|
|
@ -795,38 +796,48 @@ in
|
|||
# `ca/pem` takes no token; the TLS client certificate is the
|
||||
# store's listener's demand, not bao's, so this presents the
|
||||
# hive's reader identity and never logs in with it.
|
||||
# A hive with no reader identity has no certificate to present, so it
|
||||
# skips the fetch and the bundle goes without this root.
|
||||
if [ "$want_svc" = 0 ]; then
|
||||
echo "no swarm service domains configured — fetching only the store's pki root"
|
||||
err="$(mktemp)"
|
||||
raw="$(mktemp "$d/swarm-services-root.XXXXXX")"
|
||||
trap 'rm -f "$err" "$raw"' EXIT
|
||||
if ! curl --fail --silent --show-error \
|
||||
${storeRootCurlTls} --output "$raw" \
|
||||
"$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then
|
||||
echo "could not read the store's ${baoServicesPkiMount} root." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Re-encoded rather than copied: it fails on a body that does not
|
||||
# parse as a certificate, and ends the file in the newline the
|
||||
# bundle's `cat` relies on.
|
||||
if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then
|
||||
echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
if cmp -s "$svcroot.new" "$svcroot"; then
|
||||
rm -f "$svcroot.new"
|
||||
echo "the store's pki root is unchanged"
|
||||
exit 0
|
||||
fi
|
||||
mv -f "$svcroot.new" "$svcroot"
|
||||
chmod 0644 "$svcroot"
|
||||
if systemctl is-active --quiet hive-tls-ca.service; then
|
||||
echo "the services root changed — rebuilding the trust bundle"
|
||||
systemctl restart --no-block hive-tls-ca.service
|
||||
fi
|
||||
exit 0
|
||||
${
|
||||
if haveReaderIdentity then
|
||||
''
|
||||
echo "no swarm service domains configured — fetching only the store's pki root"
|
||||
err="$(mktemp)"
|
||||
raw="$(mktemp "$d/swarm-services-root.XXXXXX")"
|
||||
trap 'rm -f "$err" "$raw"' EXIT
|
||||
if ! curl --fail --silent --show-error \
|
||||
${storeRootCurlTls} --output "$raw" \
|
||||
"$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then
|
||||
echo "could not read the store's ${baoServicesPkiMount} root." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Re-encoded rather than copied: it fails on a body that does not
|
||||
# parse as a certificate, and ends the file in the newline the
|
||||
# bundle's `cat` relies on.
|
||||
if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then
|
||||
echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
if cmp -s "$svcroot.new" "$svcroot"; then
|
||||
rm -f "$svcroot.new"
|
||||
echo "the store's pki root is unchanged"
|
||||
exit 0
|
||||
fi
|
||||
mv -f "$svcroot.new" "$svcroot"
|
||||
chmod 0644 "$svcroot"
|
||||
if systemctl is-active --quiet hive-tls-ca.service; then
|
||||
echo "the services root changed — rebuilding the trust bundle"
|
||||
systemctl restart --no-block hive-tls-ca.service
|
||||
fi
|
||||
exit 0''
|
||||
else
|
||||
''
|
||||
echo "store pki root not staged: this hive has no store identity (services.hyperhive.deploy.bao.clientCertFile/clientKeyFile)"
|
||||
exit 0''
|
||||
}
|
||||
fi
|
||||
|
||||
# Re-issue when the file is missing, past half the lifetime the
|
||||
|
|
|
|||
|
|
@ -46,6 +46,17 @@ let
|
|||
};
|
||||
noSvcFetch = noService.systemd.services.swarm-services-cert.script;
|
||||
|
||||
# The same hive with no reader identity at all.
|
||||
noIdentity = hive { };
|
||||
noIdFetch = noIdentity.systemd.services.swarm-services-cert.script;
|
||||
|
||||
# The `want_svc=0` branch of a rendered script, up to its closing `fi`.
|
||||
noSvcBranch =
|
||||
script:
|
||||
builtins.head (
|
||||
lib.splitString "\nfi\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script))
|
||||
);
|
||||
|
||||
units = allLocal.systemd.services;
|
||||
boot = units.swarm-services-cert;
|
||||
timer = allLocal.systemd.timers.swarm-services-cert-renew;
|
||||
|
|
@ -147,6 +158,21 @@ let
|
|||
+ " --key ${lib.escapeShellArg "/etc/pki/bao-client-key.pem"}"
|
||||
) noSvcFetch;
|
||||
}
|
||||
{
|
||||
# Nothing to present means no fetch to fail: the unit stays green and
|
||||
# says why the root is missing. The control is the same hive with an
|
||||
# identity, whose branch does fetch and can fail.
|
||||
name = "a hive fronting no swarm name with no store identity skips the fetch and cannot fail on it";
|
||||
ok =
|
||||
noIdentity.services.hyperhive.swarm.localServiceDomains == [ ]
|
||||
&& noIdentity.services.hyperhive.deploy.bao.clientCertFile == null
|
||||
&& !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch)
|
||||
&& lib.hasInfix "store pki root not staged" (noSvcBranch noIdFetch)
|
||||
&& lib.hasInfix "deploy.bao.clientCertFile" (noSvcBranch noIdFetch)
|
||||
&& !(lib.hasInfix "exit 1" (noSvcBranch noIdFetch))
|
||||
&& lib.hasInfix "exit 1" (noSvcBranch noSvcFetch)
|
||||
&& lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch);
|
||||
}
|
||||
{
|
||||
# The fetch retries against a sealed store; a gateway `Requires=` on it
|
||||
# would go down with each attempt. The control is a host fronting
|
||||
|
|
|
|||
Loading…
Reference in a new issue