diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 0a8f7094..9bdfed91 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -63,7 +63,9 @@ gateway can't serve a swarm service that runs behind another. **The root's public certificate is a file, on every hive:** `/var/lib/hive-tls/swarm-services-root.pem` (0644), written beside the -leaf it signed and folded into `trust-bundle.pem`. On the host running +leaf it signed and folded into `trust-bundle.pem`. A hive fronting no +swarm name asks for no leaf and reads the root alone, from the store's +`pki/ca/pem`. On the host running the store it's also at `/var/lib/swarm-bao-services-pki/services-root.pem`. That's the file to hand a browser, and reading it needs no store login — which matters, @@ -124,6 +126,11 @@ host: scp /var/lib/swarm-ca/root.pem :/var/lib/swarm-ca/root.pem ``` +It then prints the secret store's `pki` root itself, read from +`/var/lib/hive-tls/swarm-services-root.pem`. The store issues the swarm's +service names under that root, not the swarm root, so a peer host trusting only +the swarm root can't verify them. + That's the point of the hierarchy: the operator installs the root **once per swarm host**, not once per peer, so a hive joining later needs no edit on the hives already running. A hive whose cert already chains to a public diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 8e4df7a5..327cda36 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -45,7 +45,7 @@ neither is a renaming of the other. | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | swarm root CA cert | `swarm-ca.nix` first-boot unit, when the operator sets `autoConfigure` | `/var/lib/swarm-ca/root.pem` | operator copies the **cert** in; it's public | | swarm root CA key | same unit | `/var/lib/swarm-ca/root-key.pem`, `0600` | stays on whichever host holds it — see the constraint below | -| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for | +| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name | | authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | | authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | | authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | diff --git a/hivectl/src/wg.rs b/hivectl/src/wg.rs index ab86431d..7c001566 100644 --- a/hivectl/src/wg.rs +++ b/hivectl/src/wg.rs @@ -37,6 +37,12 @@ const HIVE_TLS_TRUST_BUNDLE_PATH: &str = "/var/lib/hive-tls/trust-bundle.pem"; /// the per-hive CA pinning. const SWARM_CA_ROOT_PATH: &str = "/var/lib/swarm-ca/root.pem"; +/// Host path of the secret store's `pki` root (hive-tls.nix's +/// `$svcroot`, folded into the trust bundle). The swarm's service names +/// are issued under it, so a peer trusting only the swarm root above +/// cannot validate them. +const STORE_PKI_ROOT_PATH: &str = "/var/lib/hive-tls/swarm-services-root.pem"; + /// Attrset key for a hive in `services.hyperhive.swarm.hives`, derived /// from its domain's first DNS label. /// @@ -180,11 +186,29 @@ pub(crate) fn wg_peer(domain: &str, pubkey: &str, address: &str, endpoint: Optio println!(" # the key must be that hive's services.hyperhive.hiveName"); } +/// The `peer-config` step carrying the secret store's `pki` root: the +/// certificate itself, or where it will appear when this hive has not +/// fetched it yet. +fn store_root_step(pem: Option<&str>) -> String { + const HEAD: &str = "# 2. trust the SECRET STORE's pki root on the peer host (once, not per hive);\n\ + # the swarm's service names are issued under it:\n"; + match pem.map(str::trim_end).filter(|p| !p.is_empty()) { + Some(p) => format!( + "{HEAD}{p}\n\ + # (a self-signed hive in this swarm fetches it itself, into {STORE_PKI_ROOT_PATH})\n" + ), + None => format!( + "{HEAD}# not on this host yet: swarm-services-cert.service writes it to\n\ + # {STORE_PKI_ROOT_PATH}; rerun once that unit has succeeded\n" + ), + } +} + /// `peer-config` — print the `swarm.hives.""` block a peer -/// operator pastes to federate with THIS hive, plus the swarm-root -/// install step when this hive serves a self-signed chain. Reads local -/// state only (the TLS trust bundle's presence + the wg key); prints, -/// never mutates. +/// operator pastes to federate with THIS hive, plus the swarm-root and +/// store-root steps when this hive serves a self-signed chain. Reads +/// local state only (the TLS trust bundle, the store root + the wg key); +/// prints, never mutates. pub(crate) fn peer_config(domain: &str, wg_address: Option<&str>, wg_endpoint: Option<&str>) { let self_signed = Path::new(HIVE_TLS_TRUST_BUNDLE_PATH).exists(); let key = hive_key(domain); @@ -202,7 +226,10 @@ pub(crate) fn peer_config(domain: &str, wg_address: Option<&str>, wg_endpoint: O println!("scp {SWARM_CA_ROOT_PATH} :{SWARM_CA_ROOT_PATH}"); println!("# (skip if that host already has the swarm root)"); println!(); - println!("# 2. paste into the peer hive's NixOS config:"); + let store_root = std::fs::read_to_string(STORE_PKI_ROOT_PATH).ok(); + print!("{}", store_root_step(store_root.as_deref())); + println!(); + println!("# 3. paste into the peer hive's NixOS config:"); } else { println!("# paste into the peer hive's NixOS config:"); } @@ -243,3 +270,30 @@ pub(crate) fn wg_status() -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::{STORE_PKI_ROOT_PATH, store_root_step}; + + const PEM: &str = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----"; + + /// The certificate is printed whole, on lines of its own, so a peer + /// operator can copy it out of the output as a PEM file. + #[test] + fn store_root_is_printed_verbatim() { + let s = store_root_step(Some(&format!("{PEM}\n"))); + assert!(s.contains(&format!("\n{PEM}\n")), "{s}"); + } + + /// A hive that has not fetched the root yet says where it will be, + /// rather than printing an empty step. + #[test] + fn missing_store_root_names_the_path_and_the_unit() { + for pem in [None, Some(""), Some("\n")] { + let s = store_root_step(pem); + assert!(s.contains(STORE_PKI_ROOT_PATH), "{s}"); + assert!(s.contains("swarm-services-cert.service"), "{s}"); + assert!(!s.contains("BEGIN CERTIFICATE"), "{s}"); + } + } +} diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 8286ee7f..fdbf6d7f 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -42,7 +42,12 @@ let # path down whenever it is sealed. There the gateway starts on the hive # leaf fallback, and the issuing script re-imports the services leaf into # the running nginx once it lands. - gatewayWaitsForServicesLeaf = localServiceDomains != [ baoCfg.ui.domain ]; + # + # Nor on a host fronting no swarm name: it asks the store for the root + # alone, which nginx never reads, and a sealed store would otherwise hold + # its gateway down. + gatewayWaitsForServicesLeaf = + localServiceDomains != [ ] && localServiceDomains != [ baoCfg.ui.domain ]; # The host-managed hive CA is the trust anchor for self-signed mode. # It is only stood up when the gateway actually serves a self-signed @@ -283,6 +288,8 @@ let pkgs.diffutils # `flock`, for the lock at the top of the script. pkgs.util-linux + # The root-only fetch on a host fronting no swarm name. + pkgs.curl ]; servicesCertEnvironment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -299,6 +306,21 @@ let // lib.optionalAttrs (baoDeploy.serverCaFile != null) { BAO_CACERT = baoDeploy.serverCaFile; }; + # The root-only fetch. Same store and server CA as `servicesCertEnvironment`, + # but the hive's reader identity: a host fronting no swarm name holds no + # services-issuer pair. + storeRootCurlTls = lib.escapeShellArgs ( + lib.optionals (baoDeploy.serverCaFile != null) [ + "--cacert" + baoDeploy.serverCaFile + ] + ++ lib.optionals (baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null) [ + "--cert" + baoDeploy.clientCertFile + "--key" + baoDeploy.clientKeyFile + ] + ); storeRetry = import ./lib/store-retry.nix { }; in { @@ -768,8 +790,42 @@ in ${leafCoverage} + # No leaf to ask for, but the root still belongs in the trust + # bundle: every other hive's swarm-service names chain to it. + # `ca/pem` takes no token; the TLS client certificate is the + # store's listener's demand, not bao's, so this presents the + # hive's reader identity and never logs in with it. if [ "$want_svc" = 0 ]; then - echo "no swarm service domains configured — nothing to issue" + echo "no swarm service domains configured — fetching only the store's pki root" + err="$(mktemp)" + raw="$(mktemp "$d/swarm-services-root.XXXXXX")" + trap 'rm -f "$err" "$raw"' EXIT + if ! curl --fail --silent --show-error \ + ${storeRootCurlTls} --output "$raw" \ + "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then + echo "could not read the store's ${baoServicesPkiMount} root." >&2 + cat "$err" >&2 + exit 1 + fi + # Re-encoded rather than copied: it fails on a body that does not + # parse as a certificate, and ends the file in the newline the + # bundle's `cat` relies on. + if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then + echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2 + cat "$err" >&2 + exit 1 + fi + if cmp -s "$svcroot.new" "$svcroot"; then + rm -f "$svcroot.new" + echo "the store's pki root is unchanged" + exit 0 + fi + mv -f "$svcroot.new" "$svcroot" + chmod 0644 "$svcroot" + if systemctl is-active --quiet hive-tls-ca.service; then + echo "the services root changed — rebuilding the trust bundle" + systemctl restart --no-block hive-tls-ca.service + fi exit 0 fi diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 4e014093..6e92a050 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -2,7 +2,8 @@ # this suite exists, naming convention, "evaluates not executes"). # # The swarm-services leaf's renewal: a timer that really re-runs the -# issuance, at a threshold read off the store role's lifetime. +# issuance, at a threshold read off the store role's lifetime. And the +# store's root on a hive that asks it for no leaf. { pkgs, lib, @@ -36,6 +37,15 @@ let deploy.bao.servicesPkiLeafTtlHours = 48; }; + # Fronting none of the swarm's names, reading a store it does not run, + # with the reader identity placed by hand. + noService = hive { + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem"; + }; + noSvcFetch = noService.systemd.services.swarm-services-cert.script; + units = allLocal.systemd.services; boot = units.swarm-services-cert; timer = allLocal.systemd.timers.swarm-services-cert-renew; @@ -117,6 +127,40 @@ let && lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (48 * 3600)} '' shortBoot && lib.hasInfix "-checkend ${toString (48 * 3600)} <<<\"$root_ca\"" shortStore; } + { + # The slot the bundle reads the store's root from, filled on a hive + # that asks the store for no leaf. + name = "a hive fronting no swarm name puts the store's pki root in its trust bundle"; + ok = + noService.services.hyperhive.swarm.localServiceDomains == [ ] + && lib.hasInfix ''"$BAO_ADDR/v1/pki/ca/pem"'' noSvcFetch + && lib.hasInfix ''mv -f "$svcroot.new" "$svcroot"'' noSvcFetch + && lib.hasInfix ''svcroot="$d/swarm-services-root.pem"'' noService.systemd.services.hive-tls-ca.script + && lib.hasInfix ''if [ -s "$svcroot" ]; then anchors+=("$svcroot"); fi'' noService.systemd.services.hive-tls-ca.script; + } + { + name = "the root fetch reaches the store with the hive's own reader identity and server CA"; + ok = + lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch + && lib.hasInfix ( + "--cert ${lib.escapeShellArg "/etc/pki/bao-client.pem"}" + + " --key ${lib.escapeShellArg "/etc/pki/bao-client-key.pem"}" + ) noSvcFetch; + } + { + # The fetch retries against a sealed store; a gateway `Requires=` on it + # would go down with each attempt. The control is a host fronting + # swarm names, whose gateway does wait for its leaf. + name = "a hive fronting no swarm name does not hold its gateway on the store"; + ok = + let + holds = + m: + lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy + || lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before; + in + !(holds noService) && holds allLocal; + } ]; in runGroup "hive-tls" cases