diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 4a4aff4f..18549577 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -43,7 +43,10 @@ let # What a reader calls itself to the store. The hive's name, because a bao # cert-auth role matches on the CN — this is an interface, not a label. - clientCn = if hyperhiveCfg.hiveName != null then hyperhiveCfg.hiveName else cfg.domain; + # No fallback: `hiveName` is asserted set for every hyperhive host, which is + # the same condition this file's `config` is gated on. A fallback here reads + # as a second supported spelling and there is no such thing. + clientCn = hyperhiveCfg.hiveName; # $1 dir $2 basename $3 CN $4 SAN or "" $5 EKU signLeaf = pkgs.writeShellScript "swarm-bao-sign-leaf" '' diff --git a/nix/host-modules/glue-controller-bao-identity.nix b/nix/host-modules/glue-controller-bao-identity.nix index 049c3746..555102f3 100644 --- a/nix/host-modules/glue-controller-bao-identity.nix +++ b/nix/host-modules/glue-controller-bao-identity.nix @@ -36,6 +36,12 @@ in services.hyperhive.deploy.swarm-controller = { baoClientCertFile = lib.mkDefault "${pkiDir}/controller.pem"; baoClientKeyFile = lib.mkDefault "${pkiDir}/controller-key.pem"; + } + // lib.optionalAttrs (baoDeploy.clientCaFile != null) { + # The authority the store already trusts hives by, handed to the daemon + # that has to write it into each hive's role. One file, two readers — + # a second copy would authenticate hives the store does not. + hiveClientCaFile = lib.mkDefault baoDeploy.clientCaFile; }; }; } diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 509cb7fd..e2168bcf 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -32,6 +32,11 @@ let deployCfg.swarm-controller.baoClientCertFile != null && deployCfg.swarm-controller.baoClientKeyFile != null; + # Creating a hive's role means writing to the store, so the CA alone is not + # enough — without an identity there is nothing to write it with, and the + # file would be handed to a daemon that cannot use it. + haveHiveClientCa = haveBaoIdentity && deployCfg.swarm-controller.hiveClientCaFile != null; + # `swarm_secret_client` reads these spellings explicitly rather than # vaultrs's `VAULT_*` defaults — falling through to those builds a client # with no identity and fails at the TLS handshake, naming neither. `%d` and @@ -47,6 +52,13 @@ let # the self-signed one ./glue-bao-tls.nix mints, which is why that file # names this path rather than leaving it to a default. BAO_CACERT = "%d/bao-ca.pem"; + } + // lib.optionalAttrs haveHiveClientCa { + # Not a `BAO_` name: that family is `swarm_secret_client`'s, and + # `BAO_CACERT` above is the *server* CA. This is the authority the + # store should trust hives by, which the controller reads as a value + # to put in each hive's cert-auth role. + SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE = "%d/hive-client-ca.pem"; }; # What `swarmctl` needs in order to act on authelia from the host. @@ -529,6 +541,31 @@ in ''; }; + hiveClientCaFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-ca/root.pem"; + description = '' + Authority whose leaves are hive client certificates, so that this + daemon can create each hive's cert-auth role. The store matches a + presented certificate against the role's copy of this, which is why the + role carries the authority itself rather than a path to it. + + The same material as {option}`services.hyperhive.deploy.bao.clientCaFile`, + named separately because that option is the **store's**: it says which + readers the store trusts, on the host that runs the store. A controller + runs anywhere, so it names its own copy, and a glue module supplies this + as a `mkDefault` where the two are co-located. + + Public material — a certificate authority, not a key — so unlike every + other credential here it leaks nothing. It stays a path anyway, because + the file it names is the one the store already installs. + + `null` leaves hive roles uncreated, which is the state a swarm is in + before anyone has onboarded a hive. + ''; + }; + queue = { clientSecretFile = lib.mkOption { type = lib.types.str; @@ -686,7 +723,8 @@ in ] ++ lib.optional ( haveBaoIdentity && deployCfg.bao.serverCaFile != null - ) "bao-ca.pem:${deployCfg.bao.serverCaFile}"; + ) "bao-ca.pem:${deployCfg.bao.serverCaFile}" + ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"; # The placeholder default that makes the above non-fatal. # `LoadCredential=` takes priority over `SetCredential=`, so this is diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 5b19eb96..a229fe44 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -268,6 +268,20 @@ let # files this host will never have. controllerNoStore = hive { deploy.swarm-controller.enable = true; }; + # The two authorities told apart. A deployment that self-signs both ends + # points `clientCaFile` and `serverCaFile` at one file, so on the fixture + # above the CA a hive is issued from and the CA the store is verified by are + # the same string — and a case wiring either into the other's slot passes. + # This is the deployment where they differ, which is what makes the arm + # below able to fail at all. + controllerTwoCas = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.swarm-controller.enable = true; + deploy.bao.clientCaFile = lib.mkForce "/etc/pki/hive-clients-ca.pem"; + deploy.bao.serverCaFile = lib.mkForce "/etc/pki/store-server-ca.pem"; + }; + # The store and the token, with no CA to trust. `mkForce` because the PKI # glue supplies one by default here — this is the deployment that brings its # own certificates and has not named the authority yet, and it separates @@ -621,6 +635,40 @@ let && builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential && builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential; } + { + # A hive's cert-auth role carries the authority by value, so the daemon + # has to be handed the file rather than a path into the store's own + # directory it cannot read. + # + # ⚠️ The LoadCredential source is asserted, not just the `%d` name, for + # the reason the arm above gives — and here the wrong file is a + # *plausible* one: `deploy.bao.serverCaFile` is the CA a reader checks + # the store's certificate with, evaluates fine in this slot, and would + # make every hive role trust the wrong authority. + name = "the controller is handed the CA hives are issued from"; + ok = + let + s = controllerTwoCas.systemd.services; + m = controllerTwoCas.services.hyperhive; + in + (s.swarm-controller.environment.SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE or null) + == "%d/hive-client-ca.pem" + && builtins.elem "hive-client-ca.pem:/etc/pki/hive-clients-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile; + } + { + # Absence arm for the one above: without a store identity there is + # nothing to write a role with, so handing over the authority would be + # giving a file to a daemon that cannot act on it. + name = "a controller with no store leaf is given no hive CA either"; + ok = + let + s = controllerNoStore.systemd.services; + in + !(s.swarm-controller.environment ? SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE) + && !(lib.any (c: lib.hasPrefix "hive-client-ca" c) s.swarm-controller.serviceConfig.LoadCredential); + } { # Absence arm for the one above, and what makes it mean anything: a # controller with no leaf gets no store environment at all rather than diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 00454f5b..dbe17243 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -1623,11 +1623,15 @@ async fn main() -> Result<()> { let state_forge = keep_forge_for_state(forge_client, webhook_secret.clone()); let hives = load_hives(); - // Before serving, because a hive whose policy does not exist cannot read - // anything this daemon writes for it. Same "log and carry on" shape as - // every connect above. - read_policy::ensure_hive_policies(&hives.iter().map(|h| h.name.clone()).collect::>()) - .await; + // Before serving, because a hive whose role does not exist cannot log in, + // and one whose policy does not exist logs in able to read nothing — + // either way it cannot collect what this daemon writes for it. Policy + // first: the role names the policy, so writing it the other way round + // leaves a window where the role points at nothing. Same "log and carry + // on" shape as every connect above. + let hive_names: Vec = hives.iter().map(|h| h.name.clone()).collect(); + read_policy::ensure_hive_policies(&hive_names).await; + read_policy::ensure_hive_roles(&hive_names).await; let state = AppState { hives: Arc::new(hives), diff --git a/swarm-controller/src/read_policy.rs b/swarm-controller/src/read_policy.rs index f00297e4..70769cbb 100644 --- a/swarm-controller/src/read_policy.rs +++ b/swarm-controller/src/read_policy.rs @@ -1,22 +1,28 @@ -//! Writing the policy a hive's own certificate logs in with. +//! Provisioning a hive's access to the swarm's secret store: the policy its +//! certificate carries, and the cert-auth role that hands it that policy. //! //! The controller writes a credential; the hive fetches it with its own -//! certificate. Nothing said which paths that certificate may read, so the -//! read half of every delivery answers 403. +//! certificate. Without the role the certificate authenticates to nothing, and +//! without the policy the token it gets may read nothing — either way the read +//! half of a delivery answers 403. //! -//! A startup pass, not a hook: the document is the same for every hive and -//! does not depend on which agents exist ([`swarm_secret_client::policy`] -//! explains why it is that wide), so there is nothing to keep in step with -//! anything. Per-hive failures are reported and skipped — one unwritable -//! policy should not take down a daemon that serves everything else, and the -//! next start retries it. +//! Both are startup passes rather than hooks. The policy document is the same +//! for every hive and depends on nothing ([`swarm_secret_client::policy`] +//! explains why it is that wide), and the hive list is loaded once because a +//! config change means a redeploy. Per-hive failures are reported and skipped — +//! one hive that cannot be provisioned should not take down a daemon that +//! serves everything else, and the next start retries it. use anyhow::{Context, Result}; -use swarm_secret_client::{SecretStore, policy}; +use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, policy}; + +/// File holding the authority hives are issued from, as +/// `swarm-controller.nix` names it. +pub const ENV_HIVE_CLIENT_CA: &str = "SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE"; /// Give every hive in `hives` the read policy its certificate will carry. /// -/// Does nothing when this deployment has no store identity, which is the +/// Does nothing when this deployment has no store to reach, which is the /// shape a controller without a secret store runs in. pub async fn ensure_hive_policies(hives: &[String]) { if hives.is_empty() { @@ -46,3 +52,48 @@ async fn write_one(store: &SecretStore, hive: &str) -> Result<()> { tracing::info!(hive, policy = %name, "hive read policy in place"); Ok(()) } + +/// Create each hive's cert-auth role, so the certificate it already holds logs +/// in and carries the policy written above. +/// +/// Does nothing when no authority is configured, which is the deployment that +/// has not onboarded a hive yet. +pub async fn ensure_hive_roles(hives: &[String]) { + let Some(path) = std::env::var_os(ENV_HIVE_CLIENT_CA) else { + tracing::info!("no hive certificate authority configured; hive roles are not managed here"); + return; + }; + let path = path.to_string_lossy().into_owned(); + let ca = match std::fs::read_to_string(&path) { + Ok(ca) => ca, + Err(e) => { + tracing::warn!(path, error = %e, var = ENV_HIVE_CLIENT_CA, "reading the hive authority failed; hive roles are not created"); + return; + } + }; + let store = match crate::store::connect().await { + Ok(store) => store, + Err(e) => { + tracing::warn!(error = %e, "connecting to the swarm secret store failed; hive roles are not created"); + return; + } + }; + for hive in hives { + if let Err(e) = write_role(&store, hive, &ca).await { + tracing::warn!(hive, error = %format!("{e:#}"), "creating this hive's store role failed"); + } + } +} + +/// The role for one hive: its own certificate's subject, its own policy. +async fn write_role(store: &SecretStore, hive: &str, ca: &str) -> Result<()> { + let name = policy::hive_object_name(hive)?; + store + // The subject is the hive's name: `glue-bao-tls.nix` mints a hive's + // client leaf with its name as the CN, and cert auth matches on that. + .write_cert_role(DEFAULT_CERT_MOUNT, &name, ca, hive, &name) + .await + .with_context(|| format!("writing the cert-auth role {name}"))?; + tracing::info!(hive, role = %name, "hive store role in place"); + Ok(()) +} diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs index fcd9bda1..be8d6a9b 100644 --- a/swarm-secret-client/src/client.rs +++ b/swarm-secret-client/src/client.rs @@ -167,6 +167,39 @@ impl SecretStore { vaultrs::sys::policy::set(&self.inner, name, policy).await?; Ok(()) } + + /// Create or replace the cert-auth role `name`, so that a certificate + /// issued by `certificate` and carrying `common_name` logs in with + /// `policy`. + /// + /// `certificate` is the authority by value, not a path: the store keeps + /// its own copy inside the role and never reads a file of ours. + /// + /// # Errors + /// [`Error::Vault`] when the token's policy does not cover + /// `auth//certs/`, or the store rejects the authority. + pub async fn write_cert_role( + &self, + mount: &str, + name: &str, + certificate: &str, + common_name: &str, + policy: &str, + ) -> Result<(), Error> { + let mut opts = + vaultrs::api::auth::cert::requests::CreateCaCertificateRoleRequest::builder(); + opts.allowed_common_names(vec![common_name.to_owned()]) + .token_policies(vec![policy.to_owned()]); + vaultrs::auth::cert::ca_cert_role::set( + &self.inner, + mount, + name, + certificate, + Some(&mut opts), + ) + .await?; + Ok(()) + } } #[cfg(test)]