From 7f9e65e92366bf198ac345dbcd79890cefc8c428 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 9 Sep 2026 16:32:29 +0200 Subject: [PATCH 1/3] swarm-controller: hand the daemon the authority hives are issued from Creating a hive's cert-auth role means writing the authority into the role by value -- the store matches a presented certificate against the role's own copy -- and nothing gave this daemon that file. Named separately from deploy.bao.clientCaFile rather than read off it: that option is the store's, saying which readers the store trusts on the host that runs it, while a controller runs anywhere. The glue module supplies it where the two are co-located, which is the same split baoClientCertFile already makes against the hive reader's leaf. Gated on the identity as well as the CA. Without a leaf there is nothing to write a role with, so the file would reach a daemon that cannot act on it. The module-eval arm needed a fixture of its own: a deployment that self-signs both ends points clientCaFile and serverCaFile at one file, so on the existing fixture the two authorities are the same string and wiring either into the other's slot passes. controllerTwoCas is where they differ. --- .../glue-controller-bao-identity.nix | 6 +++ nix/host-modules/swarm-controller.nix | 40 +++++++++++++++- nix/module-eval.nix | 48 +++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/glue-controller-bao-identity.nix b/nix/host-modules/glue-controller-bao-identity.nix index 049c3746..555102f3 100644 --- a/nix/host-modules/glue-controller-bao-identity.nix +++ b/nix/host-modules/glue-controller-bao-identity.nix @@ -36,6 +36,12 @@ in services.hyperhive.deploy.swarm-controller = { baoClientCertFile = lib.mkDefault "${pkiDir}/controller.pem"; baoClientKeyFile = lib.mkDefault "${pkiDir}/controller-key.pem"; + } + // lib.optionalAttrs (baoDeploy.clientCaFile != null) { + # The authority the store already trusts hives by, handed to the daemon + # that has to write it into each hive's role. One file, two readers — + # a second copy would authenticate hives the store does not. + hiveClientCaFile = lib.mkDefault baoDeploy.clientCaFile; }; }; } diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 509cb7fd..e2168bcf 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -32,6 +32,11 @@ let deployCfg.swarm-controller.baoClientCertFile != null && deployCfg.swarm-controller.baoClientKeyFile != null; + # Creating a hive's role means writing to the store, so the CA alone is not + # enough — without an identity there is nothing to write it with, and the + # file would be handed to a daemon that cannot use it. + haveHiveClientCa = haveBaoIdentity && deployCfg.swarm-controller.hiveClientCaFile != null; + # `swarm_secret_client` reads these spellings explicitly rather than # vaultrs's `VAULT_*` defaults — falling through to those builds a client # with no identity and fails at the TLS handshake, naming neither. `%d` and @@ -47,6 +52,13 @@ let # the self-signed one ./glue-bao-tls.nix mints, which is why that file # names this path rather than leaving it to a default. BAO_CACERT = "%d/bao-ca.pem"; + } + // lib.optionalAttrs haveHiveClientCa { + # Not a `BAO_` name: that family is `swarm_secret_client`'s, and + # `BAO_CACERT` above is the *server* CA. This is the authority the + # store should trust hives by, which the controller reads as a value + # to put in each hive's cert-auth role. + SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE = "%d/hive-client-ca.pem"; }; # What `swarmctl` needs in order to act on authelia from the host. @@ -529,6 +541,31 @@ in ''; }; + hiveClientCaFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-ca/root.pem"; + description = '' + Authority whose leaves are hive client certificates, so that this + daemon can create each hive's cert-auth role. The store matches a + presented certificate against the role's copy of this, which is why the + role carries the authority itself rather than a path to it. + + The same material as {option}`services.hyperhive.deploy.bao.clientCaFile`, + named separately because that option is the **store's**: it says which + readers the store trusts, on the host that runs the store. A controller + runs anywhere, so it names its own copy, and a glue module supplies this + as a `mkDefault` where the two are co-located. + + Public material — a certificate authority, not a key — so unlike every + other credential here it leaks nothing. It stays a path anyway, because + the file it names is the one the store already installs. + + `null` leaves hive roles uncreated, which is the state a swarm is in + before anyone has onboarded a hive. + ''; + }; + queue = { clientSecretFile = lib.mkOption { type = lib.types.str; @@ -686,7 +723,8 @@ in ] ++ lib.optional ( haveBaoIdentity && deployCfg.bao.serverCaFile != null - ) "bao-ca.pem:${deployCfg.bao.serverCaFile}"; + ) "bao-ca.pem:${deployCfg.bao.serverCaFile}" + ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"; # The placeholder default that makes the above non-fatal. # `LoadCredential=` takes priority over `SetCredential=`, so this is diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 5b19eb96..a229fe44 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -268,6 +268,20 @@ let # files this host will never have. controllerNoStore = hive { deploy.swarm-controller.enable = true; }; + # The two authorities told apart. A deployment that self-signs both ends + # points `clientCaFile` and `serverCaFile` at one file, so on the fixture + # above the CA a hive is issued from and the CA the store is verified by are + # the same string — and a case wiring either into the other's slot passes. + # This is the deployment where they differ, which is what makes the arm + # below able to fail at all. + controllerTwoCas = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.swarm-controller.enable = true; + deploy.bao.clientCaFile = lib.mkForce "/etc/pki/hive-clients-ca.pem"; + deploy.bao.serverCaFile = lib.mkForce "/etc/pki/store-server-ca.pem"; + }; + # The store and the token, with no CA to trust. `mkForce` because the PKI # glue supplies one by default here — this is the deployment that brings its # own certificates and has not named the authority yet, and it separates @@ -621,6 +635,40 @@ let && builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential && builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential; } + { + # A hive's cert-auth role carries the authority by value, so the daemon + # has to be handed the file rather than a path into the store's own + # directory it cannot read. + # + # ⚠️ The LoadCredential source is asserted, not just the `%d` name, for + # the reason the arm above gives — and here the wrong file is a + # *plausible* one: `deploy.bao.serverCaFile` is the CA a reader checks + # the store's certificate with, evaluates fine in this slot, and would + # make every hive role trust the wrong authority. + name = "the controller is handed the CA hives are issued from"; + ok = + let + s = controllerTwoCas.systemd.services; + m = controllerTwoCas.services.hyperhive; + in + (s.swarm-controller.environment.SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE or null) + == "%d/hive-client-ca.pem" + && builtins.elem "hive-client-ca.pem:/etc/pki/hive-clients-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile; + } + { + # Absence arm for the one above: without a store identity there is + # nothing to write a role with, so handing over the authority would be + # giving a file to a daemon that cannot act on it. + name = "a controller with no store leaf is given no hive CA either"; + ok = + let + s = controllerNoStore.systemd.services; + in + !(s.swarm-controller.environment ? SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE) + && !(lib.any (c: lib.hasPrefix "hive-client-ca" c) s.swarm-controller.serviceConfig.LoadCredential); + } { # Absence arm for the one above, and what makes it mean anything: a # controller with no leaf gets no store environment at all rather than From 4007fc965dd700cb9379bd938ca9fb8763a52d1c Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 9 Sep 2026 16:51:39 +0200 Subject: [PATCH 2/3] swarm-controller: create each hive's cert-auth role at startup A hive holds an mTLS pair and a policy naming what it may read, and still cannot log in: nothing creates the role that maps its certificate to that policy. The one pre-shared credential in the system therefore buys no access. Minting happens here rather than in nix, which was the first plan. Nix mints from the store's own container, and that path is gated on the bootstrap token -- so onboarding a hive later would mean placing the one genuinely pre-shared secret again. Doing it from the controller costs a public certificate authority as an input and makes the bootstrap token one-time. A startup pass, not a hook: the hive list is loaded once and a config change means a redeploy, so the roles are as static as the list. Only the policy is derived from something that moves. The subject is the hive's name because glue-bao-tls.nix mints a hive's client leaf with its name as the CN, and cert auth matches on that. Per-hive failures are logged and skipped, matching the queue, bridge and forge connects above it: a controller whose store is unreachable still serves everything else, and the next start retries. Not covered by a test: ensure_hive_roles is IO from end to end, and the seam that would make it assertable is the one the read-grant sink already has. Said here rather than implied by a green suite. --- swarm-controller/src/main.rs | 14 ++++-- swarm-controller/src/read_policy.rs | 73 ++++++++++++++++++++++++----- swarm-secret-client/src/client.rs | 33 +++++++++++++ 3 files changed, 104 insertions(+), 16 deletions(-) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 00454f5b..dbe17243 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -1623,11 +1623,15 @@ async fn main() -> Result<()> { let state_forge = keep_forge_for_state(forge_client, webhook_secret.clone()); let hives = load_hives(); - // Before serving, because a hive whose policy does not exist cannot read - // anything this daemon writes for it. Same "log and carry on" shape as - // every connect above. - read_policy::ensure_hive_policies(&hives.iter().map(|h| h.name.clone()).collect::>()) - .await; + // Before serving, because a hive whose role does not exist cannot log in, + // and one whose policy does not exist logs in able to read nothing — + // either way it cannot collect what this daemon writes for it. Policy + // first: the role names the policy, so writing it the other way round + // leaves a window where the role points at nothing. Same "log and carry + // on" shape as every connect above. + let hive_names: Vec = hives.iter().map(|h| h.name.clone()).collect(); + read_policy::ensure_hive_policies(&hive_names).await; + read_policy::ensure_hive_roles(&hive_names).await; let state = AppState { hives: Arc::new(hives), diff --git a/swarm-controller/src/read_policy.rs b/swarm-controller/src/read_policy.rs index f00297e4..70769cbb 100644 --- a/swarm-controller/src/read_policy.rs +++ b/swarm-controller/src/read_policy.rs @@ -1,22 +1,28 @@ -//! Writing the policy a hive's own certificate logs in with. +//! Provisioning a hive's access to the swarm's secret store: the policy its +//! certificate carries, and the cert-auth role that hands it that policy. //! //! The controller writes a credential; the hive fetches it with its own -//! certificate. Nothing said which paths that certificate may read, so the -//! read half of every delivery answers 403. +//! certificate. Without the role the certificate authenticates to nothing, and +//! without the policy the token it gets may read nothing — either way the read +//! half of a delivery answers 403. //! -//! A startup pass, not a hook: the document is the same for every hive and -//! does not depend on which agents exist ([`swarm_secret_client::policy`] -//! explains why it is that wide), so there is nothing to keep in step with -//! anything. Per-hive failures are reported and skipped — one unwritable -//! policy should not take down a daemon that serves everything else, and the -//! next start retries it. +//! Both are startup passes rather than hooks. The policy document is the same +//! for every hive and depends on nothing ([`swarm_secret_client::policy`] +//! explains why it is that wide), and the hive list is loaded once because a +//! config change means a redeploy. Per-hive failures are reported and skipped — +//! one hive that cannot be provisioned should not take down a daemon that +//! serves everything else, and the next start retries it. use anyhow::{Context, Result}; -use swarm_secret_client::{SecretStore, policy}; +use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, policy}; + +/// File holding the authority hives are issued from, as +/// `swarm-controller.nix` names it. +pub const ENV_HIVE_CLIENT_CA: &str = "SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE"; /// Give every hive in `hives` the read policy its certificate will carry. /// -/// Does nothing when this deployment has no store identity, which is the +/// Does nothing when this deployment has no store to reach, which is the /// shape a controller without a secret store runs in. pub async fn ensure_hive_policies(hives: &[String]) { if hives.is_empty() { @@ -46,3 +52,48 @@ async fn write_one(store: &SecretStore, hive: &str) -> Result<()> { tracing::info!(hive, policy = %name, "hive read policy in place"); Ok(()) } + +/// Create each hive's cert-auth role, so the certificate it already holds logs +/// in and carries the policy written above. +/// +/// Does nothing when no authority is configured, which is the deployment that +/// has not onboarded a hive yet. +pub async fn ensure_hive_roles(hives: &[String]) { + let Some(path) = std::env::var_os(ENV_HIVE_CLIENT_CA) else { + tracing::info!("no hive certificate authority configured; hive roles are not managed here"); + return; + }; + let path = path.to_string_lossy().into_owned(); + let ca = match std::fs::read_to_string(&path) { + Ok(ca) => ca, + Err(e) => { + tracing::warn!(path, error = %e, var = ENV_HIVE_CLIENT_CA, "reading the hive authority failed; hive roles are not created"); + return; + } + }; + let store = match crate::store::connect().await { + Ok(store) => store, + Err(e) => { + tracing::warn!(error = %e, "connecting to the swarm secret store failed; hive roles are not created"); + return; + } + }; + for hive in hives { + if let Err(e) = write_role(&store, hive, &ca).await { + tracing::warn!(hive, error = %format!("{e:#}"), "creating this hive's store role failed"); + } + } +} + +/// The role for one hive: its own certificate's subject, its own policy. +async fn write_role(store: &SecretStore, hive: &str, ca: &str) -> Result<()> { + let name = policy::hive_object_name(hive)?; + store + // The subject is the hive's name: `glue-bao-tls.nix` mints a hive's + // client leaf with its name as the CN, and cert auth matches on that. + .write_cert_role(DEFAULT_CERT_MOUNT, &name, ca, hive, &name) + .await + .with_context(|| format!("writing the cert-auth role {name}"))?; + tracing::info!(hive, role = %name, "hive store role in place"); + Ok(()) +} diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs index fcd9bda1..be8d6a9b 100644 --- a/swarm-secret-client/src/client.rs +++ b/swarm-secret-client/src/client.rs @@ -167,6 +167,39 @@ impl SecretStore { vaultrs::sys::policy::set(&self.inner, name, policy).await?; Ok(()) } + + /// Create or replace the cert-auth role `name`, so that a certificate + /// issued by `certificate` and carrying `common_name` logs in with + /// `policy`. + /// + /// `certificate` is the authority by value, not a path: the store keeps + /// its own copy inside the role and never reads a file of ours. + /// + /// # Errors + /// [`Error::Vault`] when the token's policy does not cover + /// `auth//certs/`, or the store rejects the authority. + pub async fn write_cert_role( + &self, + mount: &str, + name: &str, + certificate: &str, + common_name: &str, + policy: &str, + ) -> Result<(), Error> { + let mut opts = + vaultrs::api::auth::cert::requests::CreateCaCertificateRoleRequest::builder(); + opts.allowed_common_names(vec![common_name.to_owned()]) + .token_policies(vec![policy.to_owned()]); + vaultrs::auth::cert::ca_cert_role::set( + &self.inner, + mount, + name, + certificate, + Some(&mut opts), + ) + .await?; + Ok(()) + } } #[cfg(test)] From c22db5eb579b9135b6eec33a545b3583763f4b7e Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 9 Sep 2026 18:23:58 +0200 Subject: [PATCH 3/3] swarm-bao-tls: drop the unreachable CN fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clientCn` fell back to `cfg.domain` when `hiveName` was unset. That branch cannot run: `hive-network.nix` asserts `hiveName != null` under `mkIf services.hyperhive.enable`, and this file's `config` is gated on the same predicate, so any host that evaluates the conditional has already failed the assertion. Worse than dead, it read as a second supported spelling of a hive's identity — which is what a cert-auth role matches on. It was not even the hive's own domain: `cfg` here is `services.hyperhive.swarm.bao`, so the fallback resolved to the store's address, one string shared by every hive in the swarm and the same CN the server leaf carries. Reading the option directly matches what other modules needing the name already do (`hive-c0re/environment.nix`). --- nix/host-modules/glue-bao-tls.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 4a4aff4f..18549577 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -43,7 +43,10 @@ let # What a reader calls itself to the store. The hive's name, because a bao # cert-auth role matches on the CN — this is an interface, not a label. - clientCn = if hyperhiveCfg.hiveName != null then hyperhiveCfg.hiveName else cfg.domain; + # No fallback: `hiveName` is asserted set for every hyperhive host, which is + # the same condition this file's `config` is gated on. A fallback here reads + # as a second supported spelling and there is no such thing. + clientCn = hyperhiveCfg.hiveName; # $1 dir $2 basename $3 CN $4 SAN or "" $5 EKU signLeaf = pkgs.writeShellScript "swarm-bao-sign-leaf" ''