From 4384a1fffa8bdda07cf4edf837a04ed5b4cd300b Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 2 Sep 2026 22:27:58 +0200 Subject: [PATCH 1/6] swarm-secret-client: the agreements both ends of the store must share mara ruled (a) on #3726: a thin workspace crate over `vaultrs` rather than keeping bao access in nix and having each end trigger units. The HTTP is the SDK's job; what this crate owns is the things the controller and a hive must say *identically*, and which have no other home because neither end is senior to the other. Three such agreements: `path::matrix_account` builds where a credential lives. It is fallible rather than a `format!`, because both names reach it from elsewhere -- the agent name from the topology, the account name from an agent's own config -- and a `/` or `..` in either does not produce a malformed path, it produces a valid path to a *different agent's* secret. The charset mirrors the KV bucket-name rule. `Credential`'s `value` field is not a free choice: glue-matrix-bao-token.nix reads the store with `bao kv get -field=value`, so the name is load-bearing for a consumer no Rust test can reach. A test pins the serialised shape. `client::Settings` reads BAO_ADDR / BAO_CLIENT_CERT / BAO_CLIENT_KEY / BAO_CACERT explicitly instead of letting vaultrs fall through to its own defaults, which look for VAULT_ADDR / VAULT_CLIENT_CERT / VAULT_CLIENT_KEY. Every unit in this tree sets the BAO_ spellings, so the defaults would yield a client with no identity at all -- surfacing as a TLS handshake failure, which names neither the missing variable nor the reason. The env read is split from the connect so every misconfiguration arm is testable without a reachable store and without touching process-global env. Dependency impact, measured against the lock at forge/main rather than assumed: native-tls 0 -> 0, openssl-sys 0 -> 0, one reqwest (0.13.4) which vaultrs shares, and 10 new crates that are all derive/proc-macro helpers. Refs #3726 --- Cargo.lock | 154 ++++++++++++++++++- Cargo.toml | 3 + swarm-secret-client/Cargo.toml | 19 +++ swarm-secret-client/src/client.rs | 248 ++++++++++++++++++++++++++++++ swarm-secret-client/src/lib.rs | 68 ++++++++ swarm-secret-client/src/path.rs | 98 ++++++++++++ 6 files changed, 584 insertions(+), 6 deletions(-) create mode 100644 swarm-secret-client/Cargo.toml create mode 100644 swarm-secret-client/src/client.rs create mode 100644 swarm-secret-client/src/lib.rs create mode 100644 swarm-secret-client/src/path.rs diff --git a/Cargo.lock b/Cargo.lock index 0d1fafcc..f356b36c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -894,14 +894,38 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -917,13 +941,24 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.119", ] @@ -1020,6 +1055,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -3823,7 +3889,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "783d787bf21813b285f13019adc49e11af501c658890c1e519f31f937c68b7e3" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", "serde_json", @@ -4012,6 +4078,40 @@ dependencies = [ "semver", ] +[[package]] +name = "rustify" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4800ce4c1cc2fec12c559dae2ddbf0e17fcee7569b796e6d75898efef443368b" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "http", + "reqwest", + "rustify_derive", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 1.0.69", + "tracing", + "url", +] + +[[package]] +name = "rustify_derive" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ea7fda74240f7410d0198b603a8a2f662acc7d76b6667a49f9b162cd8d9b4f" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "serde_urlencoded", + "syn 1.0.109", + "synstructure 0.12.6", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4642,6 +4742,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "swarm-secret-client" +version = "0.1.0" +dependencies = [ + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.18", + "vaultrs", +] + [[package]] name = "swarmctl" version = "0.1.0" @@ -4696,6 +4807,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -5337,6 +5460,25 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vaultrs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ffcc0e81025065dda612ec1e26a3d81bb16ef3062354873d17a35965d68522" +dependencies = [ + "async-trait", + "derive_builder", + "http", + "reqwest", + "rustify", + "rustify_derive", + "serde", + "serde_json", + "thiserror 2.0.18", + "tracing", + "url", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -5873,7 +6015,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -5914,7 +6056,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d1edfdd6..8ca89bd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "swarm-controller", "swarm-nats-auth", "swarm-queue-client", + "swarm-secret-client", "swarmctl", ] @@ -90,7 +91,9 @@ hive-sock-client = { path = "hive-sock-client" } hive-types = { path = "hive-types" } swarm-authelia-bridge-sock = { path = "swarm-authelia-bridge-sock" } swarm-queue-client = { path = "swarm-queue-client" } +swarm-secret-client = { path = "swarm-secret-client" } thiserror = "2" +vaultrs = "0.8" tower-http = { version = "0.7", features = ["fs"] } uuid = { version = "1", features = ["v4"] } rmcp = { version = "2", default-features = false, features = [ diff --git a/swarm-secret-client/Cargo.toml b/swarm-secret-client/Cargo.toml new file mode 100644 index 00000000..bb59a56e --- /dev/null +++ b/swarm-secret-client/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "swarm-secret-client" +version.workspace = true +edition.workspace = true + +[dependencies] +# `Identity` is re-exported by vaultrs, but the identity is built here (from +# the `BAO_*` files) rather than by its env defaults, so the dependency is +# direct rather than incidental. +reqwest.workspace = true +serde.workspace = true +thiserror.workspace = true +vaultrs.workspace = true + +[dev-dependencies] +serde_json.workspace = true + +[lints] +workspace = true diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs new file mode 100644 index 00000000..d2f00a14 --- /dev/null +++ b/swarm-secret-client/src/client.rs @@ -0,0 +1,248 @@ +//! A logged-in handle on the store, built from this deployment's environment. + +use serde::{Deserialize, Serialize}; +use vaultrs::client::{Client, VaultClient, VaultClientSettingsBuilder}; + +use crate::{Error, path::MOUNT}; + +/// The store's address. +pub const ENV_ADDR: &str = "BAO_ADDR"; +/// PEM client certificate presented to the store's listener. +pub const ENV_CLIENT_CERT: &str = "BAO_CLIENT_CERT"; +/// PEM private key for [`ENV_CLIENT_CERT`]. +pub const ENV_CLIENT_KEY: &str = "BAO_CLIENT_KEY"; +/// CA bundle the store's own certificate is verified against. Optional: +/// absent means the system trust store, which is what a deployment with a +/// real CA wants and what a self-signed one must not be left with. +pub const ENV_CACERT: &str = "BAO_CACERT"; + +/// The auth mount a cert login goes through, unless a caller says otherwise. +pub const DEFAULT_CERT_MOUNT: &str = "cert"; + +#[derive(Serialize, Deserialize)] +struct Credential { + /// `glue-matrix-bao-token.nix` reads the store with + /// `bao kv get -field=value`, so this name is load-bearing for a reader + /// this crate does not control. Renaming it silently breaks that unit. + value: String, +} + +/// Where the store is and which identity we present to it. +/// +/// Separate from the connect so the environment can be checked without one: +/// every arm below is a misconfiguration an operator has to read an error +/// about, and none of them needs a reachable store to happen. +#[derive(Debug, PartialEq, Eq)] +pub struct Settings { + address: String, + cert_path: String, + key_path: String, + ca_path: Option, +} + +impl Settings { + /// Read the `BAO_*` variables from the process environment. + /// + /// # Errors + /// [`Error::MissingEnv`] naming the first variable that is unset or empty. + pub fn from_env() -> Result { + Self::from_lookup(|k| std::env::var(k).ok()) + } + + /// [`Settings::from_env`] against an arbitrary lookup. + /// + /// [`vaultrs`]'s own defaults are deliberately not used: it looks for + /// `VAULT_ADDR` / `VAULT_CLIENT_CERT` / `VAULT_CLIENT_KEY`, and every unit + /// in this tree sets the `BAO_` spellings. Falling through to those + /// defaults yields a client with **no identity**, which fails at the TLS + /// handshake rather than anywhere that names the cause. + /// + /// # Errors + /// [`Error::MissingEnv`] naming the first variable that is unset or empty. + pub fn from_lookup(get: impl Fn(&str) -> Option) -> Result { + let required = |var: &'static str| -> Result { + get(var) + .filter(|v| !v.is_empty()) + .ok_or(Error::MissingEnv(var)) + }; + Ok(Self { + address: required(ENV_ADDR)?, + cert_path: required(ENV_CLIENT_CERT)?, + key_path: required(ENV_CLIENT_KEY)?, + ca_path: get(ENV_CACERT).filter(|v| !v.is_empty()), + }) + } +} + +/// A client that has already exchanged its certificate for a token. +pub struct SecretStore { + inner: VaultClient, +} + +fn read_file(var: &'static str, path: &str) -> Result, Error> { + std::fs::read(path).map_err(|source| Error::Identity { + var, + path: path.to_owned(), + source, + }) +} + +impl SecretStore { + /// Connect using the `BAO_*` environment and log in with the certificate + /// auth method, returning a handle that already holds a token. + /// + /// `cert_role` is the role configured on the store's `cert` mount; it + /// selects which policy the returned token carries. + /// + /// # Errors + /// As [`SecretStore::connect`], plus [`Error::MissingEnv`]. + pub async fn from_env(cert_role: &str) -> Result { + Self::connect(&Settings::from_env()?, cert_role, DEFAULT_CERT_MOUNT).await + } + + /// Present `settings`' identity to the store and log in at `cert_mount`. + /// + /// # Errors + /// [`Error::Identity`] when a named file cannot be read, [`Error::Tls`] + /// when the cert/key pair does not form a usable identity, + /// [`Error::Settings`] when the address will not parse, and + /// [`Error::Vault`] when the store refuses the login — which is what an + /// unconfigured `cert` mount looks like from here. + pub async fn connect( + settings: &Settings, + cert_role: &str, + cert_mount: &str, + ) -> Result { + // One PEM blob holding both, which is the shape `Identity::from_pem` + // wants; the two stay separate on disk because the key is the half + // that gets `0600`. + let mut pem = read_file(ENV_CLIENT_CERT, &settings.cert_path)?; + pem.push(b'\n'); + pem.extend_from_slice(&read_file(ENV_CLIENT_KEY, &settings.key_path)?); + let identity = reqwest::Identity::from_pem(&pem).map_err(Error::Tls)?; + + let mut builder = VaultClientSettingsBuilder::default(); + builder + .address(settings.address.clone()) + .identity(Some(identity)); + if let Some(ca) = &settings.ca_path { + builder.ca_certs(vec![ca.clone()]); + } + let built = builder + .build() + .map_err(|e| Error::Settings(e.to_string()))?; + + let mut inner = VaultClient::new(built)?; + let auth = vaultrs::auth::cert::login(&inner, cert_mount, cert_role).await?; + inner.set_token(&auth.client_token); + Ok(Self { inner }) + } + + /// Read the credential stored at `path`. + /// + /// # Errors + /// [`Error::Vault`] when the path does not exist, the token's policy does + /// not cover it, or the stored object has no `value` field. + pub async fn read(&self, path: &str) -> Result { + let c: Credential = vaultrs::kv2::read(&self.inner, MOUNT, path).await?; + Ok(c.value) + } + + /// Write `value` as the credential at `path`, creating a new version. + /// + /// # Errors + /// [`Error::Vault`] when the token's policy does not cover the path. + pub async fn write(&self, path: &str, value: &str) -> Result<(), Error> { + let body = Credential { + value: value.to_owned(), + }; + vaultrs::kv2::set(&self.inner, MOUNT, path, &body).await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A lookup standing in for a fully-configured unit's environment. + fn full(k: &str) -> Option { + match k { + ENV_ADDR => Some("https://store.invalid:8200".to_owned()), + ENV_CLIENT_CERT => Some("/run/c.pem".to_owned()), + ENV_CLIENT_KEY => Some("/run/k.pem".to_owned()), + _ => None, + } + } + + #[test] + fn a_complete_environment_is_accepted() { + // The control: without this, every assertion below could be passing + // because `from_lookup` rejects everything. + let s = Settings::from_lookup(full).expect("every required variable is set"); + assert_eq!(s.address, "https://store.invalid:8200"); + assert_eq!(s.ca_path, None, "an absent CA is the system trust store"); + } + + #[test] + fn each_required_variable_is_named_when_it_is_the_missing_one() { + for var in [ENV_ADDR, ENV_CLIENT_CERT, ENV_CLIENT_KEY] { + let e = Settings::from_lookup(|k| if k == var { None } else { full(k) }) + .expect_err("one required variable is absent"); + assert!( + matches!(e, Error::MissingEnv(v) if v == var), + "dropping {var} should name {var}, got {e:?}" + ); + } + } + + #[test] + fn an_empty_variable_is_as_absent_as_an_unset_one() { + // systemd writes `Environment=BAO_CACERT=` for an unset nix option, so + // empty is the shape this actually arrives in. + let e = Settings::from_lookup(|k| { + if k == ENV_ADDR { + Some(String::new()) + } else { + full(k) + } + }) + .expect_err("an empty address is not an address"); + assert!(matches!(e, Error::MissingEnv(ENV_ADDR)), "got {e:?}"); + + let s = Settings::from_lookup(|k| { + if k == ENV_CACERT { + Some(String::new()) + } else { + full(k) + } + }) + .expect("an empty CA is optional, not fatal"); + assert_eq!(s.ca_path, None); + } + + #[test] + fn an_unreadable_cert_names_the_file_and_the_variable() { + let e = read_file(ENV_CLIENT_CERT, "/nonexistent/cert.pem") + .expect_err("the file does not exist"); + match e { + Error::Identity { var, ref path, .. } => { + assert_eq!(var, ENV_CLIENT_CERT); + assert_eq!(path, "/nonexistent/cert.pem"); + } + other => panic!("wanted an Identity error, got {other:?}"), + } + } + + #[test] + fn the_value_field_matches_what_the_nix_reader_asks_for() { + // The literal is the point: `bao kv get -field=value` is the other end + // of this agreement and lives in a file no Rust test can reach, so the + // name is pinned here rather than derived from the struct. + let json = serde_json::to_string(&Credential { + value: "t".to_owned(), + }) + .expect("a struct of one String serialises"); + assert_eq!(json, r#"{"value":"t"}"#); + } +} diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs new file mode 100644 index 00000000..02fa1efd --- /dev/null +++ b/swarm-secret-client/src/lib.rs @@ -0,0 +1,68 @@ +//! The swarm's secret-store client: where a credential lives, and how both ends +//! reach it. +//! +//! The HTTP is [`vaultrs`]'s job. What this crate owns is the *agreements* — +//! the path a credential is written to and read from ([`path`]), the field its +//! bytes live in, and the translation from this deployment's environment into +//! a logged-in client ([`client`]). Each of those is a thing the controller and +//! a hive must say identically, so it is said once here. + +pub mod client; +pub mod path; + +pub use client::SecretStore; + +/// What can go wrong between "we have a client certificate" and "we have the +/// credential". +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// A name that would have addressed something other than what the caller + /// meant. See [`path`]. + #[error("{kind} name {value:?} is not a single path segment of [A-Za-z0-9_-]")] + PathSegment { + /// Which name was rejected — `agent` or `account`. + kind: &'static str, + /// The offending value, quoted in the message because the caller + /// usually got it from config and needs to see which one. + value: String, + }, + + /// A variable the store's address or identity comes from is unset or + /// empty. Named rather than defaulted: a wrong store address fails much + /// later and much less clearly than a missing one. + #[error("{0} is unset or empty")] + MissingEnv(&'static str), + + /// A client-certificate file named by the environment could not be read. + #[error("reading {path} (from {var}): {source}")] + Identity { + /// The variable that named the file. + var: &'static str, + /// The path it named. + path: String, + /// The underlying IO failure. + source: std::io::Error, + }, + + /// The address would not parse into a URL the client can use. + #[error("the store's settings are unusable: {0}")] + Settings(String), + + /// The store refused us, was unreachable, or answered something we could + /// not parse. + #[error(transparent)] + Vault(#[from] Box), + + /// The client certificate and key did not form a usable identity, or the + /// CA bundle did not parse. + #[error("building the TLS identity: {0}")] + Tls(#[source] reqwest::Error), +} + +impl From for Error { + fn from(e: vaultrs::error::ClientError) -> Self { + // Boxed because `ClientError` is large enough that carrying it inline + // makes every `Result` in the crate pay for the rare arm. + Self::Vault(Box::new(e)) + } +} diff --git a/swarm-secret-client/src/path.rs b/swarm-secret-client/src/path.rs new file mode 100644 index 00000000..00116422 --- /dev/null +++ b/swarm-secret-client/src/path.rs @@ -0,0 +1,98 @@ +//! Where a credential lives, for both ends of the store. +//! +//! The controller writes and a hive reads, and neither is senior to the other, +//! so the path they must agree on is built here rather than formatted at each +//! call site. + +use crate::Error; + +/// The KV v2 mount every swarm secret lives under. +/// +/// A literal because the store's existing reader already hardcodes the same +/// one (`secret/swarm/matrix/registration-token`); an option nobody sets would +/// be two ways to say one thing. +pub const MOUNT: &str = "secret"; + +/// The prefix under [`MOUNT`] owned by per-agent credentials. +pub const AGENT_PREFIX: &str = "swarm/agents"; + +/// A path segment that cannot change the path's shape. +/// +/// The charset is deliberately narrower than what the store accepts: a `/` +/// turns one agent's segment into another agent's directory, and `..` walks +/// out of the prefix entirely. Both are names this crate receives from +/// elsewhere — an agent name from the topology, an account name from an +/// agent's own config — so neither is trusted to be well-formed here. +fn checked_segment(kind: &'static str, value: &str) -> Result<(), Error> { + if value.is_empty() { + return Err(Error::PathSegment { + kind, + value: value.to_owned(), + }); + } + if !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + { + return Err(Error::PathSegment { + kind, + value: value.to_owned(), + }); + } + Ok(()) +} + +/// The path holding `agent`'s token for the external matrix account `account`. +/// +/// # Errors +/// [`Error::PathSegment`] when either name contains anything but +/// `[A-Za-z0-9_-]`, which is what keeps one agent's name from addressing +/// another agent's secret. +pub fn matrix_account(agent: &str, account: &str) -> Result { + checked_segment("agent", agent)?; + checked_segment("account", account)?; + Ok(format!("{AGENT_PREFIX}/{agent}/matrix/{account}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_well_formed_pair_lands_under_the_agent_prefix() { + let p = matrix_account("atlas", "ops-relay").expect("both segments are legal"); + assert_eq!(p, "swarm/agents/atlas/matrix/ops-relay"); + assert!(p.starts_with(AGENT_PREFIX)); + } + + #[test] + fn a_segment_cannot_escape_its_own_directory() { + // Each of these is a *different* way to address another agent's tree, + // and the last two are the ones a charset check catches but a + // `contains("..")` check does not. + for bad in [ + "../argus", + "atlas/../argus", + "atlas/matrix", + "a b", + "a.b", + "", + ] { + assert!( + matrix_account(bad, "ops-relay").is_err(), + "agent segment {bad:?} must be refused" + ); + assert!( + matrix_account("atlas", bad).is_err(), + "account segment {bad:?} must be refused" + ); + } + } + + #[test] + fn the_legal_charset_is_actually_reachable() { + // The control for the test above: if `checked_segment` rejected + // everything, the escape cases would pass for the wrong reason. + assert!(matrix_account("a-b_C9", "d-e_F0").is_ok()); + } +} From f2713486a5169834987d3c9e8d1dee2100155b8b Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 2 Sep 2026 22:33:48 +0200 Subject: [PATCH 2/6] docs: the repo map gains the crate this branch adds /knowledge/doc-standards.md lists "file added -> CLAUDE.md ## Repo map" under always-update, and "CLAUDE.md after a file rename" under most-commonly-missed. This branch added a workspace member and I missed it; a scheduled nudge to re-read the hive rules is what caught it, not a gate. The entry leads with the BAO_ vs VAULT_ env mismatch because that is the part a reader cannot derive from the crate name: vaultrs' own defaults look for VAULT_ADDR / VAULT_CLIENT_CERT / VAULT_CLIENT_KEY, no unit in this tree sets those, and falling through to them produces a client with no identity whose only symptom is a TLS handshake failure. Measured while doing it: 6 of 28 workspace members were absent from the map. Five predate this branch (hive-jobq-metrics, swarm-authelia-bridge, swarm-authelia-bridge-sock, swarm-nats-auth, swarm-queue-client) and are deliberately left alone here rather than widening this PR. Refs #3726 --- CLAUDE.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 4d6990f9..fb26720c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,6 +147,14 @@ hand-maintained per-file tree drifts out of sync with the code. unix socket the gateway's nginx proxies to — ⚠️ **the socket's directory is its access control**; the constraint that governs it is in the crate's README, and a unit test pins the path. +- **`swarm-secret-client/`** — client for the swarm's secret store, over + `vaultrs`. Owns the *agreements* both ends of the store must share rather + than the HTTP: the path a credential lives at, the field its bytes are in, + and the translation from this deployment's environment into a logged-in + client. ⚠️ Reads the **`BAO_`** env spellings explicitly — `vaultrs`'s own + defaults look for `VAULT_ADDR`/`VAULT_CLIENT_CERT`/`VAULT_CLIENT_KEY`, which + no unit in this tree sets, so falling through to them yields a client with + **no identity** and a TLS handshake failure that names no cause. - **`swarmctl/`** — swarm-level operator CLI, installed by the swarm-controller module on the host that runs the daemon. Runs as **root and acts directly** — no socket, no HTTP route, no priv helper; From 7a03ce096a074f82f5c52987e1d9fb3a82339c97 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 2 Sep 2026 23:28:31 +0200 Subject: [PATCH 3/6] hive-c0re: deliver an agent's credential from the store to its state dir mara on #4015: "not merging code without callers", and on the same PR "see issue, we decided what the first thing should be". #3726 decided it: the controller writes a token to the store and tells the hive; the hive reads it back and writes /agents//state/matrix-token- at 0600, where matrix.nix's existing systemd.paths glob re-fires the daemon. So this is the hive half of that, and the library's first caller. The notice names a credential and never carries one, and deploy_subject's own doc is why: the auth-callout responder scopes publish and leaves sub unrestricted, so a hive that wanted another's messages could subscribe to them. A secret in that payload would be readable swarm-wide. The value is read from the store under the reading hive's own certificate, where the store's policy is what actually scopes it. Two boundaries guard the two addresses, and they are not the same check. `path::matrix_account` guards the address in the store. `Ident` guards the address on disk -- `agent_state_dir` takes one, so an unvalidated name off the queue cannot reach a directory. I had written the first and assumed it covered both; the compiler refused the `&str` and was right. `token_path` now takes the newtype so a call site cannot forget. The write is atomic because the path-watcher fires on the file appearing: written in place it would be visible while partial, and the daemon would read a truncated credential exactly once, which is the hardest possible failure to reproduce. The temp name is dot-prefixed so it cannot match the `matrix-token*` glob on its way past. The publish grant is here because without it the failure is invisible. policy.rs already says why for its siblings: a refused publish reaches the client as a timeout, so the symptom is a hive that never receives a credential with nothing in either log naming a permission. Two tests: the controller may publish, a hive may not -- its own subject included. A forged notice leaks nothing, but it would make a hive fetch and overwrite a token file for a name the forger chose. Refs #3726 --- Cargo.lock | 1 + hive-c0re/Cargo.toml | 1 + hive-c0re/src/swarm_status.rs | 59 +++++++++++- hive-c0re/src/workers/credential.rs | 144 ++++++++++++++++++++++++++++ hive-c0re/src/workers/mod.rs | 1 + swarm-nats-auth/src/policy.rs | 41 ++++++++ swarm-queue-client/src/lib.rs | 36 +++++++ 7 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 hive-c0re/src/workers/credential.rs diff --git a/Cargo.lock b/Cargo.lock index f356b36c..f5a37f9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1767,6 +1767,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "swarm-queue-client", + "swarm-secret-client", "tempfile", "tokio", "tokio-stream", diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index a1c9084f..895dc27f 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -54,6 +54,7 @@ serde_json.workspace = true # the swarm controller reads it with, and `kv` for the same reason: the # bucket's name and creation config belong to neither end of it alone. swarm-queue-client = { workspace = true, features = ["kv", "notices"] } +swarm-secret-client.workspace = true tokio.workspace = true tokio-stream.workspace = true tracing.workspace = true diff --git a/hive-c0re/src/swarm_status.rs b/hive-c0re/src/swarm_status.rs index aadbffac..b0c63070 100644 --- a/hive-c0re/src/swarm_status.rs +++ b/hive-c0re/src/swarm_status.rs @@ -198,7 +198,24 @@ async fn drain_swarm_events( return; } }; - tracing::info!(%subject, %deploy_subject, "swarm events: listening"); + // Also this hive's own, and for a second reason on top of the deploy + // subject's: the payload names an agent in *this* hive's state dir, so a + // notice for another hive is not merely noise, it is unactionable here. + let credential_subject = swarm_queue_client::credential_subject(&hive); + let mut credential_sub = match client.subscribe(credential_subject.clone()).await { + Ok(sub) => sub, + Err(e) => { + tracing::warn!( + subject = %credential_subject, error = %e, + "swarm events: subscribe failed; this hive will not hear credential notices" + ); + return; + } + }; + tracing::info!( + %subject, %deploy_subject, %credential_subject, + "swarm events: listening" + ); loop { tokio::select! { @@ -223,6 +240,13 @@ async fn drain_swarm_events( }; handle_deploy_request(&coord, &msg.payload).await; } + msg = credential_sub.next() => { + let Some(msg) = msg else { + tracing::warn!(subject = %credential_subject, "swarm events: credential subscription closed"); + return; + }; + handle_credential_notice(&hive, &msg.payload).await; + } _ = shutdown.changed() => { tracing::info!("swarm events: shutdown signal received"); return; @@ -236,6 +260,39 @@ async fn drain_swarm_events( /// /// A payload that will not decode is worth a `warn`: the controller and this /// end share one type, so a decode failure means they disagree about it. +/// Deliver the credential a [`swarm_queue_client::CredentialNotice`] names. +/// +/// `hive` doubles as the cert-auth role this hive logs into the store as: +/// `glue-bao-tls.nix` mints the client certificate with the hive name as its +/// CN, and a bao cert role matches on CN — so the two share a name by +/// construction rather than by convention. +/// +/// ⚠️ Nothing here can log the secret, and that is structural rather than +/// careful: the notice carries only names, and `deliver` writes the value +/// without returning it. +async fn handle_credential_notice(hive: &str, payload: &[u8]) { + let notice: swarm_queue_client::CredentialNotice = match serde_json::from_slice(payload) { + Ok(notice) => notice, + Err(e) => { + tracing::warn!(error = %e, "swarm events: undecodable credential notice"); + return; + } + }; + if let Err(e) = crate::workers::credential::deliver(¬ice, hive).await { + // Warn rather than retry: the controller republishes, and a hive that + // spun here would hold the queue task off its other two subjects. + tracing::warn!( + agent = %notice.agent, account = %notice.account, error = ?e, + "swarm events: credential delivery failed" + ); + return; + } + tracing::info!( + agent = %notice.agent, account = %notice.account, + "swarm events: credential delivered" + ); +} + async fn handle_deploy_request( coord: &std::sync::Arc, payload: &[u8], diff --git a/hive-c0re/src/workers/credential.rs b/hive-c0re/src/workers/credential.rs new file mode 100644 index 00000000..a6f11933 --- /dev/null +++ b/hive-c0re/src/workers/credential.rs @@ -0,0 +1,144 @@ +//! Delivering an agent's external-account credential from the swarm's secret +//! store into that agent's own state dir. +//! +//! The controller publishes a [`CredentialNotice`] naming an agent and an +//! account; this reads the value out of the store and writes it where the +//! agent's matrix daemon already watches for it. Nothing here activates +//! anything: `nix/agent-modules/matrix.nix` has a `systemd.paths` unit +//! globbing `/agents/*/state/matrix-token*` that re-fires the daemon when a +//! token appears, so arrival is the whole trigger. +//! +//! 🔑 The notice carries no secret — see [`swarm_queue_client::credential_subject`] +//! for why that is a requirement rather than a preference. The value is read +//! from the store under this hive's own identity. + +use std::os::unix::fs::PermissionsExt as _; +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use hive_types::Ident; +use swarm_queue_client::CredentialNotice; +use swarm_secret_client::{SecretStore, path}; + +use crate::paths::agent_state_dir; + +/// The basename every matrix token must start with. +/// +/// `nix/agent-modules/matrix.nix` asserts the same prefix on every configured +/// `tokenFile`, because its path-watcher globs for it. A name written here +/// that did not match would land a file the daemon never notices — no error, +/// just a credential that silently never arrives. +const TOKEN_PREFIX: &str = "matrix-token"; + +/// Where `agent`'s credential for `account` is written. +/// +/// Takes an [`Ident`] rather than a `&str` because the name arrives off the +/// queue: `agent_state_dir` addresses a directory, and an unvalidated name +/// there is a path-traversal argument. The compiler refusing the `&str` is the +/// check — nothing here has to remember to perform one. +#[must_use] +pub fn token_path(agent: &Ident, account: &str) -> PathBuf { + agent_state_dir(agent).join(format!("{TOKEN_PREFIX}-{account}")) +} + +/// Read the credential `notice` names and write it into the agent's state dir. +/// +/// `cert_role` is the role on the store's `cert` auth mount whose policy scopes +/// what this hive may read. +/// +/// # Errors +/// The store refusing, being unreachable, or holding nothing at that path; a +/// name that is not a single path segment; or the write failing. +pub async fn deliver(notice: &CredentialNotice, cert_role: &str) -> Result<()> { + // Parsed before anything is read, so a malformed name costs a decode and + // not a round trip to the store. + let agent = Ident::parse(¬ice.agent) + .map_err(|e| anyhow::anyhow!("agent name {:?} off the queue: {e}", notice.agent))?; + let secret_path = path::matrix_account(¬ice.agent, ¬ice.account) + .context("building the credential's path in the store")?; + + let store = SecretStore::from_env(cert_role) + .await + .context("connecting to the swarm secret store")?; + let value = store + .read(&secret_path) + .await + .with_context(|| format!("reading {secret_path} from the store"))?; + + write_token(&token_path(&agent, ¬ice.account), &value) +} + +/// Write `value` to `dest` at `0600`, atomically. +/// +/// Atomic because the daemon's path-watcher fires on the file *appearing*: a +/// token written in place would be visible while still partial, and the daemon +/// would read a truncated credential exactly once, at the moment it is hardest +/// to reproduce. The temp name is dot-prefixed so it cannot match the +/// `matrix-token*` glob on its way past. +fn write_token(dest: &PathBuf, value: &str) -> Result<()> { + let dir = dest + .parent() + .context("a token path always has a parent state dir")?; + let name = dest + .file_name() + .context("a token path always has a file name")?; + let mut tmp = dir.join("."); + tmp.as_mut_os_string().push(name); + tmp.as_mut_os_string().push(".partial"); + + std::fs::write(&tmp, value).with_context(|| format!("writing {}", tmp.display()))?; + // Before the rename, so the file is never briefly readable by others under + // its final name. + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600)) + .with_context(|| format!("restricting {}", tmp.display()))?; + std::fs::rename(&tmp, dest).with_context(|| format!("publishing {}", dest.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_written_name_matches_the_glob_the_daemon_watches() { + // The other end of this agreement is an assertion in + // `nix/agent-modules/matrix.nix` and a `systemd.paths` glob — neither + // reachable from a Rust test, so the prefix is pinned here. + let p = token_path(&Ident::parse("dmatrix").expect("a legal agent name"), "ccc"); + let name = p.file_name().unwrap().to_str().unwrap(); + assert!(name.starts_with("matrix-token"), "got {name}"); + assert_eq!(name, "matrix-token-ccc"); + } + + #[test] + fn the_partial_file_cannot_match_that_glob() { + // A temp name starting with `matrix-token` would be picked up + // mid-write; the dot prefix is what stops it. + let p = token_path(&Ident::parse("dmatrix").expect("a legal agent name"), "ccc"); + let name = p.file_name().unwrap().to_str().unwrap(); + let tmp = format!(".{name}.partial"); + assert!(!tmp.starts_with("matrix-token"), "got {tmp}"); + } + + #[test] + fn a_name_that_could_address_another_agent_is_refused() { + // `path::matrix_account` owns this rule; asserted here because this is + // the module that feeds it names off the wire. + assert!(path::matrix_account("../argus", "ccc").is_err()); + assert!(path::matrix_account("dmatrix", "../../etc/x").is_err()); + assert!(path::matrix_account("dmatrix", "ccc").is_ok()); + } + + #[test] + fn an_agent_name_off_the_queue_must_pass_the_ident_parser_too() { + // Two independent refusals, not one restated: `path::matrix_account` + // guards the address in the *store*, `Ident` guards the address on + // *disk*. `token_path` cannot even be called without the second, + // which is why it takes an `Ident` rather than validating internally. + for bad in ["../argus", "dmatrix/../argus", "Dmatrix", "d matrix", ""] { + assert!(Ident::parse(bad).is_err(), "{bad:?} must be refused"); + } + // The control: without it, a parser that rejected everything would + // satisfy the loop above. + assert!(Ident::parse("dmatrix").is_ok()); + } +} diff --git a/hive-c0re/src/workers/mod.rs b/hive-c0re/src/workers/mod.rs index da62baaa..bc6f232b 100644 --- a/hive-c0re/src/workers/mod.rs +++ b/hive-c0re/src/workers/mod.rs @@ -8,6 +8,7 @@ pub mod agent_sockets; pub mod auto_update; pub mod crash_watch; +pub mod credential; pub mod knowledge; pub mod mcp_sockets; pub mod scheduled_prompts_worker; diff --git a/swarm-nats-auth/src/policy.rs b/swarm-nats-auth/src/policy.rs index 4a0b5520..2383f2e0 100644 --- a/swarm-nats-auth/src/policy.rs +++ b/swarm-nats-auth/src/policy.rs @@ -352,6 +352,17 @@ impl Policy { // admission). Same failure mode as the knowledge event above: a // refused publish reaches the client as a timeout. swarm_queue_client::DEPLOY_SUBJECT_WILDCARD.to_owned(), + // The credential notices: same per-hive family and same wildcard + // reasoning as the deploy events, and the same timeout-not-error + // failure if this line is missing. + // + // 🔑 Worth being explicit that this grant is not a confidentiality + // boundary, because it looks like one. `sub` is unrestricted, so + // any hive could subscribe to another's notices — which is exactly + // why the payload names a credential and never carries one. What + // scopes the secret is the store's own policy at read time, under + // the reading hive's certificate. + swarm_queue_client::CREDENTIAL_SUBJECT_WILDCARD.to_owned(), // The wanted-state buckets — one per hive, all created and written // by this single client. // @@ -581,6 +592,36 @@ mod tests { ); } + #[test] + fn a_reader_may_publish_a_credential_notice_to_any_hive() { + let p = policy().permissions("swarm-controller").expect("a reader"); + assert!( + p.publish + .contains(&swarm_queue_client::CREDENTIAL_SUBJECT_WILDCARD.to_owned()), + "without this grant the controller's publish is refused, and a \ + refusal arrives as a timeout — a hive that silently never receives \ + a credential, with nothing in either log saying why" + ); + } + + #[test] + fn a_hive_may_not_publish_a_credential_notice() { + // A forged notice cannot leak a secret — the payload carries none — but + // it can make a hive fetch and overwrite an agent's token file with + // whatever the store holds for a name the forger chose. + let p = policy() + .permissions("hive-alpha") + .expect("a hive is admitted"); + assert!( + !p.publish + .iter() + .any(|s| s == swarm_queue_client::CREDENTIAL_SUBJECT_WILDCARD + || s == &swarm_queue_client::credential_subject("hive-alpha")), + "a hive must not publish credential notices, its own included: {:?}", + p.publish + ); + } + #[test] fn a_hive_may_not_publish_a_deploy_event_to_anyone_including_itself() { // Same arm as the knowledge event's, and it matters more here: a forged diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index f1098326..809304ff 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -212,6 +212,42 @@ pub const DEPLOY_SUBJECT_WILDCARD: &str = "$SWARM.deploy.*"; /// cannot drift into naming different families. const DEPLOY_SUBJECT_PREFIX: &str = "$SWARM.deploy"; +/// The subject the controller publishes on to tell `hive` that a credential +/// for one of its agents is waiting in the secret store. Same per-hive family +/// as [`deploy_subject`], here for the same three-crate reason. +/// +/// 🔑 **The message NAMES a credential and never carries one**, and the note +/// on [`deploy_subject`] is why: the family split buys quiet, not +/// confidentiality — the auth-callout responder scopes `pub` and leaves `sub` +/// unrestricted, so any hive that wanted another's messages could subscribe to +/// them. A secret in this payload would be readable swarm-wide. The hive reads +/// the value from the store under its own identity instead, where the store's +/// policy is the thing that actually scopes it. +#[must_use] +pub fn credential_subject(hive: &str) -> String { + format!("{CREDENTIAL_SUBJECT_PREFIX}.{hive}") +} + +/// The publish grant covering every [`credential_subject`] — a wildcard for +/// the same no-roster reason as [`DEPLOY_SUBJECT_WILDCARD`]. +pub const CREDENTIAL_SUBJECT_WILDCARD: &str = "$SWARM.credential.*"; + +/// Shared by [`credential_subject`] and [`CREDENTIAL_SUBJECT_WILDCARD`] so the +/// two cannot drift into naming different families. +const CREDENTIAL_SUBJECT_PREFIX: &str = "$SWARM.credential"; + +/// What a [`credential_subject`] message says: which agent's credential +/// changed, and which account it belongs to. Deliberately the whole payload — +/// anything more would be either derivable by the reader or a secret that +/// must not be on the wire. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct CredentialNotice { + /// The agent whose state dir receives the credential. + pub agent: String, + /// The external account the credential authenticates as. + pub account: String, +} + /// What a [`deploy_subject`] message carries. /// /// Only the agent: the subject already names the hive, and repeating it here From c6eedec3edea599e59034b5aed42f6a689496bda Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 2 Sep 2026 23:37:06 +0200 Subject: [PATCH 4/6] swarm-secret-client: a README, like every other crate in the workspace The only one of the 28 without one. Records the things a reader cannot get from the source: why the field name is pinned by a literal-string test (its other end is a shell line in a nix module, unreachable from any Rust test), why the BAO_ spellings are read explicitly rather than left to vaultrs's VAULT_ defaults, and that cert_role is the hive's own name because the cert-auth role matches on the CN the glue module mints. --- swarm-secret-client/README.md | 77 +++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 swarm-secret-client/README.md diff --git a/swarm-secret-client/README.md b/swarm-secret-client/README.md new file mode 100644 index 00000000..273865aa --- /dev/null +++ b/swarm-secret-client/README.md @@ -0,0 +1,77 @@ +# swarm-secret-client + +Reading and writing a swarm credential in the secret store, over `vaultrs`. +The HTTP is that crate's job. What this one owns is the **agreements** both +ends of the store have to state identically: where a credential lives, which +field its bytes are in, and how this deployment's environment becomes a +logged-in client. + +The surrounding picture — which secret is minted where, and why delivery is a +copy rather than a bind mount — is [`docs/swarm/secrets.md`](../docs/swarm/secrets.md). + +## Why a crate and not a module per binary + +The store has two Rust ends and they are peers: the swarm controller writes a +credential, a hive reads it. Neither is senior to the other, so a path +formatted at each call site is an agreement with no owner — it holds right up +until one side is edited alone, and then it fails as a missing key rather than +as a mismatch. + +The third end is what settles it. `nix/host-modules/glue-matrix-bao-token.nix` +reads the store with `bao kv get -field=value`. That reader is a shell line in +a nix module: it cannot be renamed by the same refactor as a Rust struct, and +no Rust test reaches it. So the field name is pinned by a test against the +serialised literal rather than left to the struct definition. + +## The identity is a certificate, and the role is the hive's name + +Authentication is the store's `cert` auth method. +`nix/host-modules/glue-bao-tls.nix` mints the client certificate with its CN +set to the hive's name, because a cert-auth role matches on the CN. So +`cert_role` is not a free choice for the caller: a hive passes its own name, +and the policy attached to that role is what scopes what it may read. + +The listener's `tls_require_and_verify_client_cert` is a different thing and +not a substitute. It decides who may open a connection; it says nothing about +who the connection belongs to, and a store with the option set and no `cert` +mount configured refuses every login made here. That refusal is what +`Error::Vault` out of `connect` means. + +## Configuration + +`Settings::from_env` reads `BAO_ADDR`, `BAO_CLIENT_CERT`, `BAO_CLIENT_KEY`, +and the optional `BAO_CACERT`. + +**The `BAO_` spellings are read explicitly rather than left to `vaultrs`.** Its +own defaults look for `VAULT_ADDR` / `VAULT_CLIENT_CERT` / `VAULT_CLIENT_KEY`, +which no unit in this tree sets. Falling through to them builds a client with +**no identity at all**, and that surfaces as a TLS handshake failure — a place +that names neither the variable nor the reason. + +**Empty is as absent as unset.** systemd renders an unset nix option as +`Environment=BAO_CACERT=`, so empty is the shape a missing value arrives in. + +**The certificate and key are paths, not values**, and are read at connect +time — same rule as every other credential in this tree, for the same reason: +a value in a nix expression is rendered into the world-readable store. + +Reading the environment is separate from connecting (`Settings::from_lookup`) +because every one of those failures is a misconfiguration an operator has to +read an error about, and none of them needs a reachable store to happen. + +## Names that arrive from elsewhere + +`path::matrix_account` is fallible, which for a string formatter needs saying: +its segments are an agent name from the topology and an account name from that +agent's own config. A `/` turns one agent's segment into another agent's +directory and `..` walks out of the prefix entirely, so the charset it accepts +is deliberately narrower than what the store would. + +## What this crate does not do + +It has no opinion on **what** a caller may read. That is the policy attached to +the cert role, and it lives in the store. + +It holds the token minted at login and renews nothing. A handle is built per +credential, so the login is the cheap part of a rare operation — a caller that +wanted to keep one alive across a token's lifetime would need more than this. From ad439843f188eaf1eeee5dd385feb36a4c7537b2 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 2 Sep 2026 23:51:00 +0200 Subject: [PATCH 5/6] hive-c0re: hand the daemon the store identity it cannot open itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `credential.rs` calls `SecretStore::from_env`, and nothing set `BAO_*` for this unit — only `swarm-bao-matrix-token` had them. Fixing that alone would not have been enough: hive-c0re runs as hive-core, and glue-bao-tls mints the client key `0600` inside a `0700` directory, so the daemon cannot read the file even when it is named. Both halves go through LoadCredential, which this unit already uses for the swarm-status client secret: root reads the identity at unit start, hive-core sees it under `%d`, and no second on-disk copy exists. The gate is the identity, never `deploy.bao.enable` — a hive that reads a store on another machine holds a certificate and runs no store. Four module-eval cases: the co-located and off-host presence arms, the absence arm for a hive with no identity, and a presence/absence pair for the optional CA. --- nix/host-modules/hive-c0re/default.nix | 25 ++++++++++- nix/host-modules/hive-c0re/environment.nix | 27 +++++++++++ nix/module-eval.nix | 52 ++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 87482bc9..ee8a9f1c 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -13,6 +13,13 @@ let cfg = config.services.hyperhive.c0re; + baoDeploy = config.services.hyperhive.deploy.bao; + # Held in one place because the LoadCredential below and the `BAO_*` + # environment in ./environment.nix have to agree on when they exist: a + # credential with no reader is dead weight, and an environment naming a + # credential nobody loaded is a daemon that fails at the TLS handshake. + baoClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null; + caTrust = import ../lib/hive-ca-trust.nix { inherit lib; tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; @@ -309,7 +316,23 @@ in # delivers into a container, across a filesystem boundary.) lib.optional ( config.services.hyperhive.swarm.statusPublish.clientSecretFile != null - ) "swarm-status-client.secret:${config.services.hyperhive.swarm.statusPublish.clientSecretFile}"; + ) "swarm-status-client.secret:${config.services.hyperhive.swarm.statusPublish.clientSecretFile}" + # The secret store's client identity, on the same reasoning one + # paragraph up — with a sharper edge: ./glue-bao-tls.nix mints the + # key `0600` inside a `0700` directory, so hive-core cannot read it + # at all. Loading it as a credential is what makes the store + # reachable from an unprivileged daemon without widening either. + # The gate is the identity, never `deploy.bao.enable`: a hive that + # reads a store on another machine holds one of these and runs no + # store. ./glue-matrix-bao-token.nix gates its own reader the same + # way. + ++ lib.optionals baoClientIdentity [ + "bao-client.pem:${baoDeploy.clientCertFile}" + "bao-client-key.pem:${baoDeploy.clientKeyFile}" + ] + ++ lib.optional ( + baoClientIdentity && baoDeploy.serverCaFile != null + ) "bao-ca.pem:${baoDeploy.serverCaFile}"; # Sandboxing. hive-c0re is unprivileged (runs as hive-core, never # setuid), makes HTTP requests to forge/matrix/Anthropic (keeps INET), # and delegates all privileged ops to hive-priv via a Unix socket. diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 3345bd06..8539e6b4 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -9,6 +9,13 @@ }: let cfg = config.services.hyperhive.c0re; + baoDeploy = config.services.hyperhive.deploy.bao; + baoCfg = config.services.hyperhive.swarm.bao; + # The same test ../glue-matrix-bao-token.nix applies, and for the same + # reason: a reader is defined by holding a certificate the store accepts, + # not by sharing a host with the store. Deriving this from + # `deploy.bao.enable` would be the co-location assumption itself. + haveBaoClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null; in { # nix (the prebuild `nix build`, flake-check, and meta eval) writes @@ -263,3 +270,23 @@ in # ./default.nix. The daemon reads a path, never a value. HIVE_C0RE_OIDC_CLIENT_SECRET_FILE = "%d/swarm-status-client.secret"; } +// + # Where the swarm's secret store is, and the identity this hive presents to + # it (hive-c0re::workers::credential). `swarm_secret_client` reads these + # spellings explicitly rather than vaultrs's `VAULT_*` defaults — falling + # through to those would build a client with no identity and fail at the TLS + # handshake, naming neither. + lib.optionalAttrs haveBaoClientIdentity { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + # `%d`, not the paths themselves: the key is `0600` root-owned and this + # daemon runs as hive-core, so it never gets read access to the original. + # See the LoadCredential in ./default.nix. + BAO_CLIENT_CERT = "%d/bao-client.pem"; + BAO_CLIENT_KEY = "%d/bao-client-key.pem"; + } +// lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) { + # Absent means the system trust store — right for a deployment with a real + # CA, wrong for the self-signed one ../glue-bao-tls.nix mints, which is why + # that file names this path rather than leaving it to a default. + BAO_CACERT = "%d/bao-ca.pem"; +} diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 3ca034d1..49678d98 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -483,6 +483,58 @@ let in s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ]; } + { + # hive-c0re runs as hive-core and the client key is `0600` root-owned + # inside a `0700` directory, so the identity reaches the daemon as a + # systemd credential and the environment names `%d` rather than the + # file. Both halves are asserted together because either alone is a + # daemon that fails at the TLS handshake, naming neither. + name = "a reader hands hive-c0re a store identity the daemon cannot open itself"; + ok = + let + s = baoRemoteReader.systemd.services; + in + s ? hive-c0re + && (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem" + && (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem" + && builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential + && builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential; + } + { + # Absence arm for the case above. A hive with no client identity gets no + # store environment at all — the daemon reports a queue it cannot serve + # rather than a handshake it cannot explain. + name = "a hive with no client identity gives hive-c0re no store environment"; + ok = + let + s = matrixNoBaoIdentity.systemd.services; + in + s ? hive-c0re + && !(s.hive-c0re.environment ? BAO_ADDR) + && !(lib.any (c: lib.hasPrefix "bao-" c) s.hive-c0re.serviceConfig.LoadCredential); + } + { + # The CA is its own arm: absent means the system trust store, which is + # right for a deployment with a real CA and wrong for a self-signed one. + name = "a reader that names no store CA falls through to the system trust store"; + ok = + let + s = baoRemoteReader.systemd.services; + in + s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT); + } + { + # Presence control for the arm above: the CA is conditional, not gone. + # Co-located, ./host-modules/glue-bao-tls.nix mints one and names it. + name = "a reader beside a self-signed store is given that store's CA"; + ok = + let + s = baoWithMatrix.systemd.services; + in + s ? hive-c0re + && (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem" + && lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential; + } { # The name a reader dials has to resolve where the store runs; a # multi-host swarm resolves it upstream instead. From d6d60ff06e8ca30ce9625b4c4b3ccf5effb33e90 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 2 Sep 2026 23:51:07 +0200 Subject: [PATCH 6/6] swarm-bao: the reader's identity is not a store-runner option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The block header says `deploy.bao.*` is what the host RUNNING the store decides and that none of it means anything on a host that only reads secrets. Three options in that block are exactly what a read-only host must set, and their own descriptions say so a screen further down — so reading the options never corrects the header. It reads as though an off-host reader were inexpressible. It is already supported: glue-matrix-bao-token gates on the identity rather than on `enable`, and module-eval's baoRemoteReader is that deployment. Also drops a sentence describing what used to be undeclarable. --- nix/host-modules/swarm-bao.nix | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 6ba6e96f..f1031f53 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -245,10 +245,17 @@ in { # One service, two namespaces, and the split decides who may set what. # - # `deploy.bao.*` is what the host RUNNING the store decides: whether to run - # it (`enable`, declared in ./deploy.nix with its siblings), which build, - # how the root key is sealed, what it listens on. None of it means anything - # on a host that only reads secrets. + # `deploy.bao.*` is mostly what the host RUNNING the store decides: whether + # to run it (`enable`, declared in ./deploy.nix with its siblings), which + # build, how the root key is sealed, what it listens on. + # + # ⚠️ Except the reader's own identity. `clientCertFile`, `clientKeyFile` and + # `serverCaFile` live here too, and they are set by whoever authenticates TO + # the store — which includes a host that runs none of it. A reader is defined + # by holding a certificate the store accepts, never by sharing a host with + # one; ./glue-matrix-bao-token.nix gates on exactly that and not on `enable`. + # Reading this block as store-runner-only is what makes an off-host reader + # look inexpressible when it is already supported. # # `swarm.bao.*` below is what every host in the swarm has to agree on — the # name the store answers to, its port, its container. A host that is purely @@ -378,9 +385,7 @@ in Certificate a **reader on this machine** presents to the store. The counterpart to {option}`services.hyperhive.deploy.bao.clientCaFile`, - which is the store's side of the same handshake. Only the store's side - was declarable, so a hive that did not run the store had no way to be - pointed at a certificate even when one was placed for it. + which is the store's side of the same handshake. On a hive that runs the store, a glue module supplies the leaf it minted, as a `mkDefault`. Everywhere else this is the credential an operator