From ec06a91debc6687562d123a31761421a03d44190 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 13:02:25 +0100 Subject: [PATCH] mint per-agent proxy credentials and a per-hive squid self-check credential --- nix/host-modules/swarm-bao.nix | 4 + nix/module-eval/bao-grants.nix | 7 +- swarm-controller/src/main.rs | 1 + swarm-controller/src/proxy_credential.rs | 320 +++++++++++++++++++++++ swarm-controller/src/startup.rs | 3 +- swarm-secret-client/src/lib.rs | 3 +- swarm-secret-client/src/proxy.rs | 153 +++++++++++ 7 files changed, 487 insertions(+), 4 deletions(-) create mode 100644 swarm-controller/src/proxy_credential.rs create mode 100644 swarm-secret-client/src/proxy.rs diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index ecd6b395..4f66b668 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -486,6 +486,10 @@ let capabilities = ["create", "read", "update"] } + path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" { + capabilities = ["create", "read"] + } + ${lib.concatMapStrings readStanza ( map (leaf: "${credentialMountPath}/data/${leaf}") atticFactLeaves )} diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 2f7951d6..022a51c1 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -1724,17 +1724,20 @@ let { # `matrix_account::hive_sender` writes every hive's sender token and # `forge::hive_token` every hive's forge token, and nothing else under - # `hives/`, where each hive's queue credential also lives. `+` is one + # `hives/`, where each hive's queue credential also lives. `proxy_credential` + # creates each hive's squid self-check credential and reads it back, and + # has no `update`, so it cannot replace one. `+` is one # segment, which keeps each stanza to its one leaf; a `*` is a glob only # at the end of a path. Pinned as whole stanzas, so an added capability # fails. - name = "the controller writes each hive's sender and forge tokens and nothing else under hives"; + name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, and nothing else under hives"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/+/matrix/sender-token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s && lib.hasInfix "path \"secret/data/swarm/hives/+/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s + && lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s && !(lib.hasInfix "secret/data/swarm/hives/*" s) && !(lib.hasInfix "secret/metadata/swarm/hives" s); } diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 9d882782..f90980dd 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -60,6 +60,7 @@ mod meta_flake; mod notify_flows; mod notify_router_api; mod otel_http_client; +mod proxy_credential; mod pulls; mod queue_identity; mod read_policy; diff --git a/swarm-controller/src/proxy_credential.rs b/swarm-controller/src/proxy_credential.rs new file mode 100644 index 00000000..3b242ef1 --- /dev/null +++ b/swarm-controller/src/proxy_credential.rs @@ -0,0 +1,320 @@ +//! The credentials squid authenticates its clients with: one per agent, at +//! `swarm_secret_client::proxy::agent_credential_path`, and one per hive for +//! that hive's squid self-check, at `proxy::selfcheck_path`. +//! +//! Minted here because the agent's store policy is read-only and a hive's +//! policy cannot create under another principal's prefix. [`spawn`]'s pass runs +//! at start and every [`RECONCILE_INTERVAL`] over every agent that holds a store +//! identity, so an agent that existed before this pass was added gets one on the +//! next tick, with no restart. +//! +//! ⚠️ **Never replaced.** A path that holds anything is left alone, and the write +//! is a `cas=0` create. Replacing a credential is a rotation: squid's users file +//! is rendered from whatever the store holds, and a rotation has to reach every +//! client that carries the old one in the same step. + +use anyhow::{Context, Result}; +use serde::de::IgnoredAny; +use swarm_secret_client::{Error, SecretStore, client::DEFAULT_CERT_MOUNT, policy, proxy}; + +/// How often [`spawn`] re-checks every agent and hive. +const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5); + +/// What [`ensure`] did at one path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// The path already held a credential. + Present, + /// This pass wrote one. + Minted, + /// Another writer's create landed between this pass's read and its own. + LostRace, +} + +/// The two store calls [`ensure`] makes, so a test can stand in for the store. +pub(crate) trait CredentialStore { + /// Whether anything is stored at `path`, as [`SecretStore::read_optional`] + /// answers it. The value is decoded as [`IgnoredAny`] and never held. + fn exists(&self, path: &str) -> impl Future> + Send; + + /// As [`SecretStore::create`]. + fn create( + &self, + path: &str, + value: &proxy::Credential, + ) -> impl Future> + Send; +} + +impl CredentialStore for SecretStore { + async fn exists(&self, path: &str) -> Result { + Ok(self.read_optional::(path).await?.is_some()) + } + + async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> { + SecretStore::create(self, path, value).await + } +} + +/// Make sure `path` holds a credential, writing one only if it holds nothing. +/// +/// A create that fails is read back: if the path now holds something, a +/// concurrent writer won and that is the credential. +/// +/// # Errors +/// When the store refuses the read, or the create fails and the path is still +/// empty. +pub(crate) async fn ensure(store: &impl CredentialStore, path: &str) -> Result { + if store + .exists(path) + .await + .with_context(|| format!("reading {path}"))? + { + return Ok(Outcome::Present); + } + let credential = proxy::Credential { + value: crate::agent_identity::generate_queue_secret()?, + }; + let Err(e) = store.create(path, &credential).await else { + return Ok(Outcome::Minted); + }; + if store + .exists(path) + .await + .with_context(|| format!("reading {path} back after a failed create"))? + { + return Ok(Outcome::LostRace); + } + Err(anyhow::Error::new(e).context(format!("creating {path}"))) +} + +/// [`ensure`] at each of `paths`, which `what` names in the log. Returns how +/// many it minted. A failure at one path is logged and the rest still run. +async fn ensure_all( + store: &impl CredentialStore, + what: &str, + paths: Vec<(String, Result)>, +) -> usize { + let mut minted = 0; + for (name, path) in paths { + let path = match path { + Ok(path) => path, + Err(e) => { + tracing::warn!(name, error = %e, "{what}: unusable name"); + continue; + } + }; + match ensure(store, &path).await { + Ok(Outcome::Present) => {} + Ok(Outcome::Minted) => { + minted += 1; + tracing::info!(name, %path, "{what} minted"); + } + Ok(Outcome::LostRace) => { + tracing::info!(name, %path, "{what}: another writer created it first"); + } + Err(e) => tracing::warn!(name, error = %format!("{e:#}"), "{what}: mint failed"), + } + } + minted +} + +/// One pass: a credential for every agent in `agents` and a self-check +/// credential for every hive in `hives`. Returns how many were minted. +async fn mint_missing(store: &impl CredentialStore, agents: &[String], hives: &[String]) -> usize { + let for_agents = agents + .iter() + .map(|a| (a.clone(), proxy::agent_credential_path(a))) + .collect(); + let for_hives = hives + .iter() + .map(|h| (h.clone(), proxy::selfcheck_path(h))) + .collect(); + ensure_all(store, "agent proxy credential", for_agents).await + + ensure_all(store, "hive squid self-check credential", for_hives).await +} + +/// The agents that hold a store identity: exactly the ones whose hive reads +/// what a pass writes. +async fn roster(store: &SecretStore) -> Result> { + let roles = store + .list_cert_roles(DEFAULT_CERT_MOUNT) + .await + .context("listing the store's cert-auth roles")?; + Ok(policy::agents_from_role_names(&roles)) +} + +/// Run a pass now and every [`RECONCILE_INTERVAL`] after. A pass that fails is +/// logged and retried on the next tick; it never stops the daemon. +pub fn spawn(hives: Vec) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); + loop { + ticker.tick().await; + let pass = async { + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + let agents = roster(&store).await?; + anyhow::Ok((agents.len(), mint_missing(&store, &agents, &hives).await)) + }; + match pass.await { + Ok((agents, minted)) => { + tracing::debug!(agents, minted, "proxy credentials: pass done"); + } + Err(e) => tracing::warn!( + error = %format!("{e:#}"), + retry_in_s = RECONCILE_INTERVAL.as_secs(), + "proxy credentials: pass failed; retrying next tick" + ), + } + } + }); +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Mutex; + + use super::*; + + /// An in-memory store that refuses a create over a held path the way a + /// `cas=0` write does, and counts the creates it was asked for. + #[derive(Default)] + struct Fake { + held: Mutex>, + creates: Mutex, + /// A path whose create fails and leaves nothing behind. + broken: Option, + /// Makes the next create find the path taken by someone else. + race: Option, + } + + impl Fake { + fn holding(path: &str, value: &str) -> Self { + let fake = Self::default(); + fake.held + .lock() + .unwrap() + .insert(path.to_owned(), value.to_owned()); + fake + } + + fn value(&self, path: &str) -> Option { + self.held.lock().unwrap().get(path).cloned() + } + } + + fn refused() -> Error { + Error::MissingEnv("BAO_ADDR") + } + + impl CredentialStore for Fake { + async fn exists(&self, path: &str) -> Result { + Ok(self.held.lock().unwrap().contains_key(path)) + } + + async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> { + *self.creates.lock().unwrap() += 1; + let mut held = self.held.lock().unwrap(); + if self.broken.as_deref() == Some(path) { + return Err(refused()); + } + if let Some(winner) = &self.race { + held.insert(path.to_owned(), winner.clone()); + return Err(refused()); + } + if held.contains_key(path) { + return Err(refused()); + } + held.insert(path.to_owned(), value.value.clone()); + Ok(()) + } + } + + const ATLAS: &str = "swarm/agents/atlas/proxy-credential"; + const ARGUS: &str = "swarm/agents/argus/proxy-credential"; + const H1: &str = "swarm/hives/h1/squid-selfcheck"; + + fn names(names: &[&str]) -> Vec { + names.iter().map(|n| (*n).to_owned()).collect() + } + + #[tokio::test] + async fn an_absent_credential_is_created() { + let store = Fake::default(); + assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Minted); + let value = store.value(ATLAS).expect("written"); + // 32 bytes, base64url without padding. + assert_eq!(value.len(), 43, "{value}"); + assert!( + value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + ); + } + + #[tokio::test] + async fn a_present_credential_is_untouched_and_never_updated() { + let store = Fake::holding(ATLAS, "existing"); + assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Present); + assert_eq!(store.value(ATLAS).as_deref(), Some("existing")); + assert_eq!(*store.creates.lock().unwrap(), 0, "no write was attempted"); + } + + #[tokio::test] + async fn a_lost_race_keeps_the_winners_value() { + let store = Fake { + race: Some("winner".to_owned()), + ..Fake::default() + }; + assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::LostRace); + assert_eq!(store.value(ATLAS).as_deref(), Some("winner")); + } + + #[tokio::test] + async fn a_failed_create_with_nothing_behind_it_is_an_error() { + let store = Fake { + broken: Some(ATLAS.to_owned()), + ..Fake::default() + }; + assert!(ensure(&store, ATLAS).await.is_err()); + assert_eq!(store.value(ATLAS), None); + } + + #[tokio::test] + async fn a_pass_mints_for_every_agent_and_hive_that_lacks_one() { + let store = Fake::holding(ARGUS, "existing"); + let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await; + assert_eq!(minted, 2); + assert!(store.value(ATLAS).is_some()); + assert!(store.value(H1).is_some()); + assert_eq!(store.value(ARGUS).as_deref(), Some("existing")); + // Control: the second pass has nothing left to mint, and writes nothing. + let creates = *store.creates.lock().unwrap(); + assert_eq!( + mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await, + 0 + ); + assert_eq!(*store.creates.lock().unwrap(), creates); + } + + #[tokio::test] + async fn one_agent_failing_does_not_stop_the_rest() { + let store = Fake { + broken: Some(ATLAS.to_owned()), + ..Fake::default() + }; + let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await; + assert_eq!(minted, 2); + assert!(store.value(ARGUS).is_some()); + assert!(store.value(H1).is_some()); + } + + #[tokio::test] + async fn an_unusable_agent_name_mints_nothing() { + let store = Fake::default(); + assert_eq!(mint_missing(&store, &names(&["../argus"]), &[]).await, 0); + assert!(store.held.lock().unwrap().is_empty()); + } +} diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index b30325ac..81531948 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -368,7 +368,7 @@ fn spawn_subagent_streams(status: Option<&Arc>, hives: &[H /// The per-agent background passes: subagent and terminal streams, and /// schedule keys, and the permissions bucket and its import, and the /// archetypes agents are created from. Beside them, the -/// copy of the cache facts to each hive. +/// copy of the cache facts to each hive, and the proxy credentials squid checks. /// Lifted out of `main` for `clippy::too_many_lines`. pub(crate) fn spawn_per_agent_upkeep( status: Option<&Arc>, @@ -378,6 +378,7 @@ pub(crate) fn spawn_per_agent_upkeep( spawn_schedule_reconcile(status, hives); spawn_agent_permissions(status, hives); crate::attic_copy::spawn(hives.iter().map(|h| h.name.clone()).collect()); + crate::proxy_credential::spawn(hives.iter().map(|h| h.name.clone()).collect()); if let Some(status) = status { archetypes::spawn(status.queue_client()); } diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index a21ea5b0..502f2b84 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -7,7 +7,7 @@ //! of secret, the path it lives at together with the fields it holds //! ([`matrix`], [`queue`], [`mtls`], [`forge`], [`github`], [`grafana`], //! [`acp`], [`attic`], [`backend_env`], [`ci`], [`review_classifier`], [`builder`], [`ci_metrics`], -//! [`claude_login`]). +//! [`claude_login`], [`proxy`]). //! Each of those is a thing the controller and a hive must say identically, so //! it is said once here. //! @@ -47,6 +47,7 @@ pub mod matrix; pub mod mtls; pub mod path; pub mod policy; +pub mod proxy; pub mod queue; pub mod review_classifier; pub mod startup_read; diff --git a/swarm-secret-client/src/proxy.rs b/swarm-secret-client/src/proxy.rs new file mode 100644 index 00000000..319c6f34 --- /dev/null +++ b/swarm-secret-client/src/proxy.rs @@ -0,0 +1,153 @@ +//! The egress-proxy agreement: the credentials squid authenticates its clients +//! with, where they live in the store, and what the objects at those paths hold. +//! +//! Each agent has one ([`agent_credential_path`]), minted by `swarm-controller` +//! and read by the agent's hive, which turns it into the hash squid's users file +//! holds. Each hive has one more ([`selfcheck_path`]) for its own squid +//! self-check, so the self-check can prove an authenticated request is let +//! through. Neither end is senior, so both halves are stated once, here, beside +//! [`crate::forge`]. + +use serde::{Deserialize, Serialize}; + +use crate::{ + Error, + path::{Kind, principal_prefix}, +}; + +/// The name of an agent's leaf. +pub const AGENT_LEAF: &str = "proxy-credential"; + +/// The name of a hive's self-check leaf. +pub const SELFCHECK_LEAF: &str = "squid-selfcheck"; + +/// The squid user the self-check logs in as. +/// +/// An agent's squid user is its name, and agent names cannot hold a `.` +/// ([`crate::path::checked_segment`]), so this one cannot be an agent's. +/// `nix/host-modules/hive-squid.nix` spells the same string. +pub const SELFCHECK_USER: &str = "selfcheck.squid"; + +/// The path holding the secret `agent` authenticates to the egress proxy with. +/// +/// A flat leaf under the agent's prefix, like +/// [`crate::forge::agent_token_path`], so the agent's own read stanza +/// ([`crate::policy::render_shared_agent`]) already covers it, and so does the +/// hive's. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`, +/// which is what keeps one agent's name from addressing another agent's secret. +pub fn agent_credential_path(agent: &str) -> Result { + let prefix = principal_prefix(Kind::Agent, agent)?; + Ok(format!("{prefix}/{AGENT_LEAF}")) +} + +/// The path holding the secret `hive`'s own squid self-check authenticates with. +/// +/// Under [`Kind::Hive`], so the grant that reaches it is the hive's own +/// `swarm/hives//*` stanza: hive `alpha` reads its own and gets a 403 +/// on `beta`'s. +/// +/// # Errors +/// [`Error::PathSegment`] when `hive` contains anything but `[A-Za-z0-9_-]`. +pub fn selfcheck_path(hive: &str) -> Result { + let prefix = principal_prefix(Kind::Hive, hive)?; + Ok(format!("{prefix}/{SELFCHECK_LEAF}")) +} + +/// What [`agent_credential_path`] and [`selfcheck_path`] hold. +/// +/// No `Debug` derive: `value` is a live credential, and a derived `Debug` is +/// one `{:?}` away from a log line. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Credential { + /// The secret: 32 random bytes, base64url without padding, so it can sit in + /// a proxy URL's userinfo unescaped. + pub value: String, +} + +impl std::fmt::Debug for Credential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credential") + .field("value", &"") + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn covered_by(policy: &str, path: &str) -> bool { + policy.lines().any(|line| { + line.strip_prefix("path \"") + .and_then(|rest| rest.split_once("\" {")) + .and_then(|(p, _)| p.strip_suffix('*')) + .is_some_and(|prefix| format!("secret/data/{path}").starts_with(prefix)) + }) + } + + #[test] + fn the_paths_are_the_leaves_the_nix_side_spells() { + assert_eq!( + agent_credential_path("atlas").expect("legal"), + "swarm/agents/atlas/proxy-credential" + ); + assert_eq!( + selfcheck_path("h1").expect("legal"), + "swarm/hives/h1/squid-selfcheck" + ); + } + + #[test] + fn a_traversal_in_either_name_is_refused() { + for bad in ["../argus", "a/b", "a.b", ""] { + assert!(agent_credential_path(bad).is_err(), "{bad:?}"); + assert!(selfcheck_path(bad).is_err(), "{bad:?}"); + } + assert!(agent_credential_path("a-b_C9").is_ok()); + assert!(selfcheck_path("a-b_C9").is_ok()); + } + + #[test] + fn the_agent_reads_its_own_credential_and_a_hive_reads_every_one() { + let path = agent_credential_path("atlas").expect("legal"); + assert!(covered_by(&crate::policy::effective_agent("atlas"), &path)); + assert!(!covered_by(&crate::policy::effective_agent("argus"), &path)); + assert!(covered_by(&crate::policy::effective_hive("h1"), &path)); + } + + #[test] + fn a_hive_reads_its_own_selfcheck_credential_and_no_other_hives() { + let path = selfcheck_path("h1").expect("legal"); + assert!(covered_by(&crate::policy::effective_hive("h1"), &path)); + assert!(!covered_by(&crate::policy::effective_hive("h2"), &path)); + assert!(!covered_by(&crate::policy::effective_agent("atlas"), &path)); + } + + #[test] + fn the_selfcheck_user_is_not_a_legal_agent_name() { + assert!(crate::path::checked_segment("agent", SELFCHECK_USER).is_err()); + } + + #[test] + fn debug_hides_the_value() { + let shown = format!( + "{:?}", + Credential { + value: "s3cret".to_owned() + } + ); + assert!(!shown.contains("s3cret"), "{shown}"); + } + + #[test] + fn the_value_field_is_named_value() { + let json = serde_json::to_string(&Credential { + value: "t".to_owned(), + }) + .expect("a String serialises"); + assert_eq!(json, r#"{"value":"t"}"#); + } +}