diff --git a/Cargo.lock b/Cargo.lock index b8383b3c..8d768a19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5022,7 +5022,6 @@ dependencies = [ "axum", "backon", "data-encoding", - "futures-util", "hive-sh4re", "reqwest", "serde", @@ -5032,7 +5031,6 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tracing", - "tracing-subscriber", "utoipa", ] @@ -5062,6 +5060,7 @@ dependencies = [ name = "swarm-secret-client" version = "0.1.0" dependencies = [ + "futures-util", "reqwest", "rustify", "rustify_derive", @@ -5070,6 +5069,9 @@ dependencies = [ "serde_json", "strum", "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", "vaultrs", ] diff --git a/hive-c0re/src/swarm_queue.rs b/hive-c0re/src/swarm_queue.rs index ec221dcf..4b0e8779 100644 --- a/hive-c0re/src/swarm_queue.rs +++ b/hive-c0re/src/swarm_queue.rs @@ -14,7 +14,7 @@ //! environment. The client secret comes from the swarm's secret store, at //! `swarm_secret_client::queue::hive_client_path`, read under this hive's own //! certificate, held only in memory and read again when authelia refuses it -//! (`swarm_queue_client::stored_secret`). `swarm-secret-publish`, in authelia's +//! (`swarm_secret_client::stored_secret`). `swarm-secret-publish`, in authelia's //! container, puts it there. //! //! Connects lazily on first use rather than at boot — nothing here @@ -28,9 +28,9 @@ use std::path::PathBuf; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; -use swarm_queue_client::stored_secret::StoredSecret; use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::queue::{self, HiveClientCredential}; +use swarm_secret_client::stored_secret::StoredSecret; use tokio::sync::Mutex; use tokio::time::Instant; diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index cf325d8f..bf0cfe2b 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -23,9 +23,9 @@ use swarm_queue_client::builder::{ ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, QUEUE_GROUP, REPLY_DEADLINE, Reply, SUBJECT, }; -use swarm_queue_client::stored_secret::StoredSecret; use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::forge::Credential; +use swarm_secret_client::stored_secret::StoredSecret; mod archetypes; mod nix_build; diff --git a/swarm-ci-metrics/src/main.rs b/swarm-ci-metrics/src/main.rs index a106689d..87d5e448 100644 --- a/swarm-ci-metrics/src/main.rs +++ b/swarm-ci-metrics/src/main.rs @@ -18,11 +18,11 @@ use std::time::Duration; use anyhow::{Context as _, Result, bail}; use forgejo_api::structs::{ActionRun, ListActionRunsQuery, RepoSearchQuery}; use forgejo_api::{Auth, Forgejo}; -use swarm_queue_client::stored_secret::StoredSecret; use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::SecretStore; use swarm_secret_client::ci_metrics::{ClientSecret as StoredClientSecret, client_secret_path}; use swarm_secret_client::forge::Credential; +use swarm_secret_client::stored_secret::StoredSecret; mod cursor; mod otlp; diff --git a/swarm-claude-login/src/main.rs b/swarm-claude-login/src/main.rs index 9ad56f03..36bcdf3f 100644 --- a/swarm-claude-login/src/main.rs +++ b/swarm-claude-login/src/main.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use anyhow::{Context, Result, bail}; use futures_util::StreamExt as _; use swarm_queue_client::claude_login::{Reply, Request, SESSION_DEADLINE, STEP_DEADLINE, SUBJECT}; -use swarm_queue_client::stored_secret::StoredSecret; use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::claude_login::{Credentials, client_secret_path, credentials_path}; +use swarm_secret_client::stored_secret::StoredSecret; mod login; diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index 75199cc7..c3338acf 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -42,7 +42,7 @@ use utoipa::ToSchema; use crate::webhook::DeliveryKind; use swarm_queue_client::review_classifier::Commit; -use swarm_queue_client::stored_secret::StoredSecret; +use swarm_secret_client::stored_secret::StoredSecret; pub mod agent_token; mod api; diff --git a/swarm-controller/src/forge/api.rs b/swarm-controller/src/forge/api.rs index 8effba2a..e1198b14 100644 --- a/swarm-controller/src/forge/api.rs +++ b/swarm-controller/src/forge/api.rs @@ -12,7 +12,7 @@ use std::sync::{Arc, Mutex}; use forgejo_api::{ApiErrorKind, Auth, Forgejo, ForgejoError}; use futures_util::future::BoxFuture; use reqwest::StatusCode; -use swarm_queue_client::stored_secret::StoredSecret; +use swarm_secret_client::stored_secret::StoredSecret; /// One forge request, as [`Api::call`] runs it. pub(crate) type Call<'a, R> = BoxFuture<'a, Result>; diff --git a/swarm-controller/src/queue_identity.rs b/swarm-controller/src/queue_identity.rs index 2820e71a..1ff98e52 100644 --- a/swarm-controller/src/queue_identity.rs +++ b/swarm-controller/src/queue_identity.rs @@ -5,7 +5,7 @@ //! environment. The client secret comes from the swarm's secret store, at //! `swarm_secret_client::queue::controller_client_path`, read at start under //! this daemon's own certificate, held only in memory and read again when -//! authelia refuses it (`swarm_queue_client::stored_secret`). It is never on +//! authelia refuses it (`swarm_secret_client::stored_secret`). It is never on //! this host's disk. `swarm-secret-publish`, in authelia's //! container, puts it there. //! @@ -15,9 +15,9 @@ use std::path::PathBuf; use std::sync::OnceLock; use anyhow::{Context, Result, bail}; -use swarm_queue_client::stored_secret::StoredSecret; use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::queue::{self, ControllerCredential}; +use swarm_secret_client::stored_secret::StoredSecret; /// The environment prefix the nix module's `queueEnv` sets. const PREFIX: &str = "SWARM_CONTROLLER"; diff --git a/swarm-nats-auth/src/main.rs b/swarm-nats-auth/src/main.rs index a1ed9178..fded845c 100644 --- a/swarm-nats-auth/src/main.rs +++ b/swarm-nats-auth/src/main.rs @@ -714,7 +714,7 @@ mod tests { .with_agent_token_subjects(vec!["$SWARM.term.{agent}".to_owned()]) .expect("valid"); let client_secret = - oidc_secret::OidcSecret::new(swarm_queue_client::stored_secret::StoredSecret::new( + oidc_secret::OidcSecret::new(swarm_secret_client::stored_secret::StoredSecret::new( "swarm-nats", "unused".to_owned(), || async { Err("never re-read".into()) }, diff --git a/swarm-nats-auth/src/oidc_secret.rs b/swarm-nats-auth/src/oidc_secret.rs index 3cffcbc0..4ee30687 100644 --- a/swarm-nats-auth/src/oidc_secret.rs +++ b/swarm-nats-auth/src/oidc_secret.rs @@ -16,9 +16,9 @@ use std::future::Future; use std::sync::Arc; use anyhow::{Context, Result}; -use swarm_queue_client::stored_secret::{StoredSecret, is_credential_refusal}; use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings}; use swarm_secret_client::path::{Kind, principal_prefix}; +use swarm_secret_client::stored_secret::{StoredSecret, is_credential_refusal}; use tokio::time::Instant; use crate::introspect::{Answer, Caller}; diff --git a/swarm-queue-client/Cargo.toml b/swarm-queue-client/Cargo.toml index e43623e7..b7a590c0 100644 --- a/swarm-queue-client/Cargo.toml +++ b/swarm-queue-client/Cargo.toml @@ -68,9 +68,6 @@ agent-archetype = ["async-nats/jetstream"] # above - the connect itself needs none of them. async-nats.workspace = true backon.workspace = true -# `Shared`, so every caller refused while a secret re-read runs waits on that -# one read (`stored_secret`). -futures-util.workspace = true hive-sh4re = { workspace = true, optional = true } # `blocking` on top of the workspace default (`form`/`json`/`rustls`) — # `mint_token_for_blocking` needs `reqwest::blocking::Client` for a caller @@ -85,7 +82,8 @@ reqwest = { workspace = true, features = ["blocking"] } serde.workspace = true serde_json.workspace = true strum.workspace = true -# The rustls provider `connect` installs. +# `stored_secret::StoredSecret`, the held client secret, and the rustls +# provider `connect` installs. swarm-secret-client.workspace = true # A library, so its errors are a matchable enum rather than an opaque # `anyhow::Error`. The binaries that consume this keep anyhow; `?` converts. @@ -98,12 +96,9 @@ utoipa = { workspace = true, optional = true } # A loopback stand-in token endpoint for the client secret re-read tests. axum.workspace = true data-encoding.workspace = true -# `test-util` for `#[tokio::test(start_paused = true)]`: the re-read backoff -# tests run on paused time. +# `test-util` for `#[tokio::test(start_paused = true)]`: the startup-read +# retry tests run on paused time. tokio = { workspace = true, features = ["test-util"] } -# Capturing what a re-read logs, to check it names the client and never the -# secret. -tracing-subscriber.workspace = true [lints] workspace = true diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index f74be01d..e11f1f23 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -31,6 +31,8 @@ use std::path::PathBuf; +use swarm_secret_client::stored_secret::{self, StoredSecret}; + /// Everything that can go wrong reaching the swarm queue. /// /// `thiserror` and not `anyhow` because this is a library: a caller gets a @@ -480,8 +482,6 @@ pub mod subagent_term; /// same shapes. pub mod review_classifier; -pub mod stored_secret; - mod startup_read; pub use startup_read::read_with_startup_retry; @@ -588,13 +588,13 @@ pub enum ClientSecret { File(PathBuf), /// The plaintext, held only in this process's memory and read again /// when the token endpoint refuses it. - Stored(stored_secret::StoredSecret), + Stored(StoredSecret), } impl ClientSecret { /// The secret to read again after `error`: a held one the token endpoint /// refused as a credential ([`stored_secret::is_credential_refusal`]). - fn refused(&self, error: &Error) -> Option<&stored_secret::StoredSecret> { + fn refused(&self, error: &Error) -> Option<&StoredSecret> { match (self, error) { (Self::Stored(stored), Error::TokenRefused { status, .. }) if stored_secret::is_credential_refusal(*status) => @@ -1096,6 +1096,34 @@ mod tests { use super::*; + /// A store holding `secret`, counting how often it is read. + struct Store { + secret: &'static str, + reads: Arc, + } + + impl Store { + fn holding(secret: &'static str) -> Self { + Self { + secret, + reads: Arc::default(), + } + } + + /// `client_id`'s secret, `initial` in use, re-read from this store. + fn secret(&self, client_id: &str, initial: &str) -> StoredSecret { + let (secret, reads) = (self.secret, Arc::clone(&self.reads)); + StoredSecret::new(client_id, initial.to_owned(), move || { + reads.fetch_add(1, Ordering::SeqCst); + async move { Ok(secret.to_owned()) } + }) + } + + fn reads(&self) -> usize { + self.reads.load(Ordering::SeqCst) + } + } + /// The names the publishers, the relays and the queue's grant all spell. #[test] fn term_and_agent_state_subjects_are_the_agreed_names() { @@ -1126,7 +1154,7 @@ mod tests { async fn a_debug_print_of_a_config_does_not_carry_a_held_secret() { let cfg = QueueConfig { client_secret: ClientSecret::Stored( - stored_secret::tests::Fake::holding("new").secret("hive-alpha", "s3cret"), + Store::holding("new").secret("hive-alpha", "s3cret"), ), ..token_cfg() }; @@ -1180,7 +1208,7 @@ mod tests { #[tokio::test] async fn a_refused_secret_is_re_read_and_the_mint_retried_with_the_new_one() { let calls = Arc::new(AtomicUsize::new(0)); - let store = stored_secret::tests::Fake::holding("secret-b"); + let store = Store::holding("secret-b"); let cfg = QueueConfig { token_endpoint: stub_idp("secret-b", Arc::clone(&calls)).await, client_secret: ClientSecret::Stored(store.secret("hive-alpha", "secret-a")), @@ -1200,7 +1228,7 @@ mod tests { #[tokio::test] async fn a_permanently_refused_secret_is_not_re_read_on_every_mint() { let calls = Arc::new(AtomicUsize::new(0)); - let store = stored_secret::tests::Fake::holding("secret-a"); + let store = Store::holding("secret-a"); let cfg = QueueConfig { token_endpoint: stub_idp("secret-b", Arc::clone(&calls)).await, client_secret: ClientSecret::Stored(store.secret("hive-alpha", "secret-a")), @@ -1238,7 +1266,7 @@ mod tests { let url = format!("http://{}/token", listener.local_addr().expect("addr")); tokio::spawn(async move { axum::serve(listener, app).await }); - let store = stored_secret::tests::Fake::holding("secret-b"); + let store = Store::holding("secret-b"); let cfg = QueueConfig { token_endpoint: url, client_secret: ClientSecret::Stored(store.secret("hive-alpha", "secret-a")), @@ -1261,7 +1289,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn the_blocking_mint_re_reads_a_refused_secret() { let calls = Arc::new(AtomicUsize::new(0)); - let store = stored_secret::tests::Fake::holding("secret-b"); + let store = Store::holding("secret-b"); let cfg = QueueConfig { token_endpoint: stub_idp("secret-b", Arc::clone(&calls)).await, client_secret: ClientSecret::Stored(store.secret("hive-alpha", "secret-a")), diff --git a/swarm-review-classifier/src/main.rs b/swarm-review-classifier/src/main.rs index 6b6c6940..4d2e133c 100644 --- a/swarm-review-classifier/src/main.rs +++ b/swarm-review-classifier/src/main.rs @@ -15,8 +15,8 @@ use std::path::PathBuf; use anyhow::{Context, Result, bail}; use futures_util::StreamExt as _; use swarm_queue_client::review_classifier::{QUEUE_GROUP, Reply, SUBJECT, TryOut}; -use swarm_queue_client::stored_secret::StoredSecret; use swarm_queue_client::{ClientSecret, QueueConfig}; +use swarm_secret_client::stored_secret::StoredSecret; mod acp; mod aggregate; diff --git a/swarm-secret-client/Cargo.toml b/swarm-secret-client/Cargo.toml index 629355d6..70202ed6 100644 --- a/swarm-secret-client/Cargo.toml +++ b/swarm-secret-client/Cargo.toml @@ -4,6 +4,9 @@ version.workspace = true edition.workspace = true [dependencies] +# `Shared`, so every caller refused while a secret re-read runs waits on that +# one read (`stored_secret`). +futures-util.workspace = true # `Identity` is re-exported by vaultrs, but the identity is built here (from # the `BAO_*` files) rather than by its env defaults, so the dependency is # direct rather than incidental. @@ -21,6 +24,9 @@ serde.workspace = true serde_json.workspace = true strum.workspace = true thiserror.workspace = true +# `stored_secret` spawns its re-reads and times them out. +tokio.workspace = true +tracing.workspace = true vaultrs.workspace = true [lints] @@ -31,3 +37,9 @@ workspace = true # test would pass with `install_crypto_provider` doing nothing. [dev-dependencies] rustls = { workspace = true, features = ["ring"] } +# `test-util` for `#[tokio::test(start_paused = true)]`: the re-read backoff +# tests run on paused time. +tokio = { workspace = true, features = ["test-util"] } +# Capturing what a re-read logs, to check it names the client and never the +# secret. +tracing-subscriber.workspace = true diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 9449d446..45e70020 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -44,6 +44,7 @@ pub mod path; pub mod policy; pub mod queue; pub mod review_classifier; +pub mod stored_secret; pub use client::SecretStore; diff --git a/swarm-queue-client/src/stored_secret.rs b/swarm-secret-client/src/stored_secret.rs similarity index 97% rename from swarm-queue-client/src/stored_secret.rs rename to swarm-secret-client/src/stored_secret.rs index f00819b1..51a4ee3f 100644 --- a/swarm-queue-client/src/stored_secret.rs +++ b/swarm-secret-client/src/stored_secret.rs @@ -191,7 +191,7 @@ impl Inner { return; } Ok(Ok(_)) => "the stored client secret is the one the IdP refused".to_owned(), - Ok(Err(e)) => format!("could not re-read the client secret: {}", crate::chain(&*e)), + Ok(Err(e)) => format!("could not re-read the client secret: {}", chain(&*e)), Err(_) => format!("re-reading the client secret timed out after {REREAD_TIMEOUT:?}"), }; rereads.fruitless = rereads.fruitless.saturating_add(1); @@ -205,6 +205,18 @@ impl Inner { } } +/// `error` and its sources, joined with `: `. +fn chain(error: &dyn std::error::Error) -> String { + let mut rendered = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + rendered.push_str(": "); + rendered.push_str(&cause.to_string()); + source = cause.source(); + } + rendered +} + /// Whether an `IdP` answering `status` refused the client's credential (400 or /// 401), the one answer a re-read can fix. A 429 or a 5xx says nothing about /// the secret, and re-reading on one would spend the backoff on an outage.