//! One acked write to a key of a KV bucket the swarm-controller owns, by a //! `JetStream` publish straight to the key's subject: the same message //! `kv::Store::put` sends, without opening the bucket first, so the agent's //! credential needs only the one subject. The controller creates the bucket, so //! a write can arrive before it exists; that write and any other failed one are //! retried with backoff until one is acked. //! //! Shared by the writers of an agent's own keys: [`crate::swarm_agent_icon`] //! and [`crate::swarm_agent_commands`]. use std::time::Duration; use async_nats::jetstream::context::PublishErrorKind; use backon::{ExponentialBuilder, Retryable}; /// First retry delay after a failed write, doubled per failure up to /// [`MAX_RETRY`], with no cap on the number of tries. const FIRST_RETRY: Duration = Duration::from_secs(5); /// Longest wait between retries. const MAX_RETRY: Duration = Duration::from_mins(5); /// The KV header and value that mark an entry deleted. `kv::Store::get` /// answers `None` for an entry carrying them. const KV_OPERATION: &str = "KV-Operation"; const KV_OPERATION_DELETE: &str = "DEL"; /// What to write under a key. pub enum Put<'a> { /// These bytes as the key's value. Set(&'a [u8]), /// Delete the key. Delete, } /// Why a write did not land. pub enum Failure { /// The payload can never fit; retrying cannot help. TooLarge(async_nats::jetstream::context::PublishError), /// Anything else, rendered for the log. Retry(String), } /// One acked write of `put` to `subject`. async fn put_once( client: &async_nats::Client, js: &async_nats::jetstream::Context, subject: &str, put: &Put<'_>, ) -> Result<(), Failure> { // An unconnected client does not fail a publish, it buffers it, and the // payload limit it checks against is the library default until the // server has announced its own. swarm_queue_client::ensure_connected(client) .map_err(|e| Failure::Retry(swarm_queue_client::chain(&e)))?; let sent = match put { Put::Set(bytes) => js.publish(subject.to_owned(), bytes.to_vec().into()).await, Put::Delete => { let mut headers = async_nats::HeaderMap::new(); headers.insert(KV_OPERATION, KV_OPERATION_DELETE); js.publish_with_headers(subject.to_owned(), headers, Vec::new().into()) .await } }; let ack = sent.map_err(|e| match e.kind() { PublishErrorKind::MaxPayloadExceeded => Failure::TooLarge(e), _ => Failure::Retry(e.to_string()), })?; // `StreamNotFound` here is the bucket not existing yet. ack.await.map_err(|e| Failure::Retry(e.to_string()))?; Ok(()) } /// [`put_once`] until it is acked, or fails with [`Failure::TooLarge`], which /// ends the retries. `what` names the value in the log. pub async fn put_retrying( client: &async_nats::Client, subject: &str, put: Put<'_>, what: &str, ) -> Result<(), Failure> { let js = async_nats::jetstream::new(client.clone()); (|| put_once(client, &js, subject, &put)) .retry( ExponentialBuilder::default() .with_min_delay(FIRST_RETRY) .with_max_delay(MAX_RETRY) .without_max_times(), ) .when(|failure| matches!(failure, Failure::Retry(_))) .notify(|failure, delay| { if let Failure::Retry(e) = failure { tracing::warn!(error = e, retry_in = ?delay, "publishing {what} failed"); } }) .await }