97 lines
3.5 KiB
Rust
97 lines
3.5 KiB
Rust
//! One acked write to a key of a KV bucket the swarm-controller owns, by a
|
|
//! `JetStream` publish straight to the key's subject: the same message
|
|
//! `kv::Store::put` sends, without opening the bucket first, so the agent's
|
|
//! credential needs only the one subject. The controller creates the bucket, so
|
|
//! a write can arrive before it exists; that write and any other failed one are
|
|
//! retried with backoff until one is acked.
|
|
//!
|
|
//! Shared by the writers of an agent's own keys: [`crate::swarm_agent_icon`]
|
|
//! and [`crate::swarm_agent_commands`].
|
|
|
|
use std::time::Duration;
|
|
|
|
use async_nats::jetstream::context::PublishErrorKind;
|
|
use backon::{ExponentialBuilder, Retryable};
|
|
|
|
/// First retry delay after a failed write, doubled per failure up to
|
|
/// [`MAX_RETRY`], with no cap on the number of tries.
|
|
const FIRST_RETRY: Duration = Duration::from_secs(5);
|
|
|
|
/// Longest wait between retries.
|
|
const MAX_RETRY: Duration = Duration::from_mins(5);
|
|
|
|
/// The KV header and value that mark an entry deleted. `kv::Store::get`
|
|
/// answers `None` for an entry carrying them.
|
|
const KV_OPERATION: &str = "KV-Operation";
|
|
const KV_OPERATION_DELETE: &str = "DEL";
|
|
|
|
/// What to write under a key.
|
|
pub enum Put<'a> {
|
|
/// These bytes as the key's value.
|
|
Set(&'a [u8]),
|
|
/// Delete the key.
|
|
Delete,
|
|
}
|
|
|
|
/// Why a write did not land.
|
|
pub enum Failure {
|
|
/// The payload can never fit; retrying cannot help.
|
|
TooLarge(async_nats::jetstream::context::PublishError),
|
|
/// Anything else, rendered for the log.
|
|
Retry(String),
|
|
}
|
|
|
|
/// One acked write of `put` to `subject`.
|
|
async fn put_once(
|
|
client: &async_nats::Client,
|
|
js: &async_nats::jetstream::Context,
|
|
subject: &str,
|
|
put: &Put<'_>,
|
|
) -> Result<(), Failure> {
|
|
// An unconnected client does not fail a publish, it buffers it, and the
|
|
// payload limit it checks against is the library default until the
|
|
// server has announced its own.
|
|
swarm_queue_client::ensure_connected(client)
|
|
.map_err(|e| Failure::Retry(swarm_queue_client::chain(&e)))?;
|
|
let sent = match put {
|
|
Put::Set(bytes) => js.publish(subject.to_owned(), bytes.to_vec().into()).await,
|
|
Put::Delete => {
|
|
let mut headers = async_nats::HeaderMap::new();
|
|
headers.insert(KV_OPERATION, KV_OPERATION_DELETE);
|
|
js.publish_with_headers(subject.to_owned(), headers, Vec::new().into())
|
|
.await
|
|
}
|
|
};
|
|
let ack = sent.map_err(|e| match e.kind() {
|
|
PublishErrorKind::MaxPayloadExceeded => Failure::TooLarge(e),
|
|
_ => Failure::Retry(e.to_string()),
|
|
})?;
|
|
// `StreamNotFound` here is the bucket not existing yet.
|
|
ack.await.map_err(|e| Failure::Retry(e.to_string()))?;
|
|
Ok(())
|
|
}
|
|
|
|
/// [`put_once`] until it is acked, or fails with [`Failure::TooLarge`], which
|
|
/// ends the retries. `what` names the value in the log.
|
|
pub async fn put_retrying(
|
|
client: &async_nats::Client,
|
|
subject: &str,
|
|
put: Put<'_>,
|
|
what: &str,
|
|
) -> Result<(), Failure> {
|
|
let js = async_nats::jetstream::new(client.clone());
|
|
(|| put_once(client, &js, subject, &put))
|
|
.retry(
|
|
ExponentialBuilder::default()
|
|
.with_min_delay(FIRST_RETRY)
|
|
.with_max_delay(MAX_RETRY)
|
|
.without_max_times(),
|
|
)
|
|
.when(|failure| matches!(failure, Failure::Retry(_)))
|
|
.notify(|failure, delay| {
|
|
if let Failure::Retry(e) = failure {
|
|
tracing::warn!(error = e, retry_in = ?delay, "publishing {what} failed");
|
|
}
|
|
})
|
|
.await
|
|
}
|