Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/hive-agent/src/swarm_kv_put.rs

97 lines
3.5 KiB
Rust

//! One acked write to a key of a KV bucket the swarm-controller owns, by a
//! `JetStream` publish straight to the key's subject: the same message
//! `kv::Store::put` sends, without opening the bucket first, so the agent's
//! credential needs only the one subject. The controller creates the bucket, so
//! a write can arrive before it exists; that write and any other failed one are
//! retried with backoff until one is acked.
//!
//! Shared by the writers of an agent's own keys: [`crate::swarm_agent_icon`]
//! and [`crate::swarm_agent_commands`].
use std::time::Duration;
use async_nats::jetstream::context::PublishErrorKind;
use backon::{ExponentialBuilder, Retryable};
/// First retry delay after a failed write, doubled per failure up to
/// [`MAX_RETRY`], with no cap on the number of tries.
const FIRST_RETRY: Duration = Duration::from_secs(5);
/// Longest wait between retries.
const MAX_RETRY: Duration = Duration::from_mins(5);
/// The KV header and value that mark an entry deleted. `kv::Store::get`
/// answers `None` for an entry carrying them.
const KV_OPERATION: &str = "KV-Operation";
const KV_OPERATION_DELETE: &str = "DEL";
/// What to write under a key.
pub enum Put<'a> {
/// These bytes as the key's value.
Set(&'a [u8]),
/// Delete the key.
Delete,
}
/// Why a write did not land.
pub enum Failure {
/// The payload can never fit; retrying cannot help.
TooLarge(async_nats::jetstream::context::PublishError),
/// Anything else, rendered for the log.
Retry(String),
}
/// One acked write of `put` to `subject`.
async fn put_once(
client: &async_nats::Client,
js: &async_nats::jetstream::Context,
subject: &str,
put: &Put<'_>,
) -> Result<(), Failure> {
// An unconnected client does not fail a publish, it buffers it, and the
// payload limit it checks against is the library default until the
// server has announced its own.
swarm_queue_client::ensure_connected(client)
.map_err(|e| Failure::Retry(swarm_queue_client::chain(&e)))?;
let sent = match put {
Put::Set(bytes) => js.publish(subject.to_owned(), bytes.to_vec().into()).await,
Put::Delete => {
let mut headers = async_nats::HeaderMap::new();
headers.insert(KV_OPERATION, KV_OPERATION_DELETE);
js.publish_with_headers(subject.to_owned(), headers, Vec::new().into())
.await
}
};
let ack = sent.map_err(|e| match e.kind() {
PublishErrorKind::MaxPayloadExceeded => Failure::TooLarge(e),
_ => Failure::Retry(e.to_string()),
})?;
// `StreamNotFound` here is the bucket not existing yet.
ack.await.map_err(|e| Failure::Retry(e.to_string()))?;
Ok(())
}
/// [`put_once`] until it is acked, or fails with [`Failure::TooLarge`], which
/// ends the retries. `what` names the value in the log.
pub async fn put_retrying(
client: &async_nats::Client,
subject: &str,
put: Put<'_>,
what: &str,
) -> Result<(), Failure> {
let js = async_nats::jetstream::new(client.clone());
(|| put_once(client, &js, subject, &put))
.retry(
ExponentialBuilder::default()
.with_min_delay(FIRST_RETRY)
.with_max_delay(MAX_RETRY)
.without_max_times(),
)
.when(|failure| matches!(failure, Failure::Retry(_)))
.notify(|failure, delay| {
if let Failure::Retry(e) = failure {
tracing::warn!(error = e, retry_in = ?delay, "publishing {what} failed");
}
})
.await
}