vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

Guard the creation roll-back by revision

Browse source 
A store answers the revision it wrote and what it replaced, and restoring
compares against that revision, so a write that landed after the creation's
own is kept instead of overwritten. A write that changed nothing rolls back
nothing, and a key that held no value is removed rather than set to an empty
one. A test fails the second store after the first and checks the earlier
caps come back.
This commit is contained in:
iris 2026-10-15 11:53:19 +02:00 • committed by mara
commit 4ad9579e97
4 changed files with 237 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

73
swarm-controller/src/main.rs
View file

@ -1946,8 +1946,8 @@ async fn create_agent(
/// creation is refused after the write.
#[derive(Default)]
struct StoredSettings {
limits: Option<agent_limits::Replaced>,
inference: Option<inference::Replaced>,
limits: agent_limits::Replaced,
inference: inference::Replaced,
}
impl StoredSettings {
@ -1961,11 +1961,11 @@ impl StoredSettings {
) -> Result<Self, problem_details::ProblemDetails> {
let mut stored = Self::default();
if let Some(limits) = limits {
stored.limits = Some(agent_limits::store(client, agent, limits).await?);
stored.limits = agent_limits::store(client, agent, limits).await?;
}
if let Some(inference) = inference {
match inference::store(client, agent, inference).await {
Ok(replaced) => stored.inference = Some(replaced),
Ok(replaced) => stored.inference = replaced,
Err(e) => {
stored.roll_back(client, agent).await;
return Err(e);
@ -1976,12 +1976,8 @@ impl StoredSettings {
}
async fn roll_back(self, client: &async_nats::Client, agent: &str) {
if let Some(replaced) = self.limits {
agent_limits::restore(client, agent, replaced).await;
}
if let Some(replaced) = self.inference {
inference::restore(client, agent, replaced).await;
}
agent_limits::restore(client, agent, self.limits).await;
inference::restore(client, agent, self.inference).await;
}
}
@ -3913,6 +3909,63 @@ mod tests {
);
}
/// When the inference store fails after the caps are stored, the caps go
/// back to what they were: the earlier value, or no key. Without the
/// roll-back this leaves a cap behind for an agent that was never created.
/// The inference bucket is made with a one-byte value limit, so any write
/// to it fails.
#[tokio::test]
async fn a_failed_second_store_puts_the_first_back() {
let nats = crate::test_nats::TestNats::start();
let client = nats.client().await;
async_nats::jetstream::new(client.clone())
.create_key_value(async_nats::jetstream::kv::Config {
bucket: swarm_queue_client::agent_inference::BUCKET.to_owned(),
history: 1,
max_value_size: 1,
..Default::default()
})
.await
.expect("a bucket that refuses any value");
let caps = |json: &str| {
serde_json::from_str::<crate::agent_limits::Caps>(json)
.expect("caps")
.into_stored()
.expect("valid")
};
let inference =
serde_json::from_str::<crate::inference::InferenceSettings>(r#"{"model":"opus"}"#)
.expect("settings")
.into_stored()
.expect("valid");
crate::agent_limits::store(&client, "atlas", &caps(r#"{"cpu_quota":"100%"}"#))
.await
.expect("an earlier cap");
for agent in ["atlas", "iris"] {
let stored = super::StoredSettings::store(
&client,
agent,
Some(&caps(r#"{"cpu_quota":"400%"}"#)),
Some(&inference),
)
.await;
assert!(stored.is_err(), "{agent}: the inference write must fail");
}
assert_eq!(
crate::agent_limits::tests::published(&client, "atlas")
.await
.as_deref(),
Some(r#"{"cpu_quota":"100%"}"#),
"the earlier cap comes back"
);
assert_eq!(
crate::agent_limits::tests::published(&client, "iris").await,
None,
"a cap stored for a name with none is removed"
);
}
/// A valid cap beside an invalid model stores neither and queues nothing,
/// on a server that could have stored the cap.
#[tokio::test]