mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

a hive-side pause holds against the swarm's declared state until an operator resumes

Browse source 
the harness writes the reason into the pause marker when it pauses itself and reports it as paused_by_hive in its turn-state header. swarm-controller declares such an agent paused (only if declared up) and shows the reason on the agents status row and the swarm ui card.
This commit is contained in:
atlas 2026-11-09 23:09:14 +01:00 • committed by mara
commit e5c55257f8
2 changed files with 517 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

45
hive-agent/src/main.rs
View file

@ -945,6 +945,14 @@ async fn drive_turn_and_wake_chain<S: Surface>(
}
}
/// The pause marker's content for an auto-pause after `streak` missed todo
/// wakes. A non-empty marker is a pause by the harness itself, which the
/// turn-state header reports as `paused_by_hive`; the operator's pause writes
/// an empty one.
fn todo_miss_pause_reason(streak: u32) -> String {
format!("skipped get_loose_ends on {streak} consecutive todo wakes")
}
/// Apply a finished turn's `todo_wake_checked` signal to the miss-streak:
/// resets on a checked wake, increments (and auto-pauses at the threshold)
/// on a missed one, no-ops for a non-todo-wake turn. Split out of
@ -963,15 +971,19 @@ fn apply_todo_wake_checked(checked: Option<bool>, todo_miss_streak: &mut u32, bu
);
if streak >= TODO_MISS_PAUSE_THRESHOLD {
tracing::warn!("todo-miss streak hit the threshold — pausing the turn loop");
// Marker first: the Note triggers a turn-state header rebuild,
// which reads the reason back out of this file.
if let Err(e) =
std::fs::write(paths::paused_marker(), todo_miss_pause_reason(streak))
{
tracing::warn!(error = ?e, "failed to write pause marker");
}
bus.emit(LiveEvent::Note {
text: format!(
"auto-paused: skipped get_loose_ends on {streak} \
consecutive todo wakes — an operator needs to resume this agent"
),
});
if let Err(e) = std::fs::write(paths::paused_marker(), "") {
tracing::warn!(error = ?e, "failed to write pause marker");
}
// Fresh slate for whenever this agent gets resumed — see
// `serve_loop`'s `was_paused` reset for the mirror side.
*todo_miss_streak = 0;
@ -1254,4 +1266,31 @@ mod tests {
.await;
assert!(matches!(woke, RecvOutcome::Empty));
}
/// The third consecutive miss parks the agent with its reason in the
/// marker; an earlier miss, or a wake that did call `get_loose_ends`,
/// writes nothing.
#[test]
fn the_threshold_miss_writes_the_reason_into_the_pause_marker() {
crate::paths::with_harness_dir(|| {
let bus = crate::events::Bus::new();
let mut streak = 0;
for _ in 1..super::TODO_MISS_PAUSE_THRESHOLD {
super::apply_todo_wake_checked(Some(false), &mut streak, &bus);
assert!(
!crate::paths::paused_marker().exists(),
"below the threshold"
);
}
super::apply_todo_wake_checked(Some(true), &mut streak, &bus);
assert_eq!(streak, 0);
for _ in 0..super::TODO_MISS_PAUSE_THRESHOLD {
super::apply_todo_wake_checked(Some(false), &mut streak, &bus);
}
assert_eq!(
std::fs::read_to_string(crate::paths::paused_marker()).unwrap(),
super::todo_miss_pause_reason(super::TODO_MISS_PAUSE_THRESHOLD)
);
});
}
}

56
hive-agent/src/swarm_agent_state.rs
View file

@ -35,6 +35,7 @@ use tokio::time::{Interval, MissedTickBehavior};
use swarm_wire::agent_inference::Effective;
use swarm_wire::agent_login::Login;
use swarm_wire::agent_pause::HivePause;
use swarm_wire::hive_status::{Capabilities, TOLERANT_PERMISSIONS};
use swarm_wire::wanted::AgentState;
@ -101,6 +102,10 @@ struct AgentStateMsg {
/// `needs_login`, from this process's own [`LoginState`], so always set.
#[serde(flatten)]
login: Login,
/// `paused_by_hive`, the reason in the pause marker when this harness wrote
/// it itself; absent otherwise.
#[serde(flatten)]
hive_pause: HivePause,
/// What this harness can do, so swarm-controller grants an agent a
/// permission only when the harness running it can decode it.
#[serde(flatten)]
@ -136,6 +141,20 @@ fn observed_state() -> AgentState {
}
}
/// Longest reason reported; the marker is a file the agent can write to.
const PAUSE_REASON_MAX_CHARS: usize = 200;
/// The header's pause-by-harness field: the pause marker's content, when it has
/// any. The operator's pause writes an empty marker, so it reports nothing.
fn hive_pause() -> HivePause {
let reason = std::fs::read_to_string(crate::paths::paused_marker()).unwrap_or_default();
let reason = reason.trim();
HivePause {
paused_by_hive: (!reason.is_empty())
.then(|| reason.chars().take(PAUSE_REASON_MAX_CHARS).collect()),
}
}
/// The header's login fields for `state`.
fn login(state: LoginState) -> Login {
Login {
@ -170,6 +189,7 @@ fn snapshot(bus: &Bus, login_state: &Mutex<LoginState>) -> AgentStateMsg {
ctx_usage: bus.last_ctx_usage(),
cost_usage: bus.last_cost_usage(),
login: login(*login_state.lock().unwrap_or_else(PoisonError::into_inner)),
hive_pause: hive_pause(),
capabilities: capabilities(),
}
}
@ -346,13 +366,14 @@ async fn publish(
#[cfg(test)]
mod tests {
use super::{
AgentStateMsg, REFRESH_INTERVAL, Trigger, capabilities, due, login, next_trigger,
refresh_timer, snapshot,
AgentStateMsg, PAUSE_REASON_MAX_CHARS, REFRESH_INTERVAL, Trigger, capabilities, due,
hive_pause, login, next_trigger, refresh_timer, snapshot,
};
use crate::events::{BusEvent, LiveEvent, TurnState};
use crate::login::LoginState;
use std::sync::Mutex;
use swarm_wire::agent_inference::{Effective, Source};
use swarm_wire::agent_pause::HivePause;
use swarm_wire::hive_status::{Capabilities, TOLERANT_PERMISSIONS};
use swarm_wire::wanted::AgentState;
use tokio::sync::broadcast;
@ -403,6 +424,9 @@ mod tests {
"cache_creation_input_tokens": 88,
}))),
login: login(LoginState::NeedsLogin),
hive_pause: HivePause {
paused_by_hive: Some("skipped get_loose_ends".to_owned()),
},
capabilities: capabilities(),
}
}
@ -432,6 +456,7 @@ mod tests {
"model",
"model_source",
"needs_login",
"paused_by_hive",
"resolved_model",
"turn_state",
"turn_state_since",
@ -523,6 +548,33 @@ mod tests {
}
}
/// The marker's content is the reason; an empty marker (an operator's pause) and
/// no marker report nothing, and swarm-controller reads it back with the wire type.
#[test]
fn the_pause_marker_content_is_the_reported_reason() {
crate::paths::with_harness_dir(|| {
assert_eq!(hive_pause().paused_by_hive, None, "no marker");
std::fs::write(crate::paths::paused_marker(), "").unwrap();
assert_eq!(hive_pause().paused_by_hive, None, "operator pause");
std::fs::write(crate::paths::paused_marker(), "x".repeat(500)).unwrap();
assert_eq!(
hive_pause().paused_by_hive.unwrap().len(),
PAUSE_REASON_MAX_CHARS
);
std::fs::write(crate::paths::paused_marker(), "skipped get_loose_ends\n").unwrap();
let bytes = serde_json::to_vec(&AgentStateMsg {
hive_pause: hive_pause(),
..msg()
})
.expect("serialises");
let decoded: HivePause = serde_json::from_slice(&bytes).expect("decodes");
assert_eq!(
decoded.paused_by_hive.as_deref(),
Some("skipped get_loose_ends")
);
});
}
/// swarm-controller reads the harness's capabilities back out of a header.
#[test]
fn the_header_advertises_tolerant_permissions() {