surface rate_limited status as red badge on per-agent page and dashboard
- add rate_limited: Arc<AtomicBool> to Bus; set/cleared by emit_status - write/remove sentinel file hyperhive-rate-limited in state dir so host-side dashboard can detect it without a live socket call - api_state returns status=rate_limited when flag is set (cold-load accurate) - ALIVE_LABELS gains rate_limited entry (⊘ red chip) on per-agent page - ContainerView gains rate_limited: bool read from sentinel file - dashboard container row shows ⊘ rate limited badge (red) ahead of needs_login Closes #24
This commit is contained in:
parent
808b9cbe1a
commit
804875d670
7 changed files with 69 additions and 10 deletions
|
|
@ -231,6 +231,8 @@ pre.diff {
|
|||
.status-badge.status-loading { color: var(--muted); border-color: var(--purple-dim); }
|
||||
.status-badge.status-online { color: var(--green); border-color: var(--green);
|
||||
text-shadow: 0 0 6px rgba(166, 227, 161, 0.55); }
|
||||
.status-badge.status-rate-limited { color: var(--red); border-color: var(--red);
|
||||
text-shadow: 0 0 6px rgba(243, 139, 168, 0.55); }
|
||||
.status-badge.status-needs-login { color: var(--amber); border-color: var(--amber); }
|
||||
.status-badge.status-offline { color: var(--muted); border-color: var(--muted); }
|
||||
.btn-dashlink {
|
||||
|
|
|
|||
|
|
@ -547,8 +547,9 @@
|
|||
// class. Lives in the state row so the operator sees boot/login/
|
||||
// online without losing terminal real-estate to a paragraph.
|
||||
const ALIVE_LABELS = {
|
||||
loading: { glyph: '…', text: 'connecting', cls: 'status-loading' },
|
||||
online: { glyph: '●', text: 'alive', cls: 'status-online' },
|
||||
loading: { glyph: '…', text: 'connecting', cls: 'status-loading' },
|
||||
online: { glyph: '●', text: 'alive', cls: 'status-online' },
|
||||
rate_limited: { glyph: '⊘', text: 'rate limited', cls: 'status-rate-limited' },
|
||||
needs_login_idle: { glyph: '◌', text: 'needs login', cls: 'status-needs-login' },
|
||||
needs_login_in_progress: { glyph: '◌', text: 'logging in', cls: 'status-needs-login' },
|
||||
offline: { glyph: '○', text: 'offline', cls: 'status-offline' },
|
||||
|
|
|
|||
|
|
@ -310,6 +310,13 @@ pub struct Bus {
|
|||
/// signal — tool-heavy turns rebill the cached prompt per call and
|
||||
/// blow past the model window. `None` until the first turn completes.
|
||||
last_cost_usage: Arc<Mutex<Option<TokenUsage>>>,
|
||||
/// True while the harness is parked after a rate-limit response.
|
||||
/// Set by `emit_status("rate_limited")`, cleared by
|
||||
/// `emit_status("online")`. Also mirrored to a sentinel file at
|
||||
/// `{state_dir}/hyperhive-rate-limited` so the host-side
|
||||
/// `container_view` can surface the status on the dashboard without
|
||||
/// a live socket call.
|
||||
rate_limited: Arc<AtomicBool>,
|
||||
/// One-shot: next `run_claude` call drops `--continue`, starting
|
||||
/// a fresh claude session. Set by `POST /api/new-session` from
|
||||
/// the per-agent web UI; consumed (cleared back to false) by the
|
||||
|
|
@ -345,6 +352,11 @@ impl Bus {
|
|||
};
|
||||
let (tx, _) = broadcast::channel(CHANNEL_CAPACITY);
|
||||
let initial_model = load_model().unwrap_or_else(|| DEFAULT_MODEL.to_owned());
|
||||
// Restore rate_limited from the sentinel file — if the harness
|
||||
// crashed while parked, we should still show the right status on
|
||||
// cold load until the next turn clears it.
|
||||
let sentinel = crate::paths::state_dir().join("hyperhive-rate-limited");
|
||||
let was_rate_limited = sentinel.exists();
|
||||
Self {
|
||||
tx: Arc::new(tx),
|
||||
event_seq: Arc::new(AtomicU64::new(0)),
|
||||
|
|
@ -353,6 +365,7 @@ impl Bus {
|
|||
model: Arc::new(Mutex::new(initial_model)),
|
||||
last_ctx_usage: Arc::new(Mutex::new(None)),
|
||||
last_cost_usage: Arc::new(Mutex::new(None)),
|
||||
rate_limited: Arc::new(AtomicBool::new(was_rate_limited)),
|
||||
skip_continue_once: Arc::new(AtomicBool::new(false)),
|
||||
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
|
||||
last_turn_ended_unix: Arc::new(AtomicI64::new(0)),
|
||||
|
|
@ -508,16 +521,33 @@ impl Bus {
|
|||
});
|
||||
}
|
||||
|
||||
/// Broadcast a status flip (online / `needs_login_*`). Called by
|
||||
/// the bin entry points + `turn::wait_for_login` + the
|
||||
/// Broadcast a status flip (online / `needs_login_*` / `rate_limited`).
|
||||
/// Called by the bin entry points + `turn::wait_for_login` + the
|
||||
/// `post_login_*` handlers — every site that mutates the
|
||||
/// `Arc<Mutex<LoginState>>` should also call this so the web UI
|
||||
/// drops its periodic /api/state poll while a turn loop is
|
||||
/// running.
|
||||
/// drops its periodic /api/state poll while a turn loop is running.
|
||||
///
|
||||
/// `"rate_limited"` sets the rate-limited flag and writes a sentinel
|
||||
/// file at `{state_dir}/hyperhive-rate-limited` so the host-side
|
||||
/// dashboard can show the status without a live socket call.
|
||||
/// Any other status clears the flag and removes the sentinel.
|
||||
pub fn emit_status(&self, status: impl Into<String>) {
|
||||
self.emit(LiveEvent::StatusChanged {
|
||||
status: status.into(),
|
||||
});
|
||||
let status = status.into();
|
||||
let sentinel = crate::paths::state_dir().join("hyperhive-rate-limited");
|
||||
if status == "rate_limited" {
|
||||
self.rate_limited.store(true, Ordering::Relaxed);
|
||||
let _ = std::fs::write(&sentinel, b"");
|
||||
} else {
|
||||
self.rate_limited.store(false, Ordering::Relaxed);
|
||||
let _ = std::fs::remove_file(&sentinel);
|
||||
}
|
||||
self.emit(LiveEvent::StatusChanged { status });
|
||||
}
|
||||
|
||||
/// Returns true while the harness is parked after a rate-limit response.
|
||||
#[must_use]
|
||||
pub fn is_rate_limited(&self) -> bool {
|
||||
self.rate_limited.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Current state + since-when (unix seconds). Snapshot copy, no lock held.
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ struct StateSnapshot {
|
|||
seq: u64,
|
||||
label: String,
|
||||
dashboard_port: u16,
|
||||
/// `"online"` | `"needs_login_idle"` | `"needs_login_in_progress"`.
|
||||
/// `"online"` | `"rate_limited"` | `"needs_login_idle"` | `"needs_login_in_progress"`.
|
||||
status: &'static str,
|
||||
/// Present when `status == "needs_login_in_progress"`.
|
||||
session: Option<SessionView>,
|
||||
|
|
@ -431,6 +431,7 @@ async fn api_state(State(state): State<AppState>) -> axum::Json<StateSnapshot> {
|
|||
let login = *state.login.lock().unwrap();
|
||||
let session_snapshot = state.session.lock().unwrap().clone();
|
||||
let (status, session_view) = match (login, session_snapshot) {
|
||||
(LoginState::Online, _) if state.bus.is_rate_limited() => ("rate_limited", None),
|
||||
(LoginState::Online, _) => ("online", None),
|
||||
(LoginState::NeedsLogin, None) => ("needs_login_idle", None),
|
||||
(LoginState::NeedsLogin, Some(s)) => (
|
||||
|
|
|
|||
|
|
@ -531,6 +531,10 @@
|
|||
if (pending) {
|
||||
head.append(el('span', { class: 'pending-state' },
|
||||
el('span', { class: 'spinner' }, '◐'), ' ', pending + '…'));
|
||||
} else if (c.rate_limited) {
|
||||
head.append(el('span',
|
||||
{ class: 'badge badge-rate-limited', title: 'API rate-limited — harness is parked, will retry automatically' },
|
||||
'⊘ rate limited'));
|
||||
} else if (c.needs_login) {
|
||||
head.append(el('a',
|
||||
{ class: 'badge badge-warn', href: url, target: '_blank', rel: 'noopener' },
|
||||
|
|
|
|||
|
|
@ -115,6 +115,10 @@ a:hover {
|
|||
color: var(--amber); border-color: var(--amber);
|
||||
text-shadow: 0 0 6px rgba(250, 179, 135, 0.5);
|
||||
}
|
||||
.badge-rate-limited {
|
||||
color: var(--red); border-color: var(--red);
|
||||
text-shadow: 0 0 6px rgba(243, 139, 168, 0.5);
|
||||
}
|
||||
.badge-muted {
|
||||
color: var(--muted); border-color: var(--purple-dim);
|
||||
background: rgba(127, 132, 156, 0.08);
|
||||
|
|
|
|||
|
|
@ -44,6 +44,12 @@ pub struct ContainerView {
|
|||
/// the "which agent is close to the window?" dashboard glance.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ctx_tokens: Option<u64>,
|
||||
/// True while the harness is parked after an API rate-limit response.
|
||||
/// Detected via the sentinel file `{state_dir}/hyperhive-rate-limited`
|
||||
/// that the harness writes in `Bus::emit_status("rate_limited")` and
|
||||
/// removes when it resumes. Stale by up to one crash-watch cycle.
|
||||
#[serde(default)]
|
||||
pub rate_limited: bool,
|
||||
}
|
||||
|
||||
/// Build the full container list. Wraps `lifecycle::list()` and
|
||||
|
|
@ -83,6 +89,7 @@ pub async fn build_all(coord: &Coordinator) -> Vec<ContainerView> {
|
|||
.count_pending_reminders_for(reminder_recipient)
|
||||
.unwrap_or(0);
|
||||
let ctx_tokens = read_last_ctx_tokens(&logical);
|
||||
let rate_limited = is_rate_limited(&logical);
|
||||
out.push(ContainerView {
|
||||
port: lifecycle::agent_web_port(&logical),
|
||||
running: lifecycle::is_running(&logical).await,
|
||||
|
|
@ -94,6 +101,7 @@ pub async fn build_all(coord: &Coordinator) -> Vec<ContainerView> {
|
|||
deployed_sha,
|
||||
pending_reminders,
|
||||
ctx_tokens,
|
||||
rate_limited,
|
||||
});
|
||||
}
|
||||
out
|
||||
|
|
@ -112,6 +120,15 @@ pub fn claude_has_session(dir: &Path) -> bool {
|
|||
.any(|e| e.file_type().is_ok_and(|t| t.is_file()))
|
||||
}
|
||||
|
||||
/// Returns true if the agent's harness is currently parked after an API
|
||||
/// rate-limit response. Detected via the sentinel file written by
|
||||
/// `hive_ag3nt::events::Bus::emit_status("rate_limited")`.
|
||||
fn is_rate_limited(name: &str) -> bool {
|
||||
Coordinator::agent_notes_dir(name)
|
||||
.join("hyperhive-rate-limited")
|
||||
.exists()
|
||||
}
|
||||
|
||||
/// Read the most recent completed turn's context-window size (prompt
|
||||
/// tokens) from the agent's turn-stats SQLite. Returns `None` when
|
||||
/// the file is absent or has no rows. Best-effort — any DB error
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue