diff --git a/hive-ag3nt/assets/agent.css b/hive-ag3nt/assets/agent.css index 86f9f1a..dc468db 100644 --- a/hive-ag3nt/assets/agent.css +++ b/hive-ag3nt/assets/agent.css @@ -231,6 +231,8 @@ pre.diff { .status-badge.status-loading { color: var(--muted); border-color: var(--purple-dim); } .status-badge.status-online { color: var(--green); border-color: var(--green); text-shadow: 0 0 6px rgba(166, 227, 161, 0.55); } +.status-badge.status-rate-limited { color: var(--red); border-color: var(--red); + text-shadow: 0 0 6px rgba(243, 139, 168, 0.55); } .status-badge.status-needs-login { color: var(--amber); border-color: var(--amber); } .status-badge.status-offline { color: var(--muted); border-color: var(--muted); } .btn-dashlink { diff --git a/hive-ag3nt/assets/app.js b/hive-ag3nt/assets/app.js index 65a4d4f..8fd8fd3 100644 --- a/hive-ag3nt/assets/app.js +++ b/hive-ag3nt/assets/app.js @@ -547,8 +547,9 @@ // class. Lives in the state row so the operator sees boot/login/ // online without losing terminal real-estate to a paragraph. const ALIVE_LABELS = { - loading: { glyph: '…', text: 'connecting', cls: 'status-loading' }, - online: { glyph: '●', text: 'alive', cls: 'status-online' }, + loading: { glyph: '…', text: 'connecting', cls: 'status-loading' }, + online: { glyph: '●', text: 'alive', cls: 'status-online' }, + rate_limited: { glyph: '⊘', text: 'rate limited', cls: 'status-rate-limited' }, needs_login_idle: { glyph: '◌', text: 'needs login', cls: 'status-needs-login' }, needs_login_in_progress: { glyph: '◌', text: 'logging in', cls: 'status-needs-login' }, offline: { glyph: '○', text: 'offline', cls: 'status-offline' }, diff --git a/hive-ag3nt/src/events.rs b/hive-ag3nt/src/events.rs index 2fdda22..36b9e26 100644 --- a/hive-ag3nt/src/events.rs +++ b/hive-ag3nt/src/events.rs @@ -310,6 +310,13 @@ pub struct Bus { /// signal — tool-heavy turns rebill the cached prompt per call and /// blow past the model window. `None` until the first turn completes. last_cost_usage: Arc>>, + /// True while the harness is parked after a rate-limit response. + /// Set by `emit_status("rate_limited")`, cleared by + /// `emit_status("online")`. Also mirrored to a sentinel file at + /// `{state_dir}/hyperhive-rate-limited` so the host-side + /// `container_view` can surface the status on the dashboard without + /// a live socket call. + rate_limited: Arc, /// One-shot: next `run_claude` call drops `--continue`, starting /// a fresh claude session. Set by `POST /api/new-session` from /// the per-agent web UI; consumed (cleared back to false) by the @@ -345,6 +352,11 @@ impl Bus { }; let (tx, _) = broadcast::channel(CHANNEL_CAPACITY); let initial_model = load_model().unwrap_or_else(|| DEFAULT_MODEL.to_owned()); + // Restore rate_limited from the sentinel file — if the harness + // crashed while parked, we should still show the right status on + // cold load until the next turn clears it. + let sentinel = crate::paths::state_dir().join("hyperhive-rate-limited"); + let was_rate_limited = sentinel.exists(); Self { tx: Arc::new(tx), event_seq: Arc::new(AtomicU64::new(0)), @@ -353,6 +365,7 @@ impl Bus { model: Arc::new(Mutex::new(initial_model)), last_ctx_usage: Arc::new(Mutex::new(None)), last_cost_usage: Arc::new(Mutex::new(None)), + rate_limited: Arc::new(AtomicBool::new(was_rate_limited)), skip_continue_once: Arc::new(AtomicBool::new(false)), tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())), last_turn_ended_unix: Arc::new(AtomicI64::new(0)), @@ -508,16 +521,33 @@ impl Bus { }); } - /// Broadcast a status flip (online / `needs_login_*`). Called by - /// the bin entry points + `turn::wait_for_login` + the + /// Broadcast a status flip (online / `needs_login_*` / `rate_limited`). + /// Called by the bin entry points + `turn::wait_for_login` + the /// `post_login_*` handlers — every site that mutates the /// `Arc>` should also call this so the web UI - /// drops its periodic /api/state poll while a turn loop is - /// running. + /// drops its periodic /api/state poll while a turn loop is running. + /// + /// `"rate_limited"` sets the rate-limited flag and writes a sentinel + /// file at `{state_dir}/hyperhive-rate-limited` so the host-side + /// dashboard can show the status without a live socket call. + /// Any other status clears the flag and removes the sentinel. pub fn emit_status(&self, status: impl Into) { - self.emit(LiveEvent::StatusChanged { - status: status.into(), - }); + let status = status.into(); + let sentinel = crate::paths::state_dir().join("hyperhive-rate-limited"); + if status == "rate_limited" { + self.rate_limited.store(true, Ordering::Relaxed); + let _ = std::fs::write(&sentinel, b""); + } else { + self.rate_limited.store(false, Ordering::Relaxed); + let _ = std::fs::remove_file(&sentinel); + } + self.emit(LiveEvent::StatusChanged { status }); + } + + /// Returns true while the harness is parked after a rate-limit response. + #[must_use] + pub fn is_rate_limited(&self) -> bool { + self.rate_limited.load(Ordering::Relaxed) } /// Current state + since-when (unix seconds). Snapshot copy, no lock held. diff --git a/hive-ag3nt/src/web_ui.rs b/hive-ag3nt/src/web_ui.rs index 8c324c9..3755eed 100644 --- a/hive-ag3nt/src/web_ui.rs +++ b/hive-ag3nt/src/web_ui.rs @@ -340,7 +340,7 @@ struct StateSnapshot { seq: u64, label: String, dashboard_port: u16, - /// `"online"` | `"needs_login_idle"` | `"needs_login_in_progress"`. + /// `"online"` | `"rate_limited"` | `"needs_login_idle"` | `"needs_login_in_progress"`. status: &'static str, /// Present when `status == "needs_login_in_progress"`. session: Option, @@ -431,6 +431,7 @@ async fn api_state(State(state): State) -> axum::Json { let login = *state.login.lock().unwrap(); let session_snapshot = state.session.lock().unwrap().clone(); let (status, session_view) = match (login, session_snapshot) { + (LoginState::Online, _) if state.bus.is_rate_limited() => ("rate_limited", None), (LoginState::Online, _) => ("online", None), (LoginState::NeedsLogin, None) => ("needs_login_idle", None), (LoginState::NeedsLogin, Some(s)) => ( diff --git a/hive-c0re/assets/app.js b/hive-c0re/assets/app.js index 7a042e7..3906e84 100644 --- a/hive-c0re/assets/app.js +++ b/hive-c0re/assets/app.js @@ -531,6 +531,10 @@ if (pending) { head.append(el('span', { class: 'pending-state' }, el('span', { class: 'spinner' }, '◐'), ' ', pending + '…')); + } else if (c.rate_limited) { + head.append(el('span', + { class: 'badge badge-rate-limited', title: 'API rate-limited — harness is parked, will retry automatically' }, + '⊘ rate limited')); } else if (c.needs_login) { head.append(el('a', { class: 'badge badge-warn', href: url, target: '_blank', rel: 'noopener' }, diff --git a/hive-c0re/assets/dashboard.css b/hive-c0re/assets/dashboard.css index cb453c2..1e690d0 100644 --- a/hive-c0re/assets/dashboard.css +++ b/hive-c0re/assets/dashboard.css @@ -115,6 +115,10 @@ a:hover { color: var(--amber); border-color: var(--amber); text-shadow: 0 0 6px rgba(250, 179, 135, 0.5); } +.badge-rate-limited { + color: var(--red); border-color: var(--red); + text-shadow: 0 0 6px rgba(243, 139, 168, 0.5); +} .badge-muted { color: var(--muted); border-color: var(--purple-dim); background: rgba(127, 132, 156, 0.08); diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index 7db7328..e8d3fef 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -44,6 +44,12 @@ pub struct ContainerView { /// the "which agent is close to the window?" dashboard glance. #[serde(default, skip_serializing_if = "Option::is_none")] pub ctx_tokens: Option, + /// True while the harness is parked after an API rate-limit response. + /// Detected via the sentinel file `{state_dir}/hyperhive-rate-limited` + /// that the harness writes in `Bus::emit_status("rate_limited")` and + /// removes when it resumes. Stale by up to one crash-watch cycle. + #[serde(default)] + pub rate_limited: bool, } /// Build the full container list. Wraps `lifecycle::list()` and @@ -83,6 +89,7 @@ pub async fn build_all(coord: &Coordinator) -> Vec { .count_pending_reminders_for(reminder_recipient) .unwrap_or(0); let ctx_tokens = read_last_ctx_tokens(&logical); + let rate_limited = is_rate_limited(&logical); out.push(ContainerView { port: lifecycle::agent_web_port(&logical), running: lifecycle::is_running(&logical).await, @@ -94,6 +101,7 @@ pub async fn build_all(coord: &Coordinator) -> Vec { deployed_sha, pending_reminders, ctx_tokens, + rate_limited, }); } out @@ -112,6 +120,15 @@ pub fn claude_has_session(dir: &Path) -> bool { .any(|e| e.file_type().is_ok_and(|t| t.is_file())) } +/// Returns true if the agent's harness is currently parked after an API +/// rate-limit response. Detected via the sentinel file written by +/// `hive_ag3nt::events::Bus::emit_status("rate_limited")`. +fn is_rate_limited(name: &str) -> bool { + Coordinator::agent_notes_dir(name) + .join("hyperhive-rate-limited") + .exists() +} + /// Read the most recent completed turn's context-window size (prompt /// tokens) from the agent's turn-stats SQLite. Returns `None` when /// the file is absent or has no rows. Best-effort — any DB error