From 978a3cf3914f1f8c8cd8dfc241713c2cda8ed778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Mon, 18 May 2026 00:08:09 +0200 Subject: [PATCH 1/7] reminders: persist + surface delivery failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Broker schema gains attempt_count INTEGER + last_error TEXT columns via idempotent ALTER TABLE migration (pragma-probed so fresh + existing dbs converge). reminder_scheduler::tick calls record_reminder_failure on every deliver_reminder error, bumping the counter + stashing the message. get_due_reminders filters out rows where attempt_count >= MAX_REMINDER_ATTEMPTS (5) so the scheduler stops retrying a stuck row until the operator intervenes. new POST /retry-reminder/{id} → reset_reminder_failure clears the counters; next 5s tick re-attempts. cancel-reminder unchanged (hard-delete). dashboard renders failed rows with a red left rule, the error text inline, and a ⚠ N failed badge. ↻ R3TRY button appears when attempt_count > 0 — sits next to ✗ C4NC3L in a small actions row below the body. --- hive-c0re/assets/app.js | 35 +++++++++-- hive-c0re/assets/dashboard.css | 19 ++++++ hive-c0re/src/broker.rs | 93 ++++++++++++++++++++++++++++- hive-c0re/src/dashboard.rs | 20 +++++++ hive-c0re/src/reminder_scheduler.rs | 14 ++++- 5 files changed, 173 insertions(+), 8 deletions(-) diff --git a/hive-c0re/assets/app.js b/hive-c0re/assets/app.js index 012aeef7..0c079f78 100644 --- a/hive-c0re/assets/app.js +++ b/hive-c0re/assets/app.js @@ -1349,7 +1349,8 @@ } const ul = el('ul', { class: 'reminders' }); for (const r of rows) { - const li = el('li', { class: 'reminder-row' }); + const failed = (r.attempt_count || 0) > 0; + const li = el('li', { class: 'reminder-row' + (failed ? ' reminder-failed' : '') }); const dueIn = r.due_at - Math.floor(Date.now() / 1000); const dueLabel = dueIn <= 0 ? `overdue ${fmtAgo(r.due_at)}` @@ -1364,19 +1365,45 @@ head.append(' ', el('span', { class: 'meta' }, '· payload → ')); appendLinkified(head, r.file_path); } + if (failed) { + head.append(' ', el('span', + { + class: 'badge badge-warn', + title: 'consecutive failed delivery attempts (capped at 5; over the cap the scheduler stops retrying until you click R3TRY or cancel)', + }, + `⚠ ${r.attempt_count} failed`)); + } const body = el('div', { class: 'reminder-body' }); const previews = appendLinkified(body, r.message); li.append(head, body); for (const d of previews) li.appendChild(d); - // Cancel form omits `data-no-refresh` — the resulting refreshState - // re-fires refreshReminders so the row drops on its own. + if (r.last_error) { + li.append(el('div', { class: 'reminder-error' }, + el('span', { class: 'msg-sep' }, 'error: '), + r.last_error, + )); + } + const actions = el('div', { class: 'reminder-actions' }); + if (failed) { + // Retry resets the failure counters so the scheduler picks + // the row up again on its next 5s tick. No data-no-refresh + // — the resulting refreshState re-fires refreshReminders. + const retryForm = el('form', { + method: 'POST', action: '/retry-reminder/' + r.id, + class: 'inline', 'data-async': '', + }); + retryForm.append(el('button', + { type: 'submit', class: 'btn btn-restart' }, '↻ R3TRY')); + actions.append(retryForm); + } const cancelForm = el('form', { method: 'POST', action: '/cancel-reminder/' + r.id, class: 'inline', 'data-async': '', 'data-confirm': `cancel reminder ${r.id} for ${r.agent}? this drops the queued delivery; no undo.`, }); cancelForm.append(el('button', { type: 'submit', class: 'btn btn-deny' }, '✗ C4NC3L')); - li.append(cancelForm); + actions.append(cancelForm); + li.append(actions); ul.append(li); } root.append(ul); diff --git a/hive-c0re/assets/dashboard.css b/hive-c0re/assets/dashboard.css index 46eeed67..ea62fe3d 100644 --- a/hive-c0re/assets/dashboard.css +++ b/hive-c0re/assets/dashboard.css @@ -474,6 +474,25 @@ summary:hover { color: var(--purple); } word-break: break-word; margin: 0.3em 0; } +.reminder-row.reminder-failed { + border-left: 2px solid var(--red, #f38ba8); + padding-left: 0.5em; +} +.reminder-error { + color: var(--red, #f38ba8); + background: rgba(243, 139, 168, 0.06); + border: 1px solid rgba(243, 139, 168, 0.25); + padding: 0.3em 0.5em; + font-size: 0.85em; + white-space: pre-wrap; + word-break: break-word; + margin: 0.2em 0; +} +.reminder-actions { + display: flex; + gap: 0.4em; + margin-top: 0.3em; +} /* Path linkification — agents drop pointer strings into messages constantly; clicking the anchor expands a sibling
that diff --git a/hive-c0re/src/broker.rs b/hive-c0re/src/broker.rs index afe14f7e..fbbb2d6d 100644 --- a/hive-c0re/src/broker.rs +++ b/hive-c0re/src/broker.rs @@ -57,8 +57,26 @@ pub struct PendingReminder { pub file_path: Option, pub due_at: i64, pub created_at: i64, + /// Most recent delivery failure for this row, if any. Cleared + /// to NULL on operator retry. Surfaced inline in the dashboard + /// so a stuck reminder doesn't just silently retry forever. + #[serde(skip_serializing_if = "Option::is_none")] + pub last_error: Option, + /// Number of failed delivery attempts since the row was + /// created or last retried. After `MAX_REMINDER_ATTEMPTS` the + /// scheduler stops trying (the row stays in `pending` with the + /// error so the operator can decide between retry + cancel). + #[serde(default)] + pub attempt_count: u32, } +/// Stop retrying a row after this many consecutive failures. The +/// scheduler quits scheduling it until an operator explicitly +/// retries (which resets the counter) or cancels (which deletes +/// the row). Below the cap the existing 5s tick re-attempts each +/// time the row is due. +pub const MAX_REMINDER_ATTEMPTS: u32 = 5; + /// Intra-process broker event. `recv_blocking` listens on the same /// channel as the dashboard forwarder; the forwarder re-emits each /// event as a `DashboardEvent` with a freshly-stamped seq from the @@ -95,6 +113,7 @@ impl Broker { let conn = Connection::open(path).with_context(|| format!("open broker db {}", path.display()))?; conn.execute_batch(SCHEMA).context("apply broker schema")?; + ensure_reminder_columns(&conn).context("migrate reminders columns")?; let (events, _) = broadcast::channel(EVENT_CHANNEL); Ok(Self { conn: Mutex::new(conn), @@ -305,12 +324,14 @@ impl Broker { pub fn list_pending_reminders(&self) -> Result> { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare( - "SELECT id, agent, message, file_path, due_at, created_at \ + "SELECT id, agent, message, file_path, due_at, created_at, \ + last_error, attempt_count \ FROM reminders \ WHERE sent_at IS NULL \ ORDER BY due_at ASC", )?; let rows = stmt.query_map([], |row| { + let attempts: i64 = row.get(7)?; Ok(PendingReminder { id: row.get(0)?, agent: row.get(1)?, @@ -318,12 +339,46 @@ impl Broker { file_path: row.get(3)?, due_at: row.get(4)?, created_at: row.get(5)?, + last_error: row.get(6)?, + attempt_count: u32::try_from(attempts).unwrap_or(0), }) })?; rows.collect::>>() .context("list pending reminders") } + /// Mark a delivery attempt as failed: bump `attempt_count` and + /// stash the error string. Called by `reminder_scheduler::tick` + /// when `deliver_reminder` returns Err. Soft-cap behaviour + /// lives in `get_due_reminders` (rows over the cap drop out + /// of the due-list and stop being attempted until retry). + pub fn record_reminder_failure(&self, id: i64, reason: &str) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE reminders \ + SET attempt_count = attempt_count + 1, last_error = ?1 \ + WHERE id = ?2 AND sent_at IS NULL", + params![reason, id], + )?; + Ok(()) + } + + /// Clear the failure state on a pending reminder so the + /// scheduler picks it up again. No-op when the row is already + /// fresh (attempt_count == 0). Returns the number of rows + /// affected so callers can distinguish "retried" from "no + /// such pending reminder" (already delivered, or wrong id). + pub fn reset_reminder_failure(&self, id: i64) -> Result { + let conn = self.conn.lock().unwrap(); + let n = conn.execute( + "UPDATE reminders \ + SET attempt_count = 0, last_error = NULL \ + WHERE id = ?1 AND sent_at IS NULL", + params![id], + )?; + Ok(n) + } + /// Count this agent's still-pending (un-delivered) reminders. /// Used by the per-turn stats sink for a cheap "what was queued /// at turn-end" snapshot. @@ -357,13 +412,16 @@ impl Broker { pub fn get_due_reminders(&self, limit: u64) -> Result> { let conn = self.conn.lock().unwrap(); let limit_i = i64::try_from(limit.min(i64::MAX as u64)).unwrap_or(i64::MAX); + let max_attempts = i64::from(MAX_REMINDER_ATTEMPTS); + // attempt_count >= cap = give up; row stays pending so the + // operator sees + can retry/cancel via the dashboard. let mut stmt = conn.prepare( "SELECT agent, id, message, file_path FROM reminders \ - WHERE due_at <= ?1 AND sent_at IS NULL \ + WHERE due_at <= ?1 AND sent_at IS NULL AND attempt_count < ?3 \ ORDER BY agent, due_at ASC \ LIMIT ?2", )?; - let rows = stmt.query_map(params![now_unix(), limit_i], |row| { + let rows = stmt.query_map(params![now_unix(), limit_i, max_attempts], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, i64>(1)?, @@ -410,6 +468,35 @@ impl Broker { } } +/// Idempotent reminder-table migrations. `ALTER TABLE ADD COLUMN` +/// has no `IF NOT EXISTS` form in sqlite, so we probe +/// `pragma_table_info` per column. New deploys (table created by +/// SCHEMA in this commit cycle) skip the ALTER; pre-existing +/// broker.sqlite files get the columns added on next boot. +fn ensure_reminder_columns(conn: &Connection) -> Result<()> { + for (name, sql) in [ + ( + "attempt_count", + "ALTER TABLE reminders ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0;", + ), + ( + "last_error", + "ALTER TABLE reminders ADD COLUMN last_error TEXT;", + ), + ] { + let has: bool = conn + .prepare(&format!( + "SELECT 1 FROM pragma_table_info('reminders') WHERE name = '{name}'" + ))? + .exists([])?; + if !has { + conn.execute_batch(sql) + .with_context(|| format!("add reminders.{name} column"))?; + } + } + Ok(()) +} + fn now_unix() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/hive-c0re/src/dashboard.rs b/hive-c0re/src/dashboard.rs index da25b15b..6badad46 100644 --- a/hive-c0re/src/dashboard.rs +++ b/hive-c0re/src/dashboard.rs @@ -57,6 +57,7 @@ pub async fn serve(port: u16, coord: Arc) -> Result<()> { .route("/api/state-file", get(get_state_file)) .route("/api/reminders", get(api_reminders)) .route("/cancel-reminder/{id}", post(post_cancel_reminder)) + .route("/retry-reminder/{id}", post(post_retry_reminder)) .route("/api/agent-config/{name}", get(get_agent_config)) .route("/request-spawn", post(post_request_spawn)) .route("/op-send", post(post_op_send)) @@ -1126,6 +1127,25 @@ async fn post_cancel_reminder( } } +/// Reset a pending reminder's failure state so the scheduler +/// retries it on the next tick. Useful when the failure was +/// transient (sqlite lock contention, disk full → freed up) and +/// the operator wants delivery to resume immediately instead of +/// the row sitting in attempt-count-capped purgatory. +async fn post_retry_reminder( + State(state): State, + AxumPath(id): AxumPath, +) -> Response { + match state.coord.broker.reset_reminder_failure(id) { + Ok(0) => error_response(&format!("reminder {id} not pending (already delivered?)")), + Ok(_) => { + tracing::info!(%id, "operator reset reminder failure for retry"); + (StatusCode::OK, "ok").into_response() + } + Err(e) => error_response(&format!("retry reminder {id} failed: {e:#}")), + } +} + async fn post_purge_tombstone( State(state): State, AxumPath(name): AxumPath, diff --git a/hive-c0re/src/reminder_scheduler.rs b/hive-c0re/src/reminder_scheduler.rs index c7d83f0f..92ce4260 100644 --- a/hive-c0re/src/reminder_scheduler.rs +++ b/hive-c0re/src/reminder_scheduler.rs @@ -71,12 +71,24 @@ fn tick(coord: &Arc) { for (agent, id, message, file_path) in due { let body = prepare_body(&agent, &message, file_path.as_deref()); if let Err(e) = coord.broker.deliver_reminder(id, &agent, &body) { + let reason = format!("{e:#}"); tracing::warn!( reminder_id = id, %agent, - error = ?e, + error = %reason, "failed to deliver reminder" ); + // Persist the failure so the dashboard can surface it + + // bump attempt_count. After MAX_REMINDER_ATTEMPTS the + // row drops out of `get_due_reminders` and waits for + // operator retry / cancel. + if let Err(persist_err) = coord.broker.record_reminder_failure(id, &reason) { + tracing::warn!( + reminder_id = id, + error = ?persist_err, + "failed to persist reminder failure" + ); + } } } } From 0a75e62ffe6d3320f54436a1984446939846fb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Mon, 18 May 2026 00:08:25 +0200 Subject: [PATCH 2/7] todo: drop landed entries (reminder errors + tombstones/meta_inputs events) --- TODO.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/TODO.md b/TODO.md index 9323a415..4960a0ab 100644 --- a/TODO.md +++ b/TODO.md @@ -20,9 +20,7 @@ ## Dashboard -- **Reminder delivery-error surface**: `reminder_scheduler::tick` logs failed deliveries but doesn't persist. Add `last_error TEXT, attempt_count INTEGER` columns + a banner on the dashboard row + a "retry" affordance. Needs a sqlite migration (idempotent ALTER TABLE). -- **Per-agent reminder status / query interface**: surface pending vs. delivered counts per agent (manager + each sub-agent) as a small chip on the container row. -- **Tombstones + meta_inputs events**: not yet event-derived. PURG3 + meta-update still trigger a post-submit `/api/state` refetch on the dashboard. Add `TombstoneAdded`/`TombstoneRemoved` + `MetaInputsChanged` so those forms can drop their refetch too and the cold-load is the only `/api/state` fetch in normal operation. +- **Per-agent reminder rollups**: pending count chip + delivery-error surface landed. Still open: delivered-count chip (last 24h?), per-agent histogram of attempts-vs-successes — both readable from `turn_stats.open_reminders_count` history or via a new `Broker::count_delivered_reminders_since(agent, ts)` helper. ## Security From 63f5f9a2efdf9a4de4e3c82900d467c2715ce20d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Mon, 18 May 2026 00:11:48 +0200 Subject: [PATCH 3/7] =?UTF-8?q?todo:=20drop=20landed=20entries=20=E2=80=94?= =?UTF-8?q?=20get=5Fopen=5Fthreads,=20whoami,=20oversize-msg,=20tombstones?= =?UTF-8?q?+meta=5Finputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/TODO.md b/TODO.md index 4960a0ab..359500a1 100644 --- a/TODO.md +++ b/TODO.md @@ -9,8 +9,6 @@ - **Broadcast messaging**: allow sending messages with recipient "*" to all agents; deliver with hint "this was a broadcast and may not need any action from you" - **Multi-agent restart coordination**: when rebuilding all agents, manager should start first so it can coordinate post-restart confusion (notify agents, suppress unnecessary retries, etc) - **Shared docs/skills repo (RO)**: a single repo on the hive forge that every agent has read-only access to — common references, prompts, runbooks, "skills" the operator wants every agent to inherit without baking into the system prompt or `/shared`. Implementation likely: seed an `org-shared/docs` repo on first hive-forge boot, grant every per-agent user a read membership in the org. Agents `git clone` it (or use the API) to read; only the manager + operator can push. -- ~~**Loose-ends tracker + `get_open_threads` tool**~~ ✓ landed — new `mcp__hyperhive__get_open_threads` MCP tool on both agent + manager surfaces. Wire types in `hive-sh4re`: `AgentRequest::GetOpenThreads` / `ManagerRequest::GetOpenThreads` → `OpenThreads { threads: Vec }`. `OpenThread` is a tagged enum with `Approval { id, agent, commit_ref, description, age_seconds }` and `Question { id, asker, target, question, age_seconds }`. Shared aggregator at `hive-c0re/src/open_threads.rs`: `for_agent(coord, name)` (sub-agent surface; filters questions by asker == self OR target == self, approvals only for manager) and `hive_wide(coord)` (manager surface; everything pending in the swarm). No caching — fresh sqlite sweep per call. **Per-agent web UI rendering** is a follow-up below. - - **Follow-up: surface open-threads on the per-agent web UI** so the operator can see at a glance what each agent has hanging open — same data source as the MCP tool, just rendered into the existing per-agent dashboard page (next to inbox view / model chip / etc). ## Reminder Tool @@ -20,7 +18,7 @@ ## Dashboard -- **Per-agent reminder rollups**: pending count chip + delivery-error surface landed. Still open: delivered-count chip (last 24h?), per-agent histogram of attempts-vs-successes — both readable from `turn_stats.open_reminders_count` history or via a new `Broker::count_delivered_reminders_since(agent, ts)` helper. +- **Delivered-reminder rollups**: per-agent delivered-count chip (last 24h) + histogram of attempts-vs-successes on the container row. Needs `Broker::count_delivered_reminders_since(agent, ts)` (cheap COUNT against the `reminders` table, `WHERE agent = ?1 AND sent_at >= ?2`). ## Security @@ -32,13 +30,6 @@ Filed by damocles, who actually lives in this thing. Loosely ranked by how often the friction bites in normal use. -- ~~**Auto-attach oversize message bodies**~~ — superseded by simply - raising the inline cap from 1 KiB → 4 KiB (covers ~95% of - conversational overflow). Anything genuinely larger still needs a - state file. Blob-in-broker-sqlite was prototyped on paper - (`/agents/damocles/state/oversize-msg-proposal.md`) but rejected as - future vacuum/sync pain not worth carrying for the long-tail 5% of - cases that legitimately belong in a file. - **Inbox batching hint in the wake prompt** — when the harness pops a message and there are N more waiting, the wake prompt should say so (e.g. `"(+3 more queued; consider draining before acting)"`) so claude @@ -55,7 +46,6 @@ how often the friction bites in normal use. and `cancel_ask(id)` on the agent surface, plus `list_my_reminders()` / `cancel_reminder(id)`. Bounded by `asker == self` and `reminder.owner == self` so no cross-agent meddling. -- ~~**`whoami` introspection tool**~~ ✓ landed — new `mcp__hyperhive__whoami` on both agent + manager surfaces. Returns `{ name, role, hyperhive_rev }` from coord state (socket identity for `name`, hard-coded per surface for `role`, `auto_update::current_flake_rev`). `operator_pronouns` deliberately omitted — already substituted into every agent's system prompt at boot, so returning it again was duplicate data. `model` + `started_at` deferred — those live in the harness process not the coord, would need extra plumbing for marginal value. - **Optional `in_reply_to: ` on send** — pure wire addition; no behavioural change. The dashboard could render conversation threads (already wants this for the agent-to-agent question UI in the @@ -67,7 +57,7 @@ how often the friction bites in normal use. ## Telemetry - **Per-turn stats: host-side vacuum sweep**: the sink writes to `/state/hyperhive-turn-stats.sqlite` on each agent's state dir; needs a periodic retention sweep mirroring `events_vacuum.rs` so the table doesn't grow forever. Default keep-window: 90 days (turn-stats are denser than events but smaller per-row, ~200B each). -- **Surface per-turn stats on the agent web UI**: badges sourced from the new sink — `open_threads` count chip, `open_reminders` count chip, "N turns today" chip, rolling tool-call histogram tooltip on the model chip. Both `open_threads` and `open_reminders` are already columns on every row; the badge just reads the latest. The richer histograms read across rows. +- **Surface per-turn stats on the agent web UI**: "N turns today" chip + rolling tool-call histogram tooltip on the model chip. (`open_threads` and `open_reminders` chips already landed via other paths — open-threads section on the page + reminder count chip on the container row.) Reads the per-agent `turn_stats.sqlite`. - **Stats UI on the main dashboard**: per-agent rollups (avg turn duration, tokens-since-boot, top 5 tools) on the container row. Same data source, host-side aggregation query. ## Bugs From fd7712f5c12ed30d70a93a99c8911bddfcf662df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Mon, 18 May 2026 11:25:54 +0200 Subject: [PATCH 4/7] agent terminal: pretty-render task_started / task_notification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit claude's Task tool spawns subagents whose progress lands as stream-json events with subtype=task_started or task_notification — previously fell through to the .sys catch-all and rendered as a raw json dump that wrapped per char in the live pane. now matched by subtype before the catch-all: - task_started → cyan tool-use row, ⌁ glyph, first 8 chars of task_id, description, and optional [task_type] - task_notification → row styled by status: completed → turn-end-ok (green ✓), failed → turn-end-fail (red ✗), other → tool-result (muted ◌). output_file rendered inline if present so the operator can trace where the body landed. matching on `v.subtype` rather than a particular `v.type` so the renderer survives claude wrapping these under different top-level type fields across versions. --- hive-ag3nt/assets/app.js | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/hive-ag3nt/assets/app.js b/hive-ag3nt/assets/app.js index 34b4ca5f..f5a2afd4 100644 --- a/hive-ag3nt/assets/app.js +++ b/hive-ag3nt/assets/app.js @@ -742,12 +742,45 @@ api.details('tool-result-block', summary, txt); } } + // Pretty-render claude's background-task subagent events + // (`task_started`, `task_notification`). They share the same + // task_id so the operator can correlate start ↔ result; render + // each as a peer of tool_use / tool_result with a `⌁` glyph to + // mark "this happened in a subagent" rather than the main + // session. + function renderTaskEvent(v, api) { + const id = (v.task_id || '').slice(0, 8); + const kind = v.task_type ? ` [${v.task_type}]` : ''; + const desc = v.description || v.summary || '(no description)'; + if (v.subtype === 'task_started') { + api.row('tool-use', `⌁ task ${id} started · ${desc}${kind}`); + return true; + } + if (v.subtype === 'task_notification') { + const status = v.status || 'unknown'; + const glyph = status === 'completed' ? '✓' : status === 'failed' ? '✗' : '◌'; + const cls = status === 'completed' ? 'turn-end-ok' + : status === 'failed' ? 'turn-end-fail' + : 'tool-result'; + const out = v.output_file ? ` · → ${v.output_file}` : ''; + api.row(cls, `⌁ task ${id} ${glyph} ${status} · ${desc}${out}`); + return true; + } + return false; + } function renderStream(v, api) { // Drop session init, claude's result line, rate-limit — noise. // TurnEnd communicates pass/fail; session init isn't actionable. if (v.type === 'system' && v.subtype === 'init') return; if (v.type === 'rate_limit_event') return; if (v.type === 'result') return; + // Background-task subagent events (claude's `Task` tool spawns + // a separate session whose progress lands here as `task_*` + // subtypes). Match by subtype so we don't have to track which + // top-level `type` claude wraps them under across versions. + if (v.subtype === 'task_started' || v.subtype === 'task_notification') { + if (renderTaskEvent(v, api)) return; + } if (v.type === 'assistant' && v.message && v.message.content) { for (const c of v.message.content) { if (c.type === 'text' && c.text && c.text.trim()) api.row('text', c.text); From 5389875079a16eece6346afbeef4beddb05ea9fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Mon, 18 May 2026 11:27:35 +0200 Subject: [PATCH 5/7] =?UTF-8?q?docs:=20terminal-rendering.md=20=E2=80=94?= =?UTF-8?q?=20row=20taxonomy=20+=20inconsistencies=20+=20coherence=20propo?= =?UTF-8?q?sal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 3 ++ docs/terminal-rendering.md | 104 +++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 docs/terminal-rendering.md diff --git a/CLAUDE.md b/CLAUDE.md index 200996b4..bba3919f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,6 +138,9 @@ read them à la carte. - **"What does the dashboard look like?"** → [`docs/web-ui.md`](docs/web-ui.md). +- **"How does the per-agent terminal classify + colour + events?"** → [`docs/terminal-rendering.md`](docs/terminal-rendering.md) + (taxonomy + known inconsistencies + a proposed coherence pass). - **"How does claude get its prompt and what tools does it have?"** → [`docs/turn-loop.md`](docs/turn-loop.md). - **"How do config changes flow from manager to operator to diff --git a/docs/terminal-rendering.md b/docs/terminal-rendering.md new file mode 100644 index 00000000..77d124c1 --- /dev/null +++ b/docs/terminal-rendering.md @@ -0,0 +1,104 @@ +# Per-agent terminal: row taxonomy + inconsistencies + +Snapshot of how the per-agent web UI's live pane renders each +event kind today, written up so the next coherence pass has a +reference to work from. Source of truth lives in +`hive-ag3nt/assets/app.js` (`renderStream`, +`fmtToolUse`, `renderRichToolUse`, `renderToolResult`, +`renderTaskEvent`) + `hive-fr0nt/assets/terminal.css` (the +shared `.live .` styling). + +## Row taxonomy + +| CSS class | Prefix glyph | Color | Triggered by | Source | +|---|---|---|---|---| +| `.turn-start` | `◆ TURN ← ` | amber, bold, top-margin, amber left rule | `LiveEvent::TurnStart` | harness wake | +| `.turn-body` | (inline under turn-start) | fg dimmed 85% | same | the wake-prompt body | +| `.turn-end-ok` | `✓ turn ok` | green, green left rule | `LiveEvent::TurnEnd { ok: true }` | harness | +| `.turn-end-fail` | `✗ turn fail — note` | red, red left rule | `LiveEvent::TurnEnd { ok: false }` | harness | +| `.text` | none | fg/white, indented | claude `assistant.content[].text` | stream-json | +| `.thinking` | `·` or `· thinking …` | muted, italic | claude `assistant.content[].thinking` | stream-json | +| `.tool-use` | `→ Name args…` | cyan | `assistant.content[].tool_use` | stream-json | +| `.tool-use` `
` | `→ Name path · +N` | cyan, body is diff | rich tool_use (Write/Edit) | renderRichToolUse | +| `.tool-use` `
` | `→ send → to · headline` | cyan, body is text | mcp__hyperhive__send | renderRichToolUse | +| `.tool-result` | `← ` | muted | short `user.content[].tool_result` (≤120c) | stream-json | +| `.tool-result-block` `
` | `▸ ← Nl · headline` | muted, body is text | long `tool_result` (>120c) | stream-json | +| `.tool-use` | `⌁ task started · [type]` | cyan | claude Task-tool subagent event | renderTaskEvent | +| `.turn-end-ok` / `.turn-end-fail` / `.tool-result` | `⌁ task ✓/✗/◌ · · → ` | green / red / muted | claude Task-tool result | renderTaskEvent | +| `.note` | `· ` | muted | `LiveEvent::Note` (harness chatter, /cancel /compact /new-session, stderr lines, etc.) | harness | +| **`.sys`** | `· {json…}` | **muted** | **anything `renderStream` doesn't recognise** | catch-all | +| `.result` | (defined, never emitted today) | green | — | — | +| Banner shimmer | mauve | turn in flight (ref-counted) | `setBannerActive` | + +## Where the inconsistencies live + +1. **Glyph vocabulary drifts**: + - tool_use uses `→` + - tool_result uses `←` + - thinking uses `·` + - notes use `·` + - turn-end ok/fail use `✓ / ✗` + - turn-start uses `◆` + - task events use `⌁` + - The `·` glyph is overloaded across thinking, notes, sys. + +2. **What gets a `
` block vs a flat row** is per-tool + ad-hoc: Write/Edit always expand, send always expands, every + other tool_use is flat regardless of input size. + `tool_result` is flat if ≤120 chars otherwise `
`. + +3. **Stderr handling**: claude's stderr lines come through as + `LiveEvent::Note` with `text: "stderr: "` — they + render as muted `· stderr: …`, identical styling to harness + notes about /compact / /model. No red-tinted "this is an + error" affordance for stderr. + +4. **Catch-all `.sys` rows** are visually identical to `.note` + rows — both muted, both `·` prefix. They look like normal + notes despite usually being "an event renderStream couldn't + classify". Unmatched stream shapes (rate limit warnings under + odd type/subtype combos, future claude additions, etc.) + silently fall through. + +5. **`.tool-use` ranges from one-liner (`Read foo.md`) to + multi-page collapsed diff** — same color, same prefix glyph, + very different visual weight. A small marker on the collapsed + `
` summary would help (the `▸` is present in + `.tool-result-block` summaries but absent in tool-use + `
` summaries). + +6. **Cancel/compact/new-session notes** are styled the same as + autonomous harness chatter; nothing flags them as "operator + initiated." + +## Suggested coherence pass + +Pick one scheme and audit all renderers to match. A concrete +proposal: + +- **`→` cyan**: outbound action (tool_use, send) +- **`←` muted**: inbound result (tool_result) +- **`◆` amber**: turn framing (turn_start) +- **`✓ / ✗`**: success / failure, green / red (turn_end, + task_notification) +- **`⌁` mauve**: subagent / background event (task_*) +- **`·` muted**: ambient note, italic for thinking +- **`!` orange**: caught error (stderr lines, .sys catch-all + that landed something the renderer didn't recognise) + +Plus: every tool_use `
` summary gets `▸` so collapsed +content is visually announced. Operator-initiated notes get a +distinct prefix (`op·` or similar) so they're easier to spot in +the scrollback. + +The `.sys` catch-all should escalate visually — a louder +"unrecognised event" rendering surfaces silently-dropped event +shapes for future fix-up rather than hiding them in the muted +note stream. + +## Dashboard side (not covered here) + +The main dashboard's message-flow pane is a different beast: +broker messages render as `.msgrow` grid lines (ts / arrow / +from / → / to / body) with separate styling. The current file +focuses only on the per-agent terminal. From 487be2e1fdcd697a64cd165c2ec652f6f154e15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Mon, 18 May 2026 11:28:06 +0200 Subject: [PATCH 6/7] =?UTF-8?q?todo:=20per-agent=20terminal=20coherence=20?= =?UTF-8?q?pass=20=E2=86=92=20docs/terminal-rendering.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.md b/TODO.md index 359500a1..a033b1c5 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,7 @@ ## Dashboard +- **Per-agent terminal coherence pass**: row taxonomy + colour scheme has drifted — glyph reuse, ad-hoc `
` rules, undifferentiated stderr, `.sys` catch-all silently hiding unrecognised events. Full audit + proposed unified scheme in [`docs/terminal-rendering.md`](docs/terminal-rendering.md). - **Delivered-reminder rollups**: per-agent delivered-count chip (last 24h) + histogram of attempts-vs-successes on the container row. Needs `Broker::count_delivered_reminders_since(agent, ts)` (cheap COUNT against the `reminders` table, `WHERE agent = ?1 AND sent_at >= ?2`). ## Security From f84011abc37d259e4fdc0cdaebdd595f910d599a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Mon, 18 May 2026 11:29:49 +0200 Subject: [PATCH 7/7] =?UTF-8?q?todo:=20bug=20=E2=80=94=20token=20budget=20?= =?UTF-8?q?exhaustion=20crashes=20harness,=20leaves=20container=20up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.md b/TODO.md index a033b1c5..49e9dc9a 100644 --- a/TODO.md +++ b/TODO.md @@ -63,4 +63,5 @@ how often the friction bites in normal use. ## Bugs +- **Token-budget exhaustion crashes the harness**: when claude's account hits its rate/token cap, the in-flight `claude --print` invocation returns an error the harness doesn't recognise as recoverable, the serve loop exits, and the container stays up with a dead daemon. Operator only notices when an unrelated wake fails to drive a turn. Want: detect the budget-exceeded class of failure (likely a specific stderr line or stream-json `rate_limit_event` shape), fire a `LiveEvent::StatusChanged("rate_limited")` or new status, surface as a red badge + banner on the dashboard + per-agent UI, and have the serve loop park (sleep N minutes, retry) instead of returning Err. Operator can also see "this agent is rate-limited until ~HH:MM" if claude tells us when. Inspect `crate::turn::run_claude`'s `bail!` paths + claude's stderr conventions for the budget error string. - **Post-rebuild system-message missed wake**: at 09:13:14 the dashboard showed `system → damocles container rebuilt` as ✓ delivered, but the agent harness never ran a turn for it (no claude invocation, no operator-visible activity). A subsequent `recv()` from inside the agent returned `(empty)`, confirming the message was popped + marked delivered server-side — yet drove no turn. Most likely cause: the agent_server `serve_agent_stdio` task is up and answering MCP/socket calls, but the `hive-ag3nt::serve` long-poll loop that drives `drive_turn` either died silently during rebuild or never restarted. Investigate: (a) does hive-ag3nt's serve loop survive `nixos-container update` cleanly, or does its tokio runtime get torn down mid-loop? (b) is there an early-exit path on a transient socket error during rebuild that drops the serve task without notifying the manager? (c) compare timeline with manager's own post-rebuild wake to see if this is rebuilt-agents-only or universal. Could be related to the `recv_blocking` fix in `e423d57` if the rebuild restarts the broker mid-subscribe.