harness metrics for tool calls, bash commands, reminders and first-turn context, on the agents board

The per-agent /stats page read these from its sqlite; the swarm board had
none of them. hive-agent now counts the tools a turn called, the commands
bash tasks ran (from the bash daemon's own table, newer rows only),
reminders scheduled and delivered, and records the first turn of a fresh
session; the turns tab shows them, with fresh sessions per agent. Claude's
own export has no tool-call counter, so the docs no longer say it does.

hive-agent/src/otel_turn_metrics.rs

@@ -1,13 +1,13 @@
 //! Per-turn OTEL metric export for the fields Claude Code's own built-in
 //! OTEL integration cannot know about. Claude's own export already covers
-//! token usage, cost, and tool-call counts (see `docs/scheduler/observability.md`) —
+//! token usage and cost (see `docs/scheduler/observability.md`) —
 //! duplicating those here under different metric names would just give a
 //! collector two disagreeing series for the same number. This module emits
 //! only the harness-only concepts: wall-clock turn duration as *this harness*
-//! measures it (not Claude's own per-request latency), what woke the turn,
-//! the harness's own outcome classification, the loose-ends backlog at turn
-//! end, session boundaries (fresh vs. resumed by title), context-window fill,
-//! and the cost an ACP agent reports — claude's runtime reports none.
+//! measures it, what woke the turn, its outcome classification, the loose-ends
+//! backlog, sessions and their first turn's cost, context-window fill, the
+//! tools a turn called (claude exports no tool-call counter), bash commands,
+//! reminders, and the cost an ACP agent reports — claude's runtime reports none.
 //!
 //! Synchronous instruments, not observable ones: unlike hive-c0re's
 //! container-resource gauges (`hive-c0re/src/stats/otel_metrics.rs`), which
@@ -29,7 +29,9 @@
 //! up automatically by the SDK from the container-wide `OTEL_RESOURCE_ATTRIBUTES`
 //! env var (same mechanism `hive-metric` relies on) — nothing to set here.
 
+use std::collections::HashMap;
 use std::sync::OnceLock;
+use std::sync::atomic::{AtomicI64, Ordering};
 use std::time::Duration;
 
 use opentelemetry::KeyValue;
@@ -92,6 +94,16 @@ struct Instruments {
     /// Keyed by `model` and `currency`: an ACP agent reports cost in a
     /// currency of its choosing.
     reported_cost: Counter<f64>,
+    /// Tool calls made, keyed by `tool` (`Skill:<skill>` for a skill).
+    tool_calls: Counter<u64>,
+    /// Shell commands run by bash tasks, keyed by `command`: the normalised
+    /// head the bash daemon records, not the full command line.
+    bash_commands: Counter<u64>,
+    /// Reminders scheduled and delivered, keyed by `event`.
+    reminders: Counter<u64>,
+    /// Input tokens of the first turn of a fresh claude session: what the
+    /// system prompt and `AGENTS.md` cost before anything is cached.
+    first_turn_context: Gauge<u64>,
 }
 
 /// Lazily built on the first call to [`record`]. `None` when OTEL isn't
@@ -130,7 +142,48 @@ pub fn record(row: &TurnStatRow, fresh_session: bool) {
     if fresh_session {
         inst.session_count
             .add(1, &[KeyValue::new("model", row.model.clone())]);
+        inst.first_turn_context.record(row.input_tokens, &[]);
     }
+    for (tool, calls) in tool_call_counts(row.tool_call_breakdown_json.as_deref()) {
+        inst.tool_calls.add(calls, &[KeyValue::new("tool", tool)]);
+    }
+    record_new_bash_commands(inst, row.started_at);
+}
+
+/// The per-tool call counts a turn stored in `tool_call_breakdown_json`; a
+/// turn with no tool calls, or a value that does not parse, has none.
+fn tool_call_counts(json: Option<&str>) -> Vec<(String, u64)> {
+    json.and_then(|j| serde_json::from_str::<HashMap<String, u64>>(j).ok())
+        .map(|m| m.into_iter().collect())
+        .unwrap_or_default()
+}
+
+/// Unix seconds up to which bash commands are already counted; `0` before the
+/// first turn of this run.
+static BASH_COUNTED_UNTIL: AtomicI64 = AtomicI64::new(0);
+
+/// Count the bash commands recorded since the last call. The first call after
+/// the harness starts counts only commands newer than `turn_started_at`, so a
+/// restart does not count the whole history again.
+fn record_new_bash_commands(inst: &Instruments, turn_started_at: i64) {
+    let now = chrono::Utc::now().timestamp();
+    let mut since = BASH_COUNTED_UNTIL.swap(now, Ordering::Relaxed);
+    if since == 0 {
+        since = turn_started_at.saturating_sub(1);
+    }
+    for (command, n) in crate::stats::bash_heads_since(since) {
+        inst.bash_commands
+            .add(n, &[KeyValue::new("command", command)]);
+    }
+}
+
+/// Count one reminder event, `scheduled` or `delivered`. No-op when OTEL isn't
+/// configured, same as [`record`].
+pub fn record_reminder(event: &'static str) {
+    let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
+        return;
+    };
+    inst.reminders.add(1, &[KeyValue::new("event", event)]);
 }
 
 /// Record the current `AGENTS.md` line count. Called from
@@ -250,6 +303,13 @@ fn build() -> Option<Instruments> {
                     .with_unit("{token}")
                     .build(),
                 reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(),
+                tool_calls: meter.u64_counter("hyperhive.agent.tool.calls").build(),
+                bash_commands: meter.u64_counter("hyperhive.agent.bash.commands").build(),
+                reminders: meter.u64_counter("hyperhive.agent.reminders").build(),
+                first_turn_context: meter
+                    .u64_gauge("hyperhive.agent.first_turn.context")
+                    .with_unit("{token}")
+                    .build(),
                 _provider: provider,
             })
         }
@@ -291,7 +351,23 @@ pub fn enabled() -> bool {
 
 #[cfg(test)]
 mod tests {
-    use super::TURN_DURATION_BOUNDS_MS;
+    use super::{TURN_DURATION_BOUNDS_MS, tool_call_counts};
+
+    #[test]
+    fn a_turns_stored_tool_breakdown_becomes_per_tool_counts() {
+        let mut got = tool_call_counts(Some(r#"{"Bash":3,"Skill:base:pr-review":1}"#));
+        got.sort();
+        assert_eq!(
+            got,
+            vec![
+                ("Bash".to_owned(), 3),
+                ("Skill:base:pr-review".to_owned(), 1)
+            ]
+        );
+        // No tool calls, and a value that is not a count map, both count nothing.
+        assert!(tool_call_counts(None).is_empty());
+        assert!(tool_call_counts(Some("not json")).is_empty());
+    }
 
     #[test]
     fn the_duration_bounds_ascend_and_reach_past_the_old_ten_second_ceiling() {

hive-agent/src/reminders.rs

@@ -88,6 +88,7 @@ impl Reminders {
              VALUES (?1, ?2, ?3, ?4, NULL)",
             params![message, file_path, due_at, now],
         )?;
+        crate::otel_turn_metrics::record_reminder("scheduled");
         Ok(conn.last_insert_rowid())
     }
 
@@ -173,6 +174,7 @@ impl Reminders {
             "UPDATE reminders SET sent_at = ?1 WHERE id = ?2",
             params![Utc::now().timestamp(), id],
         )?;
+        crate::otel_turn_metrics::record_reminder("delivered");
         Ok(())
     }
 

hive-agent/src/stats.rs

@@ -449,6 +449,32 @@ fn read_bash_breakdown(conn: &Connection, from: i64) -> rusqlite::Result<Vec<Key
     Ok(top_n(totals, 10))
 }
 
+/// The shell-command heads recorded after `since` (unix seconds, exclusive),
+/// with how often each ran, for the OTEL bash-command counter. Reads the live
+/// database read-only; a missing database or `bash_commands` table (no bash
+/// task has run yet) is an empty list, not an error.
+#[must_use]
+pub fn bash_heads_since(since: i64) -> Vec<(String, u64)> {
+    let Ok(conn) = Connection::open_with_flags(default_path(), OpenFlags::SQLITE_OPEN_READ_ONLY)
+    else {
+        return Vec::new();
+    };
+    let _ = conn.busy_timeout(std::time::Duration::from_millis(500));
+    read_bash_heads_since(&conn, since).unwrap_or_default()
+}
+
+fn read_bash_heads_since(conn: &Connection, since: i64) -> rusqlite::Result<Vec<(String, u64)>> {
+    let mut stmt =
+        conn.prepare("SELECT head, COUNT(*) FROM bash_commands WHERE ts > ?1 GROUP BY head")?;
+    let rows = stmt.query_map([since], |row| {
+        Ok((
+            row.get::<_, String>(0)?,
+            u64_from_i64(row.get::<_, i64>(1)?),
+        ))
+    })?;
+    rows.collect()
+}
+
 struct Row {
     started_at: i64,
     duration_ms: i64,
@@ -782,6 +808,29 @@ mod tests {
         assert!(s.bash_breakdown.is_empty());
     }
 
+    /// The OTEL bash-command counter reads only the heads recorded after its
+    /// watermark, and a database without the table yields nothing.
+    #[test]
+    fn bash_heads_since_counts_only_newer_rows() {
+        let db = tmp_db();
+        let _ = std::fs::remove_file(&db);
+        let conn = Connection::open(&db).unwrap();
+        assert!(read_bash_heads_since(&conn, 0).is_err(), "no table yet");
+        conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
+            .unwrap();
+        for (ts, head) in [(10, "cargo"), (20, "cargo"), (30, "git"), (30, "cargo")] {
+            conn.execute(
+                "INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
+                params![ts, head],
+            )
+            .unwrap();
+        }
+        let mut got = read_bash_heads_since(&conn, 10).unwrap();
+        got.sort();
+        assert_eq!(got, vec![("cargo".to_owned(), 2), ("git".to_owned(), 1)]);
+        assert!(read_bash_heads_since(&conn, 30).unwrap().is_empty());
+    }
+
     /// With a populated `bash_commands` table, `bash_breakdown` rolls up
     /// per-head counts (busiest first) and respects the window cutoff.
     #[test]