hive-bash-mcp: count bash commands over OTEL in the daemon, drop the sqlite hand-off
This commit is contained in:
parent
9ec67d7a18
commit
076df804e4
11 changed files with 100 additions and 184 deletions
4
Cargo.lock
generated
4
Cargo.lock
generated
|
|
@ -1848,8 +1848,10 @@ dependencies = [
|
||||||
"hive-sock-client",
|
"hive-sock-client",
|
||||||
"hive-types",
|
"hive-types",
|
||||||
"libc",
|
"libc",
|
||||||
|
"opentelemetry",
|
||||||
|
"opentelemetry-otlp",
|
||||||
|
"opentelemetry_sdk",
|
||||||
"rmcp",
|
"rmcp",
|
||||||
"rusqlite",
|
|
||||||
"schemars",
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|
|
||||||
|
|
@ -43,19 +43,13 @@ Retention:
|
||||||
|
|
||||||
### `/harness/hyperhive-turn-stats.sqlite` (per agent)
|
### `/harness/hyperhive-turn-stats.sqlite` (per agent)
|
||||||
|
|
||||||
The `hive-bash-daemon` writes a `bash_commands(ts INTEGER, head TEXT)`
|
Nothing writes or reads this file now. Older harnesses wrote per-turn
|
||||||
table in it: one row per executed bash task recording the normalised command head -
|
`turn_stats` and `sessions` tables there, and an older `hive-bash-daemon` a
|
||||||
the basename of the first real command, looking past `cd repo &&`
|
`bash_commands` table; they stay in existing files, which are safe to delete
|
||||||
prefixes, env-assignments, and prefix-runners like `sudo`/`env`. It
|
by hand. The bash daemon counts executed commands itself and exports them
|
||||||
feeds the bash-command counter in the agent's OTEL metrics. Best-effort and created on first write
|
over OTEL (see
|
||||||
(`CREATE TABLE IF NOT EXISTS`), so it's absent until a bash
|
[`observability.md`](../scheduler/observability.md)), and the one value the
|
||||||
task runs.
|
harness reads back between runs lives in `hyperhive-harness.json`, below.
|
||||||
|
|
||||||
The file has **no vacuum**. Older harnesses also wrote a per-turn `turn_stats`
|
|
||||||
table and a `sessions` table there; nothing writes them now and nothing in the
|
|
||||||
harness reads them, but they stay in existing files, where the
|
|
||||||
[legacy stats viewer](https://forge.constellation.darkest.space/hyperhive/legacy-stats-viewer)
|
|
||||||
can still turn them into the old stats page.
|
|
||||||
|
|
||||||
### `/state/hyperhive-harness.json` (per agent)
|
### `/state/hyperhive-harness.json` (per agent)
|
||||||
|
|
||||||
|
|
@ -228,7 +222,6 @@ Under `/var/lib/hyperhive/agents/<name>/`:
|
||||||
swept — same "audit trail, not cache" treatment as the c0re-side
|
swept — same "audit trail, not cache" treatment as the c0re-side
|
||||||
tables above.
|
tables above.
|
||||||
|
|
||||||
- `hyperhive-turn-stats.sqlite` — the bash command heads the bash daemon records.
|
|
||||||
- `hyperhive-model` / `hyperhive-effort` — single-line model name and
|
- `hyperhive-model` / `hyperhive-effort` — single-line model name and
|
||||||
effort level files, read-only fallbacks.
|
effort level files, read-only fallbacks.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -428,7 +428,7 @@ of their own — see [ACP-reported cost](#acp-reported-cost).
|
||||||
<!-- vale write-good.Passive = YES -->
|
<!-- vale write-good.Passive = YES -->
|
||||||
|
|
||||||
| Metric | Unit | Kind | Attributes |
|
| Metric | Unit | Kind | Attributes |
|
||||||
| --------------------------------------------- | --------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
|
| --------------------------------------------- | --------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` |
|
| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` |
|
||||||
| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` |
|
| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` |
|
||||||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
||||||
|
|
@ -443,7 +443,7 @@ of their own — see [ACP-reported cost](#acp-reported-cost).
|
||||||
| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator |
|
| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator |
|
||||||
| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below |
|
| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below |
|
||||||
| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:<skill>` for a skill; counted when a turn ends |
|
| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:<skill>` for a skill; counted when a turn ends |
|
||||||
| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted when a turn ends |
|
| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted by `hive-bash-daemon` as each task starts |
|
||||||
| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) |
|
| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) |
|
||||||
| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` |
|
| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` |
|
||||||
| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds |
|
| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds |
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,6 @@ mod router_todos;
|
||||||
mod schedule_timer;
|
mod schedule_timer;
|
||||||
mod serve_common;
|
mod serve_common;
|
||||||
mod state_entry_watch;
|
mod state_entry_watch;
|
||||||
mod stats;
|
|
||||||
mod subagent_role_watch;
|
mod subagent_role_watch;
|
||||||
mod swarm_agent_commands;
|
mod swarm_agent_commands;
|
||||||
mod swarm_agent_state;
|
mod swarm_agent_state;
|
||||||
|
|
@ -1031,7 +1030,6 @@ async fn handle_turn<S: Surface>(
|
||||||
unread,
|
unread,
|
||||||
});
|
});
|
||||||
bus.set_state(TurnState::Thinking);
|
bus.set_state(TurnState::Thinking);
|
||||||
let started_at = chrono::Utc::now().timestamp();
|
|
||||||
let started_instant = std::time::Instant::now();
|
let started_instant = std::time::Instant::now();
|
||||||
let model_at_start = bus.model();
|
let model_at_start = bus.model();
|
||||||
// Read-and-clear: this wake prompt is the one turn that gets to carry
|
// Read-and-clear: this wake prompt is the one turn that gets to carry
|
||||||
|
|
@ -1064,7 +1062,6 @@ async fn handle_turn<S: Surface>(
|
||||||
let (open_threads, open_reminders) = S::post_turn_counts(socket).await;
|
let (open_threads, open_reminders) = S::post_turn_counts(socket).await;
|
||||||
let context_window = bus.api_context_window();
|
let context_window = bus.api_context_window();
|
||||||
let row = serve_common::build_row(serve_common::TurnRowArgs {
|
let row = serve_common::build_row(serve_common::TurnRowArgs {
|
||||||
started_at,
|
|
||||||
duration_ms,
|
duration_ms,
|
||||||
model: model_at_start,
|
model: model_at_start,
|
||||||
wake_from: from.clone(),
|
wake_from: from.clone(),
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@
|
||||||
//! only the harness-only concepts: wall-clock turn duration as *this harness*
|
//! only the harness-only concepts: wall-clock turn duration as *this harness*
|
||||||
//! measures it, what woke the turn, its outcome classification, the loose-ends
|
//! measures it, what woke the turn, its outcome classification, the loose-ends
|
||||||
//! backlog, sessions and their first turn's cost, context-window fill, the
|
//! backlog, sessions and their first turn's cost, context-window fill, the
|
||||||
//! tools a turn called (claude exports no tool-call counter), bash commands,
|
//! tools a turn called (claude exports no tool-call counter),
|
||||||
//! reminders, and the cost an ACP agent reports — claude's runtime reports none.
|
//! reminders, and the cost an ACP agent reports — claude's runtime reports none.
|
||||||
//!
|
//!
|
||||||
//! Synchronous instruments, not observable ones: unlike hive-c0re's
|
//! Synchronous instruments, not observable ones: unlike hive-c0re's
|
||||||
|
|
@ -31,7 +31,6 @@
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
use std::sync::atomic::{AtomicI64, Ordering};
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use opentelemetry::KeyValue;
|
use opentelemetry::KeyValue;
|
||||||
|
|
@ -96,9 +95,6 @@ struct Instruments {
|
||||||
reported_cost: Counter<f64>,
|
reported_cost: Counter<f64>,
|
||||||
/// Tool calls made, keyed by `tool` (`Skill:<skill>` for a skill).
|
/// Tool calls made, keyed by `tool` (`Skill:<skill>` for a skill).
|
||||||
tool_calls: Counter<u64>,
|
tool_calls: Counter<u64>,
|
||||||
/// Shell commands run by bash tasks, keyed by `command`: the normalised
|
|
||||||
/// head the bash daemon records, not the full command line.
|
|
||||||
bash_commands: Counter<u64>,
|
|
||||||
/// Reminders scheduled and delivered, keyed by `event`.
|
/// Reminders scheduled and delivered, keyed by `event`.
|
||||||
reminders: Counter<u64>,
|
reminders: Counter<u64>,
|
||||||
/// Input tokens of the first turn of a fresh claude session: what the
|
/// Input tokens of the first turn of a fresh claude session: what the
|
||||||
|
|
@ -153,7 +149,6 @@ pub fn record(row: &TurnStatRow, fresh_session: bool) {
|
||||||
for (tool, calls) in tool_call_counts(row.tool_call_breakdown_json.as_deref()) {
|
for (tool, calls) in tool_call_counts(row.tool_call_breakdown_json.as_deref()) {
|
||||||
inst.tool_calls.add(calls, &[KeyValue::new("tool", tool)]);
|
inst.tool_calls.add(calls, &[KeyValue::new("tool", tool)]);
|
||||||
}
|
}
|
||||||
record_new_bash_commands(inst, row.started_at);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The per-tool call counts a turn stored in `tool_call_breakdown_json`; a
|
/// The per-tool call counts a turn stored in `tool_call_breakdown_json`; a
|
||||||
|
|
@ -164,25 +159,6 @@ fn tool_call_counts(json: Option<&str>) -> Vec<(String, u64)> {
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Unix seconds up to which bash commands are already counted; `0` before the
|
|
||||||
/// first turn of this run.
|
|
||||||
static BASH_COUNTED_UNTIL: AtomicI64 = AtomicI64::new(0);
|
|
||||||
|
|
||||||
/// Count the bash commands recorded since the last call. The first call after
|
|
||||||
/// the harness starts counts only commands newer than `turn_started_at`, so a
|
|
||||||
/// restart does not count the whole history again.
|
|
||||||
fn record_new_bash_commands(inst: &Instruments, turn_started_at: i64) {
|
|
||||||
let now = chrono::Utc::now().timestamp();
|
|
||||||
let mut since = BASH_COUNTED_UNTIL.swap(now, Ordering::Relaxed);
|
|
||||||
if since == 0 {
|
|
||||||
since = turn_started_at.saturating_sub(1);
|
|
||||||
}
|
|
||||||
for (command, n) in crate::stats::bash_heads_since(since) {
|
|
||||||
inst.bash_commands
|
|
||||||
.add(n, &[KeyValue::new("command", command)]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Count one reminder event, `scheduled` or `delivered`. No-op when OTEL isn't
|
/// Count one reminder event, `scheduled` or `delivered`. No-op when OTEL isn't
|
||||||
/// configured, same as [`record`].
|
/// configured, same as [`record`].
|
||||||
pub fn record_reminder(event: &'static str) {
|
pub fn record_reminder(event: &'static str) {
|
||||||
|
|
@ -348,7 +324,6 @@ fn build() -> Option<Instruments> {
|
||||||
.build(),
|
.build(),
|
||||||
reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(),
|
reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(),
|
||||||
tool_calls: meter.u64_counter("hyperhive.agent.tool.calls").build(),
|
tool_calls: meter.u64_counter("hyperhive.agent.tool.calls").build(),
|
||||||
bash_commands: meter.u64_counter("hyperhive.agent.bash.commands").build(),
|
|
||||||
reminders: meter.u64_counter("hyperhive.agent.reminders").build(),
|
reminders: meter.u64_counter("hyperhive.agent.reminders").build(),
|
||||||
first_turn_context: meter
|
first_turn_context: meter
|
||||||
.u64_gauge("hyperhive.agent.first_turn.context")
|
.u64_gauge("hyperhive.agent.first_turn.context")
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,6 @@ pub fn format_wake_prompt(from: &str, body: &str, redelivered: bool, interrupted
|
||||||
/// (`otel_turn_metrics::record`).
|
/// (`otel_turn_metrics::record`).
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct TurnStatRow {
|
pub struct TurnStatRow {
|
||||||
pub started_at: i64,
|
|
||||||
pub duration_ms: i64,
|
pub duration_ms: i64,
|
||||||
pub model: String,
|
pub model: String,
|
||||||
pub wake_from: String,
|
pub wake_from: String,
|
||||||
|
|
@ -51,7 +50,6 @@ pub struct TurnStatRow {
|
||||||
/// `get_loose_ends` call before it's consumed — `Bus::take_tool_calls` resets
|
/// `get_loose_ends` call before it's consumed — `Bus::take_tool_calls` resets
|
||||||
/// the counter, so there's only one chance to read it per turn.
|
/// the counter, so there's only one chance to read it per turn.
|
||||||
pub struct TurnRowArgs<'a> {
|
pub struct TurnRowArgs<'a> {
|
||||||
pub started_at: i64,
|
|
||||||
pub duration_ms: i64,
|
pub duration_ms: i64,
|
||||||
pub model: String,
|
pub model: String,
|
||||||
pub wake_from: String,
|
pub wake_from: String,
|
||||||
|
|
@ -68,7 +66,6 @@ pub struct TurnRowArgs<'a> {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
|
pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
|
||||||
let TurnRowArgs {
|
let TurnRowArgs {
|
||||||
started_at,
|
|
||||||
duration_ms,
|
duration_ms,
|
||||||
model,
|
model,
|
||||||
wake_from,
|
wake_from,
|
||||||
|
|
@ -101,7 +98,6 @@ pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
|
||||||
Err(TurnError::Failed(_)) => "failed",
|
Err(TurnError::Failed(_)) => "failed",
|
||||||
};
|
};
|
||||||
TurnStatRow {
|
TurnStatRow {
|
||||||
started_at,
|
|
||||||
duration_ms,
|
duration_ms,
|
||||||
model,
|
model,
|
||||||
wake_from,
|
wake_from,
|
||||||
|
|
@ -149,7 +145,6 @@ mod build_row_tests {
|
||||||
tool_calls: &[(&str, u64)],
|
tool_calls: &[(&str, u64)],
|
||||||
) -> super::TurnStatRow {
|
) -> super::TurnStatRow {
|
||||||
build_row(TurnRowArgs {
|
build_row(TurnRowArgs {
|
||||||
started_at: 10,
|
|
||||||
duration_ms: 2_000,
|
duration_ms: 2_000,
|
||||||
model: "opus".to_owned(),
|
model: "opus".to_owned(),
|
||||||
wake_from: "todo".to_owned(),
|
wake_from: "todo".to_owned(),
|
||||||
|
|
@ -187,7 +182,7 @@ mod build_row_tests {
|
||||||
let _dir = crate::paths::harness_dir_guard();
|
let _dir = crate::paths::harness_dir_guard();
|
||||||
let row = row_for(&Ok(false), &[("Bash", 3)]);
|
let row = row_for(&Ok(false), &[("Bash", 3)]);
|
||||||
|
|
||||||
assert_eq!((row.started_at, row.duration_ms), (10, 2_000));
|
assert_eq!(row.duration_ms, 2_000);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
(row.model.as_str(), row.wake_from.as_str()),
|
(row.model.as_str(), row.wake_from.as_str()),
|
||||||
("opus", "todo")
|
("opus", "todo")
|
||||||
|
|
@ -210,7 +205,6 @@ mod build_row_tests {
|
||||||
let bus = Bus::new();
|
let bus = Bus::new();
|
||||||
bus.set_resolved_model(Some("claude-opus-4-8".to_owned()));
|
bus.set_resolved_model(Some("claude-opus-4-8".to_owned()));
|
||||||
let row = build_row(TurnRowArgs {
|
let row = build_row(TurnRowArgs {
|
||||||
started_at: 0,
|
|
||||||
duration_ms: 0,
|
duration_ms: 0,
|
||||||
model: "opus".to_owned(),
|
model: "opus".to_owned(),
|
||||||
wake_from: "todo".to_owned(),
|
wake_from: "todo".to_owned(),
|
||||||
|
|
|
||||||
|
|
@ -1,63 +0,0 @@
|
||||||
//! The shell-command heads the bash daemon records in the harness's `hyperhive-turn-stats.sqlite`,
|
|
||||||
//! read for the OTEL bash-command counter.
|
|
||||||
|
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
use rusqlite::{Connection, OpenFlags};
|
|
||||||
|
|
||||||
fn default_path() -> PathBuf {
|
|
||||||
crate::paths::harness_dir().join("hyperhive-turn-stats.sqlite")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The shell-command heads recorded after `since` (unix seconds, exclusive),
|
|
||||||
/// with how often each ran, for the OTEL bash-command counter. Reads the live
|
|
||||||
/// database read-only; a missing database or `bash_commands` table (no bash
|
|
||||||
/// task has run yet) is an empty list, not an error.
|
|
||||||
#[must_use]
|
|
||||||
pub fn bash_heads_since(since: i64) -> Vec<(String, u64)> {
|
|
||||||
let Ok(conn) = Connection::open_with_flags(default_path(), OpenFlags::SQLITE_OPEN_READ_ONLY)
|
|
||||||
else {
|
|
||||||
return Vec::new();
|
|
||||||
};
|
|
||||||
let _ = conn.busy_timeout(std::time::Duration::from_millis(500));
|
|
||||||
read_bash_heads_since(&conn, since).unwrap_or_default()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_bash_heads_since(conn: &Connection, since: i64) -> rusqlite::Result<Vec<(String, u64)>> {
|
|
||||||
let mut stmt =
|
|
||||||
conn.prepare("SELECT head, COUNT(*) FROM bash_commands WHERE ts > ?1 GROUP BY head")?;
|
|
||||||
let rows = stmt.query_map([since], |row| {
|
|
||||||
Ok((
|
|
||||||
row.get::<_, String>(0)?,
|
|
||||||
u64::try_from(row.get::<_, i64>(1)?).unwrap_or(0),
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
rows.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use rusqlite::params;
|
|
||||||
|
|
||||||
/// The OTEL bash-command counter reads only the heads recorded after its
|
|
||||||
/// watermark, and a database without the table yields nothing.
|
|
||||||
#[test]
|
|
||||||
fn bash_heads_since_counts_only_newer_rows() {
|
|
||||||
let conn = Connection::open_in_memory().unwrap();
|
|
||||||
assert!(read_bash_heads_since(&conn, 0).is_err(), "no table yet");
|
|
||||||
conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
|
|
||||||
.unwrap();
|
|
||||||
for (ts, head) in [(10, "cargo"), (20, "cargo"), (30, "git"), (30, "cargo")] {
|
|
||||||
conn.execute(
|
|
||||||
"INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
|
|
||||||
params![ts, head],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
let mut got = read_bash_heads_since(&conn, 10).unwrap();
|
|
||||||
got.sort();
|
|
||||||
assert_eq!(got, vec![("cargo".to_owned(), 2), ("git".to_owned(), 1)]);
|
|
||||||
assert!(read_bash_heads_since(&conn, 30).unwrap().is_empty());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -17,8 +17,10 @@ hive-sh4re.workspace = true
|
||||||
hive-sock-client.workspace = true
|
hive-sock-client.workspace = true
|
||||||
hive-types.workspace = true
|
hive-types.workspace = true
|
||||||
libc.workspace = true
|
libc.workspace = true
|
||||||
|
opentelemetry.workspace = true
|
||||||
|
opentelemetry-otlp.workspace = true
|
||||||
|
opentelemetry_sdk.workspace = true
|
||||||
rmcp.workspace = true
|
rmcp.workspace = true
|
||||||
rusqlite.workspace = true
|
|
||||||
# The agent's own queue connection, and what a task publishes over it
|
# The agent's own queue connection, and what a task publishes over it
|
||||||
# (`swarm_term`).
|
# (`swarm_term`).
|
||||||
async-nats.workspace = true
|
async-nats.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -32,5 +32,5 @@ Two bins from one shared lib (`src/lib.rs`):
|
||||||
returns the result.
|
returns the result.
|
||||||
|
|
||||||
Supporting modules: **`paths.rs`** (daemon socket + agent-socket
|
Supporting modules: **`paths.rs`** (daemon socket + agent-socket
|
||||||
resolution), **`stats.rs`** (the `bash_commands` favorite-tool stat
|
resolution), **`stats.rs`** (the OTEL bash-command counter, one count per
|
||||||
recorded into hyperhive-turn-stats.sqlite).
|
task keyed by the command's normalised head).
|
||||||
|
|
|
||||||
|
|
@ -20,14 +20,6 @@ pub fn tasks_dir() -> PathBuf {
|
||||||
harness_dir().join("bash-tasks")
|
harness_dir().join("bash-tasks")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Per-agent sqlite in the harness dir: the runner appends normalised
|
|
||||||
/// bash-command heads to its `bash_commands` table, which the harness reads for
|
|
||||||
/// its OTEL bash-command counter.
|
|
||||||
#[must_use]
|
|
||||||
pub fn turn_stats_db() -> PathBuf {
|
|
||||||
harness_dir().join("hyperhive-turn-stats.sqlite")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The harness-served in-agent todo socket (loose-ends v2). The runner
|
/// The harness-served in-agent todo socket (loose-ends v2). The runner
|
||||||
/// pushes bash-task todos here — `upsert_todo` while a task is active,
|
/// pushes bash-task todos here — `upsert_todo` while a task is active,
|
||||||
/// a keyless `done` todo on completion — instead of firing a direct
|
/// a keyless `done` todo on completion — instead of firing a direct
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
//! Capture of normalised shell-command "heads" per bash task into the
|
//! The OTEL bash-command counter: each bash task's normalised shell-command
|
||||||
//! per-agent turn-stats sqlite (`bash_commands` table), read by the
|
//! "head", counted as `hyperhive.agent.bash.commands` keyed by `command`.
|
||||||
//! harness for its OTEL bash-command counter.
|
|
||||||
//!
|
//!
|
||||||
//! Best-effort: a missing db, a lock, or a parse miss is logged and
|
//! Best-effort: without `OTEL_EXPORTER_OTLP_ENDPOINT`, or when the exporter
|
||||||
//! swallowed — recording a stat never affects task execution. The
|
//! fails to build, or when the command has no head, nothing is counted —
|
||||||
//! read/aggregation side is the agent harness's `stats.rs`, which reads the
|
//! recording a stat never affects task execution. The daemon exports the
|
||||||
//! same `bash_commands(ts INTEGER, head TEXT)` rows.
|
//! counter itself, on the same endpoint and resource attributes the harness
|
||||||
|
//! uses for its own metrics.
|
||||||
//!
|
//!
|
||||||
//! Normalisation is deliberately rough (quoting-naive) — it just needs
|
//! Normalisation is deliberately rough (quoting-naive) — it just needs
|
||||||
//! to name the tool an operator would recognise. The dominant agent
|
//! to name the tool an operator would recognise. The dominant agent
|
||||||
|
|
@ -13,11 +13,57 @@
|
||||||
//! `cd …`-style builtins and prefix-runners like `sudo`/`env` to find
|
//! `cd …`-style builtins and prefix-runners like `sudo`/`env` to find
|
||||||
//! the first real command, then take its basename.
|
//! the first real command, then take its basename.
|
||||||
|
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::sync::OnceLock;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use rusqlite::{Connection, params};
|
use opentelemetry::KeyValue;
|
||||||
|
use opentelemetry::metrics::{Counter, MeterProvider as _};
|
||||||
|
use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig};
|
||||||
|
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
|
||||||
|
|
||||||
use crate::paths;
|
/// Export cadence, the same as the harness's metrics.
|
||||||
|
const EXPORT_INTERVAL: Duration = Duration::from_mins(1);
|
||||||
|
|
||||||
|
struct Exporter {
|
||||||
|
/// Kept alive for the process lifetime - the `PeriodicReader` only
|
||||||
|
/// exports while its provider lives.
|
||||||
|
_provider: SdkMeterProvider,
|
||||||
|
commands: Counter<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Built on the first recorded command. `None` when OTEL isn't configured or
|
||||||
|
/// the exporter failed to build; recording is then a cheap no-op.
|
||||||
|
static EXPORTER: OnceLock<Option<Exporter>> = OnceLock::new();
|
||||||
|
|
||||||
|
fn build() -> Option<Exporter> {
|
||||||
|
if !std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok_and(|s| !s.trim().is_empty()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// http/json with the blocking client and no explicit endpoint or
|
||||||
|
// resource, as in the harness's exporter: the `PeriodicReader` exports
|
||||||
|
// from a thread with no Tokio reactor, and the SDK reads the endpoint
|
||||||
|
// and `OTEL_RESOURCE_ATTRIBUTES` from the container-wide environment.
|
||||||
|
let provider = MetricExporter::builder()
|
||||||
|
.with_http()
|
||||||
|
.with_protocol(Protocol::HttpJson)
|
||||||
|
.build()
|
||||||
|
.inspect_err(|e| tracing::warn!(error = %e, "bash stats: OTEL exporter not built"))
|
||||||
|
.ok()
|
||||||
|
.map(|exporter| {
|
||||||
|
let reader = PeriodicReader::builder(exporter)
|
||||||
|
.with_interval(EXPORT_INTERVAL)
|
||||||
|
.build();
|
||||||
|
SdkMeterProvider::builder().with_reader(reader).build()
|
||||||
|
})?;
|
||||||
|
let commands = provider
|
||||||
|
.meter("hyperhive.bash")
|
||||||
|
.u64_counter("hyperhive.agent.bash.commands")
|
||||||
|
.build();
|
||||||
|
Some(Exporter {
|
||||||
|
_provider: provider,
|
||||||
|
commands,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Builtins whose presence as a segment head means the *real* command
|
/// Builtins whose presence as a segment head means the *real* command
|
||||||
/// is in a later segment (e.g. the `cargo` in `cd repo && cargo build`),
|
/// is in a later segment (e.g. the `cargo` in `cd repo && cargo build`),
|
||||||
|
|
@ -103,38 +149,16 @@ pub fn command_head(cmd: &str) -> Option<String> {
|
||||||
segments.into_iter().find_map(segment_head)
|
segments.into_iter().find_map(segment_head)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Record one bash task's normalised command head. Best-effort: any
|
/// Count one bash task's normalised command head. No-op when OTEL isn't
|
||||||
/// error (no db yet, locked, unparseable command) is logged at debug
|
/// configured or the command has no identifiable head.
|
||||||
/// and swallowed.
|
|
||||||
pub fn record_command(cmd: &str) {
|
pub fn record_command(cmd: &str) {
|
||||||
let Some(head) = command_head(cmd) else {
|
let Some(head) = command_head(cmd) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
if let Err(e) = record_head(&head) {
|
let Some(exporter) = EXPORTER.get_or_init(build).as_ref() else {
|
||||||
tracing::debug!(error = ?e, head = %head, "bash stats: record failed");
|
return;
|
||||||
}
|
};
|
||||||
}
|
exporter.commands.add(1, &[KeyValue::new("command", head)]);
|
||||||
|
|
||||||
fn record_head(head: &str) -> rusqlite::Result<()> {
|
|
||||||
let conn = Connection::open(paths::turn_stats_db())?;
|
|
||||||
// The harness writes turn rows to this same db (rollback-journal,
|
|
||||||
// not WAL) and sibling bash tasks may write concurrently — wait out
|
|
||||||
// a transient lock rather than dropping the stat on SQLITE_BUSY.
|
|
||||||
conn.busy_timeout(std::time::Duration::from_secs(2))?;
|
|
||||||
conn.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL)",
|
|
||||||
[],
|
|
||||||
)?;
|
|
||||||
let now = SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_secs()
|
|
||||||
.cast_signed();
|
|
||||||
conn.execute(
|
|
||||||
"INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
|
|
||||||
params![now, head],
|
|
||||||
)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue