hive-bash-mcp: count bash commands over OTEL in the daemon, drop the sqlite hand-off
This commit is contained in:
parent
9ec67d7a18
commit
076df804e4
11 changed files with 100 additions and 184 deletions
4
Cargo.lock
generated
4
Cargo.lock
generated
|
|
@ -1848,8 +1848,10 @@ dependencies = [
|
|||
"hive-sock-client",
|
||||
"hive-types",
|
||||
"libc",
|
||||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
"opentelemetry_sdk",
|
||||
"rmcp",
|
||||
"rusqlite",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
|
|||
|
|
@ -43,19 +43,13 @@ Retention:
|
|||
|
||||
### `/harness/hyperhive-turn-stats.sqlite` (per agent)
|
||||
|
||||
The `hive-bash-daemon` writes a `bash_commands(ts INTEGER, head TEXT)`
|
||||
table in it: one row per executed bash task recording the normalised command head -
|
||||
the basename of the first real command, looking past `cd repo &&`
|
||||
prefixes, env-assignments, and prefix-runners like `sudo`/`env`. It
|
||||
feeds the bash-command counter in the agent's OTEL metrics. Best-effort and created on first write
|
||||
(`CREATE TABLE IF NOT EXISTS`), so it's absent until a bash
|
||||
task runs.
|
||||
|
||||
The file has **no vacuum**. Older harnesses also wrote a per-turn `turn_stats`
|
||||
table and a `sessions` table there; nothing writes them now and nothing in the
|
||||
harness reads them, but they stay in existing files, where the
|
||||
[legacy stats viewer](https://forge.constellation.darkest.space/hyperhive/legacy-stats-viewer)
|
||||
can still turn them into the old stats page.
|
||||
Nothing writes or reads this file now. Older harnesses wrote per-turn
|
||||
`turn_stats` and `sessions` tables there, and an older `hive-bash-daemon` a
|
||||
`bash_commands` table; they stay in existing files, which are safe to delete
|
||||
by hand. The bash daemon counts executed commands itself and exports them
|
||||
over OTEL (see
|
||||
[`observability.md`](../scheduler/observability.md)), and the one value the
|
||||
harness reads back between runs lives in `hyperhive-harness.json`, below.
|
||||
|
||||
### `/state/hyperhive-harness.json` (per agent)
|
||||
|
||||
|
|
@ -228,7 +222,6 @@ Under `/var/lib/hyperhive/agents/<name>/`:
|
|||
swept — same "audit trail, not cache" treatment as the c0re-side
|
||||
tables above.
|
||||
|
||||
- `hyperhive-turn-stats.sqlite` — the bash command heads the bash daemon records.
|
||||
- `hyperhive-model` / `hyperhive-effort` — single-line model name and
|
||||
effort level files, read-only fallbacks.
|
||||
|
||||
|
|
|
|||
|
|
@ -427,27 +427,27 @@ of their own — see [ACP-reported cost](#acp-reported-cost).
|
|||
|
||||
<!-- vale write-good.Passive = YES -->
|
||||
|
||||
| Metric | Unit | Kind | Attributes |
|
||||
| --------------------------------------------- | --------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
||||
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
||||
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
||||
| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds |
|
||||
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
|
||||
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
|
||||
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them), `account` (see below) — polled every 5 minutes, **not** per turn |
|
||||
| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window`, `account` — unix seconds at which that window resets; same 5-minute poll |
|
||||
| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% |
|
||||
| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator |
|
||||
| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below |
|
||||
| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:<skill>` for a skill; counted when a turn ends |
|
||||
| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted when a turn ends |
|
||||
| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) |
|
||||
| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` |
|
||||
| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds |
|
||||
| `hyperhive.agent.swarm_queue.publish_skipped` | — | counter | `site` (`terminal` or `agent_state`) — a publish dropped because the client had no connection |
|
||||
| Metric | Unit | Kind | Attributes |
|
||||
| --------------------------------------------- | --------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
||||
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
||||
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
||||
| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds |
|
||||
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
|
||||
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
|
||||
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them), `account` (see below) — polled every 5 minutes, **not** per turn |
|
||||
| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window`, `account` — unix seconds at which that window resets; same 5-minute poll |
|
||||
| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% |
|
||||
| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator |
|
||||
| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below |
|
||||
| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:<skill>` for a skill; counted when a turn ends |
|
||||
| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted by `hive-bash-daemon` as each task starts |
|
||||
| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) |
|
||||
| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` |
|
||||
| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds |
|
||||
| `hyperhive.agent.swarm_queue.publish_skipped` | — | counter | `site` (`terminal` or `agent_state`) — a publish dropped because the client had no connection |
|
||||
|
||||
An agent with no `subagent_roles/` directory exports no
|
||||
`subagent_role.lines` series. A role file deleted while the harness runs
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ mod router_todos;
|
|||
mod schedule_timer;
|
||||
mod serve_common;
|
||||
mod state_entry_watch;
|
||||
mod stats;
|
||||
mod subagent_role_watch;
|
||||
mod swarm_agent_commands;
|
||||
mod swarm_agent_state;
|
||||
|
|
@ -1031,7 +1030,6 @@ async fn handle_turn<S: Surface>(
|
|||
unread,
|
||||
});
|
||||
bus.set_state(TurnState::Thinking);
|
||||
let started_at = chrono::Utc::now().timestamp();
|
||||
let started_instant = std::time::Instant::now();
|
||||
let model_at_start = bus.model();
|
||||
// Read-and-clear: this wake prompt is the one turn that gets to carry
|
||||
|
|
@ -1064,7 +1062,6 @@ async fn handle_turn<S: Surface>(
|
|||
let (open_threads, open_reminders) = S::post_turn_counts(socket).await;
|
||||
let context_window = bus.api_context_window();
|
||||
let row = serve_common::build_row(serve_common::TurnRowArgs {
|
||||
started_at,
|
||||
duration_ms,
|
||||
model: model_at_start,
|
||||
wake_from: from.clone(),
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
//! only the harness-only concepts: wall-clock turn duration as *this harness*
|
||||
//! measures it, what woke the turn, its outcome classification, the loose-ends
|
||||
//! backlog, sessions and their first turn's cost, context-window fill, the
|
||||
//! tools a turn called (claude exports no tool-call counter), bash commands,
|
||||
//! tools a turn called (claude exports no tool-call counter),
|
||||
//! reminders, and the cost an ACP agent reports — claude's runtime reports none.
|
||||
//!
|
||||
//! Synchronous instruments, not observable ones: unlike hive-c0re's
|
||||
|
|
@ -31,7 +31,6 @@
|
|||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use opentelemetry::KeyValue;
|
||||
|
|
@ -96,9 +95,6 @@ struct Instruments {
|
|||
reported_cost: Counter<f64>,
|
||||
/// Tool calls made, keyed by `tool` (`Skill:<skill>` for a skill).
|
||||
tool_calls: Counter<u64>,
|
||||
/// Shell commands run by bash tasks, keyed by `command`: the normalised
|
||||
/// head the bash daemon records, not the full command line.
|
||||
bash_commands: Counter<u64>,
|
||||
/// Reminders scheduled and delivered, keyed by `event`.
|
||||
reminders: Counter<u64>,
|
||||
/// Input tokens of the first turn of a fresh claude session: what the
|
||||
|
|
@ -153,7 +149,6 @@ pub fn record(row: &TurnStatRow, fresh_session: bool) {
|
|||
for (tool, calls) in tool_call_counts(row.tool_call_breakdown_json.as_deref()) {
|
||||
inst.tool_calls.add(calls, &[KeyValue::new("tool", tool)]);
|
||||
}
|
||||
record_new_bash_commands(inst, row.started_at);
|
||||
}
|
||||
|
||||
/// The per-tool call counts a turn stored in `tool_call_breakdown_json`; a
|
||||
|
|
@ -164,25 +159,6 @@ fn tool_call_counts(json: Option<&str>) -> Vec<(String, u64)> {
|
|||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Unix seconds up to which bash commands are already counted; `0` before the
|
||||
/// first turn of this run.
|
||||
static BASH_COUNTED_UNTIL: AtomicI64 = AtomicI64::new(0);
|
||||
|
||||
/// Count the bash commands recorded since the last call. The first call after
|
||||
/// the harness starts counts only commands newer than `turn_started_at`, so a
|
||||
/// restart does not count the whole history again.
|
||||
fn record_new_bash_commands(inst: &Instruments, turn_started_at: i64) {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let mut since = BASH_COUNTED_UNTIL.swap(now, Ordering::Relaxed);
|
||||
if since == 0 {
|
||||
since = turn_started_at.saturating_sub(1);
|
||||
}
|
||||
for (command, n) in crate::stats::bash_heads_since(since) {
|
||||
inst.bash_commands
|
||||
.add(n, &[KeyValue::new("command", command)]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Count one reminder event, `scheduled` or `delivered`. No-op when OTEL isn't
|
||||
/// configured, same as [`record`].
|
||||
pub fn record_reminder(event: &'static str) {
|
||||
|
|
@ -348,7 +324,6 @@ fn build() -> Option<Instruments> {
|
|||
.build(),
|
||||
reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(),
|
||||
tool_calls: meter.u64_counter("hyperhive.agent.tool.calls").build(),
|
||||
bash_commands: meter.u64_counter("hyperhive.agent.bash.commands").build(),
|
||||
reminders: meter.u64_counter("hyperhive.agent.reminders").build(),
|
||||
first_turn_context: meter
|
||||
.u64_gauge("hyperhive.agent.first_turn.context")
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ pub fn format_wake_prompt(from: &str, body: &str, redelivered: bool, interrupted
|
|||
/// (`otel_turn_metrics::record`).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TurnStatRow {
|
||||
pub started_at: i64,
|
||||
pub duration_ms: i64,
|
||||
pub model: String,
|
||||
pub wake_from: String,
|
||||
|
|
@ -51,7 +50,6 @@ pub struct TurnStatRow {
|
|||
/// `get_loose_ends` call before it's consumed — `Bus::take_tool_calls` resets
|
||||
/// the counter, so there's only one chance to read it per turn.
|
||||
pub struct TurnRowArgs<'a> {
|
||||
pub started_at: i64,
|
||||
pub duration_ms: i64,
|
||||
pub model: String,
|
||||
pub wake_from: String,
|
||||
|
|
@ -68,7 +66,6 @@ pub struct TurnRowArgs<'a> {
|
|||
#[must_use]
|
||||
pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
|
||||
let TurnRowArgs {
|
||||
started_at,
|
||||
duration_ms,
|
||||
model,
|
||||
wake_from,
|
||||
|
|
@ -101,7 +98,6 @@ pub fn build_row(args: TurnRowArgs<'_>) -> TurnStatRow {
|
|||
Err(TurnError::Failed(_)) => "failed",
|
||||
};
|
||||
TurnStatRow {
|
||||
started_at,
|
||||
duration_ms,
|
||||
model,
|
||||
wake_from,
|
||||
|
|
@ -149,7 +145,6 @@ mod build_row_tests {
|
|||
tool_calls: &[(&str, u64)],
|
||||
) -> super::TurnStatRow {
|
||||
build_row(TurnRowArgs {
|
||||
started_at: 10,
|
||||
duration_ms: 2_000,
|
||||
model: "opus".to_owned(),
|
||||
wake_from: "todo".to_owned(),
|
||||
|
|
@ -187,7 +182,7 @@ mod build_row_tests {
|
|||
let _dir = crate::paths::harness_dir_guard();
|
||||
let row = row_for(&Ok(false), &[("Bash", 3)]);
|
||||
|
||||
assert_eq!((row.started_at, row.duration_ms), (10, 2_000));
|
||||
assert_eq!(row.duration_ms, 2_000);
|
||||
assert_eq!(
|
||||
(row.model.as_str(), row.wake_from.as_str()),
|
||||
("opus", "todo")
|
||||
|
|
@ -210,7 +205,6 @@ mod build_row_tests {
|
|||
let bus = Bus::new();
|
||||
bus.set_resolved_model(Some("claude-opus-4-8".to_owned()));
|
||||
let row = build_row(TurnRowArgs {
|
||||
started_at: 0,
|
||||
duration_ms: 0,
|
||||
model: "opus".to_owned(),
|
||||
wake_from: "todo".to_owned(),
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
//! The shell-command heads the bash daemon records in the harness's `hyperhive-turn-stats.sqlite`,
|
||||
//! read for the OTEL bash-command counter.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rusqlite::{Connection, OpenFlags};
|
||||
|
||||
fn default_path() -> PathBuf {
|
||||
crate::paths::harness_dir().join("hyperhive-turn-stats.sqlite")
|
||||
}
|
||||
|
||||
/// The shell-command heads recorded after `since` (unix seconds, exclusive),
|
||||
/// with how often each ran, for the OTEL bash-command counter. Reads the live
|
||||
/// database read-only; a missing database or `bash_commands` table (no bash
|
||||
/// task has run yet) is an empty list, not an error.
|
||||
#[must_use]
|
||||
pub fn bash_heads_since(since: i64) -> Vec<(String, u64)> {
|
||||
let Ok(conn) = Connection::open_with_flags(default_path(), OpenFlags::SQLITE_OPEN_READ_ONLY)
|
||||
else {
|
||||
return Vec::new();
|
||||
};
|
||||
let _ = conn.busy_timeout(std::time::Duration::from_millis(500));
|
||||
read_bash_heads_since(&conn, since).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn read_bash_heads_since(conn: &Connection, since: i64) -> rusqlite::Result<Vec<(String, u64)>> {
|
||||
let mut stmt =
|
||||
conn.prepare("SELECT head, COUNT(*) FROM bash_commands WHERE ts > ?1 GROUP BY head")?;
|
||||
let rows = stmt.query_map([since], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
u64::try_from(row.get::<_, i64>(1)?).unwrap_or(0),
|
||||
))
|
||||
})?;
|
||||
rows.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rusqlite::params;
|
||||
|
||||
/// The OTEL bash-command counter reads only the heads recorded after its
|
||||
/// watermark, and a database without the table yields nothing.
|
||||
#[test]
|
||||
fn bash_heads_since_counts_only_newer_rows() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
assert!(read_bash_heads_since(&conn, 0).is_err(), "no table yet");
|
||||
conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
|
||||
.unwrap();
|
||||
for (ts, head) in [(10, "cargo"), (20, "cargo"), (30, "git"), (30, "cargo")] {
|
||||
conn.execute(
|
||||
"INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
|
||||
params![ts, head],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let mut got = read_bash_heads_since(&conn, 10).unwrap();
|
||||
got.sort();
|
||||
assert_eq!(got, vec![("cargo".to_owned(), 2), ("git".to_owned(), 1)]);
|
||||
assert!(read_bash_heads_since(&conn, 30).unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
|
|
@ -17,8 +17,10 @@ hive-sh4re.workspace = true
|
|||
hive-sock-client.workspace = true
|
||||
hive-types.workspace = true
|
||||
libc.workspace = true
|
||||
opentelemetry.workspace = true
|
||||
opentelemetry-otlp.workspace = true
|
||||
opentelemetry_sdk.workspace = true
|
||||
rmcp.workspace = true
|
||||
rusqlite.workspace = true
|
||||
# The agent's own queue connection, and what a task publishes over it
|
||||
# (`swarm_term`).
|
||||
async-nats.workspace = true
|
||||
|
|
|
|||
|
|
@ -32,5 +32,5 @@ Two bins from one shared lib (`src/lib.rs`):
|
|||
returns the result.
|
||||
|
||||
Supporting modules: **`paths.rs`** (daemon socket + agent-socket
|
||||
resolution), **`stats.rs`** (the `bash_commands` favorite-tool stat
|
||||
recorded into hyperhive-turn-stats.sqlite).
|
||||
resolution), **`stats.rs`** (the OTEL bash-command counter, one count per
|
||||
task keyed by the command's normalised head).
|
||||
|
|
|
|||
|
|
@ -20,14 +20,6 @@ pub fn tasks_dir() -> PathBuf {
|
|||
harness_dir().join("bash-tasks")
|
||||
}
|
||||
|
||||
/// Per-agent sqlite in the harness dir: the runner appends normalised
|
||||
/// bash-command heads to its `bash_commands` table, which the harness reads for
|
||||
/// its OTEL bash-command counter.
|
||||
#[must_use]
|
||||
pub fn turn_stats_db() -> PathBuf {
|
||||
harness_dir().join("hyperhive-turn-stats.sqlite")
|
||||
}
|
||||
|
||||
/// The harness-served in-agent todo socket (loose-ends v2). The runner
|
||||
/// pushes bash-task todos here — `upsert_todo` while a task is active,
|
||||
/// a keyless `done` todo on completion — instead of firing a direct
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
//! Capture of normalised shell-command "heads" per bash task into the
|
||||
//! per-agent turn-stats sqlite (`bash_commands` table), read by the
|
||||
//! harness for its OTEL bash-command counter.
|
||||
//! The OTEL bash-command counter: each bash task's normalised shell-command
|
||||
//! "head", counted as `hyperhive.agent.bash.commands` keyed by `command`.
|
||||
//!
|
||||
//! Best-effort: a missing db, a lock, or a parse miss is logged and
|
||||
//! swallowed — recording a stat never affects task execution. The
|
||||
//! read/aggregation side is the agent harness's `stats.rs`, which reads the
|
||||
//! same `bash_commands(ts INTEGER, head TEXT)` rows.
|
||||
//! Best-effort: without `OTEL_EXPORTER_OTLP_ENDPOINT`, or when the exporter
|
||||
//! fails to build, or when the command has no head, nothing is counted —
|
||||
//! recording a stat never affects task execution. The daemon exports the
|
||||
//! counter itself, on the same endpoint and resource attributes the harness
|
||||
//! uses for its own metrics.
|
||||
//!
|
||||
//! Normalisation is deliberately rough (quoting-naive) — it just needs
|
||||
//! to name the tool an operator would recognise. The dominant agent
|
||||
|
|
@ -13,11 +13,57 @@
|
|||
//! `cd …`-style builtins and prefix-runners like `sudo`/`env` to find
|
||||
//! the first real command, then take its basename.
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::sync::OnceLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use rusqlite::{Connection, params};
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry::metrics::{Counter, MeterProvider as _};
|
||||
use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig};
|
||||
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
|
||||
|
||||
use crate::paths;
|
||||
/// Export cadence, the same as the harness's metrics.
|
||||
const EXPORT_INTERVAL: Duration = Duration::from_mins(1);
|
||||
|
||||
struct Exporter {
|
||||
/// Kept alive for the process lifetime - the `PeriodicReader` only
|
||||
/// exports while its provider lives.
|
||||
_provider: SdkMeterProvider,
|
||||
commands: Counter<u64>,
|
||||
}
|
||||
|
||||
/// Built on the first recorded command. `None` when OTEL isn't configured or
|
||||
/// the exporter failed to build; recording is then a cheap no-op.
|
||||
static EXPORTER: OnceLock<Option<Exporter>> = OnceLock::new();
|
||||
|
||||
fn build() -> Option<Exporter> {
|
||||
if !std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok_and(|s| !s.trim().is_empty()) {
|
||||
return None;
|
||||
}
|
||||
// http/json with the blocking client and no explicit endpoint or
|
||||
// resource, as in the harness's exporter: the `PeriodicReader` exports
|
||||
// from a thread with no Tokio reactor, and the SDK reads the endpoint
|
||||
// and `OTEL_RESOURCE_ATTRIBUTES` from the container-wide environment.
|
||||
let provider = MetricExporter::builder()
|
||||
.with_http()
|
||||
.with_protocol(Protocol::HttpJson)
|
||||
.build()
|
||||
.inspect_err(|e| tracing::warn!(error = %e, "bash stats: OTEL exporter not built"))
|
||||
.ok()
|
||||
.map(|exporter| {
|
||||
let reader = PeriodicReader::builder(exporter)
|
||||
.with_interval(EXPORT_INTERVAL)
|
||||
.build();
|
||||
SdkMeterProvider::builder().with_reader(reader).build()
|
||||
})?;
|
||||
let commands = provider
|
||||
.meter("hyperhive.bash")
|
||||
.u64_counter("hyperhive.agent.bash.commands")
|
||||
.build();
|
||||
Some(Exporter {
|
||||
_provider: provider,
|
||||
commands,
|
||||
})
|
||||
}
|
||||
|
||||
/// Builtins whose presence as a segment head means the *real* command
|
||||
/// is in a later segment (e.g. the `cargo` in `cd repo && cargo build`),
|
||||
|
|
@ -103,38 +149,16 @@ pub fn command_head(cmd: &str) -> Option<String> {
|
|||
segments.into_iter().find_map(segment_head)
|
||||
}
|
||||
|
||||
/// Record one bash task's normalised command head. Best-effort: any
|
||||
/// error (no db yet, locked, unparseable command) is logged at debug
|
||||
/// and swallowed.
|
||||
/// Count one bash task's normalised command head. No-op when OTEL isn't
|
||||
/// configured or the command has no identifiable head.
|
||||
pub fn record_command(cmd: &str) {
|
||||
let Some(head) = command_head(cmd) else {
|
||||
return;
|
||||
};
|
||||
if let Err(e) = record_head(&head) {
|
||||
tracing::debug!(error = ?e, head = %head, "bash stats: record failed");
|
||||
}
|
||||
}
|
||||
|
||||
fn record_head(head: &str) -> rusqlite::Result<()> {
|
||||
let conn = Connection::open(paths::turn_stats_db())?;
|
||||
// The harness writes turn rows to this same db (rollback-journal,
|
||||
// not WAL) and sibling bash tasks may write concurrently — wait out
|
||||
// a transient lock rather than dropping the stat on SQLITE_BUSY.
|
||||
conn.busy_timeout(std::time::Duration::from_secs(2))?;
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL)",
|
||||
[],
|
||||
)?;
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.cast_signed();
|
||||
conn.execute(
|
||||
"INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
|
||||
params![now, head],
|
||||
)?;
|
||||
Ok(())
|
||||
let Some(exporter) = EXPORTER.get_or_init(build).as_ref() else {
|
||||
return;
|
||||
};
|
||||
exporter.commands.add(1, &[KeyValue::new("command", head)]);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
Loading…
Reference in a new issue