fix(hive-metric): blocking OTLP client (no-reactor panic), reject negative counters, allow negative gauges
This commit is contained in:
parent
aa9f56246a
commit
d91e6eb0e2
3 changed files with 35 additions and 7 deletions
3
Cargo.lock
generated
3
Cargo.lock
generated
|
|
@ -1591,7 +1591,6 @@ dependencies = [
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"opentelemetry-otlp",
|
"opentelemetry-otlp",
|
||||||
"opentelemetry_sdk",
|
"opentelemetry_sdk",
|
||||||
"tokio",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -3425,7 +3424,9 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
|
"futures-util",
|
||||||
"http",
|
"http",
|
||||||
"http-body",
|
"http-body",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
|
|
|
||||||
|
|
@ -13,10 +13,14 @@ clap.workspace = true
|
||||||
# OTEL SDK — only used by this crate, so kept local rather than in workspace.dependencies.
|
# OTEL SDK — only used by this crate, so kept local rather than in workspace.dependencies.
|
||||||
opentelemetry = "0.32"
|
opentelemetry = "0.32"
|
||||||
opentelemetry_sdk = { version = "0.32", features = ["metrics"] }
|
opentelemetry_sdk = { version = "0.32", features = ["metrics"] }
|
||||||
|
# Blocking (not async) reqwest client on purpose: the metrics SDK drives the
|
||||||
|
# OTLP push from a `PeriodicReader` background thread that has no Tokio runtime,
|
||||||
|
# so the async client panics there with "no reactor running". The blocking
|
||||||
|
# client sends on that thread directly — and for a fire-and-forget CLI that
|
||||||
|
# records one point then `shutdown()`s, a synchronous send is exactly right.
|
||||||
opentelemetry-otlp = { version = "0.32", default-features = false, features = [
|
opentelemetry-otlp = { version = "0.32", default-features = false, features = [
|
||||||
"metrics",
|
"metrics",
|
||||||
"http-json",
|
"http-json",
|
||||||
"reqwest-client",
|
"reqwest-blocking-client",
|
||||||
"reqwest-rustls",
|
"reqwest-rustls",
|
||||||
] }
|
] }
|
||||||
tokio = { workspace = true, features = ["rt", "macros"] }
|
|
||||||
|
|
|
||||||
|
|
@ -24,13 +24,19 @@ use opentelemetry_otlp::{Protocol, WithExportConfig};
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(
|
#[command(
|
||||||
name = "hive-metric",
|
name = "hive-metric",
|
||||||
about = "Push a labeled metric to the OTEL collector (requires services.hyperhive.otel.enable = true)"
|
about = "Push a labeled metric to the OTEL collector (requires services.hyperhive.otel.enable = true)",
|
||||||
|
// Let a leading-minus value parse as the `value` positional rather than
|
||||||
|
// being rejected as an unknown flag — negative gauges are legitimate
|
||||||
|
// (e.g. a delta), and it lets the counter-sign check below produce a clear
|
||||||
|
// domain error instead of clap's "unexpected argument '-1'".
|
||||||
|
allow_negative_numbers = true
|
||||||
)]
|
)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Metric name (e.g. `tasks_completed`, `latency_ms`).
|
/// Metric name (e.g. `tasks_completed`, `latency_ms`).
|
||||||
name: String,
|
name: String,
|
||||||
|
|
||||||
/// Numeric value (f64). Floats and integers both accepted.
|
/// Numeric value (f64). Floats and integers both accepted. Gauges may be
|
||||||
|
/// negative; counters must be >= 0 (they only increase).
|
||||||
value: f64,
|
value: f64,
|
||||||
|
|
||||||
/// Metric kind: `counter` (cumulative sum, default) or `gauge` (instantaneous value).
|
/// Metric kind: `counter` (cumulative sum, default) or `gauge` (instantaneous value).
|
||||||
|
|
@ -52,8 +58,7 @@ enum MetricKind {
|
||||||
Gauge,
|
Gauge,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
fn main() -> Result<()> {
|
||||||
async fn main() -> Result<()> {
|
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
|
||||||
// Fail early with a clear message when OTEL is not configured.
|
// Fail early with a clear message when OTEL is not configured.
|
||||||
|
|
@ -66,6 +71,18 @@ async fn main() -> Result<()> {
|
||||||
|
|
||||||
let labels = parse_labels(&cli.labels)?;
|
let labels = parse_labels(&cli.labels)?;
|
||||||
|
|
||||||
|
// Counters are monotonically non-negative — the OTEL SDK silently drops a
|
||||||
|
// negative `add`, so a typo like `hive-metric tasks_done -1` would vanish
|
||||||
|
// with exit 0. Reject it loudly instead.
|
||||||
|
if matches!(cli.metric_type, MetricKind::Counter) && cli.value < 0.0 {
|
||||||
|
bail!(
|
||||||
|
"counter '{}' value must be >= 0 (got {}); counters only increase — \
|
||||||
|
use `--type gauge` for a value that can go down",
|
||||||
|
cli.name,
|
||||||
|
cli.value
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
||||||
.with_http()
|
.with_http()
|
||||||
.with_protocol(Protocol::HttpJson)
|
.with_protocol(Protocol::HttpJson)
|
||||||
|
|
@ -73,6 +90,12 @@ async fn main() -> Result<()> {
|
||||||
.context("failed to build OTLP metric exporter")?;
|
.context("failed to build OTLP metric exporter")?;
|
||||||
|
|
||||||
// SDK automatically reads OTEL_RESOURCE_ATTRIBUTES for resource labels.
|
// SDK automatically reads OTEL_RESOURCE_ATTRIBUTES for resource labels.
|
||||||
|
// The metrics SDK has no `with_simple_exporter` (that's a *traces*-only
|
||||||
|
// API) — a push exporter is always driven by a periodic reader. For this
|
||||||
|
// fire-and-forget CLI the timer never actually fires: we record one metric
|
||||||
|
// and immediately `shutdown()`, which force-flushes the pending export
|
||||||
|
// synchronously before returning. So there's no background-timer flush race
|
||||||
|
// in practice — the single point is exported exactly once on shutdown.
|
||||||
let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
|
let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
|
||||||
.with_periodic_exporter(exporter)
|
.with_periodic_exporter(exporter)
|
||||||
.build();
|
.build();
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue