move client-cert gauges behind an otel feature of swarm-secret-client, drop the crate

AGENTS.md

@@ -138,9 +138,7 @@ Swarm-level:
   the **`BAO_`** env spellings explicitly — `vaultrs`'s `VAULT_*` defaults
   are set by no unit here, and falling through to them yields a client
   with **no identity** and a TLS failure that names no cause.
-- **`swarm-cert-metrics/`** — the `bao.client_cert.not_before` / `not_after`
-  observable gauges a service registers on its existing OTLP exporter; the
-  cert file is read at each collection.
+  Feature `otel` adds `cert_metrics`, the `bao.client_cert.*` gauges.
 - **`swarm-queue-client/`** — the one authenticated connect to the swarm
   NATS queue.
 - **`swarm-wire/`** — serde-only request and reply contracts services

Cargo.lock

@@ -1691,7 +1691,6 @@ dependencies = [
  "serde_json",
  "sha1",
  "swarm-agent-verify",
- "swarm-cert-metrics",
  "swarm-queue-client",
  "swarm-secret-client",
  "swarm-wire",
@@ -1802,7 +1801,6 @@ dependencies = [
  "serde_json",
  "sha2 0.11.0",
  "strum",
- "swarm-cert-metrics",
  "swarm-matrix-client",
  "swarm-queue-client",
  "swarm-secret-client",
@@ -4839,16 +4837,6 @@ dependencies = [
  "tracing-subscriber",
 ]
 
-[[package]]
-name = "swarm-cert-metrics"
-version = "0.1.0"
-dependencies = [
- "opentelemetry",
- "opentelemetry_sdk",
- "swarm-secret-client",
- "tracing",
-]
-
 [[package]]
 name = "swarm-ci-metrics"
 version = "0.1.0"
@@ -4921,7 +4909,6 @@ dependencies = [
  "similar",
  "strum",
  "swarm-authelia-bridge-sock",
- "swarm-cert-metrics",
  "swarm-controller-api",
  "swarm-matrix-client",
  "swarm-queue-client",
@@ -5108,7 +5095,6 @@ dependencies = [
  "opentelemetry_sdk",
  "serde",
  "serde_json",
- "swarm-cert-metrics",
  "swarm-queue-client",
  "swarm-secret-client",
  "swarm-wire",
@@ -5123,6 +5109,8 @@ version = "0.1.0"
 dependencies = [
  "backon",
  "futures-util",
+ "opentelemetry",
+ "opentelemetry_sdk",
  "reqwest",
  "rustify",
  "rustify_derive",

Cargo.toml

@@ -29,7 +29,6 @@ members = [
   "swarm-agent-verify",
   "swarm-authelia-bridge",
   "swarm-builder",
-  "swarm-cert-metrics",
   "swarm-ci-metrics",
   "swarm-claude-login",
   "swarm-authelia-bridge-sock",
@@ -119,7 +118,6 @@ hive-sock-client = { path = "hive-sock-client" }
 hive-types = { path = "hive-types" }
 swarm-agent-verify = { path = "swarm-agent-verify" }
 swarm-authelia-bridge-sock = { path = "swarm-authelia-bridge-sock" }
-swarm-cert-metrics = { path = "swarm-cert-metrics" }
 swarm-controller-api = { path = "swarm-controller-api" }
 swarm-matrix-client = { path = "swarm-matrix-client" }
 swarm-queue-client = { path = "swarm-queue-client" }

docs/scheduler/observability.md

@@ -572,7 +572,7 @@ data point each, unix seconds:
 
 `cert` names the identity (`client`, `controller`, `agent`, `review-classifier`,
 `ci-metrics`); the resource labels (`service.name`, `hive`, `agent`) say who
-holds it. The `swarm-cert-metrics` crate registers both gauges on the service's
+holds it. `swarm_secret_client::cert_metrics` (cargo feature `otel`) registers both gauges on the service's
 existing exporter and reads `BAO_CLIENT_CERT` at every collection, so a
 renewed leaf shows up on the next export. A file that's missing, or that isn't a
 certificate yields no data point and a debug log line. The parser is

hive-agent/Cargo.toml

@@ -58,11 +58,9 @@ swarm-queue-client = { workspace = true, features = [
 ] }
 # The spelling of the token the queue connection presents.
 swarm-agent-verify.workspace = true
-# The `bao.client_cert.*` gauges on this exporter.
-swarm-cert-metrics.workspace = true
 swarm-wire.workspace = true
 # This agent's own queue secret, read from the store under its own certificate.
-swarm-secret-client.workspace = true
+swarm-secret-client = { workspace = true, features = ["otel"] }
 # The directory an AGENTS.md edit is applied in, and the file it is written
 # through.
 tempfile = "3"

hive-agent/src/otel_turn_metrics.rs

@@ -181,7 +181,7 @@ fn build() -> Option<Instruments> {
         Ok(provider) => {
             tracing::info!(?interval, "otel turn-metrics: exporter enabled");
             let meter = provider.meter("hyperhive.turn_stats");
-            swarm_cert_metrics::register(&meter, "agent");
+            swarm_secret_client::cert_metrics::register(&meter, "agent");
             Some(Instruments {
                 turn_duration_ms: meter
                     .u64_histogram("hyperhive.agent.turn.duration")

hive-c0re/Cargo.toml

@@ -62,10 +62,8 @@ swarm-queue-client = { workspace = true, features = [
   "agent-permissions",
   "agent-archetype",
 ] }
-# The `bao.client_cert.*` gauges on this exporter.
-swarm-cert-metrics.workspace = true
 swarm-matrix-client.workspace = true
-swarm-secret-client.workspace = true
+swarm-secret-client = { workspace = true, features = ["otel"] }
 swarm-wire.workspace = true
 tokio.workspace = true
 tokio-stream.workspace = true

hive-c0re/src/stats/otel_metrics.rs

@@ -262,7 +262,7 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc<Mutex<Vec<Con
 /// only because `gather()` is async — which `readiness()` is not.
 fn register_hive_instruments(meter: &opentelemetry::metrics::Meter) {
     // The hive's own store certificate (`client.pem`, `BAO_CLIENT_CERT`).
-    swarm_cert_metrics::register(meter, "client");
+    swarm_secret_client::cert_metrics::register(meter, "client");
 
     // Seconds since the exporter started. A hive that is up reports this every
     // interval whether or not it hosts a single agent, which is what makes

nix/host-modules/swarm-grafana.nix

@@ -1717,7 +1717,7 @@ in
                     })
                     # The fraction of a store client certificate's validity
                     # already used, from the `bao.client_cert.*` gauges the
-                    # services export (`swarm-cert-metrics`). Both gauges of a
+                    # services export (`swarm_secret_client::cert_metrics`). Both gauges of a
                     # series carry identical labels, so the division matches
                     # one-to-one on all of them without `on()`/`group_left`.
                     # `last_over_time` rides out the five-minute cadence of

swarm-cert-metrics/Cargo.toml

@@ -1,17 +0,0 @@
-[package]
-name = "swarm-cert-metrics"
-version.workspace = true
-edition.workspace = true
-
-[dependencies]
-opentelemetry.workspace = true
-# The cert file's validity window.
-swarm-secret-client.workspace = true
-tracing.workspace = true
-
-[lints]
-workspace = true
-
-[dev-dependencies]
-# `InMemoryMetricExporter`, to read back what a provider exports.
-opentelemetry_sdk = { workspace = true, features = ["testing"] }

swarm-controller/Cargo.toml

@@ -94,8 +94,6 @@ serde_json.workspace = true
 serde_norway = "0.9.42"
 strum.workspace = true
 swarm-authelia-bridge-sock.workspace = true
-# The `bao.client_cert.*` gauges on this exporter.
-swarm-cert-metrics.workspace = true
 # `schema`: the forge and schedule routes describe these bodies in the spec.
 swarm-controller-api = { workspace = true, features = ["schema"] }
 # The queue connect (token mint + auth callback + reconnect) is shared with
@@ -124,7 +122,7 @@ swarm-queue-client = { workspace = true, features = [
 # crate the hive reads it back with, which is the point: the path, the field
 # name and the object's shape are agreements between the two ends, and a
 # second spelling here would be a store this hive could not read.
-swarm-secret-client.workspace = true
+swarm-secret-client = { workspace = true, features = ["otel"] }
 # The builder, claude login and review classifier subjects, requests and
 # replies it sends. `schema`: the review classifier try-out API serves the
 # same shapes, so they derive `ToSchema`.

swarm-controller/src/main.rs

@@ -250,7 +250,10 @@ async fn main() -> Result<()> {
 fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterProvider>) {
     use opentelemetry::metrics::MeterProvider as _;
     if let Some(provider) = provider {
-        swarm_cert_metrics::register(&provider.meter("hyperhive.bao_client_cert"), "controller");
+        swarm_secret_client::cert_metrics::register(
+            &provider.meter("hyperhive.bao_client_cert"),
+            "controller",
+        );
     }
 }
 

swarm-review-classifier/Cargo.toml

@@ -25,12 +25,10 @@ opentelemetry-otlp.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 # The try-out subject and the shapes the controller sends and reads back.
-# The `bao.client_cert.*` gauges on this exporter.
-swarm-cert-metrics.workspace = true
 swarm-queue-client.workspace = true
 swarm-wire.workspace = true
 # The API key and queue client secret paths, and the store login.
-swarm-secret-client.workspace = true
+swarm-secret-client = { workspace = true, features = ["otel"] }
 tokio.workspace = true
 tracing.workspace = true
 

swarm-review-classifier/src/cost.rs

@@ -171,7 +171,7 @@ fn build() -> Option<(SdkMeterProvider, Counters)> {
         .build();
     let provider = SdkMeterProvider::builder().with_reader(reader).build();
     let counters = counters(&provider);
-    swarm_cert_metrics::register(
+    swarm_secret_client::cert_metrics::register(
         &provider.meter("hyperhive.review_classifier"),
         "review-classifier",
     );

swarm-secret-client/Cargo.toml

@@ -13,6 +13,9 @@ futures-util.workspace = true
 # the `BAO_*` files) rather than by its env defaults, so the dependency is
 # direct rather than incidental.
 reqwest.workspace = true
+# `cert_metrics`: the `bao.client_cert.*` gauges, for the services that export
+# OTLP. Off by default so the other dependents link no OTel.
+opentelemetry = { workspace = true, optional = true }
 # vaultrs's own endpoint machinery, for the one endpoint it does not
 # implement — see `client::WriteAclPolicy`.
 rustify.workspace = true
@@ -40,6 +43,8 @@ workspace = true
 # provider compiled in rustls picks it without an install, so the provider
 # test would pass with `install_crypto_provider` doing nothing.
 [dev-dependencies]
+# `InMemoryMetricExporter`, to read back what `cert_metrics` exports.
+opentelemetry_sdk = { workspace = true, features = ["testing"] }
 rustls = { workspace = true, features = ["ring"] }
 # `test-util` for `#[tokio::test(start_paused = true)]`: the re-read backoff
 # tests run on paused time.
@@ -47,3 +52,6 @@ tokio = { workspace = true, features = ["test-util"] }
 # Capturing what a re-read logs, to check it names the client and never the
 # secret.
 tracing-subscriber.workspace = true
+
+[features]
+otel = ["dep:opentelemetry"]

swarm-secret-client/src/cert.rs

@@ -46,23 +46,11 @@ pub fn file_validity(path: &Path) -> Result<(i64, i64), Error> {
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
     use super::*;
 
     /// Self-signed, CN `hive-agent-atlas`, valid 2026-01-01T00:00:00Z to
-    /// 2026-04-01T00:00:00Z. Its key was discarded.
-    pub(crate) const CERT_PEM: &str = "-----BEGIN CERTIFICATE-----
-MIIBizCCATGgAwIBAgIUSFpuYmy1UoHvq/MYiyrAudRaWXcwCgYIKoZIzj0EAwIw
-GzEZMBcGA1UEAwwQaGl2ZS1hZ2VudC1hdGxhczAeFw0yNjAxMDEwMDAwMDBaFw0y
-NjA0MDEwMDAwMDBaMBsxGTAXBgNVBAMMEGhpdmUtYWdlbnQtYXRsYXMwWTATBgcq
-hkjOPQIBBggqhkjOPQMBBwNCAAR7p2eZAxjiE1RdsuSHE3CcTjbnd4swzfnmqUPW
-9NJN7mvOtqEzxCKfOSFjaIhhyShIQ41/V6vGi80EPDB+uR56o1MwUTAdBgNVHQ4E
-FgQUxgZFsN1VY3ODrPS1NUfY/x3EnmswHwYDVR0jBBgwFoAUxgZFsN1VY3ODrPS1
-NUfY/x3EnmswDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAnU9e
-WOioNTUNK9b6FignejpK5FpyrODUuH/iL4TqCh0CIAz6GzJeLebhRPvYlfVUofdV
-hWx3sOwmUgjkRQXoxY+p
------END CERTIFICATE-----
-";
+    pub(crate) const CERT_PEM: &str = include_str!("../test-cert.pem");
     pub(crate) const NOT_BEFORE: i64 = 1_767_225_600;
     pub(crate) const NOT_AFTER: i64 = 1_775_001_600;
 

swarm-secret-client/src/cert_metrics.rs

@@ -0,0 +1,199 @@
+//! (feature `otel`) The two gauges every service holding a store client certificate exports:
+//! when that certificate became valid and when it stops being valid.
+//!
+//! `bao.client_cert.not_before` and `bao.client_cert.not_after`, unix seconds,
+//! one data point each, labelled `cert` with the identity's stable name (the
+//! file stem the identity is minted under: `client`, `controller`, `agent`).
+//! Exporting the two timestamps rather than a precomputed fraction keeps the
+//! lifetime fraction right at query time: a wedged process keeps reporting the
+//! same timestamps, not a stale low percentage.
+//!
+//! The file is read at collection time, so a renewed leaf is picked up on the
+//! next export. The two gauges read it separately (the SDK has no callback
+//! shared between instruments), so one export that straddles a renewal pairs
+//! the old `not_before` with the new `not_after`; the next export agrees.
+//! A file that cannot be read or parsed yields no data point and a debug line.
+//! The export thread never panics and waits on nothing but that local read.
+
+use std::path::Path;
+
+use crate::{cert, client::ENV_CLIENT_CERT};
+use opentelemetry::KeyValue;
+use opentelemetry::metrics::Meter;
+
+/// Register both gauges on `meter` for the certificate [`ENV_CLIENT_CERT`]
+/// names, reported under `cert`.
+///
+/// Registers nothing, with a debug line, when the variable is unset or empty.
+pub fn register(meter: &Meter, cert: &'static str) {
+    match std::env::var(ENV_CLIENT_CERT) {
+        Ok(path) if !path.is_empty() => register_at(meter, cert, Path::new(&path)),
+        _ => tracing::debug!(
+            cert,
+            "client-cert gauges: {ENV_CLIENT_CERT} is unset, none registered"
+        ),
+    }
+}
+
+/// Register both gauges on `meter` for the certificate at `path`, reported
+/// under `cert`.
+pub fn register_at(meter: &Meter, cert: &'static str, path: &Path) {
+    for (name, pick) in [
+        (
+            cert::NOT_BEFORE_METRIC,
+            Window::not_before as fn(Window) -> i64,
+        ),
+        (cert::NOT_AFTER_METRIC, Window::not_after),
+    ] {
+        let path = path.to_path_buf();
+        meter
+            .i64_observable_gauge(name)
+            .with_unit("s")
+            .with_callback(move |obs| {
+                if let Some(window) = read(cert, &path) {
+                    obs.observe(pick(window), &[KeyValue::new("cert", cert)]);
+                }
+            })
+            .build();
+    }
+}
+
+#[derive(Clone, Copy)]
+struct Window {
+    not_before: i64,
+    not_after: i64,
+}
+
+impl Window {
+    const fn not_before(self) -> i64 {
+        self.not_before
+    }
+
+    const fn not_after(self) -> i64 {
+        self.not_after
+    }
+}
+
+fn read(cert_name: &str, path: &Path) -> Option<Window> {
+    match cert::file_validity(path) {
+        Ok((not_before, not_after)) => Some(Window {
+            not_before,
+            not_after,
+        }),
+        Err(e) => {
+            tracing::debug!(cert = cert_name, error = %e, "client-cert gauges: no data point");
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use opentelemetry::metrics::MeterProvider as _;
+    use opentelemetry_sdk::metrics::data::{
+        AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics,
+    };
+    use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
+    use std::path::PathBuf;
+
+    use crate::cert::tests::{CERT_PEM, NOT_AFTER, NOT_BEFORE};
+
+    /// A provider with both gauges registered for `path`, and the exporter it
+    /// flushes into.
+    fn registered(path: &Path) -> (SdkMeterProvider, InMemoryMetricExporter) {
+        let exporter = InMemoryMetricExporter::default();
+        let reader = PeriodicReader::builder(exporter.clone()).build();
+        let provider = SdkMeterProvider::builder().with_reader(reader).build();
+        register_at(&provider.meter("test"), "controller", path);
+        (provider, exporter)
+    }
+
+    /// One collection: `(metric, cert label, value)` for every data point,
+    /// sorted.
+    fn collect(
+        provider: &SdkMeterProvider,
+        exporter: &InMemoryMetricExporter,
+    ) -> Vec<(String, String, i64)> {
+        exporter.reset();
+        provider.force_flush().expect("flush");
+        let mut points = Vec::new();
+        for metric in exporter
+            .get_finished_metrics()
+            .expect("metrics")
+            .iter()
+            .flat_map(ResourceMetrics::scope_metrics)
+            .flat_map(ScopeMetrics::metrics)
+        {
+            let AggregatedMetrics::I64(MetricData::Gauge(gauge)) = metric.data() else {
+                panic!("not an i64 gauge: {:?}", metric.data());
+            };
+            for point in gauge.data_points() {
+                let label = point
+                    .attributes()
+                    .find(|kv| kv.key.as_str() == "cert")
+                    .map(|kv| kv.value.to_string())
+                    .unwrap_or_default();
+                points.push((metric.name().to_owned(), label, point.value()));
+            }
+        }
+        points.sort();
+        points
+    }
+
+    fn scratch(name: &str) -> PathBuf {
+        let dir =
+            std::env::temp_dir().join(format!("swarm-cert-metrics-{name}-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).expect("scratch dir");
+        dir
+    }
+
+    #[test]
+    fn both_gauges_report_the_files_window_under_the_cert_label() {
+        let dir = scratch("window");
+        let path = dir.join("cert.pem");
+        std::fs::write(&path, CERT_PEM).expect("write");
+        let (provider, exporter) = registered(&path);
+        assert_eq!(
+            collect(&provider, &exporter),
+            [
+                (
+                    cert::NOT_AFTER_METRIC.to_owned(),
+                    "controller".to_owned(),
+                    NOT_AFTER
+                ),
+                (
+                    cert::NOT_BEFORE_METRIC.to_owned(),
+                    "controller".to_owned(),
+                    NOT_BEFORE
+                ),
+            ]
+        );
+        std::fs::remove_dir_all(&dir).expect("cleanup");
+    }
+
+    #[test]
+    fn a_missing_or_unparseable_file_reports_nothing() {
+        let (provider, exporter) = registered(Path::new("/nonexistent/cert.pem"));
+        assert_eq!(collect(&provider, &exporter), []);
+
+        let dir = scratch("garbage");
+        let path = dir.join("cert.pem");
+        std::fs::write(&path, "not a certificate").expect("write");
+        let (provider, exporter) = registered(&path);
+        assert_eq!(collect(&provider, &exporter), []);
+        std::fs::remove_dir_all(&dir).expect("cleanup");
+    }
+
+    /// The file is read on every collection, not once at registration.
+    #[test]
+    fn a_file_that_appears_later_is_reported_from_then_on() {
+        let dir = scratch("later");
+        let path = dir.join("cert.pem");
+        let (provider, exporter) = registered(&path);
+        assert_eq!(collect(&provider, &exporter), []);
+        std::fs::write(&path, CERT_PEM).expect("write");
+        assert_eq!(collect(&provider, &exporter).len(), 2);
+        std::fs::remove_dir_all(&dir).expect("cleanup");
+    }
+}

swarm-secret-client/src/lib.rs

@@ -32,6 +32,8 @@
 pub mod acp;
 pub mod builder;
 pub mod cert;
+#[cfg(feature = "otel")]
+pub mod cert_metrics;
 pub mod ci;
 pub mod ci_metrics;
 pub mod claude_login;

swarm-secret-client/test-cert.pem

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBizCCATGgAwIBAgIUSFpuYmy1UoHvq/MYiyrAudRaWXcwCgYIKoZIzj0EAwIw
+GzEZMBcGA1UEAwwQaGl2ZS1hZ2VudC1hdGxhczAeFw0yNjAxMDEwMDAwMDBaFw0y
+NjA0MDEwMDAwMDBaMBsxGTAXBgNVBAMMEGhpdmUtYWdlbnQtYXRsYXMwWTATBgcq
+hkjOPQIBBggqhkjOPQMBBwNCAAR7p2eZAxjiE1RdsuSHE3CcTjbnd4swzfnmqUPW
+9NJN7mvOtqEzxCKfOSFjaIhhyShIQ41/V6vGi80EPDB+uR56o1MwUTAdBgNVHQ4E
+FgQUxgZFsN1VY3ODrPS1NUfY/x3EnmswHwYDVR0jBBgwFoAUxgZFsN1VY3ODrPS1
+NUfY/x3EnmswDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAnU9e
+WOioNTUNK9b6FignejpK5FpyrODUuH/iL4TqCh0CIAz6GzJeLebhRPvYlfVUofdV
+hWx3sOwmUgjkRQXoxY+p
+-----END CERTIFICATE-----