export bao client-cert not_before/not_after gauges and alert past 80% of lifetime

swarm-cert-metrics/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "swarm-cert-metrics"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+opentelemetry.workspace = true
+# The cert file's validity window.
+swarm-secret-client.workspace = true
+tracing.workspace = true
+
+[lints]
+workspace = true
+
+[dev-dependencies]
+# `InMemoryMetricExporter`, to read back what a provider exports.
+opentelemetry_sdk = { workspace = true, features = ["testing"] }

swarm-cert-metrics/src/lib.rs

@@ -0,0 +1,202 @@
+//! The two gauges every service holding a store client certificate exports:
+//! when that certificate became valid and when it stops being valid.
+//!
+//! `bao.client_cert.not_before` and `bao.client_cert.not_after`, unix seconds,
+//! one data point each, labelled `cert` with the identity's stable name (the
+//! file stem the identity is minted under: `client`, `controller`, `agent`).
+//! Exporting the two timestamps rather than a precomputed fraction keeps the
+//! lifetime fraction right at query time: a wedged process keeps reporting the
+//! same timestamps, not a stale low percentage.
+//!
+//! The file is read at collection time, so a renewed leaf is picked up on the
+//! next export. The two gauges read it separately (the SDK has no callback
+//! shared between instruments), so one export that straddles a renewal pairs
+//! the old `not_before` with the new `not_after`; the next export agrees.
+//! A file that cannot be read or parsed yields no data point and a debug line.
+//! The export thread never panics and waits on nothing but that local read.
+
+use std::path::Path;
+
+use opentelemetry::KeyValue;
+use opentelemetry::metrics::Meter;
+use swarm_secret_client::{cert, client::ENV_CLIENT_CERT};
+
+/// Register both gauges on `meter` for the certificate [`ENV_CLIENT_CERT`]
+/// names, reported under `cert`.
+///
+/// Registers nothing, with a debug line, when the variable is unset or empty.
+pub fn register(meter: &Meter, cert: &'static str) {
+    match std::env::var(ENV_CLIENT_CERT) {
+        Ok(path) if !path.is_empty() => register_at(meter, cert, Path::new(&path)),
+        _ => tracing::debug!(
+            cert,
+            "client-cert gauges: {ENV_CLIENT_CERT} is unset, none registered"
+        ),
+    }
+}
+
+/// Register both gauges on `meter` for the certificate at `path`, reported
+/// under `cert`.
+pub fn register_at(meter: &Meter, cert: &'static str, path: &Path) {
+    for (name, pick) in [
+        (
+            cert::NOT_BEFORE_METRIC,
+            Window::not_before as fn(Window) -> i64,
+        ),
+        (cert::NOT_AFTER_METRIC, Window::not_after),
+    ] {
+        let path = path.to_path_buf();
+        meter
+            .i64_observable_gauge(name)
+            .with_unit("s")
+            .with_callback(move |obs| {
+                if let Some(window) = read(cert, &path) {
+                    obs.observe(pick(window), &[KeyValue::new("cert", cert)]);
+                }
+            })
+            .build();
+    }
+}
+
+#[derive(Clone, Copy)]
+struct Window {
+    not_before: i64,
+    not_after: i64,
+}
+
+impl Window {
+    const fn not_before(self) -> i64 {
+        self.not_before
+    }
+
+    const fn not_after(self) -> i64 {
+        self.not_after
+    }
+}
+
+fn read(cert_name: &str, path: &Path) -> Option<Window> {
+    match cert::file_validity(path) {
+        Ok((not_before, not_after)) => Some(Window {
+            not_before,
+            not_after,
+        }),
+        Err(e) => {
+            tracing::debug!(cert = cert_name, error = %e, "client-cert gauges: no data point");
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use opentelemetry::metrics::MeterProvider as _;
+    use opentelemetry_sdk::metrics::data::{
+        AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics,
+    };
+    use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
+    use std::path::PathBuf;
+
+    /// Self-signed, valid 2026-01-01T00:00:00Z to 2026-04-01T00:00:00Z.
+    const CERT_PEM: &str = include_str!("../test-cert.pem");
+    const NOT_BEFORE_SECS: i64 = 1_767_225_600;
+    const NOT_AFTER_SECS: i64 = 1_775_001_600;
+
+    /// A provider with both gauges registered for `path`, and the exporter it
+    /// flushes into.
+    fn registered(path: &Path) -> (SdkMeterProvider, InMemoryMetricExporter) {
+        let exporter = InMemoryMetricExporter::default();
+        let reader = PeriodicReader::builder(exporter.clone()).build();
+        let provider = SdkMeterProvider::builder().with_reader(reader).build();
+        register_at(&provider.meter("test"), "controller", path);
+        (provider, exporter)
+    }
+
+    /// One collection: `(metric, cert label, value)` for every data point,
+    /// sorted.
+    fn collect(
+        provider: &SdkMeterProvider,
+        exporter: &InMemoryMetricExporter,
+    ) -> Vec<(String, String, i64)> {
+        exporter.reset();
+        provider.force_flush().expect("flush");
+        let mut points = Vec::new();
+        for metric in exporter
+            .get_finished_metrics()
+            .expect("metrics")
+            .iter()
+            .flat_map(ResourceMetrics::scope_metrics)
+            .flat_map(ScopeMetrics::metrics)
+        {
+            let AggregatedMetrics::I64(MetricData::Gauge(gauge)) = metric.data() else {
+                panic!("not an i64 gauge: {:?}", metric.data());
+            };
+            for point in gauge.data_points() {
+                let label = point
+                    .attributes()
+                    .find(|kv| kv.key.as_str() == "cert")
+                    .map(|kv| kv.value.to_string())
+                    .unwrap_or_default();
+                points.push((metric.name().to_owned(), label, point.value()));
+            }
+        }
+        points.sort();
+        points
+    }
+
+    fn scratch(name: &str) -> PathBuf {
+        let dir =
+            std::env::temp_dir().join(format!("swarm-cert-metrics-{name}-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).expect("scratch dir");
+        dir
+    }
+
+    #[test]
+    fn both_gauges_report_the_files_window_under_the_cert_label() {
+        let dir = scratch("window");
+        let path = dir.join("cert.pem");
+        std::fs::write(&path, CERT_PEM).expect("write");
+        let (provider, exporter) = registered(&path);
+        assert_eq!(
+            collect(&provider, &exporter),
+            [
+                (
+                    cert::NOT_AFTER_METRIC.to_owned(),
+                    "controller".to_owned(),
+                    NOT_AFTER_SECS
+                ),
+                (
+                    cert::NOT_BEFORE_METRIC.to_owned(),
+                    "controller".to_owned(),
+                    NOT_BEFORE_SECS
+                ),
+            ]
+        );
+        std::fs::remove_dir_all(&dir).expect("cleanup");
+    }
+
+    #[test]
+    fn a_missing_or_unparseable_file_reports_nothing() {
+        let (provider, exporter) = registered(Path::new("/nonexistent/cert.pem"));
+        assert_eq!(collect(&provider, &exporter), []);
+
+        let dir = scratch("garbage");
+        let path = dir.join("cert.pem");
+        std::fs::write(&path, "not a certificate").expect("write");
+        let (provider, exporter) = registered(&path);
+        assert_eq!(collect(&provider, &exporter), []);
+        std::fs::remove_dir_all(&dir).expect("cleanup");
+    }
+
+    /// The file is read on every collection, not once at registration.
+    #[test]
+    fn a_file_that_appears_later_is_reported_from_then_on() {
+        let dir = scratch("later");
+        let path = dir.join("cert.pem");
+        let (provider, exporter) = registered(&path);
+        assert_eq!(collect(&provider, &exporter), []);
+        std::fs::write(&path, CERT_PEM).expect("write");
+        assert_eq!(collect(&provider, &exporter).len(), 2);
+        std::fs::remove_dir_all(&dir).expect("cleanup");
+    }
+}

swarm-cert-metrics/test-cert.pem

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBizCCATGgAwIBAgIUSFpuYmy1UoHvq/MYiyrAudRaWXcwCgYIKoZIzj0EAwIw
+GzEZMBcGA1UEAwwQaGl2ZS1hZ2VudC1hdGxhczAeFw0yNjAxMDEwMDAwMDBaFw0y
+NjA0MDEwMDAwMDBaMBsxGTAXBgNVBAMMEGhpdmUtYWdlbnQtYXRsYXMwWTATBgcq
+hkjOPQIBBggqhkjOPQMBBwNCAAR7p2eZAxjiE1RdsuSHE3CcTjbnd4swzfnmqUPW
+9NJN7mvOtqEzxCKfOSFjaIhhyShIQ41/V6vGi80EPDB+uR56o1MwUTAdBgNVHQ4E
+FgQUxgZFsN1VY3ODrPS1NUfY/x3EnmswHwYDVR0jBBgwFoAUxgZFsN1VY3ODrPS1
+NUfY/x3EnmswDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAnU9e
+WOioNTUNK9b6FignejpK5FpyrODUuH/iL4TqCh0CIAz6GzJeLebhRPvYlfVUofdV
+hWx3sOwmUgjkRQXoxY+p
+-----END CERTIFICATE-----