export bao client-cert not_before/not_after gauges and alert past 80% of lifetime

swarm-ci-metrics/src/main.rs

@@ -10,19 +10,23 @@
 //! The store identity comes from the `BAO_*` environment. Under it this reads
 //! the forge token swarm-controller mints for the account, and the OIDC
 //! client secret the collector's route for this service checks, once per pass.
+//!
+//! Each pass also pushes the two `bao.client_cert.*` gauges for that store
+//! certificate ([`push_cert`]).
 
 use std::collections::BTreeMap;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use anyhow::{Context as _, Result, bail};
 use forgejo_api::structs::{ActionRun, ListActionRunsQuery, RepoSearchQuery};
 use forgejo_api::{Auth, Forgejo};
 use swarm_queue_client::{ClientSecret, QueueConfig};
-use swarm_secret_client::SecretStore;
 use swarm_secret_client::ci_metrics::{ClientSecret as StoredClientSecret, client_secret_path};
+use swarm_secret_client::client::ENV_CLIENT_CERT;
 use swarm_secret_client::forge::Credential;
 use swarm_secret_client::stored_secret::StoredSecret;
+use swarm_secret_client::{SecretStore, cert};
 
 mod cursor;
 mod otlp;
@@ -86,6 +90,7 @@ async fn main() -> Result<()> {
         .timeout(Duration::from_secs(30))
         .build()
         .context("building the OTLP client")?;
+    push_cert(&cfg, &http).await;
     let mut state = cursor::State::load(&cfg.state_file)?;
     let repos = repos(&forge).await?;
     tracing::info!(repos = repos.len(), "ci metrics: pass starting");
@@ -148,6 +153,40 @@ async fn repo_pass(
     Ok(())
 }
 
+/// The `cert` label this service's certificate is reported under.
+const CERT_LABEL: &str = "ci-metrics";
+
+/// Pushes the two `bao.client_cert.*` gauges for this service's own store
+/// certificate ([`ENV_CLIENT_CERT`]), once per pass, stamped now.
+///
+/// An unreadable file or a refused push is logged and the pass carries on: the
+/// runs are the pass's work, and this is a report about the process doing it.
+async fn push_cert(cfg: &Config, http: &reqwest::Client) {
+    let Ok(path) = std::env::var(ENV_CLIENT_CERT) else {
+        tracing::debug!("ci metrics: {ENV_CLIENT_CERT} is unset, no client-cert gauges");
+        return;
+    };
+    let (not_before, not_after) = match cert::file_validity(Path::new(&path)) {
+        Ok(window) => window,
+        Err(e) => {
+            tracing::debug!(error = %e, "ci metrics: no client-cert gauges");
+            return;
+        }
+    };
+    let now = time::OffsetDateTime::now_utc().unix_timestamp_nanos();
+    let body = otlp::cert_body(CERT_LABEL, not_before, not_after, now);
+    let pushed = async {
+        let bearer =
+            swarm_queue_client::mint_token_for(&cfg.queue, Some(&cfg.queue.client_id), None)
+                .await
+                .context("minting a token for the collector")?;
+        post(http, &cfg.metrics_url, &bearer, &body).await
+    };
+    if let Err(e) = pushed.await {
+        tracing::warn!(error = %format!("{e:#}"), "ci metrics: pushing the client-cert gauges failed");
+    }
+}
+
 /// Every repo the account sees that has Actions and is neither archived nor a
 /// mirror, as `owner/name`.
 async fn repos(forge: &Forgejo) -> Result<Vec<String>> {
@@ -214,11 +253,21 @@ async fn push(
     url: &str,
     bearer: &str,
     samples: &[sample::RunSamples],
+) -> Result<()> {
+    post(http, url, bearer, &otlp::body(samples)).await
+}
+
+/// POST one OTLP JSON `body` to the collector; an error unless it answers 2xx.
+async fn post(
+    http: &reqwest::Client,
+    url: &str,
+    bearer: &str,
+    body: &serde_json::Value,
 ) -> Result<()> {
     let resp = http
         .post(url)
         .bearer_auth(bearer)
-        .json(&otlp::body(samples))
+        .json(body)
         .send()
         .await
         .with_context(|| format!("pushing to {url}"))?;

swarm-ci-metrics/src/otlp.rs

@@ -63,6 +63,32 @@ pub fn body(runs: &[RunSamples]) -> Value {
     })
 }
 
+/// The request body for the two `bao.client_cert.*` gauges of the certificate
+/// reported as `cert`, stamped `at_unix_nanos`, named as in
+/// [`swarm_secret_client::cert`] so the alert rule and every exporter agree.
+pub fn cert_body(cert: &str, not_before: i64, not_after: i64, at_unix_nanos: i128) -> Value {
+    let point = |value: i64| {
+        json!({
+            "attributes": [attribute("cert", cert)],
+            "timeUnixNano": at_unix_nanos.to_string(),
+            "asInt": value.to_string(),
+        })
+    };
+    let gauge = |name: &str, value: i64| json!({ "name": name, "unit": "s", "gauge": { "dataPoints": [point(value)] } });
+    json!({
+        "resourceMetrics": [{
+            "resource": { "attributes": [attribute("service.name", SERVICE_NAME)] },
+            "scopeMetrics": [{
+                "scope": { "name": SERVICE_NAME },
+                "metrics": [
+                    gauge(swarm_secret_client::cert::NOT_BEFORE_METRIC, not_before),
+                    gauge(swarm_secret_client::cert::NOT_AFTER_METRIC, not_after),
+                ],
+            }],
+        }],
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -135,4 +161,26 @@ mod tests {
         assert_eq!(waits[0]["asDouble"], 279.0);
         assert_eq!(waits[0]["timeUnixNano"], "1792163610000000000");
     }
+
+    #[test]
+    fn the_cert_gauges_carry_the_window_under_the_cert_label() {
+        let b = cert_body(
+            "ci-metrics",
+            1_767_225_600,
+            1_775_001_600,
+            1_792_163_610_000_000_000,
+        );
+        let metrics = &b["resourceMetrics"][0]["scopeMetrics"][0]["metrics"];
+        assert_eq!(metrics[0]["name"], "bao.client_cert.not_before");
+        assert_eq!(metrics[1]["name"], "bao.client_cert.not_after");
+        for (i, value) in [(0, "1767225600"), (1, "1775001600")] {
+            let point = &metrics[i]["gauge"]["dataPoints"][0];
+            assert_eq!(point["asInt"], value);
+            assert_eq!(point["timeUnixNano"], "1792163610000000000");
+            assert_eq!(
+                point["attributes"],
+                json!([{"key": "cert", "value": {"stringValue": "ci-metrics"}}])
+            );
+        }
+    }
 }