From 078696aaeec106a9f34f57227b059dbeaaeee74d Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 25 Oct 2026 17:41:47 +0100 Subject: [PATCH] export bao client-cert not_before/not_after gauges and alert past 80% of lifetime --- AGENTS.md | 3 + Cargo.lock | 16 +- Cargo.toml | 2 + docs/scheduler/observability.md | 53 +++++++ hive-agent/Cargo.toml | 2 + hive-agent/src/otel_turn_metrics.rs | 1 + hive-c0re/Cargo.toml | 2 + hive-c0re/src/stats/otel_metrics.rs | 3 + nix/host-modules/swarm-grafana.nix | 18 +++ nix/module-eval/grafana.nix | 36 +++++ swarm-cert-metrics/Cargo.toml | 17 +++ swarm-cert-metrics/src/lib.rs | 202 ++++++++++++++++++++++++++ swarm-cert-metrics/test-cert.pem | 11 ++ swarm-ci-metrics/src/main.rs | 55 ++++++- swarm-ci-metrics/src/otlp.rs | 48 ++++++ swarm-controller/Cargo.toml | 4 +- swarm-controller/src/agent_renewal.rs | 49 +------ swarm-controller/src/main.rs | 13 +- swarm-review-classifier/Cargo.toml | 2 + swarm-review-classifier/src/cost.rs | 4 + swarm-secret-client/Cargo.toml | 2 + swarm-secret-client/src/cert.rs | 109 ++++++++++++++ swarm-secret-client/src/lib.rs | 5 + 23 files changed, 604 insertions(+), 53 deletions(-) create mode 100644 swarm-cert-metrics/Cargo.toml create mode 100644 swarm-cert-metrics/src/lib.rs create mode 100644 swarm-cert-metrics/test-cert.pem create mode 100644 swarm-secret-client/src/cert.rs diff --git a/AGENTS.md b/AGENTS.md index 1c267da4..d9b46399 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -138,6 +138,9 @@ Swarm-level: the **`BAO_`** env spellings explicitly — `vaultrs`'s `VAULT_*` defaults are set by no unit here, and falling through to them yields a client with **no identity** and a TLS failure that names no cause. +- **`swarm-cert-metrics/`** — the `bao.client_cert.not_before` / `not_after` + observable gauges a service registers on its existing OTLP exporter; the + cert file is read at each collection. - **`swarm-queue-client/`** — the one authenticated connect to the swarm NATS queue. - **`swarm-wire/`** — serde-only request and reply contracts services diff --git a/Cargo.lock b/Cargo.lock index 998f2595..316a3015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1691,6 +1691,7 @@ dependencies = [ "serde_json", "sha1", "swarm-agent-verify", + "swarm-cert-metrics", "swarm-queue-client", "swarm-secret-client", "swarm-wire", @@ -1801,6 +1802,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "strum", + "swarm-cert-metrics", "swarm-matrix-client", "swarm-queue-client", "swarm-secret-client", @@ -4837,6 +4839,16 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "swarm-cert-metrics" +version = "0.1.0" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "swarm-secret-client", + "tracing", +] + [[package]] name = "swarm-ci-metrics" version = "0.1.0" @@ -4909,6 +4921,7 @@ dependencies = [ "similar", "strum", "swarm-authelia-bridge-sock", + "swarm-cert-metrics", "swarm-controller-api", "swarm-matrix-client", "swarm-queue-client", @@ -4922,7 +4935,6 @@ dependencies = [ "url", "utoipa", "utoipa-axum", - "x509-cert", ] [[package]] @@ -5096,6 +5108,7 @@ dependencies = [ "opentelemetry_sdk", "serde", "serde_json", + "swarm-cert-metrics", "swarm-queue-client", "swarm-secret-client", "swarm-wire", @@ -5122,6 +5135,7 @@ dependencies = [ "tracing", "tracing-subscriber", "vaultrs", + "x509-cert", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cbea8d23..a758c8a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "swarm-agent-verify", "swarm-authelia-bridge", "swarm-builder", + "swarm-cert-metrics", "swarm-ci-metrics", "swarm-claude-login", "swarm-authelia-bridge-sock", @@ -118,6 +119,7 @@ hive-sock-client = { path = "hive-sock-client" } hive-types = { path = "hive-types" } swarm-agent-verify = { path = "swarm-agent-verify" } swarm-authelia-bridge-sock = { path = "swarm-authelia-bridge-sock" } +swarm-cert-metrics = { path = "swarm-cert-metrics" } swarm-controller-api = { path = "swarm-controller-api" } swarm-matrix-client = { path = "swarm-matrix-client" } swarm-queue-client = { path = "swarm-queue-client" } diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index db526bad..21ed0f70 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -559,6 +559,59 @@ rules file, at most 10 per repo. The `review-classifier` dashboard in Grafana charts these. +## Store client-certificate metrics + +Every service that logs in to the secret store with a client certificate can +report when that certificate stops being valid. Two observable gauges, one +data point each, unix seconds: + +| Metric | Unit | Kind | Attributes | +| ---------------------------- | ---- | ----- | ---------- | +| `bao.client_cert.not_before` | `s` | gauge | `cert` | +| `bao.client_cert.not_after` | `s` | gauge | `cert` | + +`cert` names the identity (`client`, `controller`, `agent`, `review-classifier`, +`ci-metrics`); the resource labels (`service.name`, `hive`, `agent`) say who +holds it. The `swarm-cert-metrics` crate registers both gauges on the service's +existing exporter and reads `BAO_CLIENT_CERT` at every collection, so a +renewed leaf shows up on the next export. A file that's missing, or that isn't a +certificate yields no data point and a debug log line. The parser is +`swarm_secret_client::cert`. + +| Service | `cert` | Exporter it rides | +| ------------------------- | ------------------- | --------------------------------------------------- | +| `hive-c0re` | `client` | the container-stats exporter | +| `hive-agent` | `agent` | the per-turn exporter, built on its first recording | +| `swarm-controller` | `controller` | the job-graph exporter | +| `swarm-review-classifier` | `review-classifier` | the cost exporter, built on its first recording | +| `swarm-ci-metrics` | `ci-metrics` | a push of its own at the start of each timer pass | + +Not covered yet: the other seven Rust services that hold a store certificate +(`swarm-builder`, `swarm-claude-login`, `swarm-notify-router`, +`swarm-nats-auth`, `swarm-logs-auth`, `swarm-metrics-auth`, +`swarm-matrix-ctl`) have no exporter, and the ten identities that only shell +units use report nothing. Neither does any CA certificate. Each leaf's own +window is all the gauges show, and a CA that expires before its leaves is +invisible to them. + +The swarm Grafana alerts on the lifetime fraction: + +| Alert | Fires when | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `hh-bao-client-cert-lifetime` | `(time() - not_before) / (not_after - not_before)` stays above 0.8 for 10 minutes, per `service.name`, `cert`, `hive` and `agent` | + +The two gauges of a series carry identical labels, so MetricsQL divides them +one-to-one without `on()` or `group_left`. The query wraps each in +`last_over_time(...[15m])` so `swarm-ci-metrics`, which reports every five +minutes, keeps a series. A series that stops reporting resolves the alert +rather than firing it (no data counts as OK). + +An agent leaf lives 7 days and renews at half its life, so a fraction over 0.8 +means the renewal stalled about two days ago. The host leaves (`client`, +`controller`, and the others minted by `swarm-bao-pki`) live 3650 days and never +renew; for them 0.8 comes about eight years after minting. A guard on the days +left would suit them better and doesn't exist yet. + ## Agent-emitted custom metrics (`hive-metric`) Agents can push arbitrary labeled metrics to the same OTEL collector via the diff --git a/hive-agent/Cargo.toml b/hive-agent/Cargo.toml index 6db5d131..5849defc 100644 --- a/hive-agent/Cargo.toml +++ b/hive-agent/Cargo.toml @@ -58,6 +58,8 @@ swarm-queue-client = { workspace = true, features = [ ] } # The spelling of the token the queue connection presents. swarm-agent-verify.workspace = true +# The `bao.client_cert.*` gauges on this exporter. +swarm-cert-metrics.workspace = true swarm-wire.workspace = true # This agent's own queue secret, read from the store under its own certificate. swarm-secret-client.workspace = true diff --git a/hive-agent/src/otel_turn_metrics.rs b/hive-agent/src/otel_turn_metrics.rs index dcc9fd99..34185454 100644 --- a/hive-agent/src/otel_turn_metrics.rs +++ b/hive-agent/src/otel_turn_metrics.rs @@ -181,6 +181,7 @@ fn build() -> Option { Ok(provider) => { tracing::info!(?interval, "otel turn-metrics: exporter enabled"); let meter = provider.meter("hyperhive.turn_stats"); + swarm_cert_metrics::register(&meter, "agent"); Some(Instruments { turn_duration_ms: meter .u64_histogram("hyperhive.agent.turn.duration") diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index 4678d516..fa46080b 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -62,6 +62,8 @@ swarm-queue-client = { workspace = true, features = [ "agent-permissions", "agent-archetype", ] } +# The `bao.client_cert.*` gauges on this exporter. +swarm-cert-metrics.workspace = true swarm-matrix-client.workspace = true swarm-secret-client.workspace = true swarm-wire.workspace = true diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index a9d3f2a7..5f9149c1 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -261,6 +261,9 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc register_at(meter, cert, Path::new(&path)), + _ => tracing::debug!( + cert, + "client-cert gauges: {ENV_CLIENT_CERT} is unset, none registered" + ), + } +} + +/// Register both gauges on `meter` for the certificate at `path`, reported +/// under `cert`. +pub fn register_at(meter: &Meter, cert: &'static str, path: &Path) { + for (name, pick) in [ + ( + cert::NOT_BEFORE_METRIC, + Window::not_before as fn(Window) -> i64, + ), + (cert::NOT_AFTER_METRIC, Window::not_after), + ] { + let path = path.to_path_buf(); + meter + .i64_observable_gauge(name) + .with_unit("s") + .with_callback(move |obs| { + if let Some(window) = read(cert, &path) { + obs.observe(pick(window), &[KeyValue::new("cert", cert)]); + } + }) + .build(); + } +} + +#[derive(Clone, Copy)] +struct Window { + not_before: i64, + not_after: i64, +} + +impl Window { + const fn not_before(self) -> i64 { + self.not_before + } + + const fn not_after(self) -> i64 { + self.not_after + } +} + +fn read(cert_name: &str, path: &Path) -> Option { + match cert::file_validity(path) { + Ok((not_before, not_after)) => Some(Window { + not_before, + not_after, + }), + Err(e) => { + tracing::debug!(cert = cert_name, error = %e, "client-cert gauges: no data point"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::metrics::MeterProvider as _; + use opentelemetry_sdk::metrics::data::{ + AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics, + }; + use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider}; + use std::path::PathBuf; + + /// Self-signed, valid 2026-01-01T00:00:00Z to 2026-04-01T00:00:00Z. + const CERT_PEM: &str = include_str!("../test-cert.pem"); + const NOT_BEFORE_SECS: i64 = 1_767_225_600; + const NOT_AFTER_SECS: i64 = 1_775_001_600; + + /// A provider with both gauges registered for `path`, and the exporter it + /// flushes into. + fn registered(path: &Path) -> (SdkMeterProvider, InMemoryMetricExporter) { + let exporter = InMemoryMetricExporter::default(); + let reader = PeriodicReader::builder(exporter.clone()).build(); + let provider = SdkMeterProvider::builder().with_reader(reader).build(); + register_at(&provider.meter("test"), "controller", path); + (provider, exporter) + } + + /// One collection: `(metric, cert label, value)` for every data point, + /// sorted. + fn collect( + provider: &SdkMeterProvider, + exporter: &InMemoryMetricExporter, + ) -> Vec<(String, String, i64)> { + exporter.reset(); + provider.force_flush().expect("flush"); + let mut points = Vec::new(); + for metric in exporter + .get_finished_metrics() + .expect("metrics") + .iter() + .flat_map(ResourceMetrics::scope_metrics) + .flat_map(ScopeMetrics::metrics) + { + let AggregatedMetrics::I64(MetricData::Gauge(gauge)) = metric.data() else { + panic!("not an i64 gauge: {:?}", metric.data()); + }; + for point in gauge.data_points() { + let label = point + .attributes() + .find(|kv| kv.key.as_str() == "cert") + .map(|kv| kv.value.to_string()) + .unwrap_or_default(); + points.push((metric.name().to_owned(), label, point.value())); + } + } + points.sort(); + points + } + + fn scratch(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("swarm-cert-metrics-{name}-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("scratch dir"); + dir + } + + #[test] + fn both_gauges_report_the_files_window_under_the_cert_label() { + let dir = scratch("window"); + let path = dir.join("cert.pem"); + std::fs::write(&path, CERT_PEM).expect("write"); + let (provider, exporter) = registered(&path); + assert_eq!( + collect(&provider, &exporter), + [ + ( + cert::NOT_AFTER_METRIC.to_owned(), + "controller".to_owned(), + NOT_AFTER_SECS + ), + ( + cert::NOT_BEFORE_METRIC.to_owned(), + "controller".to_owned(), + NOT_BEFORE_SECS + ), + ] + ); + std::fs::remove_dir_all(&dir).expect("cleanup"); + } + + #[test] + fn a_missing_or_unparseable_file_reports_nothing() { + let (provider, exporter) = registered(Path::new("/nonexistent/cert.pem")); + assert_eq!(collect(&provider, &exporter), []); + + let dir = scratch("garbage"); + let path = dir.join("cert.pem"); + std::fs::write(&path, "not a certificate").expect("write"); + let (provider, exporter) = registered(&path); + assert_eq!(collect(&provider, &exporter), []); + std::fs::remove_dir_all(&dir).expect("cleanup"); + } + + /// The file is read on every collection, not once at registration. + #[test] + fn a_file_that_appears_later_is_reported_from_then_on() { + let dir = scratch("later"); + let path = dir.join("cert.pem"); + let (provider, exporter) = registered(&path); + assert_eq!(collect(&provider, &exporter), []); + std::fs::write(&path, CERT_PEM).expect("write"); + assert_eq!(collect(&provider, &exporter).len(), 2); + std::fs::remove_dir_all(&dir).expect("cleanup"); + } +} diff --git a/swarm-cert-metrics/test-cert.pem b/swarm-cert-metrics/test-cert.pem new file mode 100644 index 00000000..b4725512 --- /dev/null +++ b/swarm-cert-metrics/test-cert.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBizCCATGgAwIBAgIUSFpuYmy1UoHvq/MYiyrAudRaWXcwCgYIKoZIzj0EAwIw +GzEZMBcGA1UEAwwQaGl2ZS1hZ2VudC1hdGxhczAeFw0yNjAxMDEwMDAwMDBaFw0y +NjA0MDEwMDAwMDBaMBsxGTAXBgNVBAMMEGhpdmUtYWdlbnQtYXRsYXMwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAAR7p2eZAxjiE1RdsuSHE3CcTjbnd4swzfnmqUPW +9NJN7mvOtqEzxCKfOSFjaIhhyShIQ41/V6vGi80EPDB+uR56o1MwUTAdBgNVHQ4E +FgQUxgZFsN1VY3ODrPS1NUfY/x3EnmswHwYDVR0jBBgwFoAUxgZFsN1VY3ODrPS1 +NUfY/x3EnmswDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAnU9e +WOioNTUNK9b6FignejpK5FpyrODUuH/iL4TqCh0CIAz6GzJeLebhRPvYlfVUofdV +hWx3sOwmUgjkRQXoxY+p +-----END CERTIFICATE----- diff --git a/swarm-ci-metrics/src/main.rs b/swarm-ci-metrics/src/main.rs index 87d5e448..1c6e3678 100644 --- a/swarm-ci-metrics/src/main.rs +++ b/swarm-ci-metrics/src/main.rs @@ -10,19 +10,23 @@ //! The store identity comes from the `BAO_*` environment. Under it this reads //! the forge token swarm-controller mints for the account, and the OIDC //! client secret the collector's route for this service checks, once per pass. +//! +//! Each pass also pushes the two `bao.client_cert.*` gauges for that store +//! certificate ([`push_cert`]). use std::collections::BTreeMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::{Context as _, Result, bail}; use forgejo_api::structs::{ActionRun, ListActionRunsQuery, RepoSearchQuery}; use forgejo_api::{Auth, Forgejo}; use swarm_queue_client::{ClientSecret, QueueConfig}; -use swarm_secret_client::SecretStore; use swarm_secret_client::ci_metrics::{ClientSecret as StoredClientSecret, client_secret_path}; +use swarm_secret_client::client::ENV_CLIENT_CERT; use swarm_secret_client::forge::Credential; use swarm_secret_client::stored_secret::StoredSecret; +use swarm_secret_client::{SecretStore, cert}; mod cursor; mod otlp; @@ -86,6 +90,7 @@ async fn main() -> Result<()> { .timeout(Duration::from_secs(30)) .build() .context("building the OTLP client")?; + push_cert(&cfg, &http).await; let mut state = cursor::State::load(&cfg.state_file)?; let repos = repos(&forge).await?; tracing::info!(repos = repos.len(), "ci metrics: pass starting"); @@ -148,6 +153,40 @@ async fn repo_pass( Ok(()) } +/// The `cert` label this service's certificate is reported under. +const CERT_LABEL: &str = "ci-metrics"; + +/// Pushes the two `bao.client_cert.*` gauges for this service's own store +/// certificate ([`ENV_CLIENT_CERT`]), once per pass, stamped now. +/// +/// An unreadable file or a refused push is logged and the pass carries on: the +/// runs are the pass's work, and this is a report about the process doing it. +async fn push_cert(cfg: &Config, http: &reqwest::Client) { + let Ok(path) = std::env::var(ENV_CLIENT_CERT) else { + tracing::debug!("ci metrics: {ENV_CLIENT_CERT} is unset, no client-cert gauges"); + return; + }; + let (not_before, not_after) = match cert::file_validity(Path::new(&path)) { + Ok(window) => window, + Err(e) => { + tracing::debug!(error = %e, "ci metrics: no client-cert gauges"); + return; + } + }; + let now = time::OffsetDateTime::now_utc().unix_timestamp_nanos(); + let body = otlp::cert_body(CERT_LABEL, not_before, not_after, now); + let pushed = async { + let bearer = + swarm_queue_client::mint_token_for(&cfg.queue, Some(&cfg.queue.client_id), None) + .await + .context("minting a token for the collector")?; + post(http, &cfg.metrics_url, &bearer, &body).await + }; + if let Err(e) = pushed.await { + tracing::warn!(error = %format!("{e:#}"), "ci metrics: pushing the client-cert gauges failed"); + } +} + /// Every repo the account sees that has Actions and is neither archived nor a /// mirror, as `owner/name`. async fn repos(forge: &Forgejo) -> Result> { @@ -214,11 +253,21 @@ async fn push( url: &str, bearer: &str, samples: &[sample::RunSamples], +) -> Result<()> { + post(http, url, bearer, &otlp::body(samples)).await +} + +/// POST one OTLP JSON `body` to the collector; an error unless it answers 2xx. +async fn post( + http: &reqwest::Client, + url: &str, + bearer: &str, + body: &serde_json::Value, ) -> Result<()> { let resp = http .post(url) .bearer_auth(bearer) - .json(&otlp::body(samples)) + .json(body) .send() .await .with_context(|| format!("pushing to {url}"))?; diff --git a/swarm-ci-metrics/src/otlp.rs b/swarm-ci-metrics/src/otlp.rs index 942e3b7d..4cefa4fd 100644 --- a/swarm-ci-metrics/src/otlp.rs +++ b/swarm-ci-metrics/src/otlp.rs @@ -63,6 +63,32 @@ pub fn body(runs: &[RunSamples]) -> Value { }) } +/// The request body for the two `bao.client_cert.*` gauges of the certificate +/// reported as `cert`, stamped `at_unix_nanos`, named as in +/// [`swarm_secret_client::cert`] so the alert rule and every exporter agree. +pub fn cert_body(cert: &str, not_before: i64, not_after: i64, at_unix_nanos: i128) -> Value { + let point = |value: i64| { + json!({ + "attributes": [attribute("cert", cert)], + "timeUnixNano": at_unix_nanos.to_string(), + "asInt": value.to_string(), + }) + }; + let gauge = |name: &str, value: i64| json!({ "name": name, "unit": "s", "gauge": { "dataPoints": [point(value)] } }); + json!({ + "resourceMetrics": [{ + "resource": { "attributes": [attribute("service.name", SERVICE_NAME)] }, + "scopeMetrics": [{ + "scope": { "name": SERVICE_NAME }, + "metrics": [ + gauge(swarm_secret_client::cert::NOT_BEFORE_METRIC, not_before), + gauge(swarm_secret_client::cert::NOT_AFTER_METRIC, not_after), + ], + }], + }], + }) +} + #[cfg(test)] mod tests { use super::*; @@ -135,4 +161,26 @@ mod tests { assert_eq!(waits[0]["asDouble"], 279.0); assert_eq!(waits[0]["timeUnixNano"], "1792163610000000000"); } + + #[test] + fn the_cert_gauges_carry_the_window_under_the_cert_label() { + let b = cert_body( + "ci-metrics", + 1_767_225_600, + 1_775_001_600, + 1_792_163_610_000_000_000, + ); + let metrics = &b["resourceMetrics"][0]["scopeMetrics"][0]["metrics"]; + assert_eq!(metrics[0]["name"], "bao.client_cert.not_before"); + assert_eq!(metrics[1]["name"], "bao.client_cert.not_after"); + for (i, value) in [(0, "1767225600"), (1, "1775001600")] { + let point = &metrics[i]["gauge"]["dataPoints"][0]; + assert_eq!(point["asInt"], value); + assert_eq!(point["timeUnixNano"], "1792163610000000000"); + assert_eq!( + point["attributes"], + json!([{"key": "cert", "value": {"stringValue": "ci-metrics"}}]) + ); + } + } } diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml index f3959680..a17a9570 100644 --- a/swarm-controller/Cargo.toml +++ b/swarm-controller/Cargo.toml @@ -94,6 +94,8 @@ serde_json.workspace = true serde_norway = "0.9.42" strum.workspace = true swarm-authelia-bridge-sock.workspace = true +# The `bao.client_cert.*` gauges on this exporter. +swarm-cert-metrics.workspace = true # `schema`: the forge and schedule routes describe these bodies in the spec. swarm-controller-api = { workspace = true, features = ["schema"] } # The queue connect (token mint + auth callback + reconnect) is shared with @@ -148,8 +150,6 @@ tracing.workspace = true url.workspace = true utoipa.workspace = true utoipa-axum.workspace = true -# `agent_renewal.rs` reads an agent certificate's validity window. -x509-cert.workspace = true # `test-util` for `#[tokio::test(start_paused = true)]`: `agent_renewal`'s # tests wait out its five-minute bound on paused time. diff --git a/swarm-controller/src/agent_renewal.rs b/swarm-controller/src/agent_renewal.rs index 56d70dec..fadb1afc 100644 --- a/swarm-controller/src/agent_renewal.rs +++ b/swarm-controller/src/agent_renewal.rs @@ -33,9 +33,8 @@ use std::collections::BTreeSet; use std::sync::Arc; use anyhow::{Context, Result, bail}; -use swarm_secret_client::{SecretStore, mtls, queue}; +use swarm_secret_client::{SecretStore, cert, mtls, queue}; use swarm_wire::wanted::{AgentState, HiveWanted}; -use x509_cert::der::DecodePem as _; use crate::wanted::WantedWriter; @@ -101,20 +100,6 @@ pub fn apply_secret_step( }) } -/// A certificate's validity window, `(notBefore, notAfter)` in unix seconds. -/// -/// # Errors -/// When `pem` is not one PEM-encoded X.509 certificate. -pub fn cert_validity(pem: &str) -> Result<(i64, i64)> { - let cert = x509_cert::Certificate::from_pem(pem.as_bytes()) - .context("decoding the stored certificate")?; - let validity = &cert.tbs_certificate.validity; - let secs = |t: x509_cert::time::Time| { - i64::try_from(t.to_unix_duration().as_secs()).unwrap_or(i64::MAX) - }; - Ok((secs(validity.not_before), secs(validity.not_after))) -} - /// Whether a certificate valid from `not_before` to `not_after` is past half /// its life at `now`. pub fn cert_is_due(not_before: i64, not_after: i64, now: i64) -> bool { @@ -272,7 +257,7 @@ async fn observe_cert(store: &SecretStore, agent: &str, now: i64) -> Observed { return Observed::Unknown; } }; - match cert_validity(&credential.cert) { + match cert::validity(&credential.cert) { Ok((not_before, not_after)) if cert_is_due(not_before, not_after, now) => { Observed::Due(now.saturating_sub(not_before)) } @@ -434,21 +419,7 @@ mod tests { const NOW: i64 = 1_790_000_000; const DAY: i64 = 86_400; - /// Self-signed, CN `hive-agent-atlas`, valid 2026-01-01T00:00:00Z to - /// 2026-04-01T00:00:00Z: 90 days, a window the agent role no longer issues. Its key was - /// discarded. - const CERT_PEM: &str = "-----BEGIN CERTIFICATE----- -MIIBizCCATGgAwIBAgIUSFpuYmy1UoHvq/MYiyrAudRaWXcwCgYIKoZIzj0EAwIw -GzEZMBcGA1UEAwwQaGl2ZS1hZ2VudC1hdGxhczAeFw0yNjAxMDEwMDAwMDBaFw0y -NjA0MDEwMDAwMDBaMBsxGTAXBgNVBAMMEGhpdmUtYWdlbnQtYXRsYXMwWTATBgcq -hkjOPQIBBggqhkjOPQMBBwNCAAR7p2eZAxjiE1RdsuSHE3CcTjbnd4swzfnmqUPW -9NJN7mvOtqEzxCKfOSFjaIhhyShIQ41/V6vGi80EPDB+uR56o1MwUTAdBgNVHQ4E -FgQUxgZFsN1VY3ODrPS1NUfY/x3EnmswHwYDVR0jBBgwFoAUxgZFsN1VY3ODrPS1 -NUfY/x3EnmswDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAnU9e -WOioNTUNK9b6FignejpK5FpyrODUuH/iL4TqCh0CIAz6GzJeLebhRPvYlfVUofdV -hWx3sOwmUgjkRQXoxY+p ------END CERTIFICATE----- -"; + /// A 90-day window, 2026-01-01T00:00:00Z to 2026-04-01T00:00:00Z. const CERT_NOT_BEFORE: i64 = 1_767_225_600; const CERT_NOT_AFTER: i64 = 1_775_001_600; @@ -505,20 +476,6 @@ hWx3sOwmUgjkRQXoxY+p } } - #[test] - fn a_certificates_validity_is_read_from_the_certificate() { - assert_eq!( - cert_validity(CERT_PEM).expect("a well-formed certificate"), - (CERT_NOT_BEFORE, CERT_NOT_AFTER) - ); - } - - #[test] - fn something_that_is_not_a_certificate_is_an_error() { - assert!(cert_validity("not a certificate").is_err()); - assert!(cert_validity("").is_err()); - } - #[test] fn a_ninety_day_certificate_is_due_from_day_forty_five() { let (nb, na) = (CERT_NOT_BEFORE, CERT_NOT_AFTER); diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 9c99a446..1af05c73 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -171,8 +171,9 @@ async fn main() -> Result<()> { ) .ok() .map(|c| Box::new(c) as Box); - let _jobq_metrics_provider = + let jobq_metrics_provider = hive_jobq_metrics::spawn_exporter(Arc::clone(&jobq), "swarm-controller", jobq_http_client); + register_cert_gauges(jobq_metrics_provider.as_ref()); let webhook_secret = load_webhook_secret(); @@ -243,6 +244,16 @@ async fn main() -> Result<()> { serve(listener, state).await } +/// Registers the `bao.client_cert.*` gauges for the controller's own store +/// certificate (`controller.pem`) on the jobq exporter, the one provider built +/// at startup rather than on first use. Nothing when no exporter is configured. +fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterProvider>) { + use opentelemetry::metrics::MeterProvider as _; + if let Some(provider) = provider { + swarm_cert_metrics::register(&provider.meter("hyperhive.bao_client_cert"), "controller"); + } +} + /// Start taking the queued review requests and answering the notification /// router's, then serve `state` on `listener`. async fn serve(listener: tokio::net::UnixListener, state: AppState) -> Result<()> { diff --git a/swarm-review-classifier/Cargo.toml b/swarm-review-classifier/Cargo.toml index 5631287e..acf91af0 100644 --- a/swarm-review-classifier/Cargo.toml +++ b/swarm-review-classifier/Cargo.toml @@ -25,6 +25,8 @@ opentelemetry-otlp.workspace = true serde.workspace = true serde_json.workspace = true # The try-out subject and the shapes the controller sends and reads back. +# The `bao.client_cert.*` gauges on this exporter. +swarm-cert-metrics.workspace = true swarm-queue-client.workspace = true swarm-wire.workspace = true # The API key and queue client secret paths, and the store login. diff --git a/swarm-review-classifier/src/cost.rs b/swarm-review-classifier/src/cost.rs index 2db04f56..a0cc8203 100644 --- a/swarm-review-classifier/src/cost.rs +++ b/swarm-review-classifier/src/cost.rs @@ -171,6 +171,10 @@ fn build() -> Option<(SdkMeterProvider, Counters)> { .build(); let provider = SdkMeterProvider::builder().with_reader(reader).build(); let counters = counters(&provider); + swarm_cert_metrics::register( + &provider.meter("hyperhive.review_classifier"), + "review-classifier", + ); Some((provider, counters)) } diff --git a/swarm-secret-client/Cargo.toml b/swarm-secret-client/Cargo.toml index 96a37bc3..ed694402 100644 --- a/swarm-secret-client/Cargo.toml +++ b/swarm-secret-client/Cargo.toml @@ -30,6 +30,8 @@ thiserror.workspace = true tokio.workspace = true tracing.workspace = true vaultrs.workspace = true +# `cert::validity` decodes a client certificate's PEM for its validity window. +x509-cert.workspace = true [lints] workspace = true diff --git a/swarm-secret-client/src/cert.rs b/swarm-secret-client/src/cert.rs new file mode 100644 index 00000000..e27b5490 --- /dev/null +++ b/swarm-secret-client/src/cert.rs @@ -0,0 +1,109 @@ +//! A client certificate's validity window, read from the PEM itself. +//! +//! The window is the certificate's own `notBefore`/`notAfter`, the same two +//! timestamps the store checks at login, so a caller that asks "how much of +//! its life is left" and the store never disagree about the answer. + +use std::path::Path; + +use x509_cert::der::DecodePem as _; + +use crate::{Error, client::ENV_CLIENT_CERT}; + +/// The gauge a service exports for its client certificate's `notBefore`, unix +/// seconds. The alert rule `hh-bao-client-cert-lifetime` selects this name. +pub const NOT_BEFORE_METRIC: &str = "bao.client_cert.not_before"; +/// The gauge for the certificate's `notAfter`, unix seconds. +pub const NOT_AFTER_METRIC: &str = "bao.client_cert.not_after"; + +/// A certificate's validity window, `(notBefore, notAfter)` in unix seconds. +/// +/// # Errors +/// [`Error::Certificate`] when `pem` is not one PEM-encoded X.509 certificate. +pub fn validity(pem: &str) -> Result<(i64, i64), Error> { + let cert = x509_cert::Certificate::from_pem(pem.as_bytes()) + .map_err(|e| Error::Certificate(e.to_string()))?; + let validity = &cert.tbs_certificate.validity; + let secs = |t: x509_cert::time::Time| { + i64::try_from(t.to_unix_duration().as_secs()).unwrap_or(i64::MAX) + }; + Ok((secs(validity.not_before), secs(validity.not_after))) +} + +/// [`validity`] of the client certificate file at `path` (the file +/// [`ENV_CLIENT_CERT`] names). +/// +/// # Errors +/// [`Error::Identity`] when the file cannot be read, [`Error::Certificate`] +/// when it is not a certificate. +pub fn file_validity(path: &Path) -> Result<(i64, i64), Error> { + let pem = std::fs::read_to_string(path).map_err(|source| Error::Identity { + var: ENV_CLIENT_CERT, + path: path.display().to_string(), + source, + })?; + validity(&pem) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Self-signed, CN `hive-agent-atlas`, valid 2026-01-01T00:00:00Z to + /// 2026-04-01T00:00:00Z. Its key was discarded. + pub(crate) const CERT_PEM: &str = "-----BEGIN CERTIFICATE----- +MIIBizCCATGgAwIBAgIUSFpuYmy1UoHvq/MYiyrAudRaWXcwCgYIKoZIzj0EAwIw +GzEZMBcGA1UEAwwQaGl2ZS1hZ2VudC1hdGxhczAeFw0yNjAxMDEwMDAwMDBaFw0y +NjA0MDEwMDAwMDBaMBsxGTAXBgNVBAMMEGhpdmUtYWdlbnQtYXRsYXMwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAAR7p2eZAxjiE1RdsuSHE3CcTjbnd4swzfnmqUPW +9NJN7mvOtqEzxCKfOSFjaIhhyShIQ41/V6vGi80EPDB+uR56o1MwUTAdBgNVHQ4E +FgQUxgZFsN1VY3ODrPS1NUfY/x3EnmswHwYDVR0jBBgwFoAUxgZFsN1VY3ODrPS1 +NUfY/x3EnmswDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAnU9e +WOioNTUNK9b6FignejpK5FpyrODUuH/iL4TqCh0CIAz6GzJeLebhRPvYlfVUofdV +hWx3sOwmUgjkRQXoxY+p +-----END CERTIFICATE----- +"; + pub(crate) const NOT_BEFORE: i64 = 1_767_225_600; + pub(crate) const NOT_AFTER: i64 = 1_775_001_600; + + #[test] + fn the_window_is_read_from_the_certificate() { + assert_eq!( + validity(CERT_PEM).expect("a well-formed certificate"), + (NOT_BEFORE, NOT_AFTER) + ); + } + + #[test] + fn something_that_is_not_a_certificate_is_an_error() { + assert!(matches!( + validity("not a certificate"), + Err(Error::Certificate(_)) + )); + assert!(validity("").is_err()); + } + + #[test] + fn a_file_is_read_through_its_path() { + let dir = + std::env::temp_dir().join(format!("swarm-secret-client-cert-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("scratch dir"); + let path = dir.join("cert.pem"); + std::fs::write(&path, CERT_PEM).expect("write"); + assert_eq!( + file_validity(&path).expect("readable"), + (NOT_BEFORE, NOT_AFTER) + ); + std::fs::remove_dir_all(&dir).expect("cleanup"); + } + + #[test] + fn a_missing_file_names_the_variable_and_path() { + let err = file_validity(Path::new("/nonexistent/cert.pem")).expect_err("no such file"); + assert!( + matches!(&err, Error::Identity { var, path, .. } + if *var == ENV_CLIENT_CERT && path == "/nonexistent/cert.pem"), + "{err}" + ); + } +} diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 1a53f597..754683bf 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -31,6 +31,7 @@ pub mod acp; pub mod builder; +pub mod cert; pub mod ci; pub mod ci_metrics; pub mod claude_login; @@ -103,6 +104,10 @@ pub enum Error { /// identity needs. Names the field, never its value. #[error("the store issued a certificate whose {0} is empty")] IncompleteIssue(&'static str), + + /// The text was not one PEM-encoded X.509 certificate. + #[error("decoding the certificate: {0}")] + Certificate(String), } impl From for Error {