swarm-metrics reads the swarm's VictoriaMetrics from inside an agent
container: query, query-range, series, labels and label-values, each one
GET against the store's Prometheus read API through the gateway, printing
the store's JSON. It authenticates the way swarm-logs does, with the
agent's own queue secret read from bao under the agent's leaf, sent as
a swarm-agent bearer.
On the metrics store's host the read API's auth map gains an agent arm
ahead of the Grafana one. It routes to swarm-metrics-agent-auth, a
swarm-logs-auth instance logged in under the existing swarm-metrics-auth
leaf, whose bao grant now includes swarm/agents/+/queue read. The check is
rate-limited per address like the log store's
(deploy.victoriametrics.agentAuth{Rate,Burst,VerdictTtl}).
nix/agent-modules/metrics.nix installs the wrapped binary when the hive
forwards HIVE_AGENT_METRICS_URL (from swarm.victoriametrics.domain) and a
store address, the same gate as swarm-logs. Adds the base:swarm-metrics
skill and the CLI reference with its freshness check.
Fixes #5395
149 lines
5.9 KiB
Rust
149 lines
5.9 KiB
Rust
//! The credential this CLI presents to the metrics store's gateway: the
|
|
//! agent's own queue secret, read from the swarm secret store with the agent's
|
|
//! store certificate.
|
|
//!
|
|
//! The same read and the same bearer as `swarm-logs`' `auth` module: path
|
|
//! `swarm/agents/<agent>/queue`, the agent's cert-auth role, and
|
|
//! `Bearer swarm-agent.<agent>.<secret>` spelled by
|
|
//! `swarm_queue_client::agent_token`. On the metrics store's host a
|
|
//! `swarm-logs-auth` instance checks it against the store.
|
|
//!
|
|
//! 🩸 The secret is never logged and never rendered into an error.
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings};
|
|
use swarm_secret_client::queue::{self, AgentCredential};
|
|
use swarm_secret_client::{SecretStore, policy};
|
|
|
|
/// The metrics store's gateway origin; the API paths are appended to it.
|
|
const ENV_URL: &str = "HIVE_AGENT_METRICS_URL";
|
|
/// This agent's name, which the store path and the role are built from.
|
|
const ENV_AGENT_NAME: &str = "HIVE_AGENT_NAME";
|
|
/// The store's address; the rest of its coordinates are `Settings`'s.
|
|
const ENV_ADDR: &str = "BAO_ADDR";
|
|
/// The store's CA bundle, which may name an undelivered credential.
|
|
const ENV_CACERT: &str = "BAO_CACERT";
|
|
|
|
/// Where this CLI queries, and who it queries as.
|
|
pub struct Config {
|
|
/// The metrics store's gateway origin, e.g. `https://metrics.example`.
|
|
pub base_url: String,
|
|
agent: String,
|
|
settings: Settings,
|
|
role: String,
|
|
path: String,
|
|
}
|
|
|
|
impl Config {
|
|
/// Read the configuration out of the environment the wrapper sets.
|
|
///
|
|
/// Errors rather than degrading: an empty result would read as "no such
|
|
/// series".
|
|
pub fn from_env() -> Result<Self> {
|
|
Self::from_lookup(|k| std::env::var(k).ok())
|
|
}
|
|
|
|
fn from_lookup(get: impl Fn(&str) -> Option<String>) -> Result<Self> {
|
|
let set = |k: &str| get(k).filter(|v| !v.is_empty());
|
|
let Some(base_url) = set(ENV_URL) else {
|
|
bail!("this agent has no swarm metrics store configured: {ENV_URL} is unset");
|
|
};
|
|
if set(ENV_ADDR).is_none() {
|
|
bail!(
|
|
"{ENV_URL} is set but {ENV_ADDR} is not, so there is no secret store \
|
|
to read this agent's credential from — this is a deployment bug"
|
|
);
|
|
}
|
|
let agent =
|
|
set(ENV_AGENT_NAME).with_context(|| format!("{ENV_AGENT_NAME} is unset or empty"))?;
|
|
let settings =
|
|
Settings::from_lookup(|k| set(k).filter(|v| k != ENV_CACERT || ca_is_usable(v)))
|
|
.context("reading the swarm secret store's coordinates from the environment")?;
|
|
Ok(Self {
|
|
base_url,
|
|
role: policy::agent_object_name(&agent)?,
|
|
path: queue::agent_queue_path(&agent)?,
|
|
agent,
|
|
settings,
|
|
})
|
|
}
|
|
|
|
/// Read this agent's secret from the store and spell it as the bearer.
|
|
///
|
|
/// Read on every call, so a caller that was refused can call again and
|
|
/// present a secret re-minted since.
|
|
pub fn bearer(&self) -> Result<String> {
|
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
|
.enable_all()
|
|
.build()
|
|
.context("starting the runtime for the secret store read")?;
|
|
let stored: Option<AgentCredential> = runtime.block_on(async {
|
|
SecretStore::connect(&self.settings, &self.role, DEFAULT_CERT_MOUNT)
|
|
.await
|
|
.context("logging in to the swarm secret store with this agent's certificate")?
|
|
.read_optional(&self.path)
|
|
.await
|
|
.with_context(|| format!("reading {}", self.path))
|
|
})?;
|
|
let stored = stored.with_context(|| format!("nothing is stored at {}", self.path))?;
|
|
swarm_queue_client::agent_token::format_agent_token(&self.agent, &stored.value)
|
|
.with_context(|| format!("spelling the credential stored at {}", self.path))
|
|
}
|
|
}
|
|
|
|
/// Whether the CA bundle at `path` is a file with bytes in it. `BAO_CACERT`
|
|
/// names a systemd credential that may not have been delivered; absent means
|
|
/// the container's own trust store.
|
|
fn ca_is_usable(path: &str) -> bool {
|
|
std::fs::metadata(path).is_ok_and(|m| m.len() > 0)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
|
|
let pairs: Vec<(String, String)> = pairs
|
|
.iter()
|
|
.map(|(k, v)| ((*k).to_owned(), (*v).to_owned()))
|
|
.collect();
|
|
move |k| pairs.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone())
|
|
}
|
|
|
|
const FULL: &[(&str, &str)] = &[
|
|
(ENV_URL, "https://metrics.example"),
|
|
(ENV_AGENT_NAME, "atlas"),
|
|
(ENV_ADDR, "https://bao.example:8200"),
|
|
("BAO_CLIENT_CERT", "/nonexistent/cert"),
|
|
("BAO_CLIENT_KEY", "/nonexistent/key"),
|
|
];
|
|
|
|
#[test]
|
|
fn no_url_says_there_is_no_metrics_store() {
|
|
let err = Config::from_lookup(env(&[])).err().expect("refused");
|
|
assert!(err.to_string().contains("no swarm metrics store"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_url_without_a_store_is_a_deployment_bug() {
|
|
let err = Config::from_lookup(env(&FULL[..2])).err().expect("refused");
|
|
assert!(err.to_string().contains("deployment bug"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_missing_agent_name_is_named() {
|
|
let mut pairs = FULL.to_vec();
|
|
pairs.retain(|(k, _)| *k != ENV_AGENT_NAME);
|
|
let err = Config::from_lookup(env(&pairs)).err().expect("refused");
|
|
assert!(err.to_string().contains(ENV_AGENT_NAME), "{err}");
|
|
}
|
|
|
|
/// The path and role are the agent's own, the ones its policy grants.
|
|
#[test]
|
|
fn the_read_is_the_agents_own_queue_secret() {
|
|
let cfg = Config::from_lookup(env(FULL)).expect("configured");
|
|
assert_eq!(cfg.base_url, "https://metrics.example");
|
|
assert_eq!(cfg.path, "swarm/agents/atlas/queue");
|
|
assert_eq!(cfg.role, policy::agent_object_name("atlas").expect("valid"));
|
|
}
|
|
}
|