Watch
0
0
Fork
You've already forked hyperhive
0

mint per-agent proxy credentials and a per-hive squid self-check credential

This commit is contained in:
atlas 2026-10-28 13:02:25 +01:00 • committed by mara
commit ec06a91deb
7 changed files with 487 additions and 4 deletions

View file

@ -486,6 +486,10 @@ let
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" {
capabilities = ["create", "read"]
}
${lib.concatMapStrings readStanza (
map (leaf: "${credentialMountPath}/data/${leaf}") atticFactLeaves
)}

View file

@ -1724,17 +1724,20 @@ let
{
# `matrix_account::hive_sender` writes every hive's sender token and
# `forge::hive_token` every hive's forge token, and nothing else under
# `hives/`, where each hive's queue credential also lives. `+` is one
# `hives/`, where each hive's queue credential also lives. `proxy_credential`
# creates each hive's squid self-check credential and reads it back, and
# has no `update`, so it cannot replace one. `+` is one
# segment, which keeps each stanza to its one leaf; a `*` is a glob only
# at the end of a path. Pinned as whole stanzas, so an added capability
# fails.
name = "the controller writes each hive's sender and forge tokens and nothing else under hives";
name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, and nothing else under hives";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/+/matrix/sender-token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/hives/+/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives" s);
}

View file

@ -60,6 +60,7 @@ mod meta_flake;
mod notify_flows;
mod notify_router_api;
mod otel_http_client;
mod proxy_credential;
mod pulls;
mod queue_identity;
mod read_policy;

View file

@ -0,0 +1,320 @@
//! The credentials squid authenticates its clients with: one per agent, at
//! `swarm_secret_client::proxy::agent_credential_path`, and one per hive for
//! that hive's squid self-check, at `proxy::selfcheck_path`.
//!
//! Minted here because the agent's store policy is read-only and a hive's
//! policy cannot create under another principal's prefix. [`spawn`]'s pass runs
//! at start and every [`RECONCILE_INTERVAL`] over every agent that holds a store
//! identity, so an agent that existed before this pass was added gets one on the
//! next tick, with no restart.
//!
//! ⚠️ **Never replaced.** A path that holds anything is left alone, and the write
//! is a `cas=0` create. Replacing a credential is a rotation: squid's users file
//! is rendered from whatever the store holds, and a rotation has to reach every
//! client that carries the old one in the same step.
use anyhow::{Context, Result};
use serde::de::IgnoredAny;
use swarm_secret_client::{Error, SecretStore, client::DEFAULT_CERT_MOUNT, policy, proxy};
/// How often [`spawn`] re-checks every agent and hive.
const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5);
/// What [`ensure`] did at one path.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Outcome {
/// The path already held a credential.
Present,
/// This pass wrote one.
Minted,
/// Another writer's create landed between this pass's read and its own.
LostRace,
}
/// The two store calls [`ensure`] makes, so a test can stand in for the store.
pub(crate) trait CredentialStore {
/// Whether anything is stored at `path`, as [`SecretStore::read_optional`]
/// answers it. The value is decoded as [`IgnoredAny`] and never held.
fn exists(&self, path: &str) -> impl Future<Output = Result<bool, Error>> + Send;
/// As [`SecretStore::create`].
fn create(
&self,
path: &str,
value: &proxy::Credential,
) -> impl Future<Output = Result<(), Error>> + Send;
}
impl CredentialStore for SecretStore {
async fn exists(&self, path: &str) -> Result<bool, Error> {
Ok(self.read_optional::<IgnoredAny>(path).await?.is_some())
}
async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> {
SecretStore::create(self, path, value).await
}
}
/// Make sure `path` holds a credential, writing one only if it holds nothing.
///
/// A create that fails is read back: if the path now holds something, a
/// concurrent writer won and that is the credential.
///
/// # Errors
/// When the store refuses the read, or the create fails and the path is still
/// empty.
pub(crate) async fn ensure(store: &impl CredentialStore, path: &str) -> Result<Outcome> {
if store
.exists(path)
.await
.with_context(|| format!("reading {path}"))?
{
return Ok(Outcome::Present);
}
let credential = proxy::Credential {
value: crate::agent_identity::generate_queue_secret()?,
};
let Err(e) = store.create(path, &credential).await else {
return Ok(Outcome::Minted);
};
if store
.exists(path)
.await
.with_context(|| format!("reading {path} back after a failed create"))?
{
return Ok(Outcome::LostRace);
}
Err(anyhow::Error::new(e).context(format!("creating {path}")))
}
/// [`ensure`] at each of `paths`, which `what` names in the log. Returns how
/// many it minted. A failure at one path is logged and the rest still run.
async fn ensure_all(
store: &impl CredentialStore,
what: &str,
paths: Vec<(String, Result<String, Error>)>,
) -> usize {
let mut minted = 0;
for (name, path) in paths {
let path = match path {
Ok(path) => path,
Err(e) => {
tracing::warn!(name, error = %e, "{what}: unusable name");
continue;
}
};
match ensure(store, &path).await {
Ok(Outcome::Present) => {}
Ok(Outcome::Minted) => {
minted += 1;
tracing::info!(name, %path, "{what} minted");
}
Ok(Outcome::LostRace) => {
tracing::info!(name, %path, "{what}: another writer created it first");
}
Err(e) => tracing::warn!(name, error = %format!("{e:#}"), "{what}: mint failed"),
}
}
minted
}
/// One pass: a credential for every agent in `agents` and a self-check
/// credential for every hive in `hives`. Returns how many were minted.
async fn mint_missing(store: &impl CredentialStore, agents: &[String], hives: &[String]) -> usize {
let for_agents = agents
.iter()
.map(|a| (a.clone(), proxy::agent_credential_path(a)))
.collect();
let for_hives = hives
.iter()
.map(|h| (h.clone(), proxy::selfcheck_path(h)))
.collect();
ensure_all(store, "agent proxy credential", for_agents).await
+ ensure_all(store, "hive squid self-check credential", for_hives).await
}
/// The agents that hold a store identity: exactly the ones whose hive reads
/// what a pass writes.
async fn roster(store: &SecretStore) -> Result<Vec<String>> {
let roles = store
.list_cert_roles(DEFAULT_CERT_MOUNT)
.await
.context("listing the store's cert-auth roles")?;
Ok(policy::agents_from_role_names(&roles))
}
/// Run a pass now and every [`RECONCILE_INTERVAL`] after. A pass that fails is
/// logged and retried on the next tick; it never stops the daemon.
pub fn spawn(hives: Vec<String>) {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
loop {
ticker.tick().await;
let pass = async {
let store = crate::store::connect()
.await
.context("logging in to the swarm secret store")?;
let agents = roster(&store).await?;
anyhow::Ok((agents.len(), mint_missing(&store, &agents, &hives).await))
};
match pass.await {
Ok((agents, minted)) => {
tracing::debug!(agents, minted, "proxy credentials: pass done");
}
Err(e) => tracing::warn!(
error = %format!("{e:#}"),
retry_in_s = RECONCILE_INTERVAL.as_secs(),
"proxy credentials: pass failed; retrying next tick"
),
}
}
});
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::Mutex;
use super::*;
/// An in-memory store that refuses a create over a held path the way a
/// `cas=0` write does, and counts the creates it was asked for.
#[derive(Default)]
struct Fake {
held: Mutex<HashMap<String, String>>,
creates: Mutex<usize>,
/// A path whose create fails and leaves nothing behind.
broken: Option<String>,
/// Makes the next create find the path taken by someone else.
race: Option<String>,
}
impl Fake {
fn holding(path: &str, value: &str) -> Self {
let fake = Self::default();
fake.held
.lock()
.unwrap()
.insert(path.to_owned(), value.to_owned());
fake
}
fn value(&self, path: &str) -> Option<String> {
self.held.lock().unwrap().get(path).cloned()
}
}
fn refused() -> Error {
Error::MissingEnv("BAO_ADDR")
}
impl CredentialStore for Fake {
async fn exists(&self, path: &str) -> Result<bool, Error> {
Ok(self.held.lock().unwrap().contains_key(path))
}
async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> {
*self.creates.lock().unwrap() += 1;
let mut held = self.held.lock().unwrap();
if self.broken.as_deref() == Some(path) {
return Err(refused());
}
if let Some(winner) = &self.race {
held.insert(path.to_owned(), winner.clone());
return Err(refused());
}
if held.contains_key(path) {
return Err(refused());
}
held.insert(path.to_owned(), value.value.clone());
Ok(())
}
}
const ATLAS: &str = "swarm/agents/atlas/proxy-credential";
const ARGUS: &str = "swarm/agents/argus/proxy-credential";
const H1: &str = "swarm/hives/h1/squid-selfcheck";
fn names(names: &[&str]) -> Vec<String> {
names.iter().map(|n| (*n).to_owned()).collect()
}
#[tokio::test]
async fn an_absent_credential_is_created() {
let store = Fake::default();
assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Minted);
let value = store.value(ATLAS).expect("written");
// 32 bytes, base64url without padding.
assert_eq!(value.len(), 43, "{value}");
assert!(
value
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
);
}
#[tokio::test]
async fn a_present_credential_is_untouched_and_never_updated() {
let store = Fake::holding(ATLAS, "existing");
assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Present);
assert_eq!(store.value(ATLAS).as_deref(), Some("existing"));
assert_eq!(*store.creates.lock().unwrap(), 0, "no write was attempted");
}
#[tokio::test]
async fn a_lost_race_keeps_the_winners_value() {
let store = Fake {
race: Some("winner".to_owned()),
..Fake::default()
};
assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::LostRace);
assert_eq!(store.value(ATLAS).as_deref(), Some("winner"));
}
#[tokio::test]
async fn a_failed_create_with_nothing_behind_it_is_an_error() {
let store = Fake {
broken: Some(ATLAS.to_owned()),
..Fake::default()
};
assert!(ensure(&store, ATLAS).await.is_err());
assert_eq!(store.value(ATLAS), None);
}
#[tokio::test]
async fn a_pass_mints_for_every_agent_and_hive_that_lacks_one() {
let store = Fake::holding(ARGUS, "existing");
let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await;
assert_eq!(minted, 2);
assert!(store.value(ATLAS).is_some());
assert!(store.value(H1).is_some());
assert_eq!(store.value(ARGUS).as_deref(), Some("existing"));
// Control: the second pass has nothing left to mint, and writes nothing.
let creates = *store.creates.lock().unwrap();
assert_eq!(
mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await,
0
);
assert_eq!(*store.creates.lock().unwrap(), creates);
}
#[tokio::test]
async fn one_agent_failing_does_not_stop_the_rest() {
let store = Fake {
broken: Some(ATLAS.to_owned()),
..Fake::default()
};
let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await;
assert_eq!(minted, 2);
assert!(store.value(ARGUS).is_some());
assert!(store.value(H1).is_some());
}
#[tokio::test]
async fn an_unusable_agent_name_mints_nothing() {
let store = Fake::default();
assert_eq!(mint_missing(&store, &names(&["../argus"]), &[]).await, 0);
assert!(store.held.lock().unwrap().is_empty());
}
}

View file

@ -368,7 +368,7 @@ fn spawn_subagent_streams(status: Option<&Arc<status::StatusReader>>, hives: &[H
/// The per-agent background passes: subagent and terminal streams, and
/// schedule keys, and the permissions bucket and its import, and the
/// archetypes agents are created from. Beside them, the
/// copy of the cache facts to each hive.
/// copy of the cache facts to each hive, and the proxy credentials squid checks.
/// Lifted out of `main` for `clippy::too_many_lines`.
pub(crate) fn spawn_per_agent_upkeep(
status: Option<&Arc<status::StatusReader>>,
@ -378,6 +378,7 @@ pub(crate) fn spawn_per_agent_upkeep(
spawn_schedule_reconcile(status, hives);
spawn_agent_permissions(status, hives);
crate::attic_copy::spawn(hives.iter().map(|h| h.name.clone()).collect());
crate::proxy_credential::spawn(hives.iter().map(|h| h.name.clone()).collect());
if let Some(status) = status {
archetypes::spawn(status.queue_client());
}

View file

@ -7,7 +7,7 @@
//! of secret, the path it lives at together with the fields it holds
//! ([`matrix`], [`queue`], [`mtls`], [`forge`], [`github`], [`grafana`],
//! [`acp`], [`attic`], [`backend_env`], [`ci`], [`review_classifier`], [`builder`], [`ci_metrics`],
//! [`claude_login`]).
//! [`claude_login`], [`proxy`]).
//! Each of those is a thing the controller and a hive must say identically, so
//! it is said once here.
//!
@ -47,6 +47,7 @@ pub mod matrix;
pub mod mtls;
pub mod path;
pub mod policy;
pub mod proxy;
pub mod queue;
pub mod review_classifier;
pub mod startup_read;

View file

@ -0,0 +1,153 @@
//! The egress-proxy agreement: the credentials squid authenticates its clients
//! with, where they live in the store, and what the objects at those paths hold.
//!
//! Each agent has one ([`agent_credential_path`]), minted by `swarm-controller`
//! and read by the agent's hive, which turns it into the hash squid's users file
//! holds. Each hive has one more ([`selfcheck_path`]) for its own squid
//! self-check, so the self-check can prove an authenticated request is let
//! through. Neither end is senior, so both halves are stated once, here, beside
//! [`crate::forge`].
use serde::{Deserialize, Serialize};
use crate::{
Error,
path::{Kind, principal_prefix},
};
/// The name of an agent's leaf.
pub const AGENT_LEAF: &str = "proxy-credential";
/// The name of a hive's self-check leaf.
pub const SELFCHECK_LEAF: &str = "squid-selfcheck";
/// The squid user the self-check logs in as.
///
/// An agent's squid user is its name, and agent names cannot hold a `.`
/// ([`crate::path::checked_segment`]), so this one cannot be an agent's.
/// `nix/host-modules/hive-squid.nix` spells the same string.
pub const SELFCHECK_USER: &str = "selfcheck.squid";
/// The path holding the secret `agent` authenticates to the egress proxy with.
///
/// A flat leaf under the agent's prefix, like
/// [`crate::forge::agent_token_path`], so the agent's own read stanza
/// ([`crate::policy::render_shared_agent`]) already covers it, and so does the
/// hive's.
///
/// # Errors
/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`,
/// which is what keeps one agent's name from addressing another agent's secret.
pub fn agent_credential_path(agent: &str) -> Result<String, Error> {
let prefix = principal_prefix(Kind::Agent, agent)?;
Ok(format!("{prefix}/{AGENT_LEAF}"))
}
/// The path holding the secret `hive`'s own squid self-check authenticates with.
///
/// Under [`Kind::Hive`], so the grant that reaches it is the hive's own
/// `swarm/hives/<this hive>/*` stanza: hive `alpha` reads its own and gets a 403
/// on `beta`'s.
///
/// # Errors
/// [`Error::PathSegment`] when `hive` contains anything but `[A-Za-z0-9_-]`.
pub fn selfcheck_path(hive: &str) -> Result<String, Error> {
let prefix = principal_prefix(Kind::Hive, hive)?;
Ok(format!("{prefix}/{SELFCHECK_LEAF}"))
}
/// What [`agent_credential_path`] and [`selfcheck_path`] hold.
///
/// No `Debug` derive: `value` is a live credential, and a derived `Debug` is
/// one `{:?}` away from a log line.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Credential {
/// The secret: 32 random bytes, base64url without padding, so it can sit in
/// a proxy URL's userinfo unescaped.
pub value: String,
}
impl std::fmt::Debug for Credential {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Credential")
.field("value", &"<redacted>")
.finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn covered_by(policy: &str, path: &str) -> bool {
policy.lines().any(|line| {
line.strip_prefix("path \"")
.and_then(|rest| rest.split_once("\" {"))
.and_then(|(p, _)| p.strip_suffix('*'))
.is_some_and(|prefix| format!("secret/data/{path}").starts_with(prefix))
})
}
#[test]
fn the_paths_are_the_leaves_the_nix_side_spells() {
assert_eq!(
agent_credential_path("atlas").expect("legal"),
"swarm/agents/atlas/proxy-credential"
);
assert_eq!(
selfcheck_path("h1").expect("legal"),
"swarm/hives/h1/squid-selfcheck"
);
}
#[test]
fn a_traversal_in_either_name_is_refused() {
for bad in ["../argus", "a/b", "a.b", ""] {
assert!(agent_credential_path(bad).is_err(), "{bad:?}");
assert!(selfcheck_path(bad).is_err(), "{bad:?}");
}
assert!(agent_credential_path("a-b_C9").is_ok());
assert!(selfcheck_path("a-b_C9").is_ok());
}
#[test]
fn the_agent_reads_its_own_credential_and_a_hive_reads_every_one() {
let path = agent_credential_path("atlas").expect("legal");
assert!(covered_by(&crate::policy::effective_agent("atlas"), &path));
assert!(!covered_by(&crate::policy::effective_agent("argus"), &path));
assert!(covered_by(&crate::policy::effective_hive("h1"), &path));
}
#[test]
fn a_hive_reads_its_own_selfcheck_credential_and_no_other_hives() {
let path = selfcheck_path("h1").expect("legal");
assert!(covered_by(&crate::policy::effective_hive("h1"), &path));
assert!(!covered_by(&crate::policy::effective_hive("h2"), &path));
assert!(!covered_by(&crate::policy::effective_agent("atlas"), &path));
}
#[test]
fn the_selfcheck_user_is_not_a_legal_agent_name() {
assert!(crate::path::checked_segment("agent", SELFCHECK_USER).is_err());
}
#[test]
fn debug_hides_the_value() {
let shown = format!(
"{:?}",
Credential {
value: "s3cret".to_owned()
}
);
assert!(!shown.contains("s3cret"), "{shown}");
}
#[test]
fn the_value_field_is_named_value() {
let json = serde_json::to_string(&Credential {
value: "t".to_owned(),
})
.expect("a String serialises");
assert_eq!(json, r#"{"value":"t"}"#);
}
}