From 04598542b627327bb57e747eba91c279175a16c6 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 17:40:09 +0100 Subject: [PATCH 1/7] logs and metrics permissions, tolerant record decode, hive capabilities --- Cargo.lock | 1 + hive-c0re/src/swarm_status.rs | 37 +++++++++++- hive-types/Cargo.toml | 1 + hive-types/src/lib.rs | 2 +- hive-types/src/permission.rs | 94 ++++++++++++++++++++++++++++- swarm-wire/src/agent_permissions.rs | 25 +++++++- swarm-wire/src/hive_status.rs | 73 ++++++++++++++++++++++ swarm-wire/src/lib.rs | 1 + 8 files changed, 226 insertions(+), 8 deletions(-) create mode 100644 swarm-wire/src/hive_status.rs diff --git a/Cargo.lock b/Cargo.lock index 8565cce9f..d24d8d0e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2092,6 +2092,7 @@ version = "0.1.0" dependencies = [ "serde", "serde_json", + "tracing", ] [[package]] diff --git a/hive-c0re/src/swarm_status.rs b/hive-c0re/src/swarm_status.rs index 33d191c77..48fb9aa0e 100644 --- a/hive-c0re/src/swarm_status.rs +++ b/hive-c0re/src/swarm_status.rs @@ -396,15 +396,29 @@ pub(crate) fn queue_first_deploy( coord.job_queue.insert_first_deploy_once(agent) } -/// Offer one snapshot: this hive's current readiness, under its own key. +/// The snapshot a hive offers: its readiness, and what it can do (`swarm_wire::hive_status`). +#[derive(serde::Serialize)] +struct Snapshot { + #[serde(flatten)] + readiness: crate::warnings::Readiness, + capabilities: &'static [&'static str], +} + +fn snapshot() -> Snapshot { + Snapshot { + readiness: crate::warnings::readiness(), + capabilities: &[swarm_wire::hive_status::TOLERANT_PERMISSIONS], + } +} + +/// Offer one snapshot under this hive's own key. async fn publish(client: &async_nats::Client, hive: &str) -> Result<()> { // An unconnected client does not fail a JetStream request, it hangs // on it — which here would hang the loop, shutdown arm included. swarm_queue_client::ensure_connected(client)?; let store = swarm_queue_client::status::open_or_create(client).await?; - let payload = - serde_json::to_vec(&crate::warnings::readiness()).context("serialising the readiness")?; + let payload = serde_json::to_vec(&snapshot()).context("serialising the snapshot")?; store .put(hive, payload.into()) .await @@ -427,3 +441,20 @@ mod rev_marker_tests { assert!(!marker_holds(&marker, "bbb"), "another rev still deploys"); } } + +#[cfg(test)] +mod snapshot_tests { + use super::snapshot; + + /// The controller reads the capability out of the published bytes with + /// the wire type, and the readiness keeps its own top-level fields. + #[test] + fn the_snapshot_advertises_tolerant_permissions_beside_the_readiness() { + let value = serde_json::to_value(snapshot()).unwrap(); + assert!( + swarm_wire::hive_status::Capabilities::of_snapshot(&value) + .has(swarm_wire::hive_status::TOLERANT_PERMISSIONS) + ); + assert!(value.get("status").is_some() && value.get("warnings").is_some()); + } +} diff --git a/hive-types/Cargo.toml b/hive-types/Cargo.toml index cea91cd55..f796ffd06 100644 --- a/hive-types/Cargo.toml +++ b/hive-types/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] serde.workspace = true +tracing.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/hive-types/src/lib.rs b/hive-types/src/lib.rs index 84d834d42..bc7d84dcf 100644 --- a/hive-types/src/lib.rs +++ b/hive-types/src/lib.rs @@ -9,7 +9,7 @@ mod permission; -pub use permission::Permission; +pub use permission::{Permission, deserialize_known}; pub mod review; pub mod term_msg; diff --git a/hive-types/src/permission.rs b/hive-types/src/permission.rs index 83395480d..eeb796a2e 100644 --- a/hive-types/src/permission.rs +++ b/hive-types/src/permission.rs @@ -38,6 +38,15 @@ pub enum Permission { /// thing to manage. Kept under that spelling because every stored grant /// carries it. ManageRootAgent, + /// Read access to the log store: what `swarm-logs` queries. Checked by + /// the log gateway's `swarm-logs-auth` against the grant swarm-controller + /// mirrors into the secret store, so `tools()` is empty. Not in + /// [`Permission::DEFAULT`]. + Logs, + /// Read access to the metrics store: what `swarm-metrics` queries. + /// Checked like [`Permission::Logs`], for the metrics gateway. Not in + /// [`Permission::DEFAULT`]. + Metrics, } impl Permission { @@ -49,6 +58,8 @@ impl Permission { Self::Execution, Self::WebTools, Self::ManageRootAgent, + Self::Logs, + Self::Metrics, ]; /// What an agent the swarm sets nothing for holds, whatever its name. @@ -69,8 +80,13 @@ impl Permission { // — an out-of-process server has no later enforcement point, so // that gate is the actual security boundary. `WebTools` grants // Claude built-ins (`builtin_tools()`); `ManageRootAgent` grants - // bind mounts. - Self::Execution | Self::WebTools | Self::ManageRootAgent => &[], + // bind mounts; `Logs` and `Metrics` are checked by the store + // gateways, not by a tool allowlist. + Self::Execution + | Self::WebTools + | Self::ManageRootAgent + | Self::Logs + | Self::Metrics => &[], } } @@ -115,8 +131,43 @@ impl Permission { Self::Execution => "execution", Self::WebTools => "web_tools", Self::ManageRootAgent => "manage_root_agent", + Self::Logs => "logs", + Self::Metrics => "metrics", } } + + /// The permission stored under `name`; `None` for a name this build does + /// not know. + #[must_use] + pub fn from_name(name: &str) -> Option { + Self::ALL.iter().copied().find(|p| p.as_str() == name) + } +} + +/// Decodes a stored permission list, skipping names this build does not know. +/// For `#[serde(deserialize_with)]` on a stored record: a permission a newer +/// swarm adds must not cost an older reader the rest of the record. +/// +/// # Errors +/// +/// When the value is not a list of strings. +pub fn deserialize_known<'de, D>(d: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let names: Option> = Option::deserialize(d)?; + Ok(names.map(|names| { + names + .iter() + .filter_map(|name| { + let known = Permission::from_name(name); + if known.is_none() { + tracing::debug!(name, "skipping a permission this build does not know"); + } + known + }) + .collect() + })) } impl From for &'static str { @@ -142,11 +193,46 @@ mod tests { "inbox", "execution", "web_tools", - "manage_root_agent" + "manage_root_agent", + "logs", + "metrics" ] ); } + /// Reading stores a log or metric grant is never a default: an agent the + /// swarm sets nothing for does not hold them. + #[test] + fn logs_and_metrics_are_off_by_default() { + assert!(!Permission::DEFAULT.contains(&Permission::Logs)); + assert!(!Permission::DEFAULT.contains(&Permission::Metrics)); + assert!(Permission::ALL.contains(&Permission::Logs)); + assert!(Permission::ALL.contains(&Permission::Metrics)); + } + + #[derive(serde::Deserialize)] + struct Record { + #[serde(default, deserialize_with = "deserialize_known")] + permissions: Option>, + } + + #[test] + fn an_unknown_name_is_skipped_not_a_decode_failure() { + let r: Record = + serde_json::from_str(r#"{"permissions":["messaging","telepathy","logs"]}"#).unwrap(); + assert_eq!( + r.permissions, + Some(vec![Permission::Messaging, Permission::Logs]) + ); + // Presence control: an unset field stays unset, `[]` stays empty. + let r: Record = serde_json::from_str("{}").unwrap(); + assert_eq!(r.permissions, None); + let r: Record = serde_json::from_str(r#"{"permissions":[]}"#).unwrap(); + assert_eq!(r.permissions, Some(vec![])); + // A list of non-strings is still a decode failure. + assert!(serde_json::from_str::(r#"{"permissions":[1]}"#).is_err()); + } + /// The serde form of every variant, pinned independently of `as_str`: /// stored grants decode by it, so it must stay byte-identical. #[test] @@ -158,6 +244,8 @@ mod tests { (Permission::Execution, "\"execution\""), (Permission::WebTools, "\"web_tools\""), (Permission::ManageRootAgent, "\"manage_root_agent\""), + (Permission::Logs, "\"logs\""), + (Permission::Metrics, "\"metrics\""), ]; assert_eq!(want.len(), Permission::ALL.len()); for (perm, json) in want { diff --git a/swarm-wire/src/agent_permissions.rs b/swarm-wire/src/agent_permissions.rs index 8a8eb8c7f..def893d58 100644 --- a/swarm-wire/src/agent_permissions.rs +++ b/swarm-wire/src/agent_permissions.rs @@ -69,7 +69,13 @@ pub const IMPORT_REPLY_DEADLINE: std::time::Duration = std::time::Duration::from /// older hive the fields it knows. #[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct AgentPermissions { - #[serde(default, skip_serializing_if = "Option::is_none")] + /// Names this build does not know are skipped on decode, so a permission + /// a newer controller adds does not cost this reader the record. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "hive_types::deserialize_known" + )] pub permissions: Option>, /// Whether the agent's traffic leaves the host unfiltered. Absent is "on": /// an agent whose key predates the field keeps the open access it had, and @@ -155,6 +161,23 @@ mod tests { assert_eq!(ruth.permissions, Some(vec![Permission::ManageRootAgent])); } + /// A record naming a permission this build does not know still decodes, + /// with that name skipped and the other fields intact: the reader holds + /// what it knows instead of nothing. + #[test] + fn a_record_with_an_unknown_permission_still_decodes() { + use hive_types::Permission; + let record: AgentPermissions = serde_json::from_str( + r#"{"permissions":["messaging","from-the-future","logs"],"unrestricted_internet":false}"#, + ) + .unwrap(); + assert_eq!( + record.permissions, + Some(vec![Permission::Messaging, Permission::Logs]) + ); + assert_eq!(record.unrestricted_internet, Some(false)); + } + /// The flag is left out when unset, so a record that never named it /// encodes as before, and an explicit `false` is kept on the wire. #[test] diff --git a/swarm-wire/src/hive_status.rs b/swarm-wire/src/hive_status.rs new file mode 100644 index 000000000..82f8b65a0 --- /dev/null +++ b/swarm-wire/src/hive_status.rs @@ -0,0 +1,73 @@ +//! What a hive's status snapshot (`swarm_queue_client::status`) says about +//! what the hive can do, next to its readiness. +//! +//! The snapshot is opaque to the controller except for this field. A snapshot +//! from a hive that predates it has none, which reads as an empty list. + +/// The hive decodes a permission list that names a permission it does not +/// know by skipping that name, and runs agents on a harness that does too. +/// swarm-controller writes `logs` or `metrics` into an agent's record only +/// when the agent's hive advertises this. +pub const TOLERANT_PERMISSIONS: &str = "tolerant-permissions"; + +/// The part of a hive's status snapshot this module describes. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Capabilities { + /// Capability names; unknown names are ignored by every reader. + #[serde(default)] + pub capabilities: Vec, +} + +impl Capabilities { + /// Whether the hive advertises `name`. + #[must_use] + pub fn has(&self, name: &str) -> bool { + self.capabilities.iter().any(|c| c == name) + } + + /// The capabilities in a stored snapshot; none when it has no such field + /// or is not an object. + #[must_use] + pub fn of_snapshot(snapshot: &serde_json::Value) -> Self { + serde_json::from_value(snapshot.clone()).unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A snapshot from before the field existed: it parses, with nothing + /// advertised. + #[test] + fn a_snapshot_without_the_field_advertises_nothing() { + let old = serde_json::json!({"status": "ok", "warnings": []}); + let caps = Capabilities::of_snapshot(&old); + assert!(!caps.has(TOLERANT_PERMISSIONS)); + assert_eq!(caps, Capabilities::default()); + } + + #[test] + fn an_advertised_capability_is_found_among_unknown_ones() { + let new = serde_json::json!({ + "status": "ok", + "warnings": [], + "capabilities": ["from-the-future", TOLERANT_PERMISSIONS], + }); + assert!(Capabilities::of_snapshot(&new).has(TOLERANT_PERMISSIONS)); + assert!(!Capabilities::of_snapshot(&new).has("other")); + } + + /// A snapshot that is not an object, or whose field is mistyped, grants + /// nothing rather than failing the read. + #[test] + fn a_malformed_snapshot_advertises_nothing() { + for bad in [ + serde_json::json!(null), + serde_json::json!("ok"), + serde_json::json!({"capabilities": "tolerant-permissions"}), + ] { + assert!(!Capabilities::of_snapshot(&bad).has(TOLERANT_PERMISSIONS)); + } + } +} diff --git a/swarm-wire/src/lib.rs b/swarm-wire/src/lib.rs index 995fdf148..1ade532eb 100644 --- a/swarm-wire/src/lib.rs +++ b/swarm-wire/src/lib.rs @@ -19,6 +19,7 @@ pub mod builder; pub mod claude_login; pub mod egress_allow; pub mod forge_events; +pub mod hive_status; pub mod notices; pub mod notify_flows; pub mod notify_router; From 3b1d118f2571ff229744ac30f968a545cf241913 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 17:40:09 +0100 Subject: [PATCH 2/7] responder checks the logs or metrics permission per surface --- swarm-agent-verify/Cargo.toml | 1 + swarm-agent-verify/src/lib.rs | 24 +- swarm-logs-auth/src/main.rs | 273 ++++++++++++++++++++--- swarm-secret-client/src/lib.rs | 1 + swarm-secret-client/src/observability.rs | 62 +++++ 5 files changed, 324 insertions(+), 37 deletions(-) create mode 100644 swarm-secret-client/src/observability.rs diff --git a/swarm-agent-verify/Cargo.toml b/swarm-agent-verify/Cargo.toml index d32dfa0f8..7e9d26264 100644 --- a/swarm-agent-verify/Cargo.toml +++ b/swarm-agent-verify/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true [dependencies] anyhow.workspace = true +serde.workspace = true # Comparing an agent's presented secret with the stored one. subtle.workspace = true # The read of the stored credential the token is checked against. diff --git a/swarm-agent-verify/src/lib.rs b/swarm-agent-verify/src/lib.rs index 7d52d3165..e06eb94f8 100644 --- a/swarm-agent-verify/src/lib.rs +++ b/swarm-agent-verify/src/lib.rs @@ -52,11 +52,17 @@ impl Store { } } -impl CredentialSource for Store { - /// Logs in per lookup, so no token is left to expire inside a long-lived - /// process. - async fn lookup(&self, agent: &str) -> anyhow::Result> { - let path = queue::agent_queue_path(agent)?; +impl Store { + /// The object at `path`, `None` when nothing is stored there. Logs in per + /// read, so no token is left to expire inside a long-lived process. + /// + /// # Errors + /// + /// When the login or the read fails, or the object does not decode. + pub async fn read_optional( + &self, + path: &str, + ) -> anyhow::Result> { let store = swarm_secret_client::SecretStore::connect( &self.settings, &self.cert_role, @@ -65,12 +71,18 @@ impl CredentialSource for Store { .await .context("logging in to the secret store")?; store - .read_optional(&path) + .read_optional(path) .await .with_context(|| format!("reading {path}")) } } +impl CredentialSource for Store { + async fn lookup(&self, agent: &str) -> anyhow::Result> { + self.read_optional(&queue::agent_queue_path(agent)?).await + } +} + /// Whether `token`'s secret is the one stored for the agent it names. /// /// The lookup is bounded by `timeout`, after which this denies. diff --git a/swarm-logs-auth/src/main.rs b/swarm-logs-auth/src/main.rs index 9ebbd56e7..c4c40e14e 100644 --- a/swarm-logs-auth/src/main.rs +++ b/swarm-logs-auth/src/main.rs @@ -3,21 +3,29 @@ //! nginx sends it every `/select/logsql/` request (`swarm-victorialogs.nix`), //! and every metrics read API request (`swarm-victoriametrics.nix`, as //! `swarm-metrics-agent-auth`), whose `Authorization` is -//! `Bearer swarm-agent..`. It answers 200 when the secret is -//! the one stored at `swarm/agents//queue`, and 401 otherwise, store -//! failures included: `auth_request` turns any other status into a 500 for -//! the caller. +//! `Bearer swarm-agent..`. `--surface` says which store this +//! instance guards. It answers 200 when the secret is the one stored at +//! `swarm/agents//queue` and the agent holds the permission for that +//! surface (`logs` or `metrics`), 401 when the secret does not verify, and 403 +//! when it does but the permission is not held. A grant it cannot read is not +//! held. Any other status becomes a 500 for the caller in `auth_request`. +//! +//! The permission is read from the leaf swarm-controller mirrors +//! (`swarm_secret_client::observability`), because this responder has no queue +//! access. //! //! It logs in to the store as the cert-auth role `--store-cert-role` names, -//! whose policy reaches every agent's `queue` secret. It serves a unix socket -//! whose directory only the gateway's nginx may enter. +//! whose policy reaches every agent's `queue` secret and grant leaf. It serves +//! a unix socket whose directory only the gateway's nginx may enter. //! //! A grant is remembered for `--verdict-ttl` seconds ([`verdicts`]), so an //! agent's queries within that window cost no store lookup, and a secret that -//! stops verifying is still accepted for up to that long. +//! stops verifying, or a permission that is revoked, is still honoured for up +//! to that long. Only an allow is remembered; a denial is looked up again. mod verdicts; +use std::future::Future; use std::os::unix::fs::PermissionsExt as _; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -26,9 +34,10 @@ use std::time::{Duration, Instant}; use anyhow::Context as _; use axum::extract::State; use axum::http::{HeaderMap, StatusCode, header}; -use clap::Parser; +use clap::{Parser, ValueEnum}; use swarm_agent_verify::CredentialSource; use swarm_agent_verify::agent_token::parse_agent_token; +use swarm_secret_client::observability::{Grant, grant_path}; use verdicts::Verdicts; /// How long one store lookup may take before the request is denied. Well @@ -49,20 +58,57 @@ struct Args { /// Cert-auth role this responder logs in to the store as. #[arg(long, default_value = "swarm-logs-auth")] store_cert_role: String, + /// Which store this instance guards, and so which permission it checks. + #[arg(long, value_enum)] + surface: Surface, /// Seconds a verified bearer is granted again without a store lookup; /// 0 looks up every request. #[arg(long, default_value_t = 60)] verdict_ttl: u64, } +/// The store a gateway fronts. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum Surface { + /// The log store, behind the `logs` permission. + Logs, + /// The metrics store, behind the `metrics` permission. + Metrics, +} + +impl Surface { + fn allowed_by(self, grant: Grant) -> bool { + match self { + Self::Logs => grant.logs, + Self::Metrics => grant.metrics, + } + } +} + +/// Where an agent's mirrored [`Grant`] comes from. +trait GrantSource { + /// The grant mirrored for `agent`, `None` when nothing is stored, or an + /// error when the store could not answer. + fn grant(&self, agent: &str) -> impl Future>> + Send; +} + +impl GrantSource for swarm_agent_verify::Store { + async fn grant(&self, agent: &str) -> anyhow::Result> { + self.read_optional(&grant_path(agent)?).await + } +} + struct Responder { source: S, + surface: Surface, + /// How long one store lookup may take before the request is denied. + lookup_timeout: Duration, verdicts: Mutex, } /// The answer for one subrequest's `Authorization` header, at `now`. async fn decide( - responder: &Responder, + responder: &Responder, authorization: Option<&str>, now: Instant, ) -> StatusCode { @@ -85,13 +131,48 @@ async fn decide( tracing::debug!(agent = token.agent, "verified recently"); return StatusCode::OK; } - if swarm_agent_verify::verify(&responder.source, &token, LOOKUP_TIMEOUT).await { - tracing::info!(agent = token.agent, "verified"); - lock(&responder.verdicts).grant(bearer, now); - StatusCode::OK - } else { - StatusCode::UNAUTHORIZED + if !swarm_agent_verify::verify(&responder.source, &token, responder.lookup_timeout).await { + return StatusCode::UNAUTHORIZED; } + if !holds_permission(responder, token.agent).await { + return StatusCode::FORBIDDEN; + } + tracing::info!(agent = token.agent, surface = ?responder.surface, "verified"); + lock(&responder.verdicts).grant(bearer, now); + StatusCode::OK +} + +/// Whether `agent` holds the permission for the surface. A grant that is +/// absent, unreadable or slow is not held. +async fn holds_permission(responder: &Responder, agent: &str) -> bool { + let grant = + match tokio::time::timeout(responder.lookup_timeout, responder.source.grant(agent)).await { + Ok(Ok(Some(grant))) => grant, + Ok(Ok(None)) => { + tracing::warn!( + agent, + "no permission grant is mirrored for this agent; denying" + ); + return false; + } + Ok(Err(e)) => { + tracing::warn!( + agent, + error = format!("{e:#}"), + "grant lookup failed; denying" + ); + return false; + } + Err(_) => { + tracing::warn!(agent, "grant lookup timed out; denying"); + return false; + } + }; + let held = responder.surface.allowed_by(grant); + if !held { + tracing::info!(agent, surface = ?responder.surface, "agent lacks the permission; denying"); + } + held } /// The cache holds no invariant a panicking holder could break mid-way, so a @@ -129,6 +210,8 @@ async fn main() -> anyhow::Result<()> { .context("reading the secret store's address and this responder's identity")?; let responder = Arc::new(Responder { source: swarm_agent_verify::Store::new(settings, args.store_cert_role), + surface: args.surface, + lookup_timeout: LOOKUP_TIMEOUT, verdicts: Mutex::new(Verdicts::new( Duration::from_secs(args.verdict_ttl), VERDICT_CAP, @@ -161,18 +244,51 @@ mod tests { use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use swarm_secret_client::queue::AgentCredential; - /// A store holding atlas's credential until `revoke`, counting lookups. - #[derive(Default)] + /// What the mirrored grant leaf answers. + #[derive(Clone, Copy)] + enum Leaf { + Holds(Grant), + Absent, + Unreadable, + /// Accepts the read and never answers. + Hangs, + } + + /// A store holding atlas's credential until `revoke` and her grant leaf, + /// counting lookups of each. struct Atlas { lookups: AtomicUsize, revoked: AtomicBool, + leaf: Mutex, + grant_lookups: AtomicUsize, } impl Atlas { + fn with(leaf: Leaf) -> Self { + Self { + lookups: AtomicUsize::new(0), + revoked: AtomicBool::new(false), + leaf: Mutex::new(leaf), + grant_lookups: AtomicUsize::new(0), + } + } + + /// Atlas holds `logs` and `metrics`. + fn both() -> Self { + Self::with(Leaf::Holds(Grant { + logs: true, + metrics: true, + })) + } + fn revoke(&self) { self.revoked.store(true, Ordering::SeqCst); } + fn set_leaf(&self, leaf: Leaf) { + *self.leaf.lock().unwrap() = leaf; + } + fn lookups(&self) -> usize { self.lookups.load(Ordering::SeqCst) } @@ -193,30 +309,97 @@ mod tests { } } + impl GrantSource for &Atlas { + async fn grant(&self, agent: &str) -> anyhow::Result> { + self.grant_lookups.fetch_add(1, Ordering::SeqCst); + let leaf = *self.leaf.lock().unwrap(); + match (agent, leaf) { + ("atlas", Leaf::Holds(grant)) => Ok(Some(grant)), + ("atlas", Leaf::Unreadable) => anyhow::bail!("store unreachable"), + ("atlas", Leaf::Hangs) => std::future::pending().await, + _ => Ok(None), + } + } + } + const TTL: Duration = Duration::from_mins(1); const GOOD: &str = "Bearer swarm-agent.atlas.Ab9_-zSECRET"; const WRONG: &str = "Bearer swarm-agent.atlas.Ab9_-zSECREX"; - fn responder(store: &Atlas) -> Responder<&Atlas> { + fn responder(store: &Atlas, surface: Surface) -> Responder<&Atlas> { Responder { source: store, + surface, + lookup_timeout: Duration::from_millis(100), verdicts: Mutex::new(Verdicts::new(TTL, 16)), } } #[tokio::test] - async fn the_stored_secret_is_200() { - let store = Atlas::default(); + async fn the_stored_secret_with_the_grant_is_200() { + let store = Atlas::both(); + for surface in [Surface::Logs, Surface::Metrics] { + assert_eq!( + decide(&responder(&store, surface), Some(GOOD), Instant::now()).await, + StatusCode::OK, + "{surface:?}" + ); + } + } + + /// Each surface checks its own permission: a `logs` grant opens the log + /// store and not the metrics store, and the reverse. + #[tokio::test] + async fn each_surface_needs_its_own_permission() { + async fn at(store: &Atlas, surface: Surface) -> StatusCode { + decide(&responder(store, surface), Some(GOOD), Instant::now()).await + } + let logs_only = Atlas::with(Leaf::Holds(Grant { + logs: true, + metrics: false, + })); + let metrics_only = Atlas::with(Leaf::Holds(Grant { + logs: false, + metrics: true, + })); + assert_eq!(at(&logs_only, Surface::Logs).await, StatusCode::OK); assert_eq!( - decide(&responder(&store), Some(GOOD), Instant::now()).await, - StatusCode::OK + at(&logs_only, Surface::Metrics).await, + StatusCode::FORBIDDEN ); + assert_eq!(at(&metrics_only, Surface::Metrics).await, StatusCode::OK); + assert_eq!( + at(&metrics_only, Surface::Logs).await, + StatusCode::FORBIDDEN + ); + } + + /// No grant, an empty grant and a grant that cannot be read all deny. + #[tokio::test] + async fn without_a_readable_grant_the_answer_is_403() { + for leaf in [ + Leaf::Absent, + Leaf::Unreadable, + Leaf::Hangs, + Leaf::Holds(Grant::default()), + ] { + let store = Atlas::with(leaf); + assert_eq!( + decide( + &responder(&store, Surface::Logs), + Some(GOOD), + Instant::now() + ) + .await, + StatusCode::FORBIDDEN + ); + } } #[tokio::test] async fn everything_else_is_401() { - let store = Atlas::default(); - let r = responder(&store); + let store = Atlas::both(); + let r = responder(&store, Surface::Logs); for authorization in [ None, Some("swarm-agent.atlas.Ab9_-zSECRET"), @@ -232,17 +415,21 @@ mod tests { "{authorization:?}" ); } + // A wrong secret is refused before the grant is read. + assert_eq!(store.grant_lookups.load(Ordering::SeqCst), 0); } - /// A grant is reused within the TTL — a revocation the store already - /// holds goes unseen — and checked against the store again after it. + /// An allow is reused within the TTL — a revocation the store already + /// holds, of the secret or of the permission, goes unseen — and checked + /// against the store again after it. #[tokio::test] - async fn a_grant_spares_the_store_until_the_ttl() { - let store = Atlas::default(); - let r = responder(&store); + async fn an_allow_spares_the_store_until_the_ttl() { + let store = Atlas::both(); + let r = responder(&store, Surface::Logs); let t0 = Instant::now(); assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); store.revoke(); + store.set_leaf(Leaf::Holds(Grant::default())); let within = t0 + Duration::from_secs(59); assert_eq!(decide(&r, Some(GOOD), within).await, StatusCode::OK); assert_eq!(store.lookups(), 1); @@ -253,10 +440,34 @@ mod tests { assert_eq!(store.lookups(), 2); } + /// A revoked permission is denied once the allow expires; a denial is not + /// remembered, so a permission granted later is honoured at once. + #[tokio::test] + async fn a_revoked_or_added_permission_takes_effect_by_the_ttl() { + let store = Atlas::both(); + let r = responder(&store, Surface::Logs); + let t0 = Instant::now(); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); + store.set_leaf(Leaf::Holds(Grant::default())); + assert_eq!( + decide(&r, Some(GOOD), t0 + TTL).await, + StatusCode::FORBIDDEN + ); + assert_eq!( + decide(&r, Some(GOOD), t0 + TTL).await, + StatusCode::FORBIDDEN + ); + store.set_leaf(Leaf::Holds(Grant { + logs: true, + metrics: false, + })); + assert_eq!(decide(&r, Some(GOOD), t0 + TTL).await, StatusCode::OK); + } + #[tokio::test] async fn a_denial_is_looked_up_every_time() { - let store = Atlas::default(); - let r = responder(&store); + let store = Atlas::both(); + let r = responder(&store, Surface::Logs); let t0 = Instant::now(); for _ in 0..3 { assert_eq!(decide(&r, Some(WRONG), t0).await, StatusCode::UNAUTHORIZED); diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 502f2b847..cb1bae3a7 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -45,6 +45,7 @@ pub mod github; pub mod grafana; pub mod matrix; pub mod mtls; +pub mod observability; pub mod path; pub mod policy; pub mod proxy; diff --git a/swarm-secret-client/src/observability.rs b/swarm-secret-client/src/observability.rs new file mode 100644 index 000000000..a78da4785 --- /dev/null +++ b/swarm-secret-client/src/observability.rs @@ -0,0 +1,62 @@ +//! The log and metric read grants: where swarm-controller mirrors an agent's +//! `logs` and `metrics` permissions in the store, and what the object holds. +//! +//! The permissions themselves live in the queue's `agent-permissions` bucket. +//! The store gateways' `swarm-logs-auth` has no queue access, so it reads this +//! mirror instead. A leaf that is absent or unreadable is no grant. +//! +//! Under the agent's own prefix, so the agent can read its own grant and no +//! agent can write it: the shared agent policy grants `create` on one other +//! leaf only. + +use serde::{Deserialize, Serialize}; + +use crate::{ + Error, + path::{Kind, principal_prefix}, +}; + +const LEAF: &str = "observability"; + +/// The path holding `agent`'s [`Grant`]. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`, +/// which is what keeps one agent's name from addressing another agent's grant. +pub fn grant_path(agent: &str) -> Result { + let prefix = principal_prefix(Kind::Agent, agent)?; + Ok(format!("{prefix}/{LEAF}")) +} + +/// What a [`grant_path`] holds: which stores the agent may read. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Grant { + /// The agent holds the `logs` permission. + #[serde(default)] + pub logs: bool, + /// The agent holds the `metrics` permission. + #[serde(default)] + pub metrics: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_path_is_under_the_agents_own_prefix() { + assert_eq!( + grant_path("atlas").unwrap(), + "swarm/agents/atlas/observability" + ); + assert!(grant_path("../argus").is_err()); + } + + #[test] + fn a_field_left_out_is_not_granted() { + let grant: Grant = serde_json::from_str("{}").unwrap(); + assert_eq!(grant, Grant::default()); + let grant: Grant = serde_json::from_str(r#"{"logs":true}"#).unwrap(); + assert_eq!((grant.logs, grant.metrics), (true, false)); + } +} From e7e328109859bfda0da23d3853e8916e57e3b91d Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 17:40:09 +0100 Subject: [PATCH 3/7] controller gates logs and metrics on the hive capability and mirrors the grant --- swarm-controller-api/src/permissions.rs | 2 +- swarm-controller/src/agent_permissions.rs | 203 ++++++++++++++-- swarm-controller/src/main.rs | 1 + swarm-controller/src/observability_grant.rs | 248 ++++++++++++++++++++ swarm-controller/src/routes/agents.rs | 95 ++++++-- swarm-controller/src/schedules.rs | 5 + swarmctl/src/main.rs | 3 +- 7 files changed, 519 insertions(+), 38 deletions(-) create mode 100644 swarm-controller/src/observability_grant.rs diff --git a/swarm-controller-api/src/permissions.rs b/swarm-controller-api/src/permissions.rs index 0a052dc74..8b8e80304 100644 --- a/swarm-controller-api/src/permissions.rs +++ b/swarm-controller-api/src/permissions.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; #[serde(deny_unknown_fields)] pub struct Permissions { /// Permission names, `snake_case`: `messaging`, `meta`, `inbox`, - /// `execution`, `web_tools`, `manage_root_agent`. + /// `execution`, `web_tools`, `manage_root_agent`, `logs`, `metrics`. #[serde(default, skip_serializing_if = "Option::is_none")] #[cfg_attr( feature = "schema", diff --git a/swarm-controller/src/agent_permissions.rs b/swarm-controller/src/agent_permissions.rs index eeadf51d1..0b6311a86 100644 --- a/swarm-controller/src/agent_permissions.rs +++ b/swarm-controller/src/agent_permissions.rs @@ -23,11 +23,13 @@ use problem_details::ProblemDetails; use serde::Serialize; use swarm_controller_api::permissions::Permissions; use swarm_queue_client::agent_permissions; +use swarm_secret_client::observability::Grant; use swarm_wire::agent_permissions::{ AgentPermissions, Import, ImportReply, hive_from_import_subject, import_subject_wildcard, }; use utoipa::ToSchema; +use crate::observability_grant::{self, grant_of, mirror, require_tolerant_hive}; use crate::routes::{AppState, error_problem, wanted_error_status}; use crate::schedules::{live, queue, read_or_default, refused, write, write_revision}; use crate::wanted::WantedWriter; @@ -102,6 +104,31 @@ fn logged(op: &'static str, agent: &str, problem: ProblemDetails) -> ProblemDeta problem } +/// Bring the store's mirror of `agent`'s `logs`/`metrics` grant in line with +/// a record that went from `before` to `after`, after the record is written. +/// A failure is an error to the caller, who repeats the request: `mirror` +/// rewrites whenever either record holds a grant. +async fn mirrored( + agent: &str, + before: &AgentPermissions, + after: &AgentPermissions, +) -> Result<(), ProblemDetails> { + let held = |v: &AgentPermissions| v.permissions.clone().unwrap_or_default(); + mirror(&observability_grant::Store, agent, &held(before), &held(after)) + .await + .map_err(|e| { + tracing::warn!(agent, error = %format!("{e:#}"), "agent permissions: mirroring the grant failed"); + error_problem( + StatusCode::INTERNAL_SERVER_ERROR, + &format!("the permissions are stored, but mirroring logs/metrics to the secret store failed: {e:#}"), + ) + }) +} + +fn hive_names(state: &AppState) -> Vec { + state.hives.iter().map(|h| h.name.clone()).collect() +} + fn queue_problem(e: &anyhow::Error) -> ProblemDetails { tracing::warn!(error = %format!("{e:#}"), "agent permissions: queue failure"); error_problem(wanted_error_status(e), &format!("{e:#}")) @@ -185,6 +212,19 @@ pub(crate) async fn put_agent_permissions( &format!("no hive's wanted state declares an agent {agent:?} that is not destroyed"), ))); } + if let Some(perms) = &req.permissions + && grant_of(perms) != Grant::default() + { + let (Some(wanted), hives) = (state.wanted.as_ref(), hive_names(&state)) else { + return Err(refuse(error_problem( + StatusCode::SERVICE_UNAVAILABLE, + "no swarm queue is configured on this host", + ))); + }; + require_tolerant_hive(&client, wanted, &hives, &agent) + .await + .map_err(|why| refuse(error_problem(StatusCode::CONFLICT, &why)))?; + } let store = open_or_create(&client).await?; for _ in 0..MAX_ATTEMPTS { let (current, revision, undecodable) = read_or_default::(&store, &agent) @@ -200,6 +240,7 @@ pub(crate) async fn put_agent_permissions( .map_err(|e| queue_problem(&e))? { tracing::info!(%agent, ?value, "set agent permissions"); + mirrored(&agent, ¤t, &value).await?; return Ok(Json(view(value))); } } @@ -255,6 +296,7 @@ pub(crate) async fn delete_agent_permissions( continue; } tracing::info!(%agent, "cleared agent permissions"); + mirrored(&agent, ¤t, &kept).await?; return Ok(StatusCode::NO_CONTENT); } Err(queue_problem(&anyhow::anyhow!( @@ -271,10 +313,17 @@ pub(crate) async fn delete_agent_permissions( pub(crate) async fn clear_on_destroy(state: &AppState, hive: &str, agent: &str) { let cleared = async { let client = queue(state).map_err(|(_, why)| anyhow::anyhow!(why))?; - agent_permissions::open_or_create(&client) - .await? - .delete(agent) - .await?; + let store = agent_permissions::open_or_create(&client).await?; + let (current, _, _) = read_or_default::(&store, agent).await?; + store.delete(agent).await?; + // The mirrored grant goes with the record. + mirror( + &observability_grant::Store, + agent, + ¤t.permissions.unwrap_or_default(), + &[], + ) + .await?; anyhow::Ok(()) } .await; @@ -457,10 +506,15 @@ pub(crate) fn spawn(client: async_nats::Client, wanted: Arc, hives return; }; tokio::spawn({ - let (store, wanted, hives) = (store.clone(), Arc::clone(&wanted), hives.clone()); + let (client, store, wanted, hives) = ( + client.clone(), + store.clone(), + Arc::clone(&wanted), + hives.clone(), + ); async move { // Retries without end, so it answers only once it has worked. - let _ = (|| seed_ruth_grant(&store, &wanted, &hives)) + let _ = (|| seed_ruth_grant(&client, &store, &wanted, &hives)) .retry(every_five_seconds()) .notify(|e: &anyhow::Error, _| { tracing::warn!( @@ -539,6 +593,7 @@ const SEEDED_AGENT: &str = "ruth"; /// `ruth` whose permissions were unset; this writes that grant into her key /// so she keeps her recovery mounts. Idempotent: a set key is never touched. async fn seed_ruth_grant( + client: &async_nats::Client, store: &async_nats::jetstream::kv::Store, wanted: &WantedWriter, hives: &[String], @@ -549,16 +604,27 @@ async fn seed_ruth_grant( declarations.push(d); } } - seed_into(store, &crate::agent_renewal::live_agents(&declarations)).await?; - Ok(()) + let declared = crate::agent_renewal::live_agents(&declarations); + if !declared.contains(SEEDED_AGENT) { + return Ok(()); + } + let tolerant = + observability_grant::hive_is_tolerant(client, wanted, hives, SEEDED_AGENT).await?; + seed_into(store, &declared, tolerant).await?; + // At every start, so a mirror that failed to write is retried. + let (current, _, _) = read_or_default::(store, SEEDED_AGENT).await?; + let held = current.permissions.unwrap_or_default(); + mirror(&observability_grant::Store, SEEDED_AGENT, &[], &held).await } -/// Write [`Permission::DEFAULT`] plus `manage_root_agent` under -/// [`SEEDED_AGENT`]'s key when `declared` holds her and her `permissions` -/// are unset (no key, a deleted key, or `{}`). `true` when it wrote. +/// Write [`Permission::DEFAULT`] plus `manage_root_agent`, and `logs` and +/// `metrics` when her hive is `tolerant`, under [`SEEDED_AGENT`]'s key when +/// `declared` holds her and her `permissions` are unset (no key, a deleted +/// key, or `{}`). `true` when it wrote. async fn seed_into( store: &async_nats::jetstream::kv::Store, declared: &BTreeSet, + tolerant: bool, ) -> anyhow::Result { if !declared.contains(SEEDED_AGENT) { return Ok(false); @@ -570,6 +636,9 @@ async fn seed_into( } let mut perms = Permission::DEFAULT.to_vec(); perms.push(Permission::ManageRootAgent); + if tolerant { + perms.extend([Permission::Logs, Permission::Metrics]); + } let value = AgentPermissions { permissions: Some(perms), ..AgentPermissions::default() @@ -616,7 +685,7 @@ async fn import_into( } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::{ Permissions, delete_agent_permissions, get_agent_permissions, import_into, put_agent_permissions, seed_into, @@ -701,7 +770,7 @@ mod tests { ); } - async fn published(client: &async_nats::Client, agent: &str) -> Option { + pub(crate) async fn published(client: &async_nats::Client, agent: &str) -> Option { let store = swarm_queue_client::agent_permissions::open_or_create(client) .await .expect("bucket"); @@ -945,6 +1014,91 @@ mod tests { assert_eq!(published(&client, "atlas").await.as_deref(), Some(older)); } + async fn publish_snapshot(client: &async_nats::Client, json: &str) { + swarm_queue_client::status::open_or_create(client) + .await + .expect("bucket") + .put("pr1ma", json.as_bytes().to_vec().into()) + .await + .expect("snapshot"); + } + + fn observing(perms: &[Permission]) -> Permissions { + Permissions { + permissions: Some(perms.to_vec()), + unrestricted_internet: None, + internet_log: None, + } + } + + /// `logs` and `metrics` are refused, naming the hive and writing nothing, + /// while the agent's hive has published no snapshot or one without the + /// capability. Control: a grant that names neither passes the same hive. + #[tokio::test] + async fn logs_and_metrics_need_a_hive_that_advertises_tolerant_decoding() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = crate::agent_limits::tests::declared(&client).await; + + for perm in [Permission::Logs, Permission::Metrics] { + let none = put(&state, "atlas", observing(&[perm])) + .await + .expect_err("no snapshot"); + assert_eq!(none.status, Some(StatusCode::CONFLICT)); + assert!(none.detail.as_deref().unwrap_or_default().contains("pr1ma")); + } + + publish_snapshot(&client, r#"{"status":"ok","warnings":[]}"#).await; + let old = put(&state, "atlas", observing(&[Permission::Logs])) + .await + .expect_err("old hive"); + assert_eq!(old.status, Some(StatusCode::CONFLICT)); + assert_eq!(published(&client, "atlas").await, None, "nothing written"); + + let _ = put(&state, "atlas", observing(&[Permission::Messaging])) + .await + .expect("control: no log or metric grant"); + } + + /// With the capability advertised the gate passes: the record is written, + /// and the mirror (which needs the secret store, absent here) is what + /// fails. + #[tokio::test] + async fn an_advertising_hive_passes_the_gate() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = crate::agent_limits::tests::declared(&client).await; + publish_snapshot(&client, r#"{"capabilities":["tolerant-permissions"]}"#).await; + + let problem = put(&state, "atlas", observing(&[Permission::Metrics])) + .await + .expect_err("no secret store to mirror to"); + assert_eq!(problem.status, Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_eq!( + published(&client, "atlas").await.as_deref(), + Some(r#"{"permissions":["metrics"]}"#) + ); + } + + /// With her hive tolerant ruth is seeded with `logs` and `metrics`; with + /// an old hive she gets the set she always had. + #[tokio::test] + async fn the_seed_includes_logs_and_metrics_on_a_tolerant_hive() { + let nats = TestNats::start(); + let client = nats.client().await; + let store = swarm_queue_client::agent_permissions::open_or_create(&client) + .await + .expect("bucket"); + let declared = ["ruth".to_owned()].into(); + assert!(seed_into(&store, &declared, true).await.expect("seeded")); + assert_eq!( + published(&client, "ruth").await.as_deref(), + Some( + r#"{"permissions":["messaging","meta","inbox","execution","manage_root_agent","logs","metrics"]}"# + ) + ); + } + /// A name no hive declares, or a name that is not an identifier, is a 400 /// and writes nothing. #[tokio::test] @@ -1170,25 +1324,29 @@ mod tests { .expect("bucket"); let declared = ["atlas".to_owned(), "ruth".to_owned()].into(); - assert!(seed_into(&store, &declared).await.expect("seeded")); + assert!(seed_into(&store, &declared, false).await.expect("seeded")); assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED)); assert_eq!(published(&client, "atlas").await, None, "only ruth"); - assert!(!seed_into(&store, &declared).await.expect("idempotent")); + assert!( + !seed_into(&store, &declared, false) + .await + .expect("idempotent") + ); store.delete("ruth").await.expect("delete"); - assert!(seed_into(&store, &declared).await.expect("seeded")); + assert!(seed_into(&store, &declared, false).await.expect("seeded")); assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED)); store .put("ruth", b"{}".to_vec().into()) .await .expect("unset"); - assert!(seed_into(&store, &declared).await.expect("seeded")); + assert!(seed_into(&store, &declared, false).await.expect("seeded")); assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED)); } /// A ruth key the swarm already set, even to nothing, is left as it is; - /// so is one this build cannot decode. + /// so is one holding only a name this build does not know. #[tokio::test] async fn the_seed_leaves_a_set_ruth_key_alone() { let nats = TestNats::start(); @@ -1206,7 +1364,10 @@ mod tests { .put("ruth", set.as_bytes().to_vec().into()) .await .expect("set"); - assert!(!seed_into(&store, &declared).await.expect("read"), "{set}"); + assert!( + !seed_into(&store, &declared, false).await.expect("read"), + "{set}" + ); assert_eq!(published(&client, "ruth").await.as_deref(), Some(set)); } } @@ -1222,12 +1383,12 @@ mod tests { .expect("bucket"); let declared = ["atlas".to_owned()].into(); - assert!(!seed_into(&store, &declared).await.expect("read")); + assert!(!seed_into(&store, &declared, false).await.expect("read")); assert_eq!(published(&client, "ruth").await, None); assert_eq!(published(&client, "atlas").await, None); let declared = ["atlas".to_owned(), "ruth".to_owned()].into(); - assert!(seed_into(&store, &declared).await.expect("seeded")); + assert!(seed_into(&store, &declared, false).await.expect("seeded")); assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED)); } diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index dcc4d4684..bac047425 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -64,6 +64,7 @@ mod meta_fanout; mod meta_flake; mod notify_flows; mod notify_router_api; +mod observability_grant; mod otel_http_client; mod proxy_credential; mod pulls; diff --git a/swarm-controller/src/observability_grant.rs b/swarm-controller/src/observability_grant.rs new file mode 100644 index 000000000..7f9cedccc --- /dev/null +++ b/swarm-controller/src/observability_grant.rs @@ -0,0 +1,248 @@ +//! The `logs` and `metrics` permissions beyond the queue's permissions bucket: +//! which hives may be granted them, and the mirror of the grant that the store +//! gateways read. +//! +//! A hive that cannot skip a permission name it does not know would fail to +//! decode a record that holds one, so the controller grants them only on a hive +//! whose status snapshot advertises `swarm_wire::hive_status::TOLERANT_PERMISSIONS`. +//! `swarm-logs-auth` has no queue access, so each grant is also written to +//! `swarm_secret_client::observability::grant_path`; a leaf that is absent is no +//! grant. + +use std::future::Future; + +use anyhow::{Context as _, Result}; +use hive_types::Permission; +use swarm_secret_client::observability::{Grant, grant_path}; +use swarm_wire::hive_status::{Capabilities, TOLERANT_PERMISSIONS}; + +use crate::wanted::WantedWriter; + +/// The grant `perms` hold. +pub(crate) fn grant_of(perms: &[Permission]) -> Grant { + Grant { + logs: perms.contains(&Permission::Logs), + metrics: perms.contains(&Permission::Metrics), + } +} + +/// Where a mirrored grant is written. A test stands in for the store. +pub(crate) trait GrantSink { + fn write(&self, agent: &str, grant: Grant) -> impl Future> + Send; +} + +/// The swarm secret store, under the controller's own identity. +pub(crate) struct Store; + +impl GrantSink for Store { + async fn write(&self, agent: &str, grant: Grant) -> Result<()> { + let path = grant_path(agent)?; + crate::store::connect() + .await + .context("logging in to the swarm secret store")? + .write(&path, &grant) + .await + .with_context(|| format!("writing {path}")) + } +} + +/// Bring the mirror in line when `agent`'s permissions go from `before` to +/// `after`. Nothing is written when neither holds `logs` or `metrics`, which +/// leaves the leaf absent. +/// +/// # Errors +/// The write failed; the responder then keeps the grant it last read, or none. +pub(crate) async fn mirror( + sink: &impl GrantSink, + agent: &str, + before: &[Permission], + after: &[Permission], +) -> Result<()> { + if grant_of(before) == Grant::default() && grant_of(after) == Grant::default() { + return Ok(()); + } + sink.write(agent, grant_of(after)).await +} + +/// `Ok` when `hive`'s snapshot advertises tolerant decoding. +/// +/// # Errors +/// Naming the hive, when it has published nothing or does not advertise it. +pub(crate) fn check_capability( + hive: &str, + snapshot: Option<&serde_json::Value>, +) -> Result<(), String> { + match snapshot { + Some(s) if Capabilities::of_snapshot(s).has(TOLERANT_PERMISSIONS) => Ok(()), + Some(_) => Err(format!( + "hive {hive} does not advertise {TOLERANT_PERMISSIONS}: update it first, \ + then grant logs or metrics" + )), + None => Err(format!( + "hive {hive} has published no status snapshot, so it is not known to \ + advertise {TOLERANT_PERMISSIONS}" + )), + } +} + +/// The hive whose declaration keeps `agent` alive. +async fn hive_of(wanted: &WantedWriter, hives: &[String], agent: &str) -> Option { + for hive in hives { + if let Ok(d) = wanted.view(hive).await + && crate::agent_renewal::live_agents(d.as_slice()).contains(agent) + { + return Some(hive.clone()); + } + } + None +} + +/// `hive`'s last status snapshot, `None` when it has published none. +async fn snapshot(client: &async_nats::Client, hive: &str) -> Result> { + swarm_queue_client::ensure_connected(client)?; + let store = swarm_queue_client::status::open_or_create(client).await?; + let Some(entry) = store.get(hive).await? else { + return Ok(None); + }; + Ok(serde_json::from_slice(&entry).ok()) +} + +/// Whether the hive running `agent` advertises tolerant decoding, waiting +/// (an error, for a retrying caller) on a hive that has published no snapshot +/// yet. `false` when no hive declares the agent. +/// +/// # Errors +/// The queue could not be read, or the hive has not published a snapshot. +pub(crate) async fn hive_is_tolerant( + client: &async_nats::Client, + wanted: &WantedWriter, + hives: &[String], + agent: &str, +) -> Result { + let Some(hive) = hive_of(wanted, hives, agent).await else { + return Ok(false); + }; + let Some(snapshot) = snapshot(client, &hive).await? else { + anyhow::bail!("hive {hive} has published no status snapshot yet"); + }; + Ok(check_capability(&hive, Some(&snapshot)).is_ok()) +} + +/// `Ok` when the hive running `agent` advertises tolerant decoding, so the +/// agent's record may name `logs` or `metrics`. +/// +/// # Errors +/// The reason: no hive declares the agent, the hive's snapshot is missing or +/// lacks the capability, or the queue could not be read. +pub(crate) async fn require_tolerant_hive( + client: &async_nats::Client, + wanted: &WantedWriter, + hives: &[String], + agent: &str, +) -> Result<(), String> { + let hive = hive_of(wanted, hives, agent) + .await + .ok_or_else(|| format!("no hive declares agent {agent}"))?; + require_tolerant(client, &hive).await +} + +/// `Ok` when `hive`'s last status snapshot advertises tolerant decoding. +/// +/// # Errors +/// The reason, naming the hive. +pub(crate) async fn require_tolerant( + client: &async_nats::Client, + hive: &str, +) -> Result<(), String> { + let snapshot = snapshot(client, hive) + .await + .map_err(|e| format!("reading hive {hive}'s status: {e:#}"))?; + check_capability(hive, snapshot.as_ref()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + #[derive(Default)] + struct Leaves(Mutex>); + + impl GrantSink for &Leaves { + async fn write(&self, agent: &str, grant: Grant) -> Result<()> { + self.0.lock().unwrap().push((agent.to_owned(), grant)); + Ok(()) + } + } + + struct Down; + + impl GrantSink for Down { + async fn write(&self, _: &str, _: Grant) -> Result<()> { + anyhow::bail!("store unreachable") + } + } + + #[tokio::test] + async fn a_grant_and_its_revocation_are_written() { + let leaves = Leaves::default(); + let both = [Permission::Messaging, Permission::Logs, Permission::Metrics]; + mirror(&&leaves, "atlas", &[], &both).await.unwrap(); + mirror(&&leaves, "atlas", &both, &[Permission::Messaging]) + .await + .unwrap(); + let written = leaves.0.lock().unwrap().clone(); + let yes = Grant { + logs: true, + metrics: true, + }; + assert_eq!( + written, + [ + ("atlas".to_owned(), yes), + ("atlas".to_owned(), Grant::default()) + ] + ); + } + + /// An agent that never held either, before or after, gets no leaf; one + /// that holds only one of them gets exactly that one. + #[tokio::test] + async fn only_the_permissions_held_are_mirrored() { + let leaves = Leaves::default(); + let plain = [Permission::Messaging]; + mirror(&&leaves, "atlas", &plain, &plain).await.unwrap(); + assert!(leaves.0.lock().unwrap().is_empty()); + mirror(&&leaves, "atlas", &plain, &[Permission::Metrics]) + .await + .unwrap(); + let written = leaves.0.lock().unwrap().clone(); + assert_eq!(written.len(), 1); + assert_eq!((written[0].1.logs, written[0].1.metrics), (false, true)); + } + + #[tokio::test] + async fn a_failed_write_is_an_error() { + let held = [Permission::Logs]; + assert!(mirror(&Down, "atlas", &[], &held).await.is_err()); + // Control: nothing to write is not a failure, whatever the store does. + assert!(mirror(&Down, "atlas", &[], &[]).await.is_ok()); + } + + #[test] + fn a_hive_without_the_capability_is_refused_by_name() { + let old = serde_json::json!({"status": "ok", "warnings": []}); + let new = serde_json::json!({"capabilities": [TOLERANT_PERMISSIONS]}); + let err = check_capability("pr1ma", Some(&old)).unwrap_err(); + assert!( + err.contains("pr1ma") && err.contains(TOLERANT_PERMISSIONS), + "{err}" + ); + assert!( + check_capability("pr1ma", None) + .unwrap_err() + .contains("pr1ma") + ); + assert!(check_capability("pr1ma", Some(&new)).is_ok()); + } +} diff --git a/swarm-controller/src/routes/agents.rs b/swarm-controller/src/routes/agents.rs index 0272a5d46..e44c7131c 100644 --- a/swarm-controller/src/routes/agents.rs +++ b/swarm-controller/src/routes/agents.rs @@ -10,9 +10,11 @@ use axum::{ }; use serde::{Deserialize, Serialize}; use swarm_controller_api::agents::{CreateAgentRequest, CreateAgentResponse}; +use swarm_secret_client::observability::Grant; use utoipa::ToSchema; use crate::jobq::{SwarmNodeKind, SwarmResourceKind, declare_agent_job}; +use crate::observability_grant::{self, grant_of, mirror, require_tolerant}; use crate::routes::hives::HiveEntry; use crate::routes::{AppState, StatusUnavailable, error_problem, wanted_error_status}; use crate::{ @@ -623,26 +625,18 @@ pub(crate) async fn create_agent( // Checked after the roster, so a mistyped hive stays a 400 while the queue // is down. let settings_queue = settings.queue(&state).await?; + let requested = settings.permissions.permissions.clone().unwrap_or_default(); + if grant_of(&requested) != Grant::default() { + require_tolerant(&settings_queue, &hive) + .await + .map_err(|why| error_problem(axum::http::StatusCode::CONFLICT, &why))?; + } // Agent names are one swarm-wide namespace: identity, forge user, store // path and matrix id all carry the bare name. The same name on the same // hive is that agent being re-created, and goes through. let _gate = state.create_gate.lock().await; - let elsewhere = placements_elsewhere( - &agent, - &hive, - || queued_placements_now(&state), - || declarations_elsewhere(&state, &agent, &hive), - ) - .await?; - if !elsewhere.is_empty() { - let detail = format!( - "agent name {agent:?} is already placed on hive {} — agent names are unique across \ - the swarm; destroy it there first to move it, or choose another name", - elsewhere.join(", ") - ); - return Err(error_problem(axum::http::StatusCode::CONFLICT, &detail)); - } + refuse_if_placed_elsewhere(&state, &agent, &hive).await?; // Stored ahead of the job graph, not once the agent is declared: the hive // reads these by agent name when it first runs the agent. A failure from // here on puts back what was stored, so a refused creation leaves no @@ -657,6 +651,22 @@ pub(crate) async fn create_agent( settings.agents_md.as_deref(), ) .await?; + // The record this create replaced is what the mirror may still describe. + // A create that names no list keeps the stored one, and its mirror with it. + let earlier = stored + .permissions + .as_ref() + .filter(|_| settings.permissions.permissions.is_some()) + .and_then(|replaced| replaced.previous()) + .and_then(|record| record.permissions.clone()) + .unwrap_or_default(); + if let Err(e) = mirror(&observability_grant::Store, &agent, &earlier, &requested).await { + stored.roll_back(&settings_queue, &agent).await; + return Err(error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &format!("mirroring logs/metrics to the secret store failed: {e:#}"), + )); + } let queued = { let mut sched = state .jobq @@ -670,6 +680,8 @@ pub(crate) async fn create_agent( Ok(ids) => ids, Err(e) => { stored.roll_back(&settings_queue, &agent).await; + // Best effort, like the roll back: the mirror follows the record. + let _ = mirror(&observability_grant::Store, &agent, &requested, &earlier).await; return Err(error_problem( axum::http::StatusCode::INTERNAL_SERVER_ERROR, &e.to_string(), @@ -685,6 +697,30 @@ pub(crate) async fn create_agent( })) } +/// A 409 when `agent` is already placed on a hive other than `hive`. +async fn refuse_if_placed_elsewhere( + state: &AppState, + agent: &str, + hive: &str, +) -> Result<(), problem_details::ProblemDetails> { + let elsewhere = placements_elsewhere( + agent, + hive, + || queued_placements_now(state), + || declarations_elsewhere(state, agent, hive), + ) + .await?; + if elsewhere.is_empty() { + return Ok(()); + } + let detail = format!( + "agent name {agent:?} is already placed on hive {} — agent names are unique across \ + the swarm; destroy it there first to move it, or choose another name", + elsewhere.join(", ") + ); + Err(error_problem(axum::http::StatusCode::CONFLICT, &detail)) +} + /// The settings a `POST /api/agents` body gives, as they are stored. struct CreateSettings { limits: Option, @@ -1552,6 +1588,35 @@ mod tests { assert_eq!(node_count(&state.jobq), 0, "nothing may be queued"); } + /// A create naming `logs` or `metrics` on a hive that does not advertise + /// tolerant decoding is a 409 that stores and queues nothing. Control: the + /// same create without them is accepted by that hive. + #[tokio::test] + async fn a_create_granting_logs_needs_a_tolerant_hive() { + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + let state = crate::agent_limits::tests::state(&client); + let create = |perms: &str| { + let req: CreateAgentRequest = serde_json::from_str(&format!( + r#"{{"name":"atlas","hive":"pr1ma","permissions":[{perms}]}}"# + )) + .expect("body parses"); + super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) + }; + let err = create(r#""messaging","logs""#).await.expect_err("refused"); + assert_eq!( + axum::response::IntoResponse::into_response(err).status(), + axum::http::StatusCode::CONFLICT + ); + assert_eq!( + crate::agent_permissions::tests::published(&client, "atlas").await, + None + ); + let _ = create(r#""messaging""#) + .await + .expect("control: no log grant"); + } + /// `state_with_roster` with a second hive, `sec0nd`. async fn state_with_two_hives() -> ( crate::test_nats::TestNats, diff --git a/swarm-controller/src/schedules.rs b/swarm-controller/src/schedules.rs index 0d0144171..94daf218e 100644 --- a/swarm-controller/src/schedules.rs +++ b/swarm-controller/src/schedules.rs @@ -503,6 +503,11 @@ impl Replaced { Self { previous, written } } + /// The value the write replaced, `None` when the key held none. + pub(crate) fn previous(&self) -> Option<&T> { + self.previous.as_ref() + } + /// Put `previous` back, or remove the key if it held none, against the /// revision the write made. `Ok(false)` when another write has landed /// since: that one is newer than what is being undone and stays. diff --git a/swarmctl/src/main.rs b/swarmctl/src/main.rs index bfedbf5ea..4df2383c5 100644 --- a/swarmctl/src/main.rs +++ b/swarmctl/src/main.rs @@ -358,7 +358,8 @@ struct PermissionsSetArgs { /// Agent name. A hive must declare it. agent: String, /// The permissions to grant, comma-separated: any of `messaging`, - /// `meta`, `inbox`, `execution`, `web_tools`, `manage_root_agent`. + /// `meta`, `inbox`, `execution`, `web_tools`, `manage_root_agent`, `logs`, + /// `metrics`. /// `--permissions ''` grants none. #[arg( long, From a73946102fe4a830a4d3d759f5b9de069dbcf932 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 1 Nov 2026 17:40:09 +0100 Subject: [PATCH 4/7] bao reader policy, responder units and docs for the per-surface grant --- docs/process/conventions.md | 19 +++++ docs/swarm/credentials-reference.md | 83 +++++++++++----------- docs/swarm/services.md | 26 +++---- docs/tools/swarmctl-cli.md | 2 +- docs/trust-boundary/security.md | 4 +- nix/host-modules/swarm-bao.nix | 6 +- nix/host-modules/swarm-victorialogs.nix | 6 +- nix/host-modules/swarm-victoriametrics.nix | 6 +- nix/module-eval/bao-grants.nix | 12 +++- nix/module-eval/core-toggle.nix | 2 + 10 files changed, 103 insertions(+), 63 deletions(-) diff --git a/docs/process/conventions.md b/docs/process/conventions.md index a9021eb85..756816f74 100644 --- a/docs/process/conventions.md +++ b/docs/process/conventions.md @@ -289,6 +289,8 @@ host access it has. | `execution` | the `bash` MCP server (`mcp__bash__run`/`status`/`kill`), rendered into the agent's MCP config only with it; a subagent gets Claude's own `Bash` instead. See `docs/tools/bash.md`. | | `web_tools` | the Claude built-ins `WebFetch`/`WebSearch` | | `manage_root_agent` | every agent's state (rw) bind-mounted into the holder's container, plus `/meta` (ro) | +| `logs` | the log store: `swarm-logs` queries. Checked by `swarm-logs-auth` at the gateway, not by a tool list. Off by default. | +| `metrics` | the metrics store: `swarm-metrics` queries. Checked by `swarm-metrics-agent-auth`, the same binary, for the metrics gateway. Off by default. | **Always-on tools** — `Permission::ALWAYS_ON_TOOLS` exposes `set_status`, `compact`, and `mark_todos_done` to every agent regardless of its @@ -311,6 +313,23 @@ the default (`Permission::DEFAULT`): `messaging`, `meta`, `inbox`, none. swarm-controller logs every refused `PUT` or `DELETE` on that route at `warn` as `agent permissions: request refused`, with the fields `op`, `agent`, `status` and `detail`. +**`logs` and `metrics`** — off for every agent until granted, and separate +because logs are the more sensitive. Grant one with +`swarmctl agent permissions set ` (listing it with the others) or the +create-agent screen. An agent without it gets a 403 from the store's gateway +when `swarm-logs` or `swarm-metrics` queries; a bad secret is still a 401. The +controller refuses the grant, with a 409 naming the hive, unless the agent's +hive advertises `tolerant-permissions` in its status snapshot +(`swarm_wire::hive_status`): a hive that predates the permission would fail to +decode the record. It also mirrors the two grants to the store leaf +`swarm/agents//observability` (`swarm_secret_client::observability`), +which the gateways read; the gateway denies when it can't read that leaf. A +revoked grant stops working within `authVerdictTtl` (60 seconds by default) of +the agent's last allowed query. Grant it only after the agent runs a harness +that skips names it doesn't know (its next update after the release that adds +these two); an older harness that reads such a record holds no permissions at +its next start. + **Internet flag** — the record also carries `unrestricted_internet`, a boolean that isn't a `Permission`. An absent value means on, so every agent that existed before the field keeps the open access it had. `POST /api/agents` diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index f7c1bc7c5..4e8419842 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -16,47 +16,48 @@ the rest of the cell says how. -| store path | minter | reader | automatic re-mint | automatic re-pull | -| ---------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `swarm/agents//matrix/main` | `swarm-controller`, at agent creation and every 5 minutes | the agent | ✅ when missing or rejected by the homeserver | ✅ the daemon restarts within 5 minutes of a rejection | -| `swarm/agents//matrix/` | `swarm-controller`, from a credential you hand it | the agent | must be stated | ✅ retried within 2 to 16 minutes after a rejection | -| `swarm/agents//matrix-recovery/` | `swarm-controller`, a random passphrase per account | the agent | ❌ never replaced: the account's key backup opens only with this passphrase | ❌ read when the account comes up | -| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl appservice sync` in `hive-matrix`, from the registration's `as_token` | `swarm-controller` | ❌ follows the registration below | ✅ read every 5 minutes | -| `swarm/controller/swarm-controller/matrix/appservice-registration` | `swarm-matrix-ctl appservice render` in `hive-matrix`, when the store answers and holds none; before that, the tokens an earlier version kept in the container, stored unchanged | `swarm-matrix-ctl` in `hive-matrix`; the controller has no grant to it | ❌ minted once; an operator who replaces it makes `sync` render the new pair and restart the homeserver | ✅ `sync` re-reads it every 10 minutes | -| `swarm/controller/swarm-controller/oidc/client` | authelia, first boot | `swarm-controller` | ❌ minted once, straight into the store; minted again only when the store holds none | ✅ re-read when authelia refuses it | -| `swarm/controller/swarm-controller/forge/token` | the forge, once | `swarm-controller` | ❌ minted once; delete `/var/lib/forgejo/swarm-controller-token-name` in the forge container to mint a new one | ✅ re-read on a 401 | -| `swarm/controller/swarm-controller/grafana/token` | `swarm-bao-grafana-controller-token` on Grafana's host | `swarm-controller` | ✅ every 10 minutes, when Grafana refuses it or none is stored | ✅ re-read on a 401 | -| `swarm/controller/swarm-controller/forge/webhook-secret` | `swarm-controller`, at start, from its old `webhook-secret` file; a new one only when there is no file | `swarm-controller` | ❌ minted once; a rotated secret fails every delivery until the forge hooks are re-registered | ❌ read at start | -| `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation, and by the renewal pass when none is stored or its cert-auth role is missing | `hive-c0re`, then the agent | ✅ re-issued every 5 minutes once past half its 7-day validity; minted when none is stored or its cert-auth role is missing, for agents not `Destroyed` | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid | -| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent; the queue, log and metrics auth services | ✅ re-minted at 45 days; skipped for `Destroyed` agents, whose path is deleted | ✅ read on every reconnect; an open connection keeps the old secret for up to an hour; the log and metrics stores honour a replaced secret for up to `deploy.victorialogs.authVerdictTtl` and `deploy.victoriametrics.agentAuthVerdictTtl` (60 s by default) | -| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and every 5 minutes | the agent | ✅ when missing or no longer matching the forge | ✅ every 10 minutes | -| `swarm/agents//forge/