//! `/api/egress/denied-domains` — the destinations the swarm refuses agents //! through their hive's egress proxy, whatever else allows them. //! //! The list is one value under the `deny` key of the `egress-allow` bucket //! (`swarm_wire::egress_allow`), which this daemon alone writes, beside the //! allow-list's key. Every hive reads it and renders the entries for the agents //! it runs: a swarm-wide entry is refused for everyone, an entry with an agent //! for that agent alone. The proxy checks the list before any allow rule, so an //! entry beats the base list, the hive's own names, the allow-list, the //! unrestricted flag and `log` mode alike, and an agent's entry can only add to //! the swarm-wide ones. //! //! Only an operator's request writes an entry. There is no derived entry: the //! stored type has no `derived` field, and nothing in this daemon reads an //! agent's environment or a linked account into this list. //! //! Each name goes into the proxy's configuration, so it is checked with //! `swarm_wire::egress_allow::normalize_domain` here and again by the hive. //! The base list is not consulted: refusing a name the proxy otherwise always //! allows is the point of the list. //! //! A hive reports the revision of the list it rendered in its status snapshot //! (`swarm_wire::hive_status::EGRESS_DENY_APPLIED`); [`behind`] names the hives //! whose report does not match, which the reads show as `behind`. use axum::Json; use axum::extract::State; use axum::http::StatusCode; use problem_details::ProblemDetails; use swarm_controller_api::egress::{DeniedDomain, DeniedDomains}; use swarm_queue_client::egress_allow; use swarm_wire::egress_allow::{DENY_KEY, DenyEntry, DenyList, MAX_ENTRIES, normalize_domain}; use swarm_wire::hive_status::egress_deny_applied; use crate::routes::{AppState, error_problem, wanted_error_status}; use crate::schedules::{live, queue, read_or_default, refused, write_revision}; use crate::status::{Freshness, HiveStatus}; /// How many times a losing writer re-reads and re-applies, as in /// `egress_allow::MAX_ATTEMPTS`. const MAX_ATTEMPTS: usize = 5; /// The entry to store for `req`, or why it is refused. pub(crate) fn checked(req: &DeniedDomain) -> Result { let domain = normalize_domain(&req.domain)?; let agent = req .agent .as_deref() .map(|a| { hive_types::Ident::parse(a) .map(hive_types::Ident::into_string) .map_err(|reason| format!("agent {a:?}: {reason}")) }) .transpose()?; Ok(DenyEntry { domain, agent }) } /// The hives among `rows` whose proxy is not known to refuse the list at /// `revision` yet. /// /// A hive that reports a revision is behind when it differs. A hive that /// reports none (it predates the field, has not read the list, or has never /// reported) is behind only while the list holds an entry: it cannot be /// enforcing a name it was never told, but an empty list asks nothing of it. /// A hive the roster does not list is left out. pub(crate) fn behind(rows: &[HiveStatus], revision: u64, list_is_empty: bool) -> Vec { rows.iter() .filter(|row| row.freshness != Freshness::Unknown) .filter( |row| match row.snapshot.as_ref().and_then(egress_deny_applied) { Some(applied) => applied != revision, None => !list_is_empty, }, ) .map(|row| row.name.clone()) .collect() } /// The list as the API answers it. fn view(list: DenyList, revision: u64, behind: Vec) -> DeniedDomains { let mut entries = list.entries; entries.sort(); DeniedDomains { entries: entries .into_iter() .map(|e| DeniedDomain { domain: e.domain, agent: e.agent, }) .collect(), revision, behind, } } /// Which hives are behind `list` at `revision`. A status read that fails names /// every hive, since nothing then confirms any. async fn behind_hives(state: &AppState, revision: u64, list: &DenyList) -> Vec { let Some(reader) = state.status.as_ref() else { return Vec::new(); }; match reader .view(&state.hives, std::time::SystemTime::now()) .await { Ok(rows) => behind(&rows, revision, list.entries.is_empty()), Err(e) => { tracing::warn!(error = %format!("{e:#}"), "egress deny list: reading the hives' status failed"); state.hives.iter().map(|h| h.name.clone()).collect() } } } fn queue_problem(e: &anyhow::Error) -> ProblemDetails { tracing::warn!(error = %format!("{e:#}"), "egress deny list: queue failure"); error_problem(wanted_error_status(e), &format!("{e:#}")) } async fn open_or_create( client: &async_nats::Client, ) -> Result { egress_allow::open_or_create(client) .await .map_err(|e| queue_problem(&anyhow::Error::new(e))) } #[utoipa::path( get, path = "/api/egress/denied-domains", responses( (status = 200, description = "every entry, swarm-wide and per agent, sorted, the \ stored revision, and the hives that have not reported rendering it; empty when \ the swarm has not set a list", body = DeniedDomains), (status = 503, description = "no swarm queue is configured on this host, or it is \ not connected (problem+json)", body = String), (status = 500, description = "the list could not be read (problem+json)", body = String), ), tag = "egress" )] pub(crate) async fn get_denied_domains( State(state): State, ) -> Result, ProblemDetails> { let client = queue(&state).map_err(refused)?; let Some(store) = egress_allow::open(&client) .await .map_err(|e| queue_problem(&anyhow::Error::new(e)))? else { return Ok(Json(DeniedDomains::default())); }; let (list, revision, _) = read_or_default::(&store, DENY_KEY) .await .map_err(|e| queue_problem(&e))?; let revision = revision.unwrap_or(0); let behind = behind_hives(&state, revision, &list).await; Ok(Json(view(list, revision, behind))) } /// What changing a list came to. enum Changed { /// Nothing to write. Same(DenyList, u64), /// The list to write. New(DenyList), } /// `list` with `entry` added, or why it can't take another entry. fn added(mut list: DenyList, entry: &DenyEntry, revision: u64) -> Result { if list.entries.contains(entry) { return Ok(Changed::Same(list, revision)); } if list.entries.len() >= MAX_ENTRIES { return Err(format!( "the list holds {MAX_ENTRIES} entries, which is the most it takes" )); } list.entries.push(entry.clone()); list.entries.sort(); Ok(Changed::New(list)) } /// `list` without `entry`. fn removed(mut list: DenyList, entry: &DenyEntry, revision: u64) -> Changed { if !list.entries.contains(entry) { return Changed::Same(list, revision); } list.entries.retain(|e| e != entry); Changed::New(list) } /// Re-read the list and write `change` of it, retrying a lost write, and answer /// the list as it stands after. async fn apply( state: &AppState, client: &async_nats::Client, what: &str, change: impl Fn(DenyList, u64) -> Result, ) -> Result, ProblemDetails> { let store = open_or_create(client).await?; for _ in 0..MAX_ATTEMPTS { let (list, revision, _) = read_or_default::(&store, DENY_KEY) .await .map_err(|e| queue_problem(&e))?; let list = match change(list, revision.unwrap_or(0)) { Ok(Changed::Same(list, revision)) => { let behind = behind_hives(state, revision, &list).await; return Ok(Json(view(list, revision, behind))); } Ok(Changed::New(list)) => list, Err(why) => return Err(error_problem(StatusCode::BAD_REQUEST, &why)), }; if let Some(written) = write_revision(&store, DENY_KEY, &list, revision) .await .map_err(|e| queue_problem(&e))? { tracing::info!(domain = %what, revision = written, "changed the egress deny list"); let behind = behind_hives(state, written, &list).await; return Ok(Json(view(list, written, behind))); } } Err(queue_problem(&anyhow::anyhow!( "gave up changing {what} after {MAX_ATTEMPTS} conflicting writes" ))) } /// Refuse one destination, for every agent or for one. /// /// Adding an entry that is already there changes nothing. The entry is /// refused before any allow rule on every hive that has taken the list. #[utoipa::path( post, path = "/api/egress/denied-domains", request_body = DeniedDomain, responses( (status = 200, description = "the whole list after the add", body = DeniedDomains), (status = 400, description = "the domain is not a DNS name (an IP address, a \ wildcard, a single label and a port are refused), the agent is not an \ identifier or not declared on any hive, or the list is full (problem+json)", body = String), (status = 422, description = "an unknown field"), (status = 503, description = "no swarm queue is configured on this host, or it is \ not connected (problem+json)", body = String), (status = 500, description = "the list could not be written (problem+json)", body = String), ), tag = "egress" )] pub(crate) async fn add_denied_domain( State(state): State, Json(req): Json, ) -> Result, ProblemDetails> { let entry = checked(&req).map_err(|why| error_problem(StatusCode::BAD_REQUEST, &why))?; let client = queue(&state).map_err(refused)?; if let Some(agent) = &entry.agent && !live(&state).await?.contains(agent) { return Err(error_problem( StatusCode::BAD_REQUEST, &format!("no hive's wanted state declares an agent {agent:?} that is not destroyed"), )); } apply(&state, &client, &entry.domain, |list, revision| { added(list, &entry, revision) }) .await } /// Stop refusing one destination for the scope it was added with. /// /// Removing an entry that is not there changes nothing. A swarm-wide entry is /// removed only by naming it without an agent: removing an agent's entry for /// the same name leaves it refused for everyone. #[utoipa::path( delete, path = "/api/egress/denied-domains", request_body = DeniedDomain, responses( (status = 200, description = "the whole list after the removal", body = DeniedDomains), (status = 400, description = "the domain is not a DNS name, or the agent is not an \ identifier (problem+json)", body = String), (status = 422, description = "an unknown field"), (status = 503, description = "no swarm queue is configured on this host, or it is \ not connected (problem+json)", body = String), (status = 500, description = "the list could not be written (problem+json)", body = String), ), tag = "egress" )] pub(crate) async fn remove_denied_domain( State(state): State, Json(req): Json, ) -> Result, ProblemDetails> { let entry = checked(&req).map_err(|why| error_problem(StatusCode::BAD_REQUEST, &why))?; let client = queue(&state).map_err(refused)?; apply(&state, &client, &entry.domain, |list, revision| { Ok(removed(list, &entry, revision)) }) .await } /// Drop every entry of a destroyed agent: a name is free again once its agent /// is destroyed, and the next agent of that name would inherit what it left. A /// list that was never written stays unwritten. A failure is logged and does /// not stop the destroy, as in `egress_allow::clear_on_destroy`. pub(crate) async fn clear_on_destroy(state: &AppState, hive: &str, agent: &str) { let cleared = async { let client = queue(state).map_err(|(_, why)| anyhow::anyhow!(why))?; let Some(store) = egress_allow::open(&client).await? else { return Ok(()); }; for _ in 0..MAX_ATTEMPTS { let (mut list, revision, _) = read_or_default::(&store, DENY_KEY).await?; let before = list.entries.len(); list.entries.retain(|e| e.agent.as_deref() != Some(agent)); if list.entries.len() == before { return Ok(()); } if write_revision(&store, DENY_KEY, &list, revision) .await? .is_some() { return Ok(()); } } anyhow::bail!("gave up after {MAX_ATTEMPTS} conflicting writes") } .await; match cleared { Ok(()) => tracing::info!(hive, agent, "destroyed agent's denied domains cleared"), Err(e) => tracing::error!( hive, agent, error = %format!("{e:#}"), "clearing this agent's denied domains FAILED; it was destroyed anyway, so a new \ agent of the same name inherits them. Declaring the agent destroyed again re-runs this." ), } } #[cfg(test)] mod tests { use super::*; use crate::agent_limits::tests::{declared, state}; use crate::test_nats::TestNats; use swarm_wire::hive_status::EGRESS_DENY_APPLIED; fn entry(domain: &str, agent: Option<&str>) -> DeniedDomain { DeniedDomain { domain: domain.to_owned(), agent: agent.map(str::to_owned), } } async fn add( state: &AppState, domain: &str, agent: Option<&str>, ) -> Result { add_denied_domain(State(state.clone()), Json(entry(domain, agent))) .await .map(|j| j.0) } async fn remove( state: &AppState, domain: &str, agent: Option<&str>, ) -> Result { remove_denied_domain(State(state.clone()), Json(entry(domain, agent))) .await .map(|j| j.0) } async fn read(state: &AppState) -> DeniedDomains { get_denied_domains(State(state.clone())) .await .expect("the list is read") .0 } async fn listed(state: &AppState) -> Vec<(String, Option)> { read(state) .await .entries .into_iter() .map(|e| (e.domain, e.agent)) .collect() } /// The raw value a hive reads under `key`, or `None` for no key. async fn published(client: &async_nats::Client, key: &str) -> Option { egress_allow::open_or_create(client) .await .expect("bucket") .get(key) .await .expect("read") .map(|v| String::from_utf8(v.to_vec()).expect("utf-8")) } fn row(name: &str, freshness: Freshness, applied: Option) -> HiveStatus { HiveStatus { name: name.to_owned(), domain: None, dashboard: None, freshness, last_seen_unix: None, age_seconds: None, snapshot: applied.map(|r| serde_json::json!({ EGRESS_DENY_APPLIED: r })), } } #[test] fn a_name_is_stored_normalised_and_an_ip_literal_is_refused() { let stored = checked(&entry(" Evil.COM ", Some("atlas"))).expect("a name"); assert_eq!(stored.domain, "evil.com"); assert_eq!(stored.agent.as_deref(), Some("atlas")); let dotted = checked(&entry(".evil.com", None)).expect("dot form"); assert_eq!(dotted.domain, ".evil.com"); for bad in [ "1.2.3.4", "[::1]", "*.example.com", "localhost", "example.com:443", "example.com\nhttp_access allow all", ] { assert!(checked(&entry(bad, None)).is_err(), "{bad:?} was accepted"); } for agent in ["", "At Las", "a/b", "../argus"] { assert!( checked(&entry("evil.com", Some(agent))).is_err(), "agent {agent:?} was accepted" ); } } /// The base list is no reason to refuse a deny: denying a name that is /// otherwise always allowed is what the list is for. #[tokio::test] async fn a_name_the_base_list_allows_can_be_denied() { let nats = TestNats::start(); let client = nats.client().await; let state = state(&client); // Control: the allow list's own check refuses this name. assert!( crate::egress_allow::checked( &swarm_controller_api::egress::AllowedDomain { domain: "api.anthropic.com".to_owned(), agent: None, derived: false, }, &["api.anthropic.com".to_owned()], ) .is_err() ); add(&state, "api.anthropic.com", None) .await .expect("denied despite the base list"); } /// Through the real handlers and a real `nats-server`: an entry lands under /// the deny key a hive reads and never under the allow key, the two scopes /// of one name are two entries, a repeat writes nothing, and removing an /// agent's entry leaves the swarm-wide one in force. /// /// The published bytes are pinned literally because hive-c0re's /// `workers::squid_deny` tests decode this same string. #[tokio::test] async fn the_route_writes_the_list_a_hive_reads_and_a_scoped_removal_never_lifts_the_swarm_wide_entry() { let nats = TestNats::start(); let client = nats.client().await; let state = declared(&client).await; add(&state, "evil.com", None).await.expect("swarm-wide"); add(&state, "bad.io", Some("atlas")).await.expect("atlas's"); add(&state, "evil.com", Some("atlas")) .await .expect("the same name for atlas alone"); let before = published(&client, DENY_KEY).await; add(&state, "EVIL.com", None).await.expect("a repeat"); assert_eq!( published(&client, DENY_KEY).await, before, "a repeat writes nothing" ); assert_eq!( published(&client, DENY_KEY).await.as_deref(), Some( r#"{"entries":[{"domain":"bad.io","agent":"atlas"},{"domain":"evil.com"},{"domain":"evil.com","agent":"atlas"}]}"# ) ); assert_eq!( published(&client, swarm_wire::egress_allow::KEY).await, None, "a deny is not an allow entry" ); remove(&state, "evil.com", Some("atlas")) .await .expect("removes atlas's"); assert_eq!( listed(&state).await, [ ("bad.io".to_owned(), Some("atlas".to_owned())), ("evil.com".to_owned(), None), ], "the swarm-wide deny stays after atlas's is removed" ); let before = published(&client, DENY_KEY).await; remove(&state, "evil.com", Some("atlas")) .await .expect("already gone"); assert_eq!(published(&client, DENY_KEY).await, before); remove(&state, "evil.com", None).await.expect("swarm-wide"); remove(&state, "bad.io", Some("atlas")) .await .expect("atlas's"); assert_eq!( published(&client, DENY_KEY).await.as_deref(), Some(r#"{"entries":[]}"#), "the last removal leaves an empty list" ); } /// An allow-list write leaves the deny list alone and the other way round. #[tokio::test] async fn the_allow_list_and_the_deny_list_are_separate_values() { let nats = TestNats::start(); let client = nats.client().await; let state = declared(&client).await; add(&state, "evil.com", None).await.expect("a deny"); let allowed = crate::egress_allow::add_allowed_domain( State(state.clone()), Json(swarm_controller_api::egress::AllowedDomain { domain: "github.com".to_owned(), agent: None, derived: false, }), ) .await .expect("an allow"); assert_eq!(allowed.0.entries.len(), 1); assert_eq!( published(&client, DENY_KEY).await.as_deref(), Some(r#"{"entries":[{"domain":"evil.com"}]}"#) ); assert_eq!( published(&client, swarm_wire::egress_allow::KEY) .await .as_deref(), Some(r#"{"entries":[{"domain":"github.com"}]}"#) ); } #[tokio::test] async fn a_read_before_any_entry_answers_empty_and_creates_no_bucket() { let nats = TestNats::start(); let client = nats.client().await; let state = state(&client); let read = read(&state).await; assert!(read.entries.is_empty() && read.revision == 0 && read.behind.is_empty()); assert!( egress_allow::open(&client) .await .expect("the queue answers") .is_none() ); } #[tokio::test] async fn an_agent_no_hive_declares_is_refused_and_nothing_is_written() { let nats = TestNats::start(); let client = nats.client().await; let state = declared(&client).await; let problem = add(&state, "evil.com", Some("iris")) .await .expect_err("no hive declares iris"); assert_eq!(problem.status, Some(StatusCode::BAD_REQUEST), "{problem:?}"); assert_eq!(published(&client, DENY_KEY).await, None); add(&state, "evil.com", Some("atlas")) .await .expect("control: a declared agent is accepted"); } #[test] fn a_full_list_takes_no_new_entry_but_a_present_one_is_not_a_new_one() { let full = DenyList { entries: (0..MAX_ENTRIES) .map(|i| DenyEntry { domain: format!("host{i}.example.com"), agent: None, }) .collect(), }; let new = DenyEntry { domain: "evil.com".to_owned(), agent: None, }; let why = added(full.clone(), &new, 1).err().expect("a full list"); assert!(why.contains("1000"), "{why}"); let present = full.entries[0].clone(); assert!(matches!( added(full.clone(), &present, 1), Ok(Changed::Same(..)) )); let mut room = full; room.entries.pop(); let Ok(Changed::New(list)) = added(room, &new, 1) else { panic!("a list with room takes the entry"); }; assert_eq!(list.entries.len(), MAX_ENTRIES); } /// A hive is behind unless it reports the list's revision; one that reports /// nothing is behind only while the list asks something of it. #[test] fn a_hive_is_behind_unless_it_reports_the_revision_it_should_have() { let rows = [ row("current", Freshness::Fresh, Some(7)), row("older", Freshness::Fresh, Some(5)), row("silent-old", Freshness::Fresh, None), row("never", Freshness::NeverReported, None), row("stale-current", Freshness::Stale, Some(7)), row("stranger", Freshness::Unknown, None), ]; assert_eq!(behind(&rows, 7, false), ["older", "silent-old", "never"]); // An empty list asks nothing of a hive that reports nothing, but a hive // that reports a different revision still holds names the list dropped. assert_eq!(behind(&rows, 7, true), ["older"]); assert_eq!( behind(&rows, 0, true), ["current", "older", "stale-current"] ); } /// Through the status bucket: after an add every hive is behind until it /// publishes the revision the add answered. #[tokio::test] async fn a_hive_leaves_behind_when_its_snapshot_carries_the_revision() { let nats = TestNats::start(); let client = nats.client().await; let state = declared(&client).await; let added = add(&state, "evil.com", None).await.expect("a deny"); assert!(added.revision > 0); assert_eq!(added.behind, ["pr1ma"], "it has not reported"); let status = swarm_queue_client::status::open_or_create(&client) .await .expect("status bucket"); let publish = |applied: u64| { let body = serde_json::json!({ "status": "ok", "warnings": [], EGRESS_DENY_APPLIED: applied }); let status = status.clone(); async move { status .put("pr1ma", body.to_string().into()) .await .expect("publish"); } }; publish(added.revision - 1).await; assert_eq!(read(&state).await.behind, ["pr1ma"], "an older revision"); publish(added.revision).await; let now = read(&state).await; assert_eq!(now.revision, added.revision); assert!(now.behind.is_empty(), "{:?}", now.behind); } #[tokio::test] async fn destroying_an_agent_drops_its_denies_and_keeps_the_rest() { let nats = TestNats::start(); let client = nats.client().await; let state = declared(&client).await; add(&state, "evil.com", None).await.expect("swarm-wide"); add(&state, "bad.io", Some("atlas")).await.expect("atlas's"); clear_on_destroy(&state, "pr1ma", "atlas").await; assert_eq!(listed(&state).await, [("evil.com".to_owned(), None)]); } async fn call(app: &axum::Router, method: &str, body: &'static str, path: &str) -> StatusCode { use tower::ServiceExt as _; app.clone() .oneshot( axum::http::Request::builder() .method(method) .uri(path) .header(axum::http::header::CONTENT_TYPE, "application/json") .body(axum::body::Body::from(body)) .expect("request"), ) .await .expect("router answers") .status() } /// Through the real router: an IP literal or a wildcard is a 400 and writes /// nothing, an unknown field (a `derived` flag included) a 422, and the /// well-formed body after them is the control that the route reaches the /// deny key. #[tokio::test] async fn the_router_refuses_bad_names_and_a_derived_flag_before_the_queue() { let nats = TestNats::start(); let client = nats.client().await; let app = crate::startup::build_app(declared(&client).await); let path = "/api/egress/denied-domains"; for body in [ r#"{"domain":"1.2.3.4"}"#, r#"{"domain":"*.example.com"}"#, r#"{"domain":""}"#, ] { assert_eq!( call(&app, "POST", body, path).await, StatusCode::BAD_REQUEST, "{body}" ); } for body in [ r#"{"domain":"evil.com","derived":true}"#, r#"{"domain":"evil.com","hive":"pr1ma"}"#, ] { assert_eq!( call(&app, "POST", body, path).await, StatusCode::UNPROCESSABLE_ENTITY, "{body}" ); } assert_eq!(published(&client, DENY_KEY).await, None); assert_eq!( call(&app, "POST", r#"{"domain":"evil.com"}"#, path).await, StatusCode::OK ); assert_eq!( published(&client, DENY_KEY).await.as_deref(), Some(r#"{"entries":[{"domain":"evil.com"}]}"#) ); assert_eq!( call(&app, "DELETE", r#"{"domain":"evil.com"}"#, path).await, StatusCode::OK ); } }