From c5a876b8452433cb64970189718760d8901612b3 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 01:10:38 +0100 Subject: [PATCH 1/3] derive an agent's backend hosts into the egress allow list at the controller --- docs/networking/network.md | 19 +- docs/swarm/ui.md | 7 + docs/turn-loop/config.md | 7 + .../src/agent/detail/AllowedDomains.tsx | 36 +- .../src/agent/detail/allowed-domains.ts | 9 +- .../swarm-ui/test/allowed-domains.test.mjs | 10 + hive-c0re/src/workers/squid_allow.rs | 1 + nix/host-modules/hive-squid.nix | 9 +- swarm-controller-api/src/egress.rs | 6 + swarm-controller/README.md | 3 +- swarm-controller/src/backend_egress.rs | 703 ++++++++++++++++++ swarm-controller/src/backend_env.rs | 2 + swarm-controller/src/egress_allow.rs | 115 ++- swarm-controller/src/linked_accounts.rs | 2 + swarm-controller/src/main.rs | 1 + swarm-controller/src/routes/agents.rs | 3 +- swarm-controller/src/startup.rs | 15 +- swarm-wire/src/egress_allow.rs | 27 + 18 files changed, 950 insertions(+), 25 deletions(-) create mode 100644 swarm-controller/src/backend_egress.rs diff --git a/docs/networking/network.md b/docs/networking/network.md index a8ef44e63..e3d15215c 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -295,10 +295,9 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. (): `api.anthropic.com` for the model API and the usage probe (`hive-agent/src/claude_usage_watch.rs`), `claude.ai` for the account login, and `platform.claude.com` for the OAuth - token exchange and refresh. An agent whose `ANTHROPIC_BASE_URL` or ACP - provider points elsewhere needs that host added to the option: the value - sits in the agent's own environment file, which the host doesn't read. The - list omits the plugin marketplace clone from `github.com`, since a failed add + token exchange and refresh. An agent whose `ANTHROPIC_BASE_URL` points + elsewhere needs no entry here: the swarm controller adds that host to the + agent's own swarm entries (see _Swarm allow-list_ below). The list omits the plugin marketplace clone from `github.com`, since a failed add isn't fatal. The rules below still refuse an entry that resolves to a private address. - **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to empty, @@ -333,6 +332,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. you add the entry, and `hive-c0re` checks each name again and skips one that fails, or that the base list or the canary covers, or that another entry of the same list covers. A name that resolves to a private address stays a 403. + The controller also adds entries itself. When an operator saves an agent's + API-key environment, and once when the controller starts, it takes the host + from `ANTHROPIC_BASE_URL` and from every variable named `*_BASE_URL` + (the address an ACP agent that loads the same file reads), and keeps one + `derived` entry for that agent per host. Each host passes the same check as + an operator's entry, so an IP address, `localhost` and a base-list name get + none. A save or a delete of the environment replaces the agent's derived + entries, and destroying the agent drops all its entries. The controller reads only the + store's copy: an agent on the hand-placed `backendEnvironmentFile` alone, and + the `opencode` preset's `acp.opencode.provider.baseUrl`, get no derived + entry, so add one in the swarm UI. A `hive-c0re` that predates the field + renders a derived entry like any other. The swarm's list replaces the hive's `allowedDomains` as soon as it exists, even when empty: add the names the hive allowed to the swarm's list before relying on it. Until the controller has a list (it predates the diff --git a/docs/swarm/ui.md b/docs/swarm/ui.md index 8df40093b..d5670db60 100644 --- a/docs/swarm/ui.md +++ b/docs/swarm/ui.md @@ -633,6 +633,13 @@ name the proxy's private-range rules leave. **remove** drops one entry; **add** opens a dialog with a domain box and a tick for every agent in the swarm. The tick is off by default, so an entry is for this agent alone. +An entry with the badge **automatic** has no **remove**: the controller keeps +it from the host in the agent's stored API-key environment +(`ANTHROPIC_BASE_URL`, or any `*_BASE_URL` variable), so a restricted agent on +another provider reaches it. Saving or removing the environment changes it, and +the API refuses to add or remove one (`400` and `409`). An agent that only has +the hand-placed environment file gets none; add its provider's host here. + A domain is a DNS name of at least two labels, such as `github.com`. One leading dot, as in `.githubusercontent.com`, also allows every subdomain; the dot form is the only wildcard, because exact names can't list a service that diff --git a/docs/turn-loop/config.md b/docs/turn-loop/config.md index f55c6139f..305920f6e 100644 --- a/docs/turn-loop/config.md +++ b/docs/turn-loop/config.md @@ -299,6 +299,13 @@ text. An agent whose file has content creates the leaf from it again within 2 minutes; one without a file keeps its copy in `/run` until the container restarts. +A save also keeps the host of `ANTHROPIC_BASE_URL` (and of every `*_BASE_URL` +variable) as an automatic allowed-domain entry for the agent, so a restricted +agent reaches its provider through the hive's proxy; **Remove** drops it. The +controller reads the stored text only. An agent that has just the file gets no +entry, and an operator adds its provider's host in the swarm UI (see +[allowed domains](../swarm/ui.md#allowed-domains)). + Provision the file once, out of band (never through nix — an API key in the store is world-readable and travels with the flake closure): diff --git a/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx b/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx index 8d690b598..ff3851f29 100644 --- a/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx +++ b/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx @@ -16,6 +16,9 @@ // internet flag reaches any name the proxy's private-range rules leave, so an // entry matters only to a restricted one. // +// An entry the controller keeps from the agent's backend environment (`derived`) +// shows as automatic with no remove control; the route refuses to remove it. +// // Its state belongs to one agent: callers key it by agent name. import { useEffect, useState } from "preact/hooks"; import { ApiErrorPanel } from "@hive/shared/api-error-panel.js"; @@ -117,19 +120,32 @@ export function AllowedDomains({ agent }: { agent: string }) { {shown.length > 0 ? ( diff --git a/frontend/packages/swarm-ui/src/agent/detail/allowed-domains.ts b/frontend/packages/swarm-ui/src/agent/detail/allowed-domains.ts index 75820332e..4fab97a03 100644 --- a/frontend/packages/swarm-ui/src/agent/detail/allowed-domains.ts +++ b/frontend/packages/swarm-ui/src/agent/detail/allowed-domains.ts @@ -7,6 +7,11 @@ export interface AllowedDomain { domain: string; /** Absent for an entry that holds for every agent in the swarm. */ agent?: string; + /** + * Kept by the controller from the agent's backend environment: shown as + * automatic, and the route refuses to remove it. + */ + derived?: boolean; } export const ALLOWED_DOMAINS_URL = "/api/egress/allowed-domains"; @@ -28,10 +33,12 @@ export function entriesFor( export function allowedSummary(shown: AllowedDomain[]): string { if (shown.length === 0) return "none added"; const swarmWide = shown.filter((e) => e.agent === undefined).length; - const own = shown.length - swarmWide; + const automatic = shown.filter((e) => e.derived === true).length; + const own = shown.length - swarmWide - automatic; return [ swarmWide > 0 ? `${swarmWide} swarm-wide` : null, own > 0 ? `${own} for this agent` : null, + automatic > 0 ? `${automatic} automatic` : null, ] .filter((part) => part !== null) .join(", "); diff --git a/frontend/packages/swarm-ui/test/allowed-domains.test.mjs b/frontend/packages/swarm-ui/test/allowed-domains.test.mjs index 8cda747b3..c2f29d705 100644 --- a/frontend/packages/swarm-ui/test/allowed-domains.test.mjs +++ b/frontend/packages/swarm-ui/test/allowed-domains.test.mjs @@ -52,3 +52,13 @@ test("the body is swarm-wide or for the agent, with the name trimmed", () => { agent: "atlas", }); }); + +test("a derived entry is counted as automatic, not as the operator's own", () => { + const derived = { domain: "llm.example.com", agent: "atlas", derived: true }; + assert.equal( + allowedSummary([...entriesFor(all, "atlas"), derived]), + "1 swarm-wide, 2 for this agent, 1 automatic", + ); + assert.equal(allowedSummary([derived]), "1 automatic"); + assert.deepEqual(entriesFor([derived], "atlas"), [derived]); +}); diff --git a/hive-c0re/src/workers/squid_allow.rs b/hive-c0re/src/workers/squid_allow.rs index c3c95bce1..95eafc7ca 100644 --- a/hive-c0re/src/workers/squid_allow.rs +++ b/hive-c0re/src/workers/squid_allow.rs @@ -349,6 +349,7 @@ mod tests { Entry { domain: domain.to_owned(), agent: agent.map(str::to_owned), + derived: false, } } diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index a6a2dcd4e..0083363d7 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -289,10 +289,11 @@ in off the swarm to work at all. The defaults are the hosts Claude Code requires. `api.anthropic.com` serves the model API and the usage probe, `claude.ai` the account login, and `platform.claude.com` the - OAuth token exchange and refresh. An agent whose `claude` or ACP provider - points elsewhere (`ANTHROPIC_BASE_URL`, an `opencode` provider) needs - that host listed here: the value lives in the agent's own backend - environment, which this host does not read. Swarm-internal names are + OAuth token exchange and refresh. The swarm controller adds the host of an + agent's stored `ANTHROPIC_BASE_URL` to that agent's swarm entries; an + agent on the hand-placed environment file alone, or an `opencode` + provider, needs its host in the swarm UI or in `allowedDomains`. + Swarm-internal names are not listed, since agents reach them directly. The destination denies still apply to every entry, and the list is separate from `allowedDomains`. A name the swarm's allow-list holds that this list diff --git a/swarm-controller-api/src/egress.rs b/swarm-controller-api/src/egress.rs index 702580532..1b861c9be 100644 --- a/swarm-controller-api/src/egress.rs +++ b/swarm-controller-api/src/egress.rs @@ -18,6 +18,11 @@ pub struct AllowedDomain { #[serde(default, skip_serializing_if = "Option::is_none")] #[cfg_attr(feature = "schema", schema(example = "atlas"))] pub agent: Option, + /// Set on an entry the controller keeps from the agent's backend + /// environment (the host of its `ANTHROPIC_BASE_URL`). The list shows it + /// as automatic; adding or removing one is refused. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub derived: bool, } /// The swarm's allow-list: every entry, swarm-wide and per agent, sorted. @@ -36,6 +41,7 @@ mod tests { let swarm_wide = AllowedDomain { domain: "github.com".to_owned(), agent: None, + derived: false, }; let wire = r#"{"domain":"github.com"}"#; assert_eq!(serde_json::to_string(&swarm_wide).unwrap(), wire); diff --git a/swarm-controller/README.md b/swarm-controller/README.md index dca8e1cd8..213f9bf3e 100644 --- a/swarm-controller/README.md +++ b/swarm-controller/README.md @@ -67,7 +67,8 @@ The swarm's control plane. The swarm UI and `swarmctl` are its clients. `egress-allow` KV bucket, which this daemon alone writes (`GET`/`POST`/`DELETE /api/egress/allowed-domains`); every hive reads it and renders the proxy's allow-list files, and it replaces the list the hive's own - configuration names → + configuration names. It also keeps a `derived` entry per agent for the host + of its stored backend environment's `ANTHROPIC_BASE_URL` → [`docs/networking/network.md`](../docs/networking/network.md#egress-proxy-squid). - **Agent instructions** — each agent's `AGENTS.md` as its harness last published it into the `agents-md` KV bucket diff --git a/swarm-controller/src/backend_egress.rs b/swarm-controller/src/backend_egress.rs new file mode 100644 index 000000000..7ba14a581 --- /dev/null +++ b/swarm-controller/src/backend_egress.rs @@ -0,0 +1,703 @@ +//! The hosts an agent's backend environment names, kept as derived entries of +//! the egress allow list (`crate::egress_allow`), so a restricted agent whose +//! `ANTHROPIC_BASE_URL` points at a provider other than Anthropic reaches that +//! provider through its hive's proxy. +//! +//! The controller sees the environment where it is written (`backend_env`) and +//! in the store, so it derives here and a hive renders the entries like any +//! other. An entry is marked `derived`: [`follow_env`] replaces an agent's +//! derived entries on every write or delete of its environment, and +//! [`spawn_backfill`] reconciles all of them from the stored environments at +//! start. Each host goes through `egress_allow::checked`, the check an +//! operator's own entry gets. +//! +//! Only the store's copy is read. An agent still on the hand-placed +//! `backendEnvironmentFile` alone has no derived entry. The opencode preset's +//! provider URL (`acp.opencode.provider.baseUrl`) is host configuration, not +//! part of the environment, so it is never derived either. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use std::time::Duration; + +use problem_details::ProblemDetails; +use swarm_secret_client::backend_env::{self, BackendEnv}; +use swarm_wire::egress_allow::{AllowList, Entry, MAX_ENTRIES}; + +use crate::egress_allow::{base_domains, checked, modify}; +use crate::linked_accounts::AccountStore; +use crate::routes::{AppState, error_problem, wanted_error_status}; +use crate::schedules::queue; +use crate::wanted::WantedWriter; + +/// `ANTHROPIC_BASE_URL` and every variable named `_BASE_URL`, which +/// is where a provider's address sits for an ACP agent that loads the same +/// file. +fn is_url_variable(name: &str) -> bool { + name == "ANTHROPIC_BASE_URL" || name.ends_with("_BASE_URL") +} + +/// The URL variables `env` assigns, as `(name, value)`. A later assignment of a +/// name replaces an earlier one, as systemd's `EnvironmentFile=` does. +fn url_assignments(env: &str) -> BTreeMap<&str, &str> { + let mut found = BTreeMap::new(); + for line in env.lines().map(str::trim) { + if line.is_empty() || line.starts_with(['#', ';']) { + continue; + } + let Some((name, value)) = line.split_once('=') else { + continue; + }; + let name = name.trim(); + if is_url_variable(name) { + found.insert(name, unquoted(value.trim())); + } + } + found +} + +fn unquoted(value: &str) -> &str { + for quote in ['"', '\''] { + if let Some(inner) = value + .strip_prefix(quote) + .and_then(|v| v.strip_suffix(quote)) + { + return inner; + } + } + value +} + +/// The host of `url`, as written: its scheme (`http` or `https`, or none), +/// user information, port, path, query and fragment are dropped. Nothing is +/// validated; the result goes through `egress_allow::checked`. +fn host_of(url: &str) -> Option<&str> { + let rest = match url.split_once("://") { + Some(("http" | "https", rest)) => rest, + Some(_) => return None, + None => url, + }; + let authority = rest.split(['/', '?', '#']).next()?; + let authority = authority.rsplit('@').next()?; + Some(match authority.rsplit_once(':') { + Some((host, port)) if port.bytes().all(|b| b.is_ascii_digit()) => host, + _ => authority, + }) +} + +/// The derived entries for `agent`'s `env`: one per distinct host that passes +/// `egress_allow::checked`. A host that does not (an IP address, `localhost`, +/// a name the base list already allows) is logged by variable name and left +/// out. The value is never logged: a URL can carry credentials. +fn derive(agent: &str, env: &str, base: &[String]) -> Vec { + let mut entries = BTreeSet::new(); + for (name, value) in url_assignments(env) { + let Some(host) = host_of(value) else { + tracing::info!(%agent, variable = name, "backend egress: not an http(s) URL; no entry"); + continue; + }; + let request = swarm_controller_api::egress::AllowedDomain { + domain: host.to_owned(), + agent: Some(agent.to_owned()), + derived: false, + }; + match checked(&request, base) { + Ok(entry) => { + entries.insert(Entry { + derived: true, + ..entry + }); + } + Err(why) => { + tracing::info!(%agent, variable = name, %why, "backend egress: no entry for this host"); + } + } + } + entries.into_iter().collect() +} + +/// `list` with, for each agent in `desired`, its derived entries replaced by +/// the given ones. Entries of other agents and operator-added entries are +/// untouched. A full list takes no more derived entries. +fn reconciled(mut list: AllowList, desired: &BTreeMap>) -> AllowList { + list.entries + .retain(|e| !(e.derived && e.agent.as_ref().is_some_and(|a| desired.contains_key(a)))); + for entry in desired.values().flatten() { + if list.entries.len() >= MAX_ENTRIES { + tracing::warn!( + domain = %entry.domain, + "backend egress: the allow list is full; no entry for this host" + ); + break; + } + list.entries.push(entry.clone()); + } + list +} + +/// Replace `agent`'s derived entries with those of `env`, or with none when +/// `env` is `None` (the environment was deleted). A host without a queue +/// (`state.status` unset) has no allow list, so nothing happens. +/// +/// # Errors +/// A problem naming that the environment is stored but the list was not +/// updated; saving the environment again retries. +pub(crate) async fn follow_env( + state: &AppState, + agent: &str, + env: Option<&str>, +) -> Result<(), ProblemDetails> { + if state.status.is_none() { + return Ok(()); + } + let base = base_domains(); + let desired = BTreeMap::from([( + agent.to_owned(), + env.map(|text| derive(agent, text, &base)) + .unwrap_or_default(), + )]); + let updated = async { + let client = queue(state).map_err(|(_, why)| anyhow::anyhow!(why))?; + modify(&client, |list| reconciled(list, &desired)).await + } + .await; + match updated { + Ok(true) => tracing::info!(%agent, "backend egress: derived entries updated"), + Ok(false) => {} + Err(e) => { + tracing::warn!(%agent, error = %format!("{e:#}"), "backend egress: updating the allow list failed"); + return Err(error_problem( + wanted_error_status(&e), + &format!( + "the backend environment is stored, but the allowed domains were not \ + updated ({e:#}); saving it again retries" + ), + )); + } + } + Ok(()) +} + +/// Reconcile every agent's derived entries from the environments in `store`: +/// the live agents, and every agent that has a derived entry now. An agent +/// whose environment cannot be read keeps its entries. +/// +/// Answers how many agents could not be read. +/// +/// # Errors +/// A queue failure reading or writing the list. +pub(crate) async fn reconcile( + client: &async_nats::Client, + store: &impl AccountStore, + live: &BTreeSet, +) -> anyhow::Result { + let mut agents = live.clone(); + if let Some(list) = crate::egress_allow::current(client).await? { + agents.extend( + list.entries + .iter() + .filter(|e| e.derived) + .filter_map(|e| e.agent.clone()), + ); + } + let base = base_domains(); + let mut desired = BTreeMap::new(); + let mut unreadable = 0; + for agent in agents { + let Ok(path) = backend_env::env_path(&agent) else { + continue; + }; + match store.read_optional::(&path).await { + Ok(stored) => { + let entries = stored.map_or_else(Vec::new, |s| derive(&agent, &s.env, &base)); + desired.insert(agent, entries); + } + Err(e) => { + tracing::warn!(%agent, error = %e, "backend egress: reading the backend env failed; its entries stay"); + unreadable += 1; + } + } + } + if modify(client, |list| reconciled(list, &desired)).await? { + let derived: usize = desired.values().map(Vec::len).sum(); + tracing::info!( + agents = desired.len(), + derived, + "backend egress: backfilled the allow list" + ); + } + Ok(unreadable) +} + +/// Run [`reconcile`] once the queue and the store answer, retrying with a +/// growing wait until a pass reads every agent. One pass at start: the +/// routes keep the entries after it. +pub(crate) fn spawn_backfill( + client: async_nats::Client, + wanted: Arc, + hives: Vec, +) { + tokio::spawn(async move { + let mut wait = Duration::from_secs(5); + loop { + let pass = async { + swarm_queue_client::ensure_connected(&client)?; + let store = crate::store::connect().await?; + let live = crate::subagent_term::live_agents(&wanted, &hives).await; + anyhow::ensure!(!live.is_empty(), "no live agent is declared yet"); + reconcile(&client, &store, &live).await + } + .await; + match pass { + Ok(0) => return, + Ok(n) => tracing::warn!( + unreadable = n, + "backend egress: backfill incomplete; retrying" + ), + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "backend egress: backfill failed; retrying"); + } + } + tokio::time::sleep(wait).await; + wait = (wait * 2).min(Duration::from_mins(5)); + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::agent_limits::tests::declared; + use crate::linked_accounts::tests::FakeStore; + use crate::test_nats::TestNats; + use serde_json::json; + + fn base() -> Vec { + vec!["api.anthropic.com".to_owned(), ".claude.ai".to_owned()] + } + + fn hosts(env: &str) -> Vec { + derive("atlas", env, &base()) + .into_iter() + .map(|e| e.domain) + .collect() + } + + #[test] + fn a_url_is_reduced_to_its_host() { + for (value, host) in [ + ("https://openrouter.ai/api/v1", "openrouter.ai"), + ("http://llm.example.com", "llm.example.com"), + ("https://LLM.Example.com:8443/v1?x=1#f", "llm.example.com"), + ("llm.example.com/v1", "llm.example.com"), + ("llm.example.com:8080", "llm.example.com"), + ("\"https://llm.example.com/v1\"", "llm.example.com"), + ("'https://llm.example.com/v1'", "llm.example.com"), + ("https://user:pw@llm.example.com/v1", "llm.example.com"), + ] { + assert_eq!( + hosts(&format!("ANTHROPIC_BASE_URL={value}\n")), + [host], + "{value}" + ); + } + } + + #[test] + fn a_host_the_swarm_would_refuse_gets_no_entry() { + for value in [ + "http://10.0.0.5:8080/v1", + "https://127.0.0.1", + "http://[::1]:11434", + "http://localhost:11434/v1", + "https://metadata", + "ftp://llm.example.com", + "https://*.example.com", + "", + "https://", + // The base list already allows it. + "https://api.anthropic.com/v1", + "https://a.claude.ai", + ] { + assert_eq!( + hosts(&format!("ANTHROPIC_BASE_URL={value}\n")), + Vec::::new(), + "{value:?}" + ); + } + // Control: the same shape with a public name gets one. + assert_eq!( + hosts("ANTHROPIC_BASE_URL=https://llm.example.com\n"), + ["llm.example.com"] + ); + } + + #[test] + fn only_url_variables_count_and_the_last_assignment_wins() { + let env = "# ANTHROPIC_BASE_URL=https://commented.example.com\n\ + ANTHROPIC_API_KEY=https://not-a-url-var.example.com\n\ + OPENAI_BASE_URL=https://acp.example.org/v1\n\ + ANTHROPIC_BASE_URL=https://first.example.com\n\ + ANTHROPIC_BASE_URL=https://second.example.com\n"; + assert_eq!(hosts(env), ["acp.example.org", "second.example.com"]); + assert_eq!(hosts("ANTHROPIC_API_KEY=sk-x\n"), Vec::::new()); + } + + #[test] + fn a_derived_entry_is_marked_and_scoped_to_the_agent() { + let entries = derive( + "atlas", + "ANTHROPIC_BASE_URL=https://llm.example.com\n", + &base(), + ); + assert_eq!( + entries, + [Entry { + domain: "llm.example.com".to_owned(), + agent: Some("atlas".to_owned()), + derived: true, + }] + ); + } + + fn manual(domain: &str, agent: &str) -> Entry { + Entry { + domain: domain.to_owned(), + agent: Some(agent.to_owned()), + derived: false, + } + } + + fn derived(domain: &str, agent: &str) -> Entry { + Entry { + derived: true, + ..manual(domain, agent) + } + } + + #[test] + fn reconciling_replaces_only_the_named_agents_derived_entries() { + let list = AllowList { + entries: vec![ + derived("old.example.com", "atlas"), + manual("old.example.com", "atlas"), + manual("github.com", "atlas"), + derived("other.example.com", "iris"), + ], + }; + let desired = BTreeMap::from([( + "atlas".to_owned(), + vec![derived("new.example.com", "atlas")], + )]); + let after = reconciled(list, &desired); + assert_eq!( + after.entries, + [ + manual("old.example.com", "atlas"), + manual("github.com", "atlas"), + derived("other.example.com", "iris"), + derived("new.example.com", "atlas"), + ] + ); + } + + /// The state of a test swarm whose `pr1ma` declares `atlas`, with `store` + /// as the secret store. + async fn swarm(client: &async_nats::Client, store: &Arc) -> AppState { + AppState { + accounts: crate::linked_accounts::Accounts::Fake(Arc::clone(store)), + ..declared(client).await + } + } + + async fn list_of(client: &async_nats::Client) -> Vec { + crate::egress_allow::current(client) + .await + .expect("the queue answers") + .map(|l| { + let mut e = l.entries; + e.sort(); + e + }) + .unwrap_or_default() + } + + async fn put(state: &AppState, agent: &str, env: &str) { + crate::backend_env::put_backend_env( + axum::extract::State(state.clone()), + axum::extract::Path(("pr1ma".to_owned(), agent.to_owned())), + axum::Json(serde_json::from_value(json!({"env": env})).expect("a request body")), + ) + .await + .expect("stored"); + } + + #[tokio::test] + async fn saving_changing_and_deleting_the_env_moves_the_derived_entries() { + let nats = TestNats::start(); + let client = nats.client().await; + let store = Arc::default(); + let state = swarm(&client, &store).await; + + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://one.example.com/v1\n", + ) + .await; + assert_eq!( + list_of(&client).await, + [derived("one.example.com", "atlas")] + ); + + // A change replaces the host, an operator's own entry stays. + let _ = crate::egress_allow::add_allowed_domain( + axum::extract::State(state.clone()), + axum::Json(swarm_controller_api::egress::AllowedDomain { + domain: "github.com".to_owned(), + agent: Some("atlas".to_owned()), + derived: false, + }), + ) + .await + .expect("manual entry"); + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://two.example.com/v1\n", + ) + .await; + assert_eq!( + list_of(&client).await, + [ + manual("github.com", "atlas"), + derived("two.example.com", "atlas") + ] + ); + + // Removing the variable removes the entry. + put(&state, "atlas", "ANTHROPIC_API_KEY=sk-live\n").await; + assert_eq!(list_of(&client).await, [manual("github.com", "atlas")]); + + // Deleting the env takes the entries of the env it held. + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://three.example.com\n", + ) + .await; + crate::linked_accounts::delete_backend_env( + axum::extract::State(state.clone()), + axum::extract::Path(("pr1ma".to_owned(), "atlas".to_owned())), + ) + .await + .expect("deleted"); + assert_eq!(list_of(&client).await, [manual("github.com", "atlas")]); + } + + #[tokio::test] + async fn an_env_with_no_host_never_creates_the_list() { + let nats = TestNats::start(); + let client = nats.client().await; + let store = Arc::default(); + let state = swarm(&client, &store).await; + + put(&state, "atlas", "ANTHROPIC_API_KEY=sk-live\n").await; + assert!( + swarm_queue_client::egress_allow::open(&client) + .await + .expect("the queue answers") + .is_none(), + "no bucket: each hive keeps its own list" + ); + // Control: a host does create it. + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://llm.example.com\n", + ) + .await; + assert_eq!(list_of(&client).await.len(), 1); + } + + #[tokio::test] + async fn the_backfill_derives_from_stored_envs_and_a_second_pass_changes_nothing() { + let nats = TestNats::start(); + let client = nats.client().await; + let store = Arc::new( + FakeStore::default() + .with( + "swarm/agents/atlas/backend-env", + json!({"env": "ANTHROPIC_BASE_URL=https://llm.example.com/v1\n"}), + ) + .with( + "swarm/agents/iris/backend-env", + json!({"env": "ANTHROPIC_API_KEY=sk-live\n"}), + ), + ); + let live: BTreeSet = ["atlas", "iris", "scribe"].map(str::to_owned).into(); + + assert_eq!(reconcile(&client, &*store, &live).await.expect("pass"), 0); + assert_eq!( + list_of(&client).await, + [derived("llm.example.com", "atlas")] + ); + + let published = published(&client).await; + assert_eq!(reconcile(&client, &*store, &live).await.expect("again"), 0); + assert_eq!( + self::published(&client).await, + published, + "the second pass writes nothing" + ); + + // An agent that stops being live but keeps its leaf keeps its entry; + // one whose leaf is gone loses it. + let gone: BTreeSet = BTreeSet::new(); + reconcile(&client, &*store, &gone).await.expect("pass"); + assert_eq!( + list_of(&client).await, + [derived("llm.example.com", "atlas")] + ); + let empty = FakeStore::default(); + reconcile(&client, &empty, &gone).await.expect("pass"); + assert_eq!(list_of(&client).await, []); + } + + #[tokio::test] + async fn an_unreadable_env_keeps_its_entries() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = swarm(&client, &Arc::default()).await; + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://llm.example.com\n", + ) + .await; + + let live: BTreeSet = ["atlas"].map(str::to_owned).into(); + let unreadable = reconcile(&client, &FakeStore::denying(), &live) + .await + .expect("pass"); + assert_eq!(unreadable, 1); + assert_eq!( + list_of(&client).await, + [derived("llm.example.com", "atlas")] + ); + } + + async fn published(client: &async_nats::Client) -> Option { + swarm_queue_client::egress_allow::open_or_create(client) + .await + .expect("bucket") + .get(swarm_wire::egress_allow::KEY) + .await + .expect("read") + .map(|v| String::from_utf8(v.to_vec()).expect("utf-8")) + } + + #[tokio::test] + async fn an_operator_cannot_add_or_remove_a_derived_entry() { + let nats = TestNats::start(); + let client = nats.client().await; + let store = Arc::default(); + let state = swarm(&client, &store).await; + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://llm.example.com\n", + ) + .await; + + let request = |derived: bool| swarm_controller_api::egress::AllowedDomain { + domain: "llm.example.com".to_owned(), + agent: Some("atlas".to_owned()), + derived, + }; + for flagged in [true, false] { + let problem = crate::egress_allow::remove_allowed_domain( + axum::extract::State(state.clone()), + axum::Json(request(flagged)), + ) + .await + .expect_err("refused"); + assert_eq!( + problem.status, + Some(axum::http::StatusCode::CONFLICT), + "{problem:?}" + ); + } + let problem = crate::egress_allow::add_allowed_domain( + axum::extract::State(state.clone()), + axum::Json(request(true)), + ) + .await + .expect_err("refused"); + assert_eq!(problem.status, Some(axum::http::StatusCode::BAD_REQUEST)); + assert_eq!( + list_of(&client).await, + [derived("llm.example.com", "atlas")] + ); + + // Control: the same name added by the operator is a separate entry that + // the operator can remove, and the derived one stays. + let manual_request = request(false); + let _ = crate::egress_allow::add_allowed_domain( + axum::extract::State(state.clone()), + axum::Json(manual_request.clone()), + ) + .await + .expect("a manual entry"); + let _ = crate::egress_allow::remove_allowed_domain( + axum::extract::State(state.clone()), + axum::Json(manual_request), + ) + .await + .expect("removes the manual one"); + assert_eq!( + list_of(&client).await, + [derived("llm.example.com", "atlas")] + ); + } + + #[tokio::test] + async fn destroying_an_agent_drops_all_its_entries_and_only_its_own() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = swarm(&client, &Arc::default()).await; + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://llm.example.com\n", + ) + .await; + modify(&client, |mut list| { + list.entries.push(manual("github.com", "atlas")); + list.entries.push(manual("crates.io", "iris")); + list + }) + .await + .expect("seed"); + + crate::egress_allow::clear_on_destroy(&state, "pr1ma", "atlas").await; + assert_eq!(list_of(&client).await, [manual("crates.io", "iris")]); + } + + #[tokio::test] + async fn a_host_without_a_queue_stores_the_env_and_skips_the_list() { + let nats = TestNats::start(); + let client = nats.client().await; + let store = Arc::new(FakeStore::default()); + let mut state = swarm(&client, &store).await; + state.status = None; + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://llm.example.com\n", + ) + .await; + assert_eq!(store.written(), ["swarm/agents/atlas/backend-env"]); + assert_eq!(list_of(&client).await, []); + } +} diff --git a/swarm-controller/src/backend_env.rs b/swarm-controller/src/backend_env.rs index 7e9017d49..173fccaa3 100644 --- a/swarm-controller/src/backend_env.rs +++ b/swarm-controller/src/backend_env.rs @@ -76,6 +76,8 @@ pub async fn put_backend_env( error_problem(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()) })?; + crate::backend_egress::follow_env(&state, &agent, Some(&stored.env)).await?; + tracing::info!(%hive, %agent, "backend env stored"); Ok(StatusCode::NO_CONTENT) } diff --git a/swarm-controller/src/egress_allow.rs b/swarm-controller/src/egress_allow.rs index 5e927a6e1..ec1e11471 100644 --- a/swarm-controller/src/egress_allow.rs +++ b/swarm-controller/src/egress_allow.rs @@ -28,6 +28,10 @@ use crate::schedules::{live, queue, read_or_default, refused, write_revision}; /// `wanted::MAX_ATTEMPTS`. const MAX_ATTEMPTS: usize = 5; +/// Why an operator can neither add nor remove an entry the controller derived. +const DERIVED_REFUSED: &str = "this entry is kept automatically from the agent's backend environment; \ +change or remove that variable instead"; + /// Set from the hive module's `baseAllowedDomains` default by /// `nix/host-modules/swarm-controller.nix`: the names, separated by spaces, /// that every hive's proxy lets every authenticated agent reach. Unset means @@ -35,7 +39,7 @@ const MAX_ATTEMPTS: usize = 5; /// the entries its own base list covers. pub(crate) const ENV_BASE_DOMAINS: &str = "SWARM_CONTROLLER_SQUID_BASE_DOMAINS"; -fn base_domains() -> Vec { +pub(crate) fn base_domains() -> Vec { std::env::var(ENV_BASE_DOMAINS) .unwrap_or_default() .split_whitespace() @@ -60,7 +64,11 @@ pub(crate) fn checked(req: &AllowedDomain, base: &[String]) -> Result AllowedDomains { .map(|e| AllowedDomain { domain: e.domain, agent: e.agent, + derived: e.derived, }) .collect(), } @@ -158,7 +167,7 @@ pub(crate) async fn get_allowed_domains( (status = 400, description = "the domain is not a DNS name (an IP address, a \ wildcard, a single label and a port are refused), the proxy already allows \ it to everyone, the agent is not an identifier or not declared on any hive, \ - or the list is full (problem+json)", body = String), + or the list is full, or the entry is marked derived (problem+json)", body = String), (status = 422, description = "an unknown field"), (status = 503, description = "no swarm queue is configured on this host, or it is \ not connected (problem+json)", body = String), @@ -171,6 +180,9 @@ pub(crate) async fn add_allowed_domain( State(state): State, Json(req): Json, ) -> Result, ProblemDetails> { + if req.derived { + return Err(error_problem(StatusCode::BAD_REQUEST, DERIVED_REFUSED)); + } let entry = checked(&req, &base_domains()) .map_err(|why| error_problem(StatusCode::BAD_REQUEST, &why))?; let client = queue(&state).map_err(refused)?; @@ -219,6 +231,9 @@ pub(crate) async fn add_allowed_domain( (status = 200, description = "the whole list after the removal", body = AllowedDomains), (status = 400, description = "the domain is not a DNS name, or the agent is not an \ identifier (problem+json)", body = String), + (status = 409, description = "the entry is one the controller keeps from the agent's \ + backend environment, which an operator cannot remove (problem+json)", + body = String), (status = 422, description = "an unknown field"), (status = 503, description = "no swarm queue is configured on this host, or it is \ not connected (problem+json)", body = String), @@ -231,6 +246,9 @@ pub(crate) async fn remove_allowed_domain( State(state): State, Json(req): Json, ) -> Result, ProblemDetails> { + if req.derived { + return Err(error_problem(StatusCode::CONFLICT, DERIVED_REFUSED)); + } // The base list is not consulted: an entry stored before a name joined it // can still be removed. let entry = checked(&req, &[]).map_err(|why| error_problem(StatusCode::BAD_REQUEST, &why))?; @@ -241,6 +259,13 @@ pub(crate) async fn remove_allowed_domain( .await .map_err(|e| queue_problem(&e))?; if !list.entries.contains(&entry) { + if list + .entries + .iter() + .any(|e| e.derived && e.domain == entry.domain && e.agent == entry.agent) + { + return Err(error_problem(StatusCode::CONFLICT, DERIVED_REFUSED)); + } return Ok(Json(view(list))); } list.entries.retain(|e| *e != entry); @@ -259,6 +284,87 @@ pub(crate) async fn remove_allowed_domain( ))) } +/// The stored list, or `None` when no one has written it. +/// +/// # Errors +/// A queue failure. +pub(crate) async fn current(client: &async_nats::Client) -> anyhow::Result> { + let Some(store) = egress_allow::open(client).await? else { + return Ok(None); + }; + let (list, _, _) = read_or_default::(&store, KEY).await?; + Ok(Some(list)) +} + +/// Re-read the list and write `change` of it, retrying a lost write, until it +/// is written or `change` leaves it as it is. Answers whether a write landed. +/// +/// A list that was never written is written only when `change` makes it +/// non-empty: the key's existence is what makes the swarm's list replace each +/// hive's own, so a change that adds nothing must not create it. +/// +/// # Errors +/// A queue failure, or `MAX_ATTEMPTS` conflicting writes. +pub(crate) async fn modify( + client: &async_nats::Client, + change: impl Fn(AllowList) -> AllowList, +) -> anyhow::Result { + let mut store = egress_allow::open(client).await?; + for _ in 0..MAX_ATTEMPTS { + let (mut list, revision) = match &store { + Some(store) => { + let (list, revision, _) = read_or_default::(store, KEY).await?; + (list, revision) + } + None => (AllowList::default(), None), + }; + list.entries.sort(); + let mut changed = change(list.clone()); + changed.entries.sort(); + changed.entries.dedup(); + if changed == list && (revision.is_some() || changed.entries.is_empty()) { + return Ok(false); + } + if store.is_none() { + store = Some(egress_allow::open_or_create(client).await?); + } + let target = store.as_ref().expect("opened above"); + if write_revision(target, KEY, &changed, revision) + .await? + .is_some() + { + return Ok(true); + } + } + anyhow::bail!("gave up after {MAX_ATTEMPTS} conflicting writes") +} + +/// Drop every entry of a destroyed agent, derived and operator-added: a name +/// is free again once its agent is destroyed, and the next agent of that name +/// would inherit what it left. A failure is logged and does not stop the +/// destroy, as in `agent_permissions::clear_on_destroy`. +pub(crate) async fn clear_on_destroy(state: &AppState, hive: &str, agent: &str) { + let cleared = async { + let client = queue(state).map_err(|(_, why)| anyhow::anyhow!(why))?; + modify(&client, |mut list| { + list.entries.retain(|e| e.agent.as_deref() != Some(agent)); + list + }) + .await + } + .await; + match cleared { + Ok(_) => tracing::info!(hive, agent, "destroyed agent's allowed domains cleared"), + Err(e) => tracing::error!( + hive, + agent, + error = %format!("{e:#}"), + "clearing this agent's allowed domains FAILED; it was destroyed anyway, so a new \ + agent of the same name inherits them. Declaring the agent destroyed again re-runs this." + ), + } +} + #[cfg(test)] mod tests { use super::*; @@ -269,6 +375,7 @@ mod tests { AllowedDomain { domain: domain.to_owned(), agent: agent.map(str::to_owned), + derived: false, } } @@ -445,12 +552,14 @@ mod tests { .map(|i| Entry { domain: format!("host{i}.example.com"), agent: None, + derived: false, }) .collect(), }; let new = Entry { domain: "github.com".to_owned(), agent: None, + derived: false, }; let why = added(full.clone(), &new).err().expect("a full list"); assert!(why.contains("1000"), "{why}"); diff --git a/swarm-controller/src/linked_accounts.rs b/swarm-controller/src/linked_accounts.rs index 4dba4a6cb..4be9f719e 100644 --- a/swarm-controller/src/linked_accounts.rs +++ b/swarm-controller/src/linked_accounts.rs @@ -621,6 +621,8 @@ pub async fn delete_backend_env( e.problem() })?; + crate::backend_egress::follow_env(&state, &agent, None).await?; + tracing::info!(%hive, %agent, "backend env deleted"); Ok(StatusCode::NO_CONTENT) } diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 28d46b51d..2a70d58d1 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -39,6 +39,7 @@ mod alerts; mod archetypes; mod attic_copy; mod auth; +mod backend_egress; mod backend_env; mod builder; mod cache_miss; diff --git a/swarm-controller/src/routes/agents.rs b/swarm-controller/src/routes/agents.rs index 7f9836fdc..0272a5d46 100644 --- a/swarm-controller/src/routes/agents.rs +++ b/swarm-controller/src/routes/agents.rs @@ -17,7 +17,7 @@ use crate::routes::hives::HiveEntry; use crate::routes::{AppState, StatusUnavailable, error_problem, wanted_error_status}; use crate::{ agent_archetype, agent_identity, agent_limits, agent_permissions, agent_status, - agents_md_initial, auth, inference, schedules, wanted, + agents_md_initial, auth, egress_allow, inference, schedules, wanted, }; /// The swarm's agent roster — every agent the swarm holds an identity for. @@ -214,6 +214,7 @@ pub(crate) async fn set_agent_state( agent_permissions::clear_on_destroy(&state, &hive, &agent).await; agent_archetype::clear_on_destroy(&state, &hive, &agent).await; agents_md_initial::clear_on_destroy(&state, &hive, &agent).await; + egress_allow::clear_on_destroy(&state, &hive, &agent).await; } Ok(Json(render(&declaration))) } diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index 307a0ba8e..bf0fa26d7 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -375,7 +375,8 @@ fn spawn_subagent_streams(status: Option<&Arc>, hives: &[H } /// The per-agent background passes: subagent and terminal streams, and -/// schedule keys, and the permissions bucket and its import, and the +/// schedule keys, and the permissions bucket and its import, the allowed +/// domains derived from backend environments, and the /// archetypes agents are created from. Beside them, the /// copy of the cache facts to each hive, and the proxy credentials squid checks. /// Lifted out of `main` for `clippy::too_many_lines`. @@ -386,6 +387,7 @@ pub(crate) fn spawn_per_agent_upkeep( spawn_subagent_streams(status, hives); spawn_schedule_reconcile(status, hives); spawn_agent_permissions(status, hives); + spawn_backend_egress_backfill(status, hives); crate::attic_copy::spawn(hives.iter().map(|h| h.name.clone()).collect()); crate::proxy_credential::spawn(hives.iter().map(|h| h.name.clone()).collect()); if let Some(status) = status { @@ -403,6 +405,17 @@ fn spawn_agent_permissions(status: Option<&Arc>, hives: &[ agent_permissions::spawn(status.queue_client(), wanted, hives); } +/// Derive each agent's allowed-domain entries from the backend environments in +/// the store when a swarm queue is wired up. Same shape as +/// [`spawn_subagent_streams`]. +fn spawn_backend_egress_backfill(status: Option<&Arc>, hives: &[HiveEntry]) { + let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { + return; + }; + let hives = hives.iter().map(|h| h.name.clone()).collect(); + crate::backend_egress::spawn_backfill(status.queue_client(), wanted, hives); +} + /// Start rewriting every agent's schedule key from the definitions when a /// swarm queue is wired up. Same shape as [`spawn_subagent_streams`]. fn spawn_schedule_reconcile(status: Option<&Arc>, hives: &[HiveEntry]) { diff --git a/swarm-wire/src/egress_allow.rs b/swarm-wire/src/egress_allow.rs index 195c83e09..bb5847f94 100644 --- a/swarm-wire/src/egress_allow.rs +++ b/swarm-wire/src/egress_allow.rs @@ -44,6 +44,12 @@ pub struct Entry { /// The agent this entry is for; absent means every agent in the swarm. #[serde(default, skip_serializing_if = "Option::is_none")] pub agent: Option, + /// The controller derived this entry from the agent's backend environment + /// and replaces it whenever that changes; an operator can neither add nor + /// remove one. A hive renders it like any other entry, and a reader that + /// predates the field ignores it. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub derived: bool, } /// The value under [`BUCKET`]/[`KEY`]. No `deny_unknown_fields`: a field a @@ -222,10 +228,12 @@ mod tests { Entry { domain: "github.com".to_owned(), agent: None, + derived: false, }, Entry { domain: "crates.io".to_owned(), agent: Some("atlas".to_owned()), + derived: false, }, ], }; @@ -238,4 +246,23 @@ mod tests { AllowList::default() ); } + + /// A hive that predates `derived` decodes the entry through this same + /// tolerance: it keeps `domain` and `agent` and drops the rest. + #[test] + fn a_derived_entry_round_trips_and_an_entry_tolerates_fields_it_does_not_know() { + let derived = Entry { + domain: "llm.example.com".to_owned(), + agent: Some("atlas".to_owned()), + derived: true, + }; + let wire = r#"{"domain":"llm.example.com","agent":"atlas","derived":true}"#; + assert_eq!(serde_json::to_string(&derived).unwrap(), wire); + assert_eq!(serde_json::from_str::(wire).unwrap(), derived); + // Absent means a manual entry; an unknown field is dropped. + let manual: Entry = + serde_json::from_str(r#"{"domain":"a.io","agent":"atlas","later":1}"#).unwrap(); + assert!(!manual.derived); + assert_eq!(manual.domain, "a.io"); + } } From 7e2f4ce7c8af5b95347f170dd58a38d1d9e6eb77 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 01:47:25 +0100 Subject: [PATCH 2/3] always include a hive's own allowedDomains in the rendered swarm-wide list --- docs/networking/network.md | 14 ++- docs/swarm/ui.md | 5 +- .../src/agent/detail/AllowedDomains.tsx | 10 +- hive-c0re/src/workers/squid_allow.rs | 118 ++++++++++++++---- nix/host-modules/hive-squid.nix | 31 ++--- nix/vm-tests/hive-squid.nix | 8 +- swarm-controller/README.md | 4 +- swarm-controller/src/backend_egress.rs | 2 +- swarm-controller/src/egress_allow.rs | 24 ++-- 9 files changed, 141 insertions(+), 75 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index e3d15215c..e7e10c283 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -307,8 +307,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. which the rules below refuse, and agents reach them directly. Each entry matches that exact name, or with a leading dot the name and its subdomains. A user on the unrestricted list reaches any other name too, on ports 80 and 443. Everyone else gets a 403 for it. - `allowedDomains` is the hive's own list. Squid uses it until the swarm has a - list of its own (next bullet). + `allowedDomains` is the hive's own list, always allowed whether or not the + swarm has a list of its own (next bullet). - **Swarm allow-list.** The operator adds domains in the swarm UI ([allowed domains](../swarm/ui.md#allowed-domains)); the swarm controller keeps them in one value of the `egress-allow` bucket, each for every agent @@ -344,11 +344,13 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. the `opencode` preset's `acp.opencode.provider.baseUrl`, get no derived entry, so add one in the swarm UI. A `hive-c0re` that predates the field renders a derived entry like any other. - The swarm's list replaces the hive's `allowedDomains` as soon as it exists, - even when empty: add the names the hive allowed to the swarm's list - before relying on it. Until the controller has a list (it predates the + The swarm's list adds to the hive's `allowedDomains`: `allowedFile` holds + those names plus the swarm-wide entries, so removing an entry never removes a + name the hive lists itself. Until the controller has a list (it predates the feature, or nobody added an entry) `allowedFile` holds `allowedDomains` and - `agent-allow.conf` is empty. A queue that stops answering for the bucket + `agent-allow.conf` is empty. A `hive-c0re` older than this rule replaces + `allowedDomains` with the swarm's list once the controller has one, so every + hive must run the new `hive-c0re` before the controller's first entry exists. A queue that stops answering for the bucket leaves a file the swarm wrote untouched, and a failed read leaves both files untouched. The first line of a file written from the swarm's list is a `# source: swarm` comment. The queue grant a hive needs to read the list diff --git a/docs/swarm/ui.md b/docs/swarm/ui.md index d5670db60..c198a1896 100644 --- a/docs/swarm/ui.md +++ b/docs/swarm/ui.md @@ -655,9 +655,8 @@ an entry that exists changes nothing. A saved change isn't instant. Each hive reads the list about every 30 seconds and reloads its proxy, so the name is reachable within a minute. The proxy's private-range rules still apply: an entry whose name resolves to a -loopback, private or bridge address stays blocked. The first entry makes the -swarm's list replace the names a hive's own `allowedDomains` lists, so add -those names here too; a hive keeps its own list until then. See [the egress +loopback, private or bridge address stays blocked. An entry adds to the names a +hive's own `allowedDomains` lists, which stay allowed. See [the egress proxy](../networking/network.md#egress-proxy-squid) for what each hive does when the controller is old or unreachable. diff --git a/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx b/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx index ff3851f29..7341e405c 100644 --- a/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx +++ b/frontend/packages/swarm-ui/src/agent/detail/AllowedDomains.tsx @@ -10,9 +10,8 @@ // allows everyone) shows as it arrives, next to the box. The hive reads a // change within about 30 seconds. // -// The first entry makes the swarm's list replace the names the hive's own -// configuration allows, so a name the hive allowed and the swarm's list lacks -// stops being allowed once hives read it. An agent that holds the unrestricted +// An entry adds to the names the hive's own configuration allows, which stay +// allowed. An agent that holds the unrestricted // internet flag reaches any name the proxy's private-range rules leave, so an // entry matters only to a restricted one. // @@ -223,9 +222,8 @@ function AddForm({ such as github.com; a leading dot, as in{" "} .githubusercontent.com, also allows every subdomain. IP addresses and wildcards are refused. A name that resolves to a private - address stays blocked. The first entry makes the swarm's list replace - the names each hive's own configuration allows. The hive reads a change - within about 30 seconds. + address stays blocked. The entry adds to the names each hive's own + configuration allows. The hive reads a change within about 30 seconds.

void save(e)}> Result { let always: Vec<&str> = always.split_whitespace().collect(); let rendered = match read { - Read::Current(list) => render(&list, agents, &always), + Read::Current(list) => render(&list, agents, &always, &seed_names(seed)?), Read::NotSet => { if super::squid_users::read_or_empty(files.allowed)?.starts_with(SWARM_MARKER) { return Ok(Source::Kept); @@ -251,18 +251,27 @@ struct Rendered { counts: Option, } -/// The files for the seed: its names as they are, and no per-agent rules. -fn seeded(seed: Option<&Path>) -> Result { - let names = match seed { +/// The names in the seed file, trimmed, without blank lines and comments. The +/// file is this hive's own configuration, so its names are not re-checked. +fn seed_names(seed: Option<&Path>) -> Result> { + let text = match seed { Some(path) => super::squid_users::read_or_empty(path)?, None => String::new(), }; + Ok(text + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_owned) + .collect()) +} + +/// The files for the seed: its names as they are, and no per-agent rules. +fn seeded(seed: Option<&Path>) -> Result { let mut allowed = SEED_MARKER.to_owned(); - for line in names.lines().map(str::trim) { - if !line.is_empty() && !line.starts_with('#') { - allowed.push_str(line); - allowed.push('\n'); - } + for name in seed_names(seed)? { + allowed.push_str(&name); + allowed.push('\n'); } Ok(Rendered { allowed, @@ -283,8 +292,9 @@ fn reduced(mut names: Vec) -> Vec { names } -/// The files for `list`, for the agents this hive runs. -fn render(list: &AllowList, agents: &[String], always: &[&str]) -> Rendered { +/// The files for `list` and the seed's `own` names, for the agents this hive +/// runs. +fn render(list: &AllowList, agents: &[String], always: &[&str], own: &[String]) -> Rendered { let mut skipped = Vec::new(); let mut swarm_wide = Vec::new(); let mut per_agent: std::collections::BTreeMap<&str, Vec> = @@ -305,6 +315,7 @@ fn render(list: &AllowList, agents: &[String], always: &[&str]) -> Rendered { } } } + swarm_wide.extend(own.iter().map(|name| name.to_ascii_lowercase())); swarm_wide.retain(|d| !covered_by(d, always)); let swarm_wide = reduced(swarm_wide); @@ -366,7 +377,7 @@ mod tests { const ALWAYS: &[&str] = &["api.anthropic.com", ".claude.ai", "cache.nixos.org"]; fn rendered(entries: &[(&str, Option<&str>)], mine: &[&str]) -> Rendered { - render(&list(entries), &agents(mine), ALWAYS) + render(&list(entries), &agents(mine), ALWAYS, &[]) } #[test] @@ -464,6 +475,7 @@ mod tests { &list(&[("api.anthropic.com", None), ("a.claude.ai", None)]), &agents(&["atlas"]), &[], + &[], ); assert_eq!( bare.allowed, @@ -545,18 +557,74 @@ mod tests { ); } + /// The seed is the hive's own additive list: a swarm list, empty or not, + /// renders the seed's names with it. #[test] - fn a_swarm_list_replaces_the_seed_even_when_empty() { + fn a_swarm_list_adds_to_the_seed_even_when_empty() { let d = dir("forge.example\n"); sync(&d, Read::NotSet).expect("seeded"); assert!(text(&d.allowed()).contains("forge.example"), "control"); let empty = Read::Current(AllowList::default()); assert_eq!(sync(&d, empty).expect("renders"), Source::Swarm); - assert_eq!(text(&d.allowed()), SWARM_MARKER); - let named = Read::Current(list(&[("github.com", None), ("pypi.org", Some("atlas"))])); + assert_eq!(text(&d.allowed()), "# source: swarm\nforge.example\n"); + let named = Read::Current(list(&[("github.com", None)])); sync(&d, named).expect("renders"); + assert_eq!( + text(&d.allowed()), + "# source: swarm\nforge.example\ngithub.com\n" + ); + assert_eq!(text(&d.agent_allow()), SWARM_MARKER); + } + + #[test] + fn a_per_agent_entry_renders_on_top_of_the_seed() { + let d = dir("forge.example\n"); + let read = Read::Current(list(&[("pypi.org", Some("atlas"))])); + sync(&d, read).expect("renders"); + assert_eq!(text(&d.allowed()), "# source: swarm\nforge.example\n"); + assert_eq!( + text(&d.agent_allow()), + "# source: swarm\n\ + acl agent_allow_atlas proxy_auth atlas\n\ + acl agent_allow_atlas_dst dstdomain pypi.org\n\ + http_access allow agent_allow_atlas agent_allow_atlas_dst\n" + ); + } + + #[test] + fn a_seed_name_the_swarm_repeats_or_squid_allows_anyway_is_written_once() { + // `cache.nixos.org` is always allowed; `.example.com` covers the + // swarm's `api.example.com`. + let d = dir("Forge.Example\n.example.com\ncache.nixos.org\n"); + let read = Read::Current(list(&[ + ("forge.example", None), + ("api.example.com", None), + ("github.com", None), + ])); + sync(&d, read).expect("renders"); + assert_eq!( + text(&d.allowed()), + "# source: swarm\n.example.com\nforge.example\ngithub.com\n" + ); + } + + #[test] + fn no_seed_file_is_no_seed_names() { + let d = dir(""); + std::fs::remove_file(d.seed()).expect("remove"); + let read = Read::Current(list(&[("github.com", None)])); + sync_with( + read, + &agents(&["atlas"]), + None, + "", + &Files { + allowed: &d.allowed(), + agent_allow: &d.agent_allow(), + }, + ) + .expect("renders"); assert_eq!(text(&d.allowed()), "# source: swarm\ngithub.com\n"); - assert!(text(&d.agent_allow()).contains("acl agent_allow_atlas proxy_auth atlas")); } #[test] diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 0083363d7..3909a9fea 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -5,8 +5,8 @@ # hive-c0re renders. A credential's agent either has the swarm's # `unrestricted_internet` flag, and may then reach any destination the deny # rules leave, or reaches only the listed names: the base list, the allow-list -# the swarm keeps (rendered by hive-c0re beside the users file, with this -# host's `allowedDomains` as the list until the swarm has one) and the names +# the swarm keeps plus this host's `allowedDomains` (rendered by hive-c0re +# beside the users file) and the names # the swarm lists for that agent alone. An agent sends its credential # once it has fetched it (../agent-modules/egress-proxy.nix); the exporter, # journal forwarding and the self-check below show the proxy works. @@ -32,8 +32,8 @@ let # The names squid allows every authenticated user whatever the swarm lists. # hive-c0re skips an allow-list entry that one of them already covers. alwaysAllowed = base ++ [ cfg.selfcheck.canary ]; - # What `allowedFile` holds until the swarm has an allow-list: this hive's - # own `allowedDomains`, one per line. The canary is in the configuration. + # What `allowedFile` starts as and always includes: this hive's own + # `allowedDomains`, one per line. The canary is in the configuration. seedFile = pkgs.writeText "squid-allowed-seed" ( lib.concatMapStrings (d: d + "\n") ( lib.unique (lib.subtractLists [ cfg.selfcheck.canary ] cfg.allowedDomains) @@ -254,12 +254,13 @@ in "crates.io" ]; description = '' - Hostnames squid lets a client reach, by CONNECT or plain HTTP, until the - swarm has an allow-list of its own: the list the swarm's controller - keeps (`/api/egress/allowed-domains`) replaces this one as soon as it - holds an entry or has had one, and this option is the list again only - while the controller has none, for instance because it predates the - feature. hive-c0re writes whichever list applies to `allowedFile`. + Hostnames squid always lets a client reach, by CONNECT or plain HTTP, on + this hive. The entries the swarm's controller keeps + (`/api/egress/allowed-domains`) add to this list and never replace it: + hive-c0re writes these names and the swarm-wide entries to + `allowedFile`. A hive-c0re that predates this rule replaced this list + with the controller's once the controller held an entry, so every hive + must run the new one before the controller creates its first entry. Each entry matches that exact name, or that name and its subdomains with a leading dot. Everything else gets a 403, except for an unrestricted user. The swarm's own service names are not @@ -340,9 +341,9 @@ in description = '' The names squid lets every restricted user reach, one per line, read as part of its allow ACL beside the self-check canary. It starts as a copy - of `allowedDomains`. hive-c0re rewrites it every pass: with the swarm's - swarm-wide entries once the controller has a list, otherwise with - `allowedDomains`. Squid is reloaded when it changes. Owned by + of `allowedDomains`. hive-c0re rewrites it every pass: with + `allowedDomains` and the swarm-wide entries once the controller has a + list, otherwise with `allowedDomains` alone. Squid is reloaded when it changes. Owned by `hive-core` and readable by the `squid` group. ''; }; @@ -369,8 +370,8 @@ in readOnly = true; defaultText = lib.literalMD "`allowedDomains`, one per line, minus the self-check canary"; description = '' - The names `allowedFile` starts as and hive-c0re writes to it again while - the swarm has no allow-list. Read-only. + The names `allowedFile` starts as and hive-c0re writes into it on every + pass, with or without a swarm allow-list. Read-only. ''; }; diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 37c4d3c13..f0f71599c 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -520,13 +520,13 @@ pkgs.testers.runNixOSTest { wait_status("other.t.local", RUTH, "403") assert connect("other.t.local", UMA) == "200", "control: uma is still listed" - with subtest("the swarm's allow-list replaces the hive's names, opens one to every restricted user, and a per-agent rule opens one to that agent alone"): + with subtest("the swarm's allow-list adds to the hive's names, opens one to every restricted user, and a per-agent rule opens one to that agent alone"): seeded = proxy.succeed(f"cat {ALLOWED}") assert connect("swarm.t.local", RUTH) == "403", "control: not allowed yet" assert connect("agentonly.t.local", ATLAS) == "403", "control: not allowed yet" assert connect("forge.t.local", RUTH) == "200", "control: the hive's own name" write_allow( - "# source: swarm\nswarm.t.local\n", + "# source: swarm\n" + seeded + "swarm.t.local\n", "# source: swarm\n" "acl agent_allow_atlas proxy_auth atlas\n" "acl agent_allow_atlas_dst dstdomain agentonly.t.local priv10.t.local\n" @@ -537,13 +537,13 @@ pkgs.testers.runNixOSTest { assert connect("swarm.t.local", ATLAS) == "200", "swarm-wide, another user" assert connect("agentonly.t.local", RUTH) == "403", "the per-agent name, another restricted user" assert connect("agentonly.t.local") == "407", "no credential" - assert connect("forge.t.local", RUTH) == "403", "the hive's own name is replaced" + assert connect("forge.t.local", RUTH) == "200", "the hive's own name stays" assert connect("priv10.t.local", ATLAS) == "403", "a per-agent name that resolves privately" assert connect("agentonly.t.local", UMA) == "200", "control: unrestricted reaches it anyway" write_allow(seeded, "") wait_status("swarm.t.local", RUTH, "403") wait_status("agentonly.t.local", ATLAS, "403") - assert connect("forge.t.local", RUTH) == "200", "the hive's own name is back" + assert connect("forge.t.local", RUTH) == "200", "the hive's own name after the swarm list is gone" with subtest("squid starts on a missing list, treats every user as restricted, and recovers when it appears"): proxy.succeed(f"rm {LIST}") diff --git a/swarm-controller/README.md b/swarm-controller/README.md index 213f9bf3e..46047ecea 100644 --- a/swarm-controller/README.md +++ b/swarm-controller/README.md @@ -66,8 +66,8 @@ The swarm's control plane. The swarm UI and `swarmctl` are its clients. hive's egress proxy, each for every agent or for one, as one value in the `egress-allow` KV bucket, which this daemon alone writes (`GET`/`POST`/`DELETE /api/egress/allowed-domains`); every hive reads it and - renders the proxy's allow-list files, and it replaces the list the hive's own - configuration names. It also keeps a `derived` entry per agent for the host + renders the proxy's allow-list files, which add to the names the hive's own + configuration lists. It also keeps a `derived` entry per agent for the host of its stored backend environment's `ANTHROPIC_BASE_URL` → [`docs/networking/network.md`](../docs/networking/network.md#egress-proxy-squid). - **Agent instructions** — each agent's `AGENTS.md` as its harness last diff --git a/swarm-controller/src/backend_egress.rs b/swarm-controller/src/backend_egress.rs index 7ba14a581..90d917edd 100644 --- a/swarm-controller/src/backend_egress.rs +++ b/swarm-controller/src/backend_egress.rs @@ -508,7 +508,7 @@ mod tests { .await .expect("the queue answers") .is_none(), - "no bucket: each hive keeps its own list" + "no bucket: nothing was written" ); // Control: a host does create it. put( diff --git a/swarm-controller/src/egress_allow.rs b/swarm-controller/src/egress_allow.rs index ec1e11471..9d16b9ad1 100644 --- a/swarm-controller/src/egress_allow.rs +++ b/swarm-controller/src/egress_allow.rs @@ -5,8 +5,9 @@ //! (`swarm_wire::egress_allow`), which this daemon alone writes. Every hive //! reads it and renders the entries for the agents it runs: a swarm-wide entry //! joins the proxy's allow list for everyone, an entry with an agent is that -//! agent's alone. Once the key exists it replaces the list a hive's own nix -//! configuration names, so an empty list is a list. +//! agent's alone. A hive always allows the names its own nix configuration +//! lists (`services.hyperhive.squid.allowedDomains`) as well, so these entries +//! only add to them. //! //! Each name goes into the proxy's configuration, so it is checked here with //! `swarm_wire::egress_allow::normalize_domain` (the hive checks again) and @@ -127,7 +128,7 @@ async fn open_or_create( path = "/api/egress/allowed-domains", responses( (status = 200, description = "every entry, swarm-wide and per agent, sorted; empty \ - when the swarm has not set a list, in which case each hive keeps its own", + when the swarm has not set a list", body = AllowedDomains), (status = 503, description = "no swarm queue is configured on this host, or it is \ not connected (problem+json)", body = String), @@ -154,10 +155,8 @@ pub(crate) async fn get_allowed_domains( /// Allow one destination, for every agent or for one. /// -/// Adding an entry that is already there changes nothing. The first entry -/// makes the swarm's list replace each hive's own, so a name the hive's nix -/// configuration allows and this list lacks stops being allowed once hives read -/// it. +/// Adding an entry that is already there changes nothing. The entry adds to +/// the names each hive's nix configuration already allows. #[utoipa::path( post, path = "/api/egress/allowed-domains", @@ -221,8 +220,8 @@ pub(crate) async fn add_allowed_domain( /// Stop allowing one destination for the scope it was added with. /// -/// Removing an entry that is not there changes nothing. Removing the last one -/// leaves an empty list, which still replaces each hive's own. +/// Removing an entry that is not there changes nothing. A name a hive's nix +/// configuration lists stays allowed on that hive. #[utoipa::path( delete, path = "/api/egress/allowed-domains", @@ -300,8 +299,7 @@ pub(crate) async fn current(client: &async_nats::Client) -> anyhow::Result Date: Thu, 29 Oct 2026 02:26:47 +0100 Subject: [PATCH 3/3] derive backend hosts only from the operator's save; the backfill only drops --- docs/networking/network.md | 23 +- docs/turn-loop/config.md | 6 +- swarm-controller/src/backend_egress.rs | 293 +++++++++++++++++-------- swarm-controller/src/startup.rs | 17 +- 4 files changed, 230 insertions(+), 109 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index e7e10c283..4c889e70e 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -332,18 +332,23 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. you add the entry, and `hive-c0re` checks each name again and skips one that fails, or that the base list or the canary covers, or that another entry of the same list covers. A name that resolves to a private address stays a 403. - The controller also adds entries itself. When an operator saves an agent's - API-key environment, and once when the controller starts, it takes the host - from `ANTHROPIC_BASE_URL` and from every variable named `*_BASE_URL` - (the address an ACP agent that loads the same file reads), and keeps one + The controller also adds entries itself, and only from the operator's save + of an agent's API-key environment in the swarm UI: it takes the host from + `ANTHROPIC_BASE_URL` and from every variable named `*_BASE_URL` (the + address an ACP agent that loads the same file reads), and keeps one `derived` entry for that agent per host. Each host passes the same check as an operator's entry, so an IP address, `localhost` and a base-list name get none. A save or a delete of the environment replaces the agent's derived - entries, and destroying the agent drops all its entries. The controller reads only the - store's copy: an agent on the hand-placed `backendEnvironmentFile` alone, and - the `opencode` preset's `acp.opencode.provider.baseUrl`, get no derived - entry, so add one in the swarm UI. A `hive-c0re` that predates the field - renders a derived entry like any other. + entries, and destroying the agent drops all its entries. The agent can + create its own `backend-env` leaf while the leaf is empty, so text found in + the store never creates an entry: when the controller starts, it only + drops derived entries whose exact host the stored environment no longer + names, and logs a host it didn't derive. An environment saved before this + existed gets its entries when the operator saves it once more. An agent on + the hand-placed `backendEnvironmentFile` alone, and the `opencode` preset's + `acp.opencode.provider.baseUrl`, get no derived entry, so add one in the + swarm UI. A `hive-c0re` that predates the field renders a derived entry + like any other. The swarm's list adds to the hive's `allowedDomains`: `allowedFile` holds those names plus the swarm-wide entries, so removing an entry never removes a name the hive lists itself. Until the controller has a list (it predates the diff --git a/docs/turn-loop/config.md b/docs/turn-loop/config.md index 305920f6e..b4f22e0cc 100644 --- a/docs/turn-loop/config.md +++ b/docs/turn-loop/config.md @@ -302,8 +302,10 @@ restarts. A save also keeps the host of `ANTHROPIC_BASE_URL` (and of every `*_BASE_URL` variable) as an automatic allowed-domain entry for the agent, so a restricted agent reaches its provider through the hive's proxy; **Remove** drops it. The -controller reads the stored text only. An agent that has just the file gets no -entry, and an operator adds its provider's host in the swarm UI (see +controller derives only from an operator's save, never from text it finds in the +store, so an environment saved before this existed needs one more save. An +agent that has just the file gets no entry, and an operator adds its +provider's host in the swarm UI (see [allowed domains](../swarm/ui.md#allowed-domains)). Provision the file once, out of band (never through nix — an API key in diff --git a/swarm-controller/src/backend_egress.rs b/swarm-controller/src/backend_egress.rs index 90d917edd..1ee1f0fad 100644 --- a/swarm-controller/src/backend_egress.rs +++ b/swarm-controller/src/backend_egress.rs @@ -3,21 +3,22 @@ //! `ANTHROPIC_BASE_URL` points at a provider other than Anthropic reaches that //! provider through its hive's proxy. //! -//! The controller sees the environment where it is written (`backend_env`) and -//! in the store, so it derives here and a hive renders the entries like any -//! other. An entry is marked `derived`: [`follow_env`] replaces an agent's -//! derived entries on every write or delete of its environment, and -//! [`spawn_backfill`] reconciles all of them from the stored environments at -//! start. Each host goes through `egress_allow::checked`, the check an -//! operator's own entry gets. +//! An entry is marked `derived` and a hive renders it like any other. Only the +//! operator's save or delete of the environment through the controller creates +//! one ([`follow_env`]): the agent can create its own `backend-env` leaf while +//! it is empty, so what the store holds is not evidence that an operator chose +//! it. [`spawn_backfill`] therefore only drops: it keeps a derived entry while +//! the stored environment still names its exact host, and never adds one. Each +//! host goes through `egress_allow::checked`, the check an operator's own +//! entry gets. //! -//! Only the store's copy is read. An agent still on the hand-placed -//! `backendEnvironmentFile` alone has no derived entry. The opencode preset's -//! provider URL (`acp.opencode.provider.baseUrl`) is host configuration, not -//! part of the environment, so it is never derived either. +//! An agent still on the hand-placed `backendEnvironmentFile` alone has no +//! derived entry, and neither does an environment saved before this existed +//! until the operator saves it once more. The opencode preset's provider URL +//! (`acp.opencode.provider.baseUrl`) is host configuration, not part of the +//! environment, so it is never derived either. use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; use std::time::Duration; use problem_details::ProblemDetails; @@ -28,7 +29,6 @@ use crate::egress_allow::{base_domains, checked, modify}; use crate::linked_accounts::AccountStore; use crate::routes::{AppState, error_problem, wanted_error_status}; use crate::schedules::queue; -use crate::wanted::WantedWriter; /// `ANTHROPIC_BASE_URL` and every variable named `_BASE_URL`, which /// is where a provider's address sits for an ACP agent that loads the same @@ -90,7 +90,18 @@ fn host_of(url: &str) -> Option<&str> { /// a name the base list already allows) is logged by variable name and left /// out. The value is never logged: a URL can carry credentials. fn derive(agent: &str, env: &str, base: &[String]) -> Vec { - let mut entries = BTreeSet::new(); + let mut entries: Vec = derive_named(agent, env, base) + .into_iter() + .map(|(_, entry)| entry) + .collect(); + entries.sort(); + entries.dedup(); + entries +} + +/// [`fn@derive`] with the variable each entry came from, one per variable. +fn derive_named<'a>(agent: &str, env: &'a str, base: &[String]) -> Vec<(&'a str, Entry)> { + let mut entries = Vec::new(); for (name, value) in url_assignments(env) { let Some(host) = host_of(value) else { tracing::info!(%agent, variable = name, "backend egress: not an http(s) URL; no entry"); @@ -102,24 +113,25 @@ fn derive(agent: &str, env: &str, base: &[String]) -> Vec { derived: false, }; match checked(&request, base) { - Ok(entry) => { - entries.insert(Entry { + Ok(entry) => entries.push(( + name, + Entry { derived: true, ..entry - }); - } + }, + )), Err(why) => { tracing::info!(%agent, variable = name, %why, "backend egress: no entry for this host"); } } } - entries.into_iter().collect() + entries } /// `list` with, for each agent in `desired`, its derived entries replaced by /// the given ones. Entries of other agents and operator-added entries are /// untouched. A full list takes no more derived entries. -fn reconciled(mut list: AllowList, desired: &BTreeMap>) -> AllowList { +fn replaced(mut list: AllowList, desired: &BTreeMap>) -> AllowList { list.entries .retain(|e| !(e.derived && e.agent.as_ref().is_some_and(|a| desired.contains_key(a)))); for entry in desired.values().flatten() { @@ -158,7 +170,7 @@ pub(crate) async fn follow_env( )]); let updated = async { let client = queue(state).map_err(|(_, why)| anyhow::anyhow!(why))?; - modify(&client, |list| reconciled(list, &desired)).await + modify(&client, |list| replaced(list, &desired)).await } .await; match updated { @@ -178,9 +190,26 @@ pub(crate) async fn follow_env( Ok(()) } -/// Reconcile every agent's derived entries from the environments in `store`: -/// the live agents, and every agent that has a derived entry now. An agent -/// whose environment cannot be read keeps its entries. +/// `list` without the derived entries of the agents in `still_named` whose host +/// is not in that agent's set. Nothing is added; entries of agents not in the +/// map and operator-added entries are untouched. +fn pruned(mut list: AllowList, still_named: &BTreeMap>) -> AllowList { + list.entries.retain(|e| { + let Some(hosts) = e.agent.as_ref().and_then(|a| still_named.get(a)) else { + return true; + }; + !e.derived || hosts.contains(&e.domain) + }); + list +} + +/// Drop the derived entries whose host the stored environment no longer names. +/// +/// Only the agents that have a derived entry are looked at. An agent whose +/// stored environment is gone loses all of them; one whose environment cannot +/// be read keeps them. A host the stored environment names that is not +/// already derived is left out and logged by variable name, since whoever +/// wrote the store may be the agent itself. /// /// Answers how many agents could not be read. /// @@ -189,28 +218,45 @@ pub(crate) async fn follow_env( pub(crate) async fn reconcile( client: &async_nats::Client, store: &impl AccountStore, - live: &BTreeSet, ) -> anyhow::Result { - let mut agents = live.clone(); - if let Some(list) = crate::egress_allow::current(client).await? { - agents.extend( - list.entries - .iter() - .filter(|e| e.derived) - .filter_map(|e| e.agent.clone()), - ); + let Some(current) = crate::egress_allow::current(client).await? else { + return Ok(0); + }; + let mut derived: BTreeMap> = BTreeMap::new(); + for entry in current.entries.iter().filter(|e| e.derived) { + if let Some(agent) = &entry.agent { + derived + .entry(agent.clone()) + .or_default() + .insert(entry.domain.clone()); + } } let base = base_domains(); - let mut desired = BTreeMap::new(); + let mut still_named = BTreeMap::new(); let mut unreadable = 0; - for agent in agents { + for (agent, kept) in derived { let Ok(path) = backend_env::env_path(&agent) else { continue; }; match store.read_optional::(&path).await { Ok(stored) => { - let entries = stored.map_or_else(Vec::new, |s| derive(&agent, &s.env, &base)); - desired.insert(agent, entries); + let named = stored.map_or_else(Vec::new, |s| { + derive_named(&agent, &s.env, &base) + .into_iter() + .map(|(variable, entry)| (variable.to_owned(), entry.domain)) + .collect() + }); + for (variable, domain) in &named { + if !kept.contains(domain) { + tracing::warn!( + %agent, + %variable, + "backend egress: the agent's stored env names a host not derived \ + (only an operator save derives)" + ); + } + } + still_named.insert(agent, named.into_iter().map(|(_, d)| d).collect()); } Err(e) => { tracing::warn!(%agent, error = %e, "backend egress: reading the backend env failed; its entries stay"); @@ -218,34 +264,25 @@ pub(crate) async fn reconcile( } } } - if modify(client, |list| reconciled(list, &desired)).await? { - let derived: usize = desired.values().map(Vec::len).sum(); + if modify(client, |list| pruned(list, &still_named)).await? { tracing::info!( - agents = desired.len(), - derived, - "backend egress: backfilled the allow list" + agents = still_named.len(), + "backend egress: dropped derived entries the stored env no longer names" ); } Ok(unreadable) } /// Run [`reconcile`] once the queue and the store answer, retrying with a -/// growing wait until a pass reads every agent. One pass at start: the -/// routes keep the entries after it. -pub(crate) fn spawn_backfill( - client: async_nats::Client, - wanted: Arc, - hives: Vec, -) { +/// growing wait until a pass reads every agent that has an entry. +pub(crate) fn spawn_backfill(client: async_nats::Client) { tokio::spawn(async move { let mut wait = Duration::from_secs(5); loop { let pass = async { swarm_queue_client::ensure_connected(&client)?; let store = crate::store::connect().await?; - let live = crate::subagent_term::live_agents(&wanted, &hives).await; - anyhow::ensure!(!live.is_empty(), "no live agent is declared yet"); - reconcile(&client, &store, &live).await + reconcile(&client, &store).await } .await; match pass { @@ -266,6 +303,8 @@ pub(crate) fn spawn_backfill( #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; use crate::agent_limits::tests::declared; use crate::linked_accounts::tests::FakeStore; @@ -376,7 +415,7 @@ mod tests { } #[test] - fn reconciling_replaces_only_the_named_agents_derived_entries() { + fn replacing_replaces_only_the_named_agents_derived_entries() { let list = AllowList { entries: vec![ derived("old.example.com", "atlas"), @@ -389,7 +428,7 @@ mod tests { "atlas".to_owned(), vec![derived("new.example.com", "atlas")], )]); - let after = reconciled(list, &desired); + let after = replaced(list, &desired); assert_eq!( after.entries, [ @@ -520,47 +559,126 @@ mod tests { assert_eq!(list_of(&client).await.len(), 1); } + const OPERATOR: &str = "ANTHROPIC_BASE_URL=https://llm.example.com/v1\n"; + + fn leaf(agent: &str) -> String { + format!("swarm/agents/{agent}/backend-env") + } + + async fn write_leaf(store: &FakeStore, agent: &str, env: &str) { + store + .write( + &leaf(agent), + &BackendEnv { + env: env.to_owned(), + }, + ) + .await + .expect("written"); + } + + /// The agent creates its own leaf while it is empty, so a stored value is + /// not evidence of an operator's choice: the backfill adds nothing for it. #[tokio::test] - async fn the_backfill_derives_from_stored_envs_and_a_second_pass_changes_nothing() { + async fn a_leaf_created_outside_the_route_yields_no_entry() { let nats = TestNats::start(); let client = nats.client().await; - let store = Arc::new( - FakeStore::default() - .with( - "swarm/agents/atlas/backend-env", - json!({"env": "ANTHROPIC_BASE_URL=https://llm.example.com/v1\n"}), - ) - .with( - "swarm/agents/iris/backend-env", - json!({"env": "ANTHROPIC_API_KEY=sk-live\n"}), - ), - ); - let live: BTreeSet = ["atlas", "iris", "scribe"].map(str::to_owned).into(); + let store = Arc::new(FakeStore::default()); + let state = swarm(&client, &store).await; + write_leaf( + &store, + "atlas", + "ANTHROPIC_BASE_URL=https://evil.example.net\n", + ) + .await; - assert_eq!(reconcile(&client, &*store, &live).await.expect("pass"), 0); + // With no list at all, the pass creates nothing. + assert_eq!(reconcile(&client, &*store).await.expect("pass"), 0); + assert!( + swarm_queue_client::egress_allow::open(&client) + .await + .expect("the queue answers") + .is_none() + ); + + // With a list that holds only an operator's entry, nothing is added. + modify(&client, |mut list| { + list.entries.push(manual("github.com", "atlas")); + list + }) + .await + .expect("seed"); + assert_eq!(reconcile(&client, &*store).await.expect("pass"), 0); + assert_eq!(list_of(&client).await, [manual("github.com", "atlas")]); + + // Control: the operator's save of the same text derives the host. + put( + &state, + "atlas", + "ANTHROPIC_BASE_URL=https://evil.example.net\n", + ) + .await; assert_eq!( list_of(&client).await, - [derived("llm.example.com", "atlas")] + [ + derived("evil.example.net", "atlas"), + manual("github.com", "atlas") + ] ); + } - let published = published(&client).await; - assert_eq!(reconcile(&client, &*store, &live).await.expect("again"), 0); - assert_eq!( - self::published(&client).await, - published, - "the second pass writes nothing" - ); + /// A pass keeps an entry whose exact host the stored env still names, + /// drops one it no longer names, and never adds the new host. + #[tokio::test] + async fn the_backfill_keeps_a_named_host_drops_a_stale_one_and_adds_nothing() { + let nats = TestNats::start(); + let client = nats.client().await; + let store = Arc::new(FakeStore::default()); + let state = swarm(&client, &store).await; + put(&state, "atlas", OPERATOR).await; + let entry = derived("llm.example.com", "atlas"); - // An agent that stops being live but keeps its leaf keeps its entry; - // one whose leaf is gone loses it. - let gone: BTreeSet = BTreeSet::new(); - reconcile(&client, &*store, &gone).await.expect("pass"); - assert_eq!( - list_of(&client).await, - [derived("llm.example.com", "atlas")] - ); - let empty = FakeStore::default(); - reconcile(&client, &empty, &gone).await.expect("pass"); + // Control: unchanged content keeps the entry, and a second pass writes + // nothing. + assert_eq!(reconcile(&client, &*store).await.expect("pass"), 0); + assert_eq!(list_of(&client).await, [entry]); + let before = published(&client).await; + reconcile(&client, &*store).await.expect("again"); + assert_eq!(published(&client).await, before); + + // The stored text changed outside the route: the old host goes, the + // new one does not come. + write_leaf( + &store, + "atlas", + "ANTHROPIC_BASE_URL=https://other.example.net\n", + ) + .await; + reconcile(&client, &*store).await.expect("pass"); + assert_eq!(list_of(&client).await, []); + } + + /// The leaf is deleted outside the controller and created again by the + /// agent with another host before the pass: the old entry goes and the new + /// host is not derived. A leaf that is simply gone drops its entries too. + #[tokio::test] + async fn a_leaf_deleted_and_recreated_by_the_agent_loses_the_entry_and_gains_none() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = swarm(&client, &Arc::default()).await; + put(&state, "atlas", OPERATOR).await; + put(&state, "iris", OPERATOR).await; + assert_eq!(list_of(&client).await.len(), 2, "control: both derived"); + + let recreated = FakeStore::default(); + write_leaf( + &recreated, + "atlas", + "ANTHROPIC_BASE_URL=https://evil.example.net\n", + ) + .await; + // iris's leaf is gone, atlas's was created again by atlas. + reconcile(&client, &recreated).await.expect("pass"); assert_eq!(list_of(&client).await, []); } @@ -576,8 +694,7 @@ mod tests { ) .await; - let live: BTreeSet = ["atlas"].map(str::to_owned).into(); - let unreadable = reconcile(&client, &FakeStore::denying(), &live) + let unreadable = reconcile(&client, &FakeStore::denying()) .await .expect("pass"); assert_eq!(unreadable, 1); diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index bf0fa26d7..8d08f40da 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -387,7 +387,7 @@ pub(crate) fn spawn_per_agent_upkeep( spawn_subagent_streams(status, hives); spawn_schedule_reconcile(status, hives); spawn_agent_permissions(status, hives); - spawn_backend_egress_backfill(status, hives); + spawn_backend_egress_backfill(status); crate::attic_copy::spawn(hives.iter().map(|h| h.name.clone()).collect()); crate::proxy_credential::spawn(hives.iter().map(|h| h.name.clone()).collect()); if let Some(status) = status { @@ -405,15 +405,12 @@ fn spawn_agent_permissions(status: Option<&Arc>, hives: &[ agent_permissions::spawn(status.queue_client(), wanted, hives); } -/// Derive each agent's allowed-domain entries from the backend environments in -/// the store when a swarm queue is wired up. Same shape as -/// [`spawn_subagent_streams`]. -fn spawn_backend_egress_backfill(status: Option<&Arc>, hives: &[HiveEntry]) { - let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { - return; - }; - let hives = hives.iter().map(|h| h.name.clone()).collect(); - crate::backend_egress::spawn_backfill(status.queue_client(), wanted, hives); +/// Drop the allowed-domain entries derived from backend environments that no +/// longer name their host, when a swarm queue is wired up. +fn spawn_backend_egress_backfill(status: Option<&Arc>) { + if let Some(status) = status { + crate::backend_egress::spawn_backfill(status.queue_client()); + } } /// Start rewriting every agent's schedule key from the definitions when a