make squid's allowed domains editable at runtime in the swarm ui

hive-c0re/Cargo.toml

@@ -62,6 +62,7 @@ swarm-queue-client = { workspace = true, features = [
   "kv",
   "notices",
   "agent-limits",
+  "egress-allow",
   "agent-permissions",
   "agent-archetype",
   "agent-closures",

hive-c0re/src/swarm_status.rs

@@ -116,6 +116,12 @@ pub fn spawn(
             shutdown.clone(),
         ));
 
+        // The swarm's egress allow-list, on the same connection.
+        tokio::spawn(crate::workers::squid_allow::follow(
+            client.clone(),
+            shutdown.clone(),
+        ));
+
         // The permissions the swarm grants, and the import of the meta repo's.
         tokio::spawn(crate::workers::swarm_permissions::follow(
             client.clone(),

hive-c0re/src/workers/mod.rs

@@ -11,6 +11,7 @@ pub mod crash_watch;
 pub mod knowledge;
 pub mod mcp_sockets;
 pub mod permissions_import;
+pub mod squid_allow;
 pub mod squid_users;
 pub mod swarm_limits;
 pub mod swarm_permissions;

hive-c0re/src/workers/squid_allow.rs

@@ -0,0 +1,596 @@
+//! Squid's allow-list files: the destination names the swarm lets this hive's
+//! agents reach through the egress proxy.
+//!
+//! swarm-controller keeps one list for the whole swarm
+//! ([`swarm_wire::egress_allow`]); each entry is for every agent or for one.
+//! Every [`INTERVAL`] this reads it and renders two files for squid:
+//!
+//! - [`ENV_ALLOWED_FILE`]: the swarm-wide names, one per line, which
+//!   `nix/host-modules/hive-squid.nix` reads as part of `allowed_dst`.
+//! - [`ENV_AGENT_ALLOW_FILE`]: squid configuration `include`d where the base
+//!   list is allowed: for each of this hive's agents with entries, an `acl`
+//!   for its squid user, an `acl` for its names and the `http_access allow`
+//!   joining them. Entries of agents this hive does not run are left out.
+//!
+//! Squid reads both when it parses its configuration, so the path unit in
+//! `nix/host-modules/hive-squid.nix` reloads it when either changes.
+//!
+//! The swarm's list replaces the names `services.hyperhive.squid.allowedDomains`
+//! lists ([`ENV_SEED_FILE`]), which stay the list until the swarm has one. A
+//! file rendered from the swarm's list starts with [`SWARM_MARKER`]. The queue
+//! not answering for the bucket does not show that the swarm withdrew its list,
+//! so a file with the marker is left alone then.
+//!
+//! Every entry is checked again with
+//! [`swarm_wire::egress_allow::normalize_domain`], since the names end up in
+//! squid's configuration, and skipped when it fails. An entry that
+//! [`ENV_ALWAYS_ALLOWED`] or another entry of its list covers is dropped,
+//! because squid ignores the narrower of two overlapping names in one ACL.
+//!
+//! A failed read leaves both files as they are.
+
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use anyhow::{Context, Result};
+use swarm_queue_client::egress_allow::{self, Read};
+use swarm_wire::egress_allow::{AllowList, covered_by, normalize_domain};
+
+use super::squid_users::write_if_changed;
+use crate::stats::sweep_health::{self, SweepHealth};
+
+/// How often the list is read.
+pub const INTERVAL: Duration = Duration::from_secs(30);
+
+/// At [`INTERVAL`] this is ~90 seconds of failing reads.
+const FAILURES_BEFORE_WARNING: u32 = 3;
+
+/// The swarm-wide names file. Set by `nix/host-modules/hive-c0re` from
+/// `services.hyperhive.squid.allowedFile`.
+pub const ENV_ALLOWED_FILE: &str = "HIVE_SQUID_ALLOWED_FILE";
+
+/// The per-agent squid configuration file. Set from
+/// `services.hyperhive.squid.agentAllowFile`.
+pub const ENV_AGENT_ALLOW_FILE: &str = "HIVE_SQUID_AGENT_ALLOW_FILE";
+
+/// The file of names `services.hyperhive.squid.allowedDomains` lists, one per
+/// line, which the files hold until the swarm has a list.
+pub const ENV_SEED_FILE: &str = "HIVE_SQUID_ALLOWED_SEED_FILE";
+
+/// The names, separated by spaces, that squid allows every authenticated user
+/// regardless of this list: the base list and the self-check canary.
+pub const ENV_ALWAYS_ALLOWED: &str = "HIVE_SQUID_ALWAYS_ALLOWED";
+
+/// The first line of a file rendered from the swarm's list.
+const SWARM_MARKER: &str = "# source: swarm\n";
+
+/// The first line of a file rendered from the seed.
+const SEED_MARKER: &str = "# source: seed\n";
+
+/// The two files a pass writes.
+struct Files<'a> {
+    allowed: &'a Path,
+    agent_allow: &'a Path,
+}
+
+/// What a pass left in the files.
+#[derive(Debug, PartialEq, Eq)]
+enum Source {
+    /// The swarm's list.
+    Swarm,
+    /// The seed, since the swarm has no list.
+    Seed,
+    /// The swarm has no list now, but the files hold the one it had.
+    Kept,
+}
+
+fn env_path(name: &str) -> Option<PathBuf> {
+    std::env::var_os(name)
+        .filter(|v| !v.is_empty())
+        .map(PathBuf::from)
+}
+
+/// Read the list now and every [`INTERVAL`] until shutdown, when this host has
+/// squid's two files.
+pub async fn follow(client: async_nats::Client, mut shutdown: tokio::sync::watch::Receiver<bool>) {
+    let (Some(allowed), Some(agent_allow)) =
+        (env_path(ENV_ALLOWED_FILE), env_path(ENV_AGENT_ALLOW_FILE))
+    else {
+        return;
+    };
+    let seed = env_path(ENV_SEED_FILE);
+    let always = std::env::var(ENV_ALWAYS_ALLOWED).unwrap_or_default();
+    let mut health = SweepHealth::new("squid_allow_render", "warn", FAILURES_BEFORE_WARNING);
+    // The last `Source` that was logged, so a list that stays unset is
+    // announced once rather than every pass.
+    let mut announced: Option<&'static str> = None;
+    loop {
+        let files = Files {
+            allowed: &allowed,
+            agent_allow: &agent_allow,
+        };
+        match pass(&client, &files, seed.as_deref(), &always).await {
+            Ok(source) => {
+                health.record_ok();
+                announce(&source, &mut announced);
+            }
+            Err(e) => {
+                tracing::warn!(error = ?e, "squid allow list: the pass failed");
+                let err = format!("{e:#}");
+                health.record_err(|ctx| {
+                    let age = ctx.since_last_ok.map_or_else(
+                        || "no success this session".to_owned(),
+                        |d| format!("last ok {} ago", sweep_health::fmt_age(d)),
+                    );
+                    format!(
+                        "reading the swarm's egress allow-list is failing ({} consecutive, \
+                         {age}) — squid keeps the list last written: {err}",
+                        ctx.consecutive
+                    )
+                });
+            }
+        }
+        tokio::select! {
+            () = tokio::time::sleep(INTERVAL) => {}
+            _ = shutdown.changed() => return,
+        }
+    }
+}
+
+/// Log the first pass of a run that left the seed or the kept files.
+fn announce(source: &Source, announced: &mut Option<&'static str>) {
+    let (key, message) = match source {
+        Source::Swarm => {
+            *announced = None;
+            return;
+        }
+        Source::Seed => (
+            "seed",
+            "squid allow list: the swarm has no list, so squid keeps this hive's own names",
+        ),
+        Source::Kept => (
+            "kept",
+            "squid allow list: the swarm's list is not readable now, so squid keeps the last one written",
+        ),
+    };
+    if *announced != Some(key) {
+        tracing::info!("{message}");
+        *announced = Some(key);
+    }
+}
+
+async fn pass(
+    client: &async_nats::Client,
+    files: &Files<'_>,
+    seed: Option<&Path>,
+    always: &str,
+) -> Result<Source> {
+    // An unconnected client does not fail a JetStream request, it hangs on it.
+    swarm_queue_client::ensure_connected(client)?;
+    let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
+        .await
+        .context("enumerating this hive's agents")?
+        .into_iter()
+        .map(|spec| spec.name)
+        .collect();
+    let read = egress_allow::read(client)
+        .await
+        .map_err(|e| anyhow::anyhow!(e))
+        .context("reading the swarm's egress allow-list")?;
+    sync_with(read, &agents, seed, always, files)
+}
+
+/// [`pass`] with the read, the agents and the files given. Each file is only
+/// written when its rendering differs from what it holds.
+fn sync_with(
+    read: Read,
+    agents: &[String],
+    seed: Option<&Path>,
+    always: &str,
+    files: &Files<'_>,
+) -> Result<Source> {
+    let always: Vec<&str> = always.split_whitespace().collect();
+    let rendered = match read {
+        Read::Current(list) => render(&list, agents, &always),
+        Read::NotSet => {
+            if super::squid_users::read_or_empty(files.allowed)?.starts_with(SWARM_MARKER) {
+                return Ok(Source::Kept);
+            }
+            seeded(seed)?
+        }
+    };
+    let changed_allowed = write_if_changed(files.allowed, &rendered.allowed)?;
+    let changed_agents = write_if_changed(files.agent_allow, &rendered.agent_allow)?;
+    if changed_allowed || changed_agents {
+        if let Some(counts) = &rendered.counts {
+            tracing::info!(file = %files.allowed.display(), "{}", counts.summary());
+            if !counts.skipped.is_empty() {
+                tracing::warn!(
+                    skipped = ?counts.skipped,
+                    "squid allow list: entries the swarm holds were skipped as invalid"
+                );
+            }
+        } else {
+            tracing::info!(
+                file = %files.allowed.display(),
+                "squid allow list: wrote this hive's own names, no swarm list"
+            );
+        }
+    }
+    Ok(if rendered.counts.is_some() {
+        Source::Swarm
+    } else {
+        Source::Seed
+    })
+}
+
+/// What the swarm's list rendered as.
+struct Counts {
+    /// Swarm-wide names written.
+    swarm: usize,
+    /// Names written across this hive's agents.
+    agent: usize,
+    /// Entries left out because they are not valid names or agents.
+    skipped: Vec<String>,
+}
+
+impl Counts {
+    /// The info line a pass that changed a file logs.
+    fn summary(&self) -> String {
+        format!(
+            "squid allow list: {} swarm, {} agent entries",
+            self.swarm, self.agent
+        )
+    }
+}
+
+struct Rendered {
+    allowed: String,
+    agent_allow: String,
+    /// `None` for the seed.
+    counts: Option<Counts>,
+}
+
+/// The files for the seed: its names as they are, and no per-agent rules.
+fn seeded(seed: Option<&Path>) -> Result<Rendered> {
+    let names = match seed {
+        Some(path) => super::squid_users::read_or_empty(path)?,
+        None => String::new(),
+    };
+    let mut allowed = SEED_MARKER.to_owned();
+    for line in names.lines().map(str::trim) {
+        if !line.is_empty() && !line.starts_with('#') {
+            allowed.push_str(line);
+            allowed.push('\n');
+        }
+    }
+    Ok(Rendered {
+        allowed,
+        agent_allow: SEED_MARKER.to_owned(),
+        counts: None,
+    })
+}
+
+/// `names` sorted, with each one that another covers dropped.
+fn reduced(mut names: Vec<String>) -> Vec<String> {
+    names.sort();
+    names.dedup();
+    let all = names.clone();
+    names.retain(|n| {
+        let others: Vec<&String> = all.iter().filter(|o| *o != n).collect();
+        !covered_by(n, &others)
+    });
+    names
+}
+
+/// The files for `list`, for the agents this hive runs.
+fn render(list: &AllowList, agents: &[String], always: &[&str]) -> Rendered {
+    let mut skipped = Vec::new();
+    let mut swarm_wide = Vec::new();
+    let mut per_agent: std::collections::BTreeMap<&str, Vec<String>> =
+        std::collections::BTreeMap::new();
+    for entry in &list.entries {
+        let Ok(domain) = normalize_domain(&entry.domain) else {
+            skipped.push(entry.domain.clone());
+            continue;
+        };
+        match entry.agent.as_deref() {
+            None => swarm_wide.push(domain),
+            Some(agent) => {
+                if hive_types::Ident::parse(agent).is_err() {
+                    skipped.push(format!("{} for {agent}", entry.domain));
+                } else if let Some(mine) = agents.iter().find(|a| *a == agent) {
+                    per_agent.entry(mine.as_str()).or_default().push(domain);
+                }
+            }
+        }
+    }
+    swarm_wide.retain(|d| !covered_by(d, always));
+    let swarm_wide = reduced(swarm_wide);
+
+    let mut allowed = SWARM_MARKER.to_owned();
+    for name in &swarm_wide {
+        allowed.push_str(name);
+        allowed.push('\n');
+    }
+    let mut agent_rules = Vec::new();
+    let mut agent_count = 0;
+    for (agent, mut names) in per_agent {
+        names.retain(|d| !covered_by(d, always) && !covered_by(d, &swarm_wide));
+        let names = reduced(names);
+        if names.is_empty() {
+            continue;
+        }
+        agent_count += names.len();
+        agent_rules.push(format!(
+            "acl agent_allow_{agent} proxy_auth {agent}\n\
+             acl agent_allow_{agent}_dst dstdomain {}\n\
+             http_access allow agent_allow_{agent} agent_allow_{agent}_dst\n",
+            names.join(" ")
+        ));
+    }
+    Rendered {
+        allowed,
+        agent_allow: format!("{SWARM_MARKER}{}", agent_rules.concat()),
+        counts: Some(Counts {
+            swarm: swarm_wide.len(),
+            agent: agent_count,
+            skipped,
+        }),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use swarm_wire::egress_allow::Entry;
+
+    fn entry(domain: &str, agent: Option<&str>) -> Entry {
+        Entry {
+            domain: domain.to_owned(),
+            agent: agent.map(str::to_owned),
+        }
+    }
+
+    fn list(entries: &[(&str, Option<&str>)]) -> AllowList {
+        AllowList {
+            entries: entries.iter().map(|(d, a)| entry(d, *a)).collect(),
+        }
+    }
+
+    fn agents(names: &[&str]) -> Vec<String> {
+        names.iter().map(|n| (*n).to_owned()).collect()
+    }
+
+    const ALWAYS: &[&str] = &["api.anthropic.com", ".claude.ai", "cache.nixos.org"];
+
+    fn rendered(entries: &[(&str, Option<&str>)], mine: &[&str]) -> Rendered {
+        render(&list(entries), &agents(mine), ALWAYS)
+    }
+
+    #[test]
+    fn swarm_wide_names_are_sorted_one_per_line_after_the_marker() {
+        let r = rendered(
+            &[
+                ("github.com", None),
+                ("crates.io", None),
+                ("GitHub.com", None),
+            ],
+            &["atlas"],
+        );
+        assert_eq!(r.allowed, "# source: swarm\ncrates.io\ngithub.com\n");
+        let counts = r.counts.expect("a swarm list");
+        assert_eq!((counts.swarm, counts.agent), (2, 0));
+    }
+
+    #[test]
+    fn an_empty_list_renders_an_empty_set_of_names_and_no_rules() {
+        let r = rendered(&[], &["atlas"]);
+        assert_eq!(r.allowed, SWARM_MARKER);
+        assert_eq!(r.agent_allow, SWARM_MARKER);
+        let counts = r.counts.expect("an empty swarm list is still a swarm list");
+        assert_eq!((counts.swarm, counts.agent), (0, 0));
+    }
+
+    #[test]
+    fn an_agents_names_become_a_user_rule_and_other_hives_agents_are_left_out() {
+        let r = rendered(
+            &[
+                ("pypi.org", Some("atlas")),
+                ("crates.io", Some("atlas")),
+                ("github.com", Some("iris")),
+                ("github.com", Some("../argus")),
+            ],
+            &["atlas", "argus"],
+        );
+        assert_eq!(
+            r.agent_allow,
+            "# source: swarm\n\
+             acl agent_allow_atlas proxy_auth atlas\n\
+             acl agent_allow_atlas_dst dstdomain crates.io pypi.org\n\
+             http_access allow agent_allow_atlas agent_allow_atlas_dst\n"
+        );
+        let counts = r.counts.expect("a swarm list");
+        assert_eq!((counts.swarm, counts.agent), (0, 2));
+        // The name that could break out of the line is reported, not rendered.
+        assert_eq!(counts.skipped, ["github.com for ../argus"]);
+        assert!(!r.agent_allow.contains("iris"), "another hive's agent");
+        assert!(!r.agent_allow.contains("argus"), "an invalid name");
+    }
+
+    #[test]
+    fn an_entry_that_is_not_a_name_never_reaches_squid_configuration() {
+        let r = rendered(
+            &[
+                ("github.com", None),
+                ("1.2.3.4", None),
+                ("*.example.com", Some("atlas")),
+                ("example.com\nhttp_access allow all", Some("atlas")),
+                ("example.com\nhttp_access allow all", None),
+            ],
+            &["atlas"],
+        );
+        assert_eq!(r.allowed, "# source: swarm\ngithub.com\n");
+        assert_eq!(r.agent_allow, SWARM_MARKER);
+        assert!(!r.allowed.contains("http_access") && !r.agent_allow.contains("http_access"));
+        assert_eq!(r.counts.expect("a swarm list").skipped.len(), 4);
+    }
+
+    #[test]
+    fn a_name_squid_allows_everyone_or_the_swarm_already_lists_is_dropped() {
+        let r = rendered(
+            &[
+                ("api.anthropic.com", None),
+                ("a.claude.ai", None),
+                ("cache.nixos.org", Some("atlas")),
+                (".github.com", None),
+                ("api.github.com", None),
+                ("api.github.com", Some("atlas")),
+                ("crates.io", Some("atlas")),
+            ],
+            &["atlas"],
+        );
+        assert_eq!(r.allowed, "# source: swarm\n.github.com\n");
+        assert_eq!(
+            r.agent_allow,
+            "# source: swarm\n\
+             acl agent_allow_atlas proxy_auth atlas\n\
+             acl agent_allow_atlas_dst dstdomain crates.io\n\
+             http_access allow agent_allow_atlas agent_allow_atlas_dst\n"
+        );
+        // Control: the same entries survive when nothing is always allowed.
+        let bare = render(
+            &list(&[("api.anthropic.com", None), ("a.claude.ai", None)]),
+            &agents(&["atlas"]),
+            &[],
+        );
+        assert_eq!(
+            bare.allowed,
+            "# source: swarm\na.claude.ai\napi.anthropic.com\n"
+        );
+    }
+
+    #[test]
+    fn the_log_line_carries_the_counts() {
+        let r = rendered(
+            &[
+                ("github.com", None),
+                ("crates.io", None),
+                ("pypi.org", Some("atlas")),
+                ("npmjs.org", Some("atlas")),
+                ("docs.rs", Some("argus")),
+            ],
+            &["atlas", "argus"],
+        );
+        let counts = r.counts.expect("a swarm list");
+        assert_eq!(
+            counts.summary(),
+            "squid allow list: 2 swarm, 3 agent entries"
+        );
+    }
+
+    struct Dir(tempfile::TempDir);
+
+    impl Dir {
+        fn allowed(&self) -> PathBuf {
+            self.0.path().join("allowed")
+        }
+        fn agent_allow(&self) -> PathBuf {
+            self.0.path().join("agent-allow.conf")
+        }
+        fn seed(&self) -> PathBuf {
+            self.0.path().join("seed")
+        }
+    }
+
+    fn dir(seed: &str) -> Dir {
+        let dir = Dir(tempfile::tempdir().expect("tempdir"));
+        std::fs::write(dir.seed(), seed).expect("seed");
+        dir
+    }
+
+    fn sync(d: &Dir, read: Read) -> Result<Source> {
+        sync_with(
+            read,
+            &agents(&["atlas"]),
+            Some(&d.seed()),
+            "api.anthropic.com cache.nixos.org",
+            &Files {
+                allowed: &d.allowed(),
+                agent_allow: &d.agent_allow(),
+            },
+        )
+    }
+
+    fn text(path: &Path) -> String {
+        std::fs::read_to_string(path).expect("the file is there")
+    }
+
+    #[test]
+    fn without_a_swarm_list_the_files_hold_the_seed() {
+        let d = dir("# a comment\nforge.example\n\n  loop.example  \n");
+        assert_eq!(sync(&d, Read::NotSet).expect("renders"), Source::Seed);
+        assert_eq!(
+            text(&d.allowed()),
+            "# source: seed\nforge.example\nloop.example\n"
+        );
+        assert_eq!(text(&d.agent_allow()), SEED_MARKER);
+        // A second pass writes nothing.
+        let ino = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(d.allowed()).unwrap());
+        sync(&d, Read::NotSet).expect("renders");
+        assert_eq!(
+            std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(d.allowed()).unwrap()),
+            ino
+        );
+    }
+
+    #[test]
+    fn a_swarm_list_replaces_the_seed_even_when_empty() {
+        let d = dir("forge.example\n");
+        sync(&d, Read::NotSet).expect("seeded");
+        assert!(text(&d.allowed()).contains("forge.example"), "control");
+        let empty = Read::Current(AllowList::default());
+        assert_eq!(sync(&d, empty).expect("renders"), Source::Swarm);
+        assert_eq!(text(&d.allowed()), SWARM_MARKER);
+        let named = Read::Current(list(&[("github.com", None), ("pypi.org", Some("atlas"))]));
+        sync(&d, named).expect("renders");
+        assert_eq!(text(&d.allowed()), "# source: swarm\ngithub.com\n");
+        assert!(text(&d.agent_allow()).contains("acl agent_allow_atlas proxy_auth atlas"));
+    }
+
+    #[test]
+    fn a_swarm_list_that_stops_answering_is_kept_and_a_seed_is_refreshed() {
+        let d = dir("forge.example\n");
+        sync(&d, Read::Current(list(&[("github.com", None)]))).expect("renders");
+        let before = (text(&d.allowed()), text(&d.agent_allow()));
+        assert_eq!(sync(&d, Read::NotSet).expect("renders"), Source::Kept);
+        assert_eq!((text(&d.allowed()), text(&d.agent_allow())), before);
+        // Control: a file that never held the swarm's list follows the seed.
+        let d = dir("forge.example\n");
+        std::fs::write(d.allowed(), "old.example\n").expect("a tmpfiles-seeded file");
+        assert_eq!(sync(&d, Read::NotSet).expect("renders"), Source::Seed);
+        assert_eq!(text(&d.allowed()), "# source: seed\nforge.example\n");
+    }
+
+    #[test]
+    fn announce_logs_a_run_of_unset_passes_once() {
+        let mut announced = None;
+        announce(&Source::Seed, &mut announced);
+        assert_eq!(announced, Some("seed"));
+        announce(&Source::Swarm, &mut announced);
+        assert_eq!(announced, None, "a list that appears resets the run");
+        announce(&Source::Kept, &mut announced);
+        assert_eq!(announced, Some("kept"));
+    }
+
+    #[test]
+    fn the_files_are_group_readable() {
+        use std::os::unix::fs::PermissionsExt as _;
+        let d = dir("");
+        sync(&d, Read::Current(list(&[("github.com", None)]))).expect("renders");
+        for file in [d.allowed(), d.agent_allow()] {
+            let mode = std::fs::metadata(file).expect("stat").permissions().mode() & 0o777;
+            assert_eq!(mode, 0o640);
+        }
+    }
+}

hive-c0re/src/workers/squid_users.rs

@@ -233,7 +233,7 @@ where
 }
 
 /// The file's text, or the empty string when it does not exist.
-fn read_or_empty(file: &Path) -> Result<String> {
+pub(super) fn read_or_empty(file: &Path) -> Result<String> {
     match std::fs::read_to_string(file) {
         Ok(text) => Ok(text),
         Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(String::new()),
@@ -243,7 +243,7 @@ fn read_or_empty(file: &Path) -> Result<String> {
 
 /// Write `text` to `file` unless it already holds exactly that. Returns
 /// whether it wrote.
-fn write_if_changed(file: &Path, text: &str) -> Result<bool> {
+pub(super) fn write_if_changed(file: &Path, text: &str) -> Result<bool> {
     if read_or_empty(file)? == text {
         return Ok(false);
     }