squid blocks a runtime deny list from the controller in every egress mode, ahead of the allow list

hive-c0re/src/swarm_status.rs

@@ -122,6 +122,12 @@ pub fn spawn(
             shutdown.clone(),
         ));
 
+        // The swarm's egress deny list, on the same connection.
+        tokio::spawn(crate::workers::squid_deny::follow(
+            client.clone(),
+            shutdown.clone(),
+        ));
+
         // The permissions the swarm grants, and the import of the meta repo's.
         tokio::spawn(crate::workers::swarm_permissions::follow(
             client.clone(),
@@ -375,15 +381,24 @@ struct Snapshot {
     #[serde(flatten)]
     readiness: crate::warnings::Readiness,
     capabilities: &'static [&'static str],
+    /// The deny-list revision this hive rendered into squid's include
+    /// (`swarm_wire::hive_status::EGRESS_DENY_APPLIED`); absent until a pass has.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    egress_deny_applied: Option<u64>,
 }
 
 fn snapshot() -> Snapshot {
+    snapshot_with(crate::workers::squid_deny::applied())
+}
+
+fn snapshot_with(egress_deny_applied: Option<u64>) -> Snapshot {
     Snapshot {
         readiness: crate::warnings::readiness(),
         capabilities: &[
             swarm_wire::hive_status::TOLERANT_PERMISSIONS,
             swarm_wire::hive_status::NO_CONFIG_CHECKOUT,
         ],
+        egress_deny_applied,
     }
 }
 
@@ -404,7 +419,7 @@ async fn publish(client: &async_nats::Client, hive: &str) -> Result<()> {
 
 #[cfg(test)]
 mod snapshot_tests {
-    use super::snapshot;
+    use super::{snapshot, snapshot_with};
 
     /// The controller reads the capability out of the published bytes with
     /// the wire type, and the readiness keeps its own top-level fields.
@@ -421,4 +436,22 @@ mod snapshot_tests {
         );
         assert!(value.get("status").is_some() && value.get("warnings").is_some());
     }
+
+    /// Before a pass has rendered the list the field is absent, as on a hive that
+    /// predates it; after, the controller reads the revision with the wire type.
+    #[test]
+    fn the_snapshot_carries_the_applied_deny_revision_once_there_is_one() {
+        let fresh = serde_json::to_value(snapshot_with(None)).unwrap();
+        assert!(
+            fresh
+                .get(swarm_wire::hive_status::EGRESS_DENY_APPLIED)
+                .is_none()
+        );
+        assert_eq!(swarm_wire::hive_status::egress_deny_applied(&fresh), None);
+        let rendered = serde_json::to_value(snapshot_with(Some(4))).unwrap();
+        assert_eq!(
+            swarm_wire::hive_status::egress_deny_applied(&rendered),
+            Some(4)
+        );
+    }
 }

hive-c0re/src/workers/mod.rs

@@ -12,6 +12,7 @@ pub mod knowledge;
 pub mod mcp_sockets;
 pub mod permissions_import;
 pub mod squid_allow;
+pub mod squid_deny;
 pub mod squid_users;
 pub mod swarm_limits;
 pub mod swarm_permissions;

hive-c0re/src/workers/squid_allow.rs

@@ -294,7 +294,7 @@ fn seeded(seed: Option<&Path>, logging: &[String]) -> Result<Rendered> {
 }
 
 /// `names` sorted, with each one that another covers dropped.
-fn reduced(mut names: Vec<String>) -> Vec<String> {
+pub(super) fn reduced(mut names: Vec<String>) -> Vec<String> {
     names.sort();
     names.dedup();
     let all = names.clone();

hive-c0re/src/workers/squid_deny.rs

@@ -0,0 +1,452 @@
+//! Squid's deny-list file: the destination names the swarm refuses this hive's
+//! agents, whatever allows them.
+//!
+//! swarm-controller keeps the deny list beside the allow-list
+//! ([`swarm_wire::egress_allow::DenyList`], its own key of the same bucket); each
+//! entry is for every agent or for one. Every [`INTERVAL`] this reads it and
+//! renders [`ENV_DENY_FILE`], squid configuration that
+//! `nix/host-modules/hive-squid.nix` `include`s after the destination denies
+//! and before `deny !authed` and every allow, so a denied name is refused for
+//! every user in every mode. A swarm-wide entry is one `dstdomain` ACL and a
+//! `deny` that names no user, so even a client without a credential gets a 403;
+//! an agent's entry also names that agent's squid user. Each rule ends in the
+//! [`MARK`] annotation, which the proxy logs as its own `egress_denied` line.
+//!
+//! No entry renders an empty file, which squid reads as no rules, so a hive
+//! whose swarm has no deny list configures squid as it did before this existed.
+//!
+//! The revision of the list last rendered is what [`applied`] answers and the
+//! hive's status snapshot carries, so the swarm can see a hive that is behind.
+//! A hive that cannot read the list keeps the file it has, and reports nothing
+//! it did not render.
+//!
+//! Every entry is checked again with
+//! [`swarm_wire::egress_allow::normalize_domain`], since the names end up in
+//! squid's configuration, and skipped when it fails. An entry another entry of
+//! its list covers is dropped, because squid ignores the narrower of two
+//! overlapping names in one ACL.
+
+use std::fmt::Write as _;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::Duration;
+
+use anyhow::{Context, Result};
+use swarm_queue_client::egress_allow::{self, DenyRead};
+use swarm_wire::egress_allow::{DenyList, covered_by, normalize_domain};
+
+use super::squid_allow::reduced;
+use super::squid_users::{read_or_empty, write_if_changed};
+use crate::stats::sweep_health::{self, SweepHealth};
+
+/// How often the list is read.
+pub const INTERVAL: Duration = Duration::from_secs(30);
+
+/// At [`INTERVAL`] this is ~90 seconds of failing reads.
+const FAILURES_BEFORE_WARNING: u32 = 3;
+
+/// The squid configuration file this renders. Set by `nix/host-modules/hive-c0re`
+/// from `services.hyperhive.deploy.hive-controller.squid.denyFile`.
+pub const ENV_DENY_FILE: &str = "HIVE_SQUID_DENY_FILE";
+
+/// The annotation a deny rule puts on a request, as `key=value`. The `note` ACL
+/// in `nix/host-modules/hive-squid.nix` reads the key.
+const MARK: &str = "egress_deny=1";
+
+/// The ACL that applies [`MARK`].
+const MARK_ACL: &str = "deny_hit_mark";
+
+/// The revision last rendered plus one; `0` before the first pass has rendered
+/// or confirmed anything.
+static APPLIED: AtomicU64 = AtomicU64::new(0);
+
+/// The revision of the swarm's deny list this hive's file holds, `0` when the
+/// swarm has none, or `None` until a pass has said so.
+#[must_use]
+pub fn applied() -> Option<u64> {
+    APPLIED.load(Ordering::Relaxed).checked_sub(1)
+}
+
+fn set_applied(revision: u64) {
+    APPLIED.store(revision.saturating_add(1), Ordering::Relaxed);
+}
+
+/// Read the list now and every [`INTERVAL`] until shutdown, when this host has
+/// squid's deny file.
+pub async fn follow(client: async_nats::Client, mut shutdown: tokio::sync::watch::Receiver<bool>) {
+    let Some(file) = std::env::var_os(ENV_DENY_FILE)
+        .filter(|v| !v.is_empty())
+        .map(PathBuf::from)
+    else {
+        return;
+    };
+    let mut health = SweepHealth::new("squid_deny_render", "warn", FAILURES_BEFORE_WARNING);
+    loop {
+        match pass(&client, &file).await {
+            Ok(()) => health.record_ok(),
+            Err(e) => {
+                tracing::warn!(error = ?e, "squid deny list: the pass failed");
+                let err = format!("{e:#}");
+                health.record_err(|ctx| {
+                    let age = ctx.since_last_ok.map_or_else(
+                        || "no success this session".to_owned(),
+                        |d| format!("last ok {} ago", sweep_health::fmt_age(d)),
+                    );
+                    format!(
+                        "reading the swarm's egress deny list is failing ({} consecutive, \
+                         {age}) — squid keeps the denies last written: {err}",
+                        ctx.consecutive
+                    )
+                });
+            }
+        }
+        tokio::select! {
+            () = tokio::time::sleep(INTERVAL) => {}
+            _ = shutdown.changed() => return,
+        }
+    }
+}
+
+async fn pass(client: &async_nats::Client, file: &Path) -> Result<()> {
+    // An unconnected client does not fail a JetStream request, it hangs on it.
+    swarm_queue_client::ensure_connected(client)?;
+    let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
+        .await
+        .context("enumerating this hive's agents")?
+        .into_iter()
+        .map(|spec| spec.name)
+        .collect();
+    let read = egress_allow::read_deny(client)
+        .await
+        .map_err(|e| anyhow::anyhow!(e))
+        .context("reading the swarm's egress deny list")?;
+    if let Some(revision) = sync_with(read, &agents, file)? {
+        set_applied(revision);
+    }
+    Ok(())
+}
+
+/// [`pass`] with the read and the agents given. The file is only written when
+/// its rendering differs from what it holds. Answers the revision the file now
+/// holds, or `None` when it cannot say.
+fn sync_with(read: DenyRead, agents: &[String], file: &Path) -> Result<Option<u64>> {
+    match read {
+        DenyRead::Current { list, revision } => {
+            let rendered = render(&list, agents);
+            if write_if_changed(file, &rendered.text)? {
+                tracing::info!(
+                    file = %file.display(), revision,
+                    "squid deny list: {} swarm, {} agent entries", rendered.swarm, rendered.agent
+                );
+            }
+            if !rendered.skipped.is_empty() {
+                tracing::warn!(
+                    skipped = ?rendered.skipped,
+                    "squid deny list: entries the swarm holds were skipped as invalid"
+                );
+            }
+            Ok(Some(revision))
+        }
+        // Nothing to read says nothing about what the swarm wants, so a file
+        // that holds denies stays. An empty file is the true "no list".
+        DenyRead::NotSet => Ok(read_or_empty(file)?.is_empty().then_some(0)),
+    }
+}
+
+struct Rendered {
+    text: String,
+    /// Swarm-wide names written.
+    swarm: usize,
+    /// Names written across this hive's agents.
+    agent: usize,
+    /// Entries left out because they are not valid names or agents.
+    skipped: Vec<String>,
+}
+
+/// The squid configuration for `list`, for the agents this hive runs. Empty for
+/// no entry that applies.
+///
+/// The destination ACL comes first in each rule: squid checks ACLs left to
+/// right, so a swarm-wide rule never reaches a `proxy_auth` ACL, which would
+/// answer an anonymous client 407 instead of the 403.
+fn render(list: &DenyList, agents: &[String]) -> Rendered {
+    let mut skipped = Vec::new();
+    let mut swarm_wide = Vec::new();
+    let mut per_agent: std::collections::BTreeMap<&str, Vec<String>> =
+        std::collections::BTreeMap::new();
+    for entry in &list.entries {
+        let Ok(domain) = normalize_domain(&entry.domain) else {
+            skipped.push(entry.domain.clone());
+            continue;
+        };
+        match entry.agent.as_deref() {
+            None => swarm_wide.push(domain),
+            Some(agent) => {
+                if hive_types::Ident::parse(agent).is_err() {
+                    skipped.push(format!("{} for {agent}", entry.domain));
+                } else if let Some(mine) = agents.iter().find(|a| *a == agent) {
+                    per_agent.entry(mine.as_str()).or_default().push(domain);
+                }
+            }
+        }
+    }
+    let swarm_wide = reduced(swarm_wide);
+
+    let mut rules = String::new();
+    if !swarm_wide.is_empty() {
+        let _ = write!(
+            rules,
+            "acl deny_swarm_dst dstdomain {}\n\
+             http_access deny deny_swarm_dst {MARK_ACL}\n",
+            swarm_wide.join(" ")
+        );
+    }
+    let mut agent_count = 0;
+    for (agent, names) in per_agent {
+        let names = reduced(
+            names
+                .into_iter()
+                .filter(|d| !covered_by(d, &swarm_wide))
+                .collect(),
+        );
+        if names.is_empty() {
+            continue;
+        }
+        agent_count += names.len();
+        let _ = write!(
+            rules,
+            "acl deny_agent_{agent}_dst dstdomain {}\n\
+             acl deny_agent_{agent} proxy_auth {agent}\n\
+             http_access deny deny_agent_{agent}_dst deny_agent_{agent} {MARK_ACL}\n",
+            names.join(" ")
+        );
+    }
+    let text = if rules.is_empty() {
+        String::new()
+    } else {
+        format!("acl {MARK_ACL} annotate_transaction {MARK}\n{rules}")
+    };
+    Rendered {
+        text,
+        swarm: swarm_wide.len(),
+        agent: agent_count,
+        skipped,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use swarm_wire::egress_allow::DenyEntry;
+
+    fn list(entries: &[(&str, Option<&str>)]) -> DenyList {
+        DenyList {
+            entries: entries
+                .iter()
+                .map(|(d, a)| DenyEntry {
+                    domain: (*d).to_owned(),
+                    agent: a.map(str::to_owned),
+                })
+                .collect(),
+        }
+    }
+
+    fn agents(names: &[&str]) -> Vec<String> {
+        names.iter().map(|n| (*n).to_owned()).collect()
+    }
+
+    const MARK_LINE: &str = "acl deny_hit_mark annotate_transaction egress_deny=1\n";
+
+    /// The one place a rule's shape is pinned: the destination ACL before any
+    /// user ACL, and the annotation last.
+    #[test]
+    fn a_swarm_wide_name_is_one_destination_rule_that_names_no_user() {
+        let r = render(
+            &list(&[("evil.com", None), (".bad.io", None), ("EVIL.com", None)]),
+            &agents(&["atlas"]),
+        );
+        assert_eq!(
+            r.text,
+            format!(
+                "{MARK_LINE}acl deny_swarm_dst dstdomain .bad.io evil.com\n\
+                 http_access deny deny_swarm_dst deny_hit_mark\n"
+            )
+        );
+        assert!(!r.text.contains("proxy_auth"), "no user, so no 407");
+        assert_eq!((r.swarm, r.agent), (2, 0));
+    }
+
+    #[test]
+    fn an_agents_name_is_a_destination_then_user_rule_and_other_hives_agents_are_left_out() {
+        let r = render(
+            &list(&[
+                ("pypi.org", Some("atlas")),
+                ("crates.io", Some("atlas")),
+                ("github.com", Some("iris")),
+                ("github.com", Some("../argus")),
+            ]),
+            &agents(&["atlas", "argus"]),
+        );
+        assert_eq!(
+            r.text,
+            format!(
+                "{MARK_LINE}acl deny_agent_atlas_dst dstdomain crates.io pypi.org\n\
+                 acl deny_agent_atlas proxy_auth atlas\n\
+                 http_access deny deny_agent_atlas_dst deny_agent_atlas deny_hit_mark\n"
+            )
+        );
+        assert_eq!((r.swarm, r.agent), (0, 2));
+        assert_eq!(r.skipped, ["github.com for ../argus"]);
+        assert!(!r.text.contains("iris"), "another hive's agent");
+        assert!(!r.text.contains("argus"), "an invalid name");
+    }
+
+    /// An agent's entry adds to the swarm-wide rule and never replaces it: the
+    /// swarm-wide rule renders whole whatever the agent lists, and a name it
+    /// already covers is not repeated for the agent.
+    #[test]
+    fn an_agents_entry_adds_to_the_swarm_wide_rule_and_never_narrows_it() {
+        let r = render(
+            &list(&[
+                (".evil.com", None),
+                ("api.evil.com", Some("atlas")),
+                ("bad.io", Some("atlas")),
+            ]),
+            &agents(&["atlas"]),
+        );
+        assert!(
+            r.text.contains(
+                "acl deny_swarm_dst dstdomain .evil.com\nhttp_access deny deny_swarm_dst"
+            )
+        );
+        assert!(r.text.contains("deny_agent_atlas_dst dstdomain bad.io\n"));
+        assert!(!r.text.contains("api.evil.com"), "{}", r.text);
+        assert_eq!((r.swarm, r.agent), (1, 1));
+    }
+
+    /// No entry renders no byte, which is what keeps a hive with no deny list
+    /// configured as it was.
+    #[test]
+    fn no_entry_that_applies_renders_an_empty_file() {
+        assert_eq!(render(&DenyList::default(), &agents(&["atlas"])).text, "");
+        // An agent this hive does not run, and an invalid name, render nothing.
+        let r = render(
+            &list(&[("evil.com", Some("iris")), ("1.2.3.4", None)]),
+            &agents(&["atlas"]),
+        );
+        assert_eq!(r.text, "");
+        assert_eq!(r.skipped, ["1.2.3.4"]);
+        // Control: the same list with a name that applies is not empty.
+        assert!(
+            !render(&list(&[("evil.com", None)]), &agents(&[]))
+                .text
+                .is_empty()
+        );
+    }
+
+    #[test]
+    fn an_entry_that_is_not_a_name_never_reaches_squid_configuration() {
+        let r = render(
+            &list(&[
+                ("evil.com", None),
+                ("*.example.com", Some("atlas")),
+                ("example.com\nhttp_access allow all", Some("atlas")),
+                ("example.com\nhttp_access allow all", None),
+            ]),
+            &agents(&["atlas"]),
+        );
+        assert!(!r.text.contains("allow"), "{}", r.text);
+        assert!(r.text.contains("dstdomain evil.com\n"));
+        assert_eq!(r.skipped.len(), 3);
+    }
+
+    /// The value swarm-controller's `egress_deny` route publishes, byte for byte
+    /// as its own test pins it, renders the rules for the agents named in it.
+    #[test]
+    fn the_controllers_published_value_renders_the_rules_it_names() {
+        let wire = r#"{"entries":[{"domain":"bad.io","agent":"atlas"},{"domain":"evil.com"},{"domain":"evil.com","agent":"atlas"}]}"#;
+        let decoded: DenyList = serde_json::from_str(wire).expect("decodes");
+        let r = render(&decoded, &agents(&["atlas"]));
+        assert!(r.text.contains("acl deny_swarm_dst dstdomain evil.com\n"));
+        assert!(
+            r.text
+                .contains("acl deny_agent_atlas_dst dstdomain bad.io\n")
+        );
+        assert_eq!((r.swarm, r.agent), (1, 1));
+    }
+
+    struct Dir(tempfile::TempDir);
+
+    impl Dir {
+        fn file(&self) -> PathBuf {
+            self.0.path().join("deny.conf")
+        }
+    }
+
+    fn dir() -> Dir {
+        Dir(tempfile::tempdir().expect("tempdir"))
+    }
+
+    fn text(d: &Dir) -> String {
+        std::fs::read_to_string(d.file()).expect("the file is there")
+    }
+
+    fn current(entries: &[(&str, Option<&str>)], revision: u64) -> DenyRead {
+        DenyRead::Current {
+            list: list(entries),
+            revision,
+        }
+    }
+
+    #[test]
+    fn the_file_follows_the_list_and_the_claimed_revision_follows_the_file() {
+        let d = dir();
+        let mine = agents(&["atlas"]);
+        let sync = |read| sync_with(read, &mine, &d.file()).expect("renders");
+
+        assert_eq!(sync(current(&[("evil.com", None)], 7)), Some(7));
+        assert!(text(&d).contains("dstdomain evil.com"));
+
+        // A second pass of the same list writes nothing.
+        let ino = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(d.file()).unwrap());
+        assert_eq!(sync(current(&[("evil.com", None)], 7)), Some(7));
+        assert_eq!(
+            std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(d.file()).unwrap()),
+            ino
+        );
+
+        // No answer keeps the denies in force and claims no revision.
+        assert_eq!(sync(DenyRead::NotSet), None);
+        assert!(text(&d).contains("dstdomain evil.com"), "denies are kept");
+
+        // The list emptied: the file is empty and the revision moves.
+        assert_eq!(sync(current(&[], 9)), Some(9));
+        assert_eq!(text(&d), "");
+
+        // An empty file with nothing to read is the true "no list": revision 0.
+        assert_eq!(sync(DenyRead::NotSet), Some(0));
+    }
+
+    /// The only test that touches the process-wide value.
+    #[test]
+    fn nothing_is_applied_until_a_pass_says_so() {
+        assert_eq!(applied(), None);
+        set_applied(0);
+        assert_eq!(applied(), Some(0));
+        set_applied(12);
+        assert_eq!(applied(), Some(12));
+    }
+
+    #[test]
+    fn the_file_is_group_readable() {
+        use std::os::unix::fs::PermissionsExt as _;
+        let d = dir();
+        sync_with(current(&[("evil.com", None)], 1), &agents(&[]), &d.file()).expect("renders");
+        let mode = std::fs::metadata(d.file())
+            .expect("stat")
+            .permissions()
+            .mode()
+            & 0o777;
+        assert_eq!(mode, 0o640);
+    }
+}