add a per-agent egress log mode

hive-c0re/src/agent_config/permissions.rs

@@ -17,6 +17,7 @@ use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 
 use hive_types::Permission;
+pub use swarm_queue_client::agent_permissions::InternetMode;
 use swarm_wire::agent_permissions::AgentPermissions;
 
 const FILE: &str = "swarm-agent-permissions.json";
@@ -55,6 +56,16 @@ pub fn unrestricted_internet(record: Option<&AgentPermissions>) -> bool {
     record.is_none_or(swarm_queue_client::agent_permissions::unrestricted)
 }
 
+/// How squid treats the agent's requests to names its lists do not allow under
+/// `record`: [`InternetMode::Unrestricted`] when there is no record.
+#[must_use]
+pub fn internet_mode(record: Option<&AgentPermissions>) -> InternetMode {
+    record.map_or(
+        InternetMode::Unrestricted,
+        swarm_queue_client::agent_permissions::internet_mode,
+    )
+}
+
 /// Whether `agent` holds `perm` per the file at `path` (normally [`path`]).
 ///
 /// # Errors
@@ -96,6 +107,22 @@ mod tests {
         ))));
     }
 
+    /// `log` is the pair `false` + `internet_log`, and nothing else reads as it.
+    #[test]
+    fn internet_mode_reads_the_log_pair() {
+        let mode = |json: &str| internet_mode(Some(&record(json)));
+        assert_eq!(internet_mode(None), InternetMode::Unrestricted);
+        assert_eq!(
+            mode(r#"{"unrestricted_internet":false,"internet_log":true}"#),
+            InternetMode::Log
+        );
+        assert_eq!(
+            mode(r#"{"unrestricted_internet":false}"#),
+            InternetMode::Restricted
+        );
+        assert_eq!(mode(r#"{"internet_log":true}"#), InternetMode::Unrestricted);
+    }
+
     /// No name holds more than the default without an entry: `ruth` with no
     /// entry holds `DEFAULT`, and only an entry grants her `manage_root_agent`.
     #[test]

hive-c0re/src/workers/squid_allow.rs

@@ -10,7 +10,7 @@
 //! - [`ENV_AGENT_ALLOW_FILE`]: squid configuration `include`d where the base
 //!   list is allowed: for each of this hive's agents with entries, an `acl`
 //!   for its squid user, an `acl` for its names and the `http_access allow`
-//!   joining them. Entries of agents this hive does not run are left out.
+//!   joining them, then the rules of the agents in `log` mode ([`log_rules`]).
 //!
 //! Squid reads both when it parses its configuration, so the path unit in
 //! `nix/host-modules/hive-squid.nix` reloads it when either changes.
@@ -29,6 +29,7 @@
 //! because squid ignores the narrower of two overlapping names in one ACL. A
 //! failed read leaves both files as they are.
 
+use std::fmt::Write as _;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
@@ -37,6 +38,7 @@ use swarm_queue_client::egress_allow::{self, Read};
 use swarm_wire::egress_allow::{AllowList, covered_by, normalize_domain};
 
 use super::squid_users::write_if_changed;
+use crate::permissions::{self, InternetMode};
 use crate::stats::sweep_health::{self, SweepHealth};
 
 /// How often the list is read.
@@ -67,6 +69,13 @@ const SWARM_MARKER: &str = "# source: swarm\n";
 /// The first line of a file rendered from the seed.
 const SEED_MARKER: &str = "# source: seed\n";
 
+/// The annotation `log` rules put on a request, as `key=value`. The `note` ACL
+/// in `nix/host-modules/hive-squid.nix` reads the key.
+const MARK: &str = "would_deny=1";
+
+/// The ACL that applies [`MARK`].
+const MARK_ACL: &str = "log_mark";
+
 /// The two files a pass writes.
 struct Files<'a> {
     allowed: &'a Path,
@@ -173,11 +182,14 @@ async fn pass(
         .into_iter()
         .map(|spec| spec.name)
         .collect();
+    let perms = permissions::read_at(&permissions::path())
+        .context("reading the stored swarm permissions")?;
+    let logging = log_agents(&agents, &perms);
     let read = egress_allow::read(client)
         .await
         .map_err(|e| anyhow::anyhow!(e))
         .context("reading the swarm's egress allow-list")?;
-    sync_with(read, &agents, seed, always, files)
+    sync_with(read, &agents, &logging, seed, always, files)
 }
 
 /// [`pass`] with the read, the agents and the files given. Each file is only
@@ -185,18 +197,19 @@ async fn pass(
 fn sync_with(
     read: Read,
     agents: &[String],
+    logging: &[String],
     seed: Option<&Path>,
     always: &str,
     files: &Files<'_>,
 ) -> Result<Source> {
     let always: Vec<&str> = always.split_whitespace().collect();
     let rendered = match read {
-        Read::Current(list) => render(&list, agents, &always, &seed_names(seed)?),
+        Read::Current(list) => render(&list, agents, logging, &always, &seed_names(seed)?),
         Read::NotSet => {
             if super::squid_users::read_or_empty(files.allowed)?.starts_with(SWARM_MARKER) {
                 return Ok(Source::Kept);
             }
-            seeded(seed)?
+            seeded(seed, logging)?
         }
     };
     let changed_allowed = write_if_changed(files.allowed, &rendered.allowed)?;
@@ -266,8 +279,8 @@ fn seed_names(seed: Option<&Path>) -> Result<Vec<String>> {
         .collect())
 }
 
-/// The files for the seed: its names as they are, and no per-agent rules.
-fn seeded(seed: Option<&Path>) -> Result<Rendered> {
+/// The files for the seed: its names as they are, and no per-agent allow rules.
+fn seeded(seed: Option<&Path>, logging: &[String]) -> Result<Rendered> {
     let mut allowed = SEED_MARKER.to_owned();
     for name in seed_names(seed)? {
         allowed.push_str(&name);
@@ -275,7 +288,7 @@ fn seeded(seed: Option<&Path>) -> Result<Rendered> {
     }
     Ok(Rendered {
         allowed,
-        agent_allow: SEED_MARKER.to_owned(),
+        agent_allow: format!("{SEED_MARKER}{}", log_rules(logging)),
         counts: None,
     })
 }
@@ -292,9 +305,54 @@ fn reduced(mut names: Vec<String>) -> Vec<String> {
     names
 }
 
+/// The agents among `agents` whose record is in `log` mode, sorted. A name that
+/// is not an identifier is left out, since the rules put it in squid's
+/// configuration.
+fn log_agents(agents: &[String], perms: &permissions::Stored) -> Vec<String> {
+    let mut logging: Vec<String> = agents
+        .iter()
+        .filter(|a| hive_types::Ident::parse(a).is_ok())
+        .filter(|a| permissions::internet_mode(perms.get(*a)) == InternetMode::Log)
+        .cloned()
+        .collect();
+    logging.sort();
+    logging.dedup();
+    logging
+}
+
+/// The rules that let each of `logging` reach a name `allowed_dst` lacks and
+/// mark the request `would_deny=1` for `access_log` (`nix/host-modules/hive-squid.nix`).
+/// Empty for no agents.
+///
+/// These follow every `http_access allow` the agent's own names get, so a name
+/// the agent is allowed never carries the mark. The mark is the last ACL of its
+/// rule: squid applies `annotate_transaction` when it is reached, whatever the
+/// rule decides after it. Everything in squid's configuration before the
+/// include is a deny that applies to these agents as to any other.
+fn log_rules(logging: &[String]) -> String {
+    if logging.is_empty() {
+        return String::new();
+    }
+    let mut rules = format!("acl {MARK_ACL} annotate_transaction {MARK}\n");
+    for agent in logging {
+        let _ = write!(
+            rules,
+            "acl agent_log_{agent} proxy_auth {agent}\n\
+             http_access allow agent_log_{agent} !allowed_dst {MARK_ACL}\n"
+        );
+    }
+    rules
+}
+
 /// The files for `list` and the seed's `own` names, for the agents this hive
-/// runs.
-fn render(list: &AllowList, agents: &[String], always: &[&str], own: &[String]) -> Rendered {
+/// runs, with `logging` of them in `log` mode.
+fn render(
+    list: &AllowList,
+    agents: &[String],
+    logging: &[String],
+    always: &[&str],
+    own: &[String],
+) -> Rendered {
     let mut skipped = Vec::new();
     let mut swarm_wide = Vec::new();
     let mut per_agent: std::collections::BTreeMap<&str, Vec<String>> =
@@ -342,7 +400,11 @@ fn render(list: &AllowList, agents: &[String], always: &[&str], own: &[String])
     }
     Rendered {
         allowed,
-        agent_allow: format!("{SWARM_MARKER}{}", agent_rules.concat()),
+        agent_allow: format!(
+            "{SWARM_MARKER}{}{}",
+            agent_rules.concat(),
+            log_rules(logging)
+        ),
         counts: Some(Counts {
             swarm: swarm_wide.len(),
             agent: agent_count,
@@ -377,7 +439,7 @@ mod tests {
     const ALWAYS: &[&str] = &["api.anthropic.com", ".claude.ai", "cache.nixos.org"];
 
     fn rendered(entries: &[(&str, Option<&str>)], mine: &[&str]) -> Rendered {
-        render(&list(entries), &agents(mine), ALWAYS, &[])
+        render(&list(entries), &agents(mine), &[], ALWAYS, &[])
     }
 
     #[test]
@@ -476,6 +538,7 @@ mod tests {
             &agents(&["atlas"]),
             &[],
             &[],
+            &[],
         );
         assert_eq!(
             bare.allowed,
@@ -502,6 +565,141 @@ mod tests {
         );
     }
 
+    fn log_list(logging: &[&str]) -> String {
+        render(
+            &list(&[("pypi.org", Some("atlas"))]),
+            &agents(&["atlas", "uma", "iris"]),
+            &agents(logging),
+            ALWAYS,
+            &[],
+        )
+        .agent_allow
+    }
+
+    /// A `log` agent's rule allows what `allowed_dst` lacks and carries the
+    /// mark last; it comes after every agent's own allow rule.
+    #[test]
+    fn a_log_agent_is_allowed_past_the_list_and_marked_after_all_own_rules() {
+        let text = log_list(&["uma"]);
+        assert_eq!(
+            text,
+            "# source: swarm\n\
+             acl agent_allow_atlas proxy_auth atlas\n\
+             acl agent_allow_atlas_dst dstdomain pypi.org\n\
+             http_access allow agent_allow_atlas agent_allow_atlas_dst\n\
+             acl log_mark annotate_transaction would_deny=1\n\
+             acl agent_log_uma proxy_auth uma\n\
+             http_access allow agent_log_uma !allowed_dst log_mark\n"
+        );
+        // The rule of an agent with its own names also follows them.
+        let both = log_list(&["atlas"]);
+        let own = both
+            .find("http_access allow agent_allow_atlas ")
+            .expect("own rule");
+        let log = both
+            .find("http_access allow agent_log_atlas ")
+            .expect("log rule");
+        assert!(own < log, "{both}");
+        // Control: with no log agent there is no rule and no mark.
+        let none = log_list(&[]);
+        assert!(
+            !none.contains("log_mark") && !none.contains("would_deny"),
+            "{none}"
+        );
+    }
+
+    /// Every log rule ends in the mark, which squid applies when it is reached;
+    /// a rule that put it earlier would mark requests the agent is allowed.
+    #[test]
+    fn the_mark_is_the_last_acl_of_every_log_rule() {
+        let text = log_list(&["atlas", "iris", "uma"]);
+        let rules: Vec<&str> = text
+            .lines()
+            .filter(|l| l.starts_with("http_access allow agent_log_"))
+            .collect();
+        assert_eq!(rules.len(), 3, "{text}");
+        for rule in rules {
+            assert!(rule.ends_with(" !allowed_dst log_mark"), "{rule}");
+        }
+        assert_eq!(
+            text.matches("annotate_transaction").count(),
+            1,
+            "defined once"
+        );
+    }
+
+    /// Only `deny`-free, `allow`-only rules reach the include: the safety
+    /// denies that precede it in squid's configuration stay the only ones.
+    #[test]
+    fn a_log_agent_adds_allow_rules_and_no_deny() {
+        let text = log_list(&["atlas", "uma"]);
+        assert!(!text.contains("http_access deny"), "{text}");
+        assert!(text.contains("http_access allow"), "control");
+    }
+
+    /// The self-check user cannot be named by a log rule: it is not an agent
+    /// name, so `squid-selfcheck`'s deny canary stays refused.
+    #[test]
+    fn the_selfcheck_users_are_never_log_agents() {
+        for user in [
+            swarm_secret_client::proxy::SELFCHECK_USER,
+            swarm_secret_client::proxy::SELFCHECK_UNRESTRICTED_USER,
+        ] {
+            assert!(hive_types::Ident::parse(user).is_err(), "{user}");
+            let perms = permissions::Stored::from([(
+                user.to_owned(),
+                serde_json::from_str(r#"{"unrestricted_internet":false,"internet_log":true}"#)
+                    .expect("decodes"),
+            )]);
+            assert!(log_agents(&agents(&[user]), &perms).is_empty(), "{user}");
+        }
+        // Control: an agent in the same record shape is a log agent.
+        let perms = permissions::Stored::from([(
+            "uma".to_owned(),
+            serde_json::from_str(r#"{"unrestricted_internet":false,"internet_log":true}"#)
+                .expect("decodes"),
+        )]);
+        assert_eq!(log_agents(&agents(&["uma", "atlas"]), &perms), ["uma"]);
+    }
+
+    /// Only `log` records give rules; restricted, unrestricted and unrecorded
+    /// agents render as they did.
+    #[test]
+    fn only_log_mode_agents_get_rules() {
+        let perms: permissions::Stored = serde_json::from_str(
+            r#"{
+                "uma": {"unrestricted_internet": false, "internet_log": true},
+                "atlas": {"unrestricted_internet": false},
+                "iris": {"unrestricted_internet": true, "internet_log": true}
+            }"#,
+        )
+        .expect("decodes");
+        let all = agents(&["uma", "atlas", "iris", "argus"]);
+        assert_eq!(log_agents(&all, &perms), ["uma"]);
+    }
+
+    /// The seed's file carries the rules too: logging does not depend on the
+    /// swarm having an allow list.
+    #[test]
+    fn the_seed_files_carry_the_log_rules() {
+        let d = dir("forge.example\n");
+        sync_with(
+            Read::NotSet,
+            &agents(&["uma"]),
+            &agents(&["uma"]),
+            Some(&d.seed()),
+            "",
+            &Files {
+                allowed: &d.allowed(),
+                agent_allow: &d.agent_allow(),
+            },
+        )
+        .expect("renders");
+        let text = text(&d.agent_allow());
+        assert!(text.starts_with(SEED_MARKER), "{text}");
+        assert!(text.contains("http_access allow agent_log_uma !allowed_dst log_mark"));
+    }
+
     struct Dir(tempfile::TempDir);
 
     impl Dir {
@@ -526,6 +724,7 @@ mod tests {
         sync_with(
             read,
             &agents(&["atlas"]),
+            &[],
             Some(&d.seed()),
             "api.anthropic.com cache.nixos.org",
             &Files {
@@ -616,6 +815,7 @@ mod tests {
         sync_with(
             read,
             &agents(&["atlas"]),
+            &[],
             None,
             "",
             &Files {