From e46ad07226f0a83f6404c2578baa1e8ab76413ea Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 30 Oct 2026 02:18:39 +0100 Subject: [PATCH] add a per-agent egress log mode --- docs/networking/network.md | 19 +- docs/process/conventions.md | 8 + docs/swarm/ui.md | 15 +- hive-c0re/src/agent_config/permissions.rs | 27 +++ hive-c0re/src/workers/squid_allow.rs | 222 +++++++++++++++++++- nix/host-modules/hive-squid.nix | 27 ++- nix/module-eval/core-toggle.nix | 34 ++- nix/vm-tests/hive-squid.nix | 28 +++ swarm-controller-api/src/permissions.rs | 18 ++ swarm-controller/src/agent_permissions.rs | 160 +++++++++++++- swarm-queue-client/src/agent_permissions.rs | 53 +++++ swarm-wire/src/agent_permissions.rs | 34 +++ swarmctl/src/permissions.rs | 23 +- 13 files changed, 637 insertions(+), 31 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index cb0b2214..33f1bce7 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -289,6 +289,22 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. restricted. Agents must not start sending credentials until every hive runs the `hive-c0re` that writes it, or a legacy agent loses general internet. +- **Log mode.** The operator can put an agent in `log` mode in the swarm UI. + Its record keeps `unrestricted_internet` at `false` and adds + `internet_log: true`, so a `hive-c0re` that doesn't know the field still + enforces the list. For such an agent `hive-c0re` adds to `agent-allow.conf`, + after the allow rules for every agent's own names, + `http_access allow !allowed_dst log_mark`, where `log_mark` is an + `annotate_transaction` ACL that sets `would_deny=1`. The rule comes last in + the include, so every destination deny and `deny !authed` before it still + applies, and a name the agent may reach never carries the mark. A request the + rule lets through gets a second log line, + `would_deny user= method= host= port=

result=`, from the + `would_denied` `access_log` in `hive-squid.nix`; the usual request line keeps + its format. Only the operator changes the mode: nothing moves an agent to + `restricted` after a time or a count. The self-check's users aren't agents, + so `deny-canary.invalid` stays a 403. + - **Base list.** `services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains` is open to every authenticated user, whatever its flag: what the agent harness needs off the swarm. The default is the hosts Claude Code requires @@ -384,7 +400,8 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. - **Logs.** One syslog line per request reaches the journal as `squid.service`, and `services.hyperhive.otel.journalUnits` forwards that unit and the self-check's to the swarm's log store. A refusal shows as `TCP_DENIED/403` with the - client's source address. + client's source address. A request that a `log`-mode agent's rule let + through also logs a `would_deny` line (see _Log mode_). - **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service` every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT` to `services.hyperhive.deploy.hive-controller.squid.selfcheck.canary` (default: `cache.nixos.org`, diff --git a/docs/process/conventions.md b/docs/process/conventions.md index 37271b96..7e36f4dd 100644 --- a/docs/process/conventions.md +++ b/docs/process/conventions.md @@ -325,6 +325,14 @@ the agent's squid policy: [the egress proxy](../networking/network.md#egress-pro lets an unrestricted agent's credential reach any destination, and holds the others to the base list and the allow-list. An agent sends its credential once it has fetched it, from its next start. +**Log mode** — `internet_log` is a second boolean, meaningful only beside an +explicit `unrestricted_internet: false`; read the pair with +`swarm_queue_client::agent_permissions::internet_mode`. A `PUT` that sets +`unrestricted_internet` without `internet_log` turns it off, the route refuses +`internet_log: true` unless the stored or requested flag is `false`, and a +`DELETE` keeps it. A hive that doesn't read the field sees `false` and enforces +the list. The proxy's handling is under _Log mode_ in [the egress proxy](../networking/network.md#egress-proxy-squid). + **How a change applies** — - The agent's harness reads its own key every 10 seconds and builds every turn's `--tools`, `--allowedTools` and MCP config from the last value, so a diff --git a/docs/swarm/ui.md b/docs/swarm/ui.md index c57085d6..a0156f89 100644 --- a/docs/swarm/ui.md +++ b/docs/swarm/ui.md @@ -583,11 +583,16 @@ refuses, for example, a percentage with a leading zero. The dialog shows the refusal's reason under the fields and leaves them as you typed them. Below the list, **internet** shows whether the agent's traffic may leave the -host unfiltered: `unrestricted`, `restricted`, or `unrestricted (legacy)` when -the swarm never set the flag, which counts as on. The **restrict** or -**allow unrestricted** badge flips it with a `PUT` that carries the stored -permission list along with the flag, because a `PUT` without the list resets a -stored list to the default. An agent with no stored list sends the flag alone. +host unfiltered: `unrestricted`, `restricted`, `log`, or `unrestricted (legacy)` +when the swarm never set the flag, which counts as on. `log` is `restricted` +that allows what the list would refuse and logs each such request as +`would_deny`; it stays until you change it. The badges for the other two +modes (**allow unrestricted**, **log**, **restrict**) set the mode with a +`PUT` that carries the stored permission list along with the flags, because a +`PUT` without the list resets a stored list to the default. An agent with no +stored list sends the flags alone. `log` sends `unrestricted_internet: false` +with `internet_log: true`; the other two leave `internet_log` out, which ends +logging. The hive's egress proxy applies the flag to each agent's proxy credential (see [the egress proxy](../networking/network.md#egress-proxy-squid)). An agent sends its credential from its next start after it has fetched it, so a flag change applies diff --git a/hive-c0re/src/agent_config/permissions.rs b/hive-c0re/src/agent_config/permissions.rs index 53ee0220..c6fdb172 100644 --- a/hive-c0re/src/agent_config/permissions.rs +++ b/hive-c0re/src/agent_config/permissions.rs @@ -17,6 +17,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use hive_types::Permission; +pub use swarm_queue_client::agent_permissions::InternetMode; use swarm_wire::agent_permissions::AgentPermissions; const FILE: &str = "swarm-agent-permissions.json"; @@ -55,6 +56,16 @@ pub fn unrestricted_internet(record: Option<&AgentPermissions>) -> bool { record.is_none_or(swarm_queue_client::agent_permissions::unrestricted) } +/// How squid treats the agent's requests to names its lists do not allow under +/// `record`: [`InternetMode::Unrestricted`] when there is no record. +#[must_use] +pub fn internet_mode(record: Option<&AgentPermissions>) -> InternetMode { + record.map_or( + InternetMode::Unrestricted, + swarm_queue_client::agent_permissions::internet_mode, + ) +} + /// Whether `agent` holds `perm` per the file at `path` (normally [`path`]). /// /// # Errors @@ -96,6 +107,22 @@ mod tests { )))); } + /// `log` is the pair `false` + `internet_log`, and nothing else reads as it. + #[test] + fn internet_mode_reads_the_log_pair() { + let mode = |json: &str| internet_mode(Some(&record(json))); + assert_eq!(internet_mode(None), InternetMode::Unrestricted); + assert_eq!( + mode(r#"{"unrestricted_internet":false,"internet_log":true}"#), + InternetMode::Log + ); + assert_eq!( + mode(r#"{"unrestricted_internet":false}"#), + InternetMode::Restricted + ); + assert_eq!(mode(r#"{"internet_log":true}"#), InternetMode::Unrestricted); + } + /// No name holds more than the default without an entry: `ruth` with no /// entry holds `DEFAULT`, and only an entry grants her `manage_root_agent`. #[test] diff --git a/hive-c0re/src/workers/squid_allow.rs b/hive-c0re/src/workers/squid_allow.rs index 4e97cdf9..9ae8a790 100644 --- a/hive-c0re/src/workers/squid_allow.rs +++ b/hive-c0re/src/workers/squid_allow.rs @@ -10,7 +10,7 @@ //! - [`ENV_AGENT_ALLOW_FILE`]: squid configuration `include`d where the base //! list is allowed: for each of this hive's agents with entries, an `acl` //! for its squid user, an `acl` for its names and the `http_access allow` -//! joining them. Entries of agents this hive does not run are left out. +//! joining them, then the rules of the agents in `log` mode ([`log_rules`]). //! //! Squid reads both when it parses its configuration, so the path unit in //! `nix/host-modules/hive-squid.nix` reloads it when either changes. @@ -29,6 +29,7 @@ //! because squid ignores the narrower of two overlapping names in one ACL. A //! failed read leaves both files as they are. +use std::fmt::Write as _; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -37,6 +38,7 @@ use swarm_queue_client::egress_allow::{self, Read}; use swarm_wire::egress_allow::{AllowList, covered_by, normalize_domain}; use super::squid_users::write_if_changed; +use crate::permissions::{self, InternetMode}; use crate::stats::sweep_health::{self, SweepHealth}; /// How often the list is read. @@ -67,6 +69,13 @@ const SWARM_MARKER: &str = "# source: swarm\n"; /// The first line of a file rendered from the seed. const SEED_MARKER: &str = "# source: seed\n"; +/// The annotation `log` rules put on a request, as `key=value`. The `note` ACL +/// in `nix/host-modules/hive-squid.nix` reads the key. +const MARK: &str = "would_deny=1"; + +/// The ACL that applies [`MARK`]. +const MARK_ACL: &str = "log_mark"; + /// The two files a pass writes. struct Files<'a> { allowed: &'a Path, @@ -173,11 +182,14 @@ async fn pass( .into_iter() .map(|spec| spec.name) .collect(); + let perms = permissions::read_at(&permissions::path()) + .context("reading the stored swarm permissions")?; + let logging = log_agents(&agents, &perms); let read = egress_allow::read(client) .await .map_err(|e| anyhow::anyhow!(e)) .context("reading the swarm's egress allow-list")?; - sync_with(read, &agents, seed, always, files) + sync_with(read, &agents, &logging, seed, always, files) } /// [`pass`] with the read, the agents and the files given. Each file is only @@ -185,18 +197,19 @@ async fn pass( fn sync_with( read: Read, agents: &[String], + logging: &[String], seed: Option<&Path>, always: &str, files: &Files<'_>, ) -> Result { let always: Vec<&str> = always.split_whitespace().collect(); let rendered = match read { - Read::Current(list) => render(&list, agents, &always, &seed_names(seed)?), + Read::Current(list) => render(&list, agents, logging, &always, &seed_names(seed)?), Read::NotSet => { if super::squid_users::read_or_empty(files.allowed)?.starts_with(SWARM_MARKER) { return Ok(Source::Kept); } - seeded(seed)? + seeded(seed, logging)? } }; let changed_allowed = write_if_changed(files.allowed, &rendered.allowed)?; @@ -266,8 +279,8 @@ fn seed_names(seed: Option<&Path>) -> Result> { .collect()) } -/// The files for the seed: its names as they are, and no per-agent rules. -fn seeded(seed: Option<&Path>) -> Result { +/// The files for the seed: its names as they are, and no per-agent allow rules. +fn seeded(seed: Option<&Path>, logging: &[String]) -> Result { let mut allowed = SEED_MARKER.to_owned(); for name in seed_names(seed)? { allowed.push_str(&name); @@ -275,7 +288,7 @@ fn seeded(seed: Option<&Path>) -> Result { } Ok(Rendered { allowed, - agent_allow: SEED_MARKER.to_owned(), + agent_allow: format!("{SEED_MARKER}{}", log_rules(logging)), counts: None, }) } @@ -292,9 +305,54 @@ fn reduced(mut names: Vec) -> Vec { names } +/// The agents among `agents` whose record is in `log` mode, sorted. A name that +/// is not an identifier is left out, since the rules put it in squid's +/// configuration. +fn log_agents(agents: &[String], perms: &permissions::Stored) -> Vec { + let mut logging: Vec = agents + .iter() + .filter(|a| hive_types::Ident::parse(a).is_ok()) + .filter(|a| permissions::internet_mode(perms.get(*a)) == InternetMode::Log) + .cloned() + .collect(); + logging.sort(); + logging.dedup(); + logging +} + +/// The rules that let each of `logging` reach a name `allowed_dst` lacks and +/// mark the request `would_deny=1` for `access_log` (`nix/host-modules/hive-squid.nix`). +/// Empty for no agents. +/// +/// These follow every `http_access allow` the agent's own names get, so a name +/// the agent is allowed never carries the mark. The mark is the last ACL of its +/// rule: squid applies `annotate_transaction` when it is reached, whatever the +/// rule decides after it. Everything in squid's configuration before the +/// include is a deny that applies to these agents as to any other. +fn log_rules(logging: &[String]) -> String { + if logging.is_empty() { + return String::new(); + } + let mut rules = format!("acl {MARK_ACL} annotate_transaction {MARK}\n"); + for agent in logging { + let _ = write!( + rules, + "acl agent_log_{agent} proxy_auth {agent}\n\ + http_access allow agent_log_{agent} !allowed_dst {MARK_ACL}\n" + ); + } + rules +} + /// The files for `list` and the seed's `own` names, for the agents this hive -/// runs. -fn render(list: &AllowList, agents: &[String], always: &[&str], own: &[String]) -> Rendered { +/// runs, with `logging` of them in `log` mode. +fn render( + list: &AllowList, + agents: &[String], + logging: &[String], + always: &[&str], + own: &[String], +) -> Rendered { let mut skipped = Vec::new(); let mut swarm_wide = Vec::new(); let mut per_agent: std::collections::BTreeMap<&str, Vec> = @@ -342,7 +400,11 @@ fn render(list: &AllowList, agents: &[String], always: &[&str], own: &[String]) } Rendered { allowed, - agent_allow: format!("{SWARM_MARKER}{}", agent_rules.concat()), + agent_allow: format!( + "{SWARM_MARKER}{}{}", + agent_rules.concat(), + log_rules(logging) + ), counts: Some(Counts { swarm: swarm_wide.len(), agent: agent_count, @@ -377,7 +439,7 @@ mod tests { const ALWAYS: &[&str] = &["api.anthropic.com", ".claude.ai", "cache.nixos.org"]; fn rendered(entries: &[(&str, Option<&str>)], mine: &[&str]) -> Rendered { - render(&list(entries), &agents(mine), ALWAYS, &[]) + render(&list(entries), &agents(mine), &[], ALWAYS, &[]) } #[test] @@ -476,6 +538,7 @@ mod tests { &agents(&["atlas"]), &[], &[], + &[], ); assert_eq!( bare.allowed, @@ -502,6 +565,141 @@ mod tests { ); } + fn log_list(logging: &[&str]) -> String { + render( + &list(&[("pypi.org", Some("atlas"))]), + &agents(&["atlas", "uma", "iris"]), + &agents(logging), + ALWAYS, + &[], + ) + .agent_allow + } + + /// A `log` agent's rule allows what `allowed_dst` lacks and carries the + /// mark last; it comes after every agent's own allow rule. + #[test] + fn a_log_agent_is_allowed_past_the_list_and_marked_after_all_own_rules() { + let text = log_list(&["uma"]); + assert_eq!( + text, + "# source: swarm\n\ + acl agent_allow_atlas proxy_auth atlas\n\ + acl agent_allow_atlas_dst dstdomain pypi.org\n\ + http_access allow agent_allow_atlas agent_allow_atlas_dst\n\ + acl log_mark annotate_transaction would_deny=1\n\ + acl agent_log_uma proxy_auth uma\n\ + http_access allow agent_log_uma !allowed_dst log_mark\n" + ); + // The rule of an agent with its own names also follows them. + let both = log_list(&["atlas"]); + let own = both + .find("http_access allow agent_allow_atlas ") + .expect("own rule"); + let log = both + .find("http_access allow agent_log_atlas ") + .expect("log rule"); + assert!(own < log, "{both}"); + // Control: with no log agent there is no rule and no mark. + let none = log_list(&[]); + assert!( + !none.contains("log_mark") && !none.contains("would_deny"), + "{none}" + ); + } + + /// Every log rule ends in the mark, which squid applies when it is reached; + /// a rule that put it earlier would mark requests the agent is allowed. + #[test] + fn the_mark_is_the_last_acl_of_every_log_rule() { + let text = log_list(&["atlas", "iris", "uma"]); + let rules: Vec<&str> = text + .lines() + .filter(|l| l.starts_with("http_access allow agent_log_")) + .collect(); + assert_eq!(rules.len(), 3, "{text}"); + for rule in rules { + assert!(rule.ends_with(" !allowed_dst log_mark"), "{rule}"); + } + assert_eq!( + text.matches("annotate_transaction").count(), + 1, + "defined once" + ); + } + + /// Only `deny`-free, `allow`-only rules reach the include: the safety + /// denies that precede it in squid's configuration stay the only ones. + #[test] + fn a_log_agent_adds_allow_rules_and_no_deny() { + let text = log_list(&["atlas", "uma"]); + assert!(!text.contains("http_access deny"), "{text}"); + assert!(text.contains("http_access allow"), "control"); + } + + /// The self-check user cannot be named by a log rule: it is not an agent + /// name, so `squid-selfcheck`'s deny canary stays refused. + #[test] + fn the_selfcheck_users_are_never_log_agents() { + for user in [ + swarm_secret_client::proxy::SELFCHECK_USER, + swarm_secret_client::proxy::SELFCHECK_UNRESTRICTED_USER, + ] { + assert!(hive_types::Ident::parse(user).is_err(), "{user}"); + let perms = permissions::Stored::from([( + user.to_owned(), + serde_json::from_str(r#"{"unrestricted_internet":false,"internet_log":true}"#) + .expect("decodes"), + )]); + assert!(log_agents(&agents(&[user]), &perms).is_empty(), "{user}"); + } + // Control: an agent in the same record shape is a log agent. + let perms = permissions::Stored::from([( + "uma".to_owned(), + serde_json::from_str(r#"{"unrestricted_internet":false,"internet_log":true}"#) + .expect("decodes"), + )]); + assert_eq!(log_agents(&agents(&["uma", "atlas"]), &perms), ["uma"]); + } + + /// Only `log` records give rules; restricted, unrestricted and unrecorded + /// agents render as they did. + #[test] + fn only_log_mode_agents_get_rules() { + let perms: permissions::Stored = serde_json::from_str( + r#"{ + "uma": {"unrestricted_internet": false, "internet_log": true}, + "atlas": {"unrestricted_internet": false}, + "iris": {"unrestricted_internet": true, "internet_log": true} + }"#, + ) + .expect("decodes"); + let all = agents(&["uma", "atlas", "iris", "argus"]); + assert_eq!(log_agents(&all, &perms), ["uma"]); + } + + /// The seed's file carries the rules too: logging does not depend on the + /// swarm having an allow list. + #[test] + fn the_seed_files_carry_the_log_rules() { + let d = dir("forge.example\n"); + sync_with( + Read::NotSet, + &agents(&["uma"]), + &agents(&["uma"]), + Some(&d.seed()), + "", + &Files { + allowed: &d.allowed(), + agent_allow: &d.agent_allow(), + }, + ) + .expect("renders"); + let text = text(&d.agent_allow()); + assert!(text.starts_with(SEED_MARKER), "{text}"); + assert!(text.contains("http_access allow agent_log_uma !allowed_dst log_mark")); + } + struct Dir(tempfile::TempDir); impl Dir { @@ -526,6 +724,7 @@ mod tests { sync_with( read, &agents(&["atlas"]), + &[], Some(&d.seed()), "api.anthropic.com cache.nixos.org", &Files { @@ -616,6 +815,7 @@ mod tests { sync_with( read, &agents(&["atlas"]), + &[], None, "", &Files { diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 6631b879..c7dbc73c 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -7,7 +7,10 @@ # rules leave, or reaches only the listed names: the base list, the allow-list # the swarm keeps plus this host's `allowedDomains` (rendered by hive-c0re # beside the users file) and the names -# the swarm lists for that agent alone. An agent sends its credential +# the swarm lists for that agent alone. A third kind, set per agent +# (`internet_log`), reaches the same names but is also let through to any other +# destination the deny rules leave; the request is logged as `would_deny`. +# An agent sends its credential # once it has fetched it (../agent-modules/egress-proxy.nix); the exporter, # journal forwarding and the self-check below show the proxy works. { @@ -81,8 +84,11 @@ let # user is then let through to the base names, a user the swarm lists names # for to those names (the include, whose rules hive-c0re generates and whose # file may be empty), an unrestricted user to any other destination on a - # safe port, and every other user to the listed names. The `dst` denies cost an - # unlisted name a lookup on the host's resolver before it is refused. + # safe port, and every other user to the listed names. A `log`-mode agent's + # rule sits in the include after the agent's own names and before + # `deny !allowed_dst`, so it sees the same denies and only turns that last + # refusal into an allow. The `dst` denies cost an unlisted name a lookup on + # the host's resolver before it is refused. squidConf = '' http_port ${listen} pid_filename /run/squid/squid.pid @@ -95,6 +101,15 @@ let # Cachemgr scrapes are left out. access_log syslog:daemon.info squid !manager + # A request that a `log`-mode agent's rule (hive-c0re's include, below) + # let through carries the annotation `would_deny=1`. It gets a second line + # in its own format, so the first line's format stays what the + # dashboards and lints parse, and `swarm-logs query '"would_deny"'` finds + # exactly these requests. + logformat would_deny would_deny user=%[un method=%rm host=%>rd port=%>rP result=%Ss/%03>Hs + acl would_denied note would_deny 1 + access_log syslog:daemon.info logformat=would_deny would_denied + # The helper reads the users file on every request and reloads it when its # mtime moves. It exits when the file is missing, which is why the file is # created before squid starts. A positive answer is cached for a minute, so @@ -358,8 +373,10 @@ in includes after every destination deny and `deny !authed`, where the base list is allowed: for each agent of this hive the swarm lists names for, an ACL for its user, an ACL for its names and the - `http_access allow` joining the two. Empty until the controller has a - list. Squid is reloaded when it changes. Owned by `hive-core` and + `http_access allow` joining the two. After those, for each agent in + `log` mode, an `http_access allow` for names `allowed_dst` lacks that + annotates the request `would_deny=1`. Empty until the controller has a + list or an agent is in `log` mode. Squid is reloaded when it changes. Owned by `hive-core` and readable by the `squid` group. ''; }; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 745260b9..a9b5a805 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -1670,7 +1670,13 @@ let # The default render, pinned so moving the options changes none of it: # the listen address squid binds and hive-c0re hands out, and the users # file hive-c0re writes, on a hive holding a store identity. - name = "squid's default listen address and users file reach squid.conf and hive-c0re unchanged"; + # + # Also the order hive-c0re's `log`-mode rules rely on: the include of + # `agent-allow.conf` follows every safety deny and `deny !authed`, and + # precedes `deny !allowed_dst`, so a log rule in it cannot let through + # what those refuse. The `would_deny` log line sits beside the unchanged + # request line. + name = "squid's default listen address and users file reach squid.conf and hive-c0re unchanged, and the include sits after the safety denies"; ok = let withIdentity = hiveWithController { @@ -1679,12 +1685,36 @@ let }; e = withIdentity.systemd.services.hive-c0re.environment; conf = withIdentity.services.squid.configText; + lines = map lib.strings.trim (lib.splitString "\n" conf); + at = line: lib.lists.findFirstIndex (l: l == line) null lines; + include = at "include /var/lib/hive-squid/agent-allow.conf"; + before = [ + "http_access deny !Safe_ports" + "http_access deny CONNECT !SSL_ports" + "http_access deny ip_v4_host" + "http_access deny ip_v6_host" + "http_access deny to_localhost" + "http_access deny to_linklocal" + "http_access deny to_bridge" + "http_access deny to_private" + "http_access deny !authed" + ]; in e.HIVE_SQUID_PROXY == "${withIdentity.services.hyperhive.network.bridgeIp}:3128" && e.HIVE_SQUID_USERS_FILE == "/var/lib/hive-squid/users" && lib.hasInfix "http_port ${e.HIVE_SQUID_PROXY}" conf && lib.hasInfix "basic_ncsa_auth /var/lib/hive-squid/users" conf - && lib.hasInfix "acl allowed_dst dstdomain cache.nixos.org" conf; + && lib.hasInfix "acl allowed_dst dstdomain cache.nixos.org" conf + # Presence controls: the include and the policy deny are found at all. + && include != null + && at "http_access deny !allowed_dst" != null + && include < at "http_access deny !allowed_dst" + && lib.all (rule: at rule != null && at rule < include) before + # The request line is unchanged; the would_deny line is a second one. + && at "access_log syslog:daemon.info squid !manager" != null + && at "acl would_denied note would_deny 1" != null + && at "access_log syslog:daemon.info logformat=would_deny would_denied" != null + && lib.hasInfix "logformat would_deny would_deny user=%[un " conf; } { # A hive's rendered identity, pinned so moving the option changes none of diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index adc0cfda..4607042c 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -545,6 +545,34 @@ pkgs.testers.runNixOSTest { wait_status("agentonly.t.local", ATLAS, "403") assert connect("forge.t.local", RUTH) == "200", "the hive's own name after the swarm list is gone" + with subtest("a log-mode agent reaches an unlisted name and is logged as would_deny, and nothing else changes"): + # What hive-c0re renders for `ruth` in log mode, after the agent rules. + write_allow( + seeded, + "# source: swarm\n" + "acl log_mark annotate_transaction would_deny=1\n" + "acl agent_log_ruth proxy_auth ruth\n" + "http_access allow agent_log_ruth !allowed_dst log_mark\n", + ) + wait_status("other.t.local", RUTH, "200") + proxy.wait_until_succeeds( + "journalctl -u squid.service -o cat | grep -E " + "'would_deny user=ruth method=CONNECT host=other\\.t\\.local port=443 result='" + ) + assert connect("forge.t.local", RUTH) == "200", "a listed name" + assert connect("other.t.local", ATLAS) == "403", "control: a restricted agent is still refused" + assert connect("other.t.local") == "407", "no credential" + assert connect("other.t.local", "ruth:" + secrets.token_urlsafe(24)) == "407", "wrong password" + assert connect("deny-canary.invalid", SELFCHECK) == "403", "the self-check's deny canary" + for name in ["loop.t.local", "bridge.t.local", "meta.t.local", "127.0.0.1", "203.0.113.2", "priv10.t.local", "other.t.local:22", "other.t.local:80"]: + assert connect(name, RUTH) == "403", f"log mode still refuses {name}" + assert squid_saw(r"would_deny user=ruth method=CONNECT host=other\.t\.local port=443 "), "the unlisted name is marked" + assert not squid_saw(r"would_deny user=ruth .* host=forge\.t\.local"), "a listed name is not marked" + assert not squid_saw(r"would_deny user=atlas "), "a restricted agent is not marked" + assert not squid_saw(r"would_deny .* host=loop\.t\.local"), "a refused request is not marked" + write_allow(seeded, "") + wait_status("other.t.local", RUTH, "403") + with subtest("squid starts on a missing list, treats every user as restricted, and recovers when it appears"): proxy.succeed(f"rm {LIST}") proxy.succeed("systemctl restart squid.service") diff --git a/swarm-controller-api/src/permissions.rs b/swarm-controller-api/src/permissions.rs index 94d2ee9e..0a052dc7 100644 --- a/swarm-controller-api/src/permissions.rs +++ b/swarm-controller-api/src/permissions.rs @@ -24,6 +24,13 @@ pub struct Permissions { /// `PUT` keeps the stored value. #[serde(default, skip_serializing_if = "Option::is_none")] pub unrestricted_internet: Option, + /// With `unrestricted_internet` `false`: requests to names the agent's lists + /// do not allow go through and squid logs each as `would_deny`. `true` + /// needs `unrestricted_internet: false`, in the request or stored. Setting + /// `unrestricted_internet` without this field turns it off; leaving both + /// out keeps the stored value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub internet_log: Option, } #[cfg(test)] @@ -39,6 +46,7 @@ mod tests { Permissions { permissions: Some(vec![]), unrestricted_internet: None, + internet_log: None, }, r#"{"permissions":[]}"#, ), @@ -46,13 +54,23 @@ mod tests { Permissions { permissions: None, unrestricted_internet: Some(false), + internet_log: None, }, r#"{"unrestricted_internet":false}"#, ), + ( + Permissions { + permissions: None, + unrestricted_internet: Some(false), + internet_log: Some(true), + }, + r#"{"unrestricted_internet":false,"internet_log":true}"#, + ), ( Permissions { permissions: Some(vec![Permission::Messaging, Permission::ManageRootAgent]), unrestricted_internet: None, + internet_log: None, }, r#"{"permissions":["messaging","manage_root_agent"]}"#, ), diff --git a/swarm-controller/src/agent_permissions.rs b/swarm-controller/src/agent_permissions.rs index 7aa6d95f..eeadf51d 100644 --- a/swarm-controller/src/agent_permissions.rs +++ b/swarm-controller/src/agent_permissions.rs @@ -37,14 +37,31 @@ use crate::wanted::WantedWriter; const MAX_ATTEMPTS: usize = 5; /// The permissions to store for `req`, whose list is already checked. The list -/// replaces the stored one; `unrestricted_internet` left out keeps the one in `current`. +/// replaces the stored one; `unrestricted_internet` left out keeps the one in +/// `current`. `internet_log` is stored only as `true`, and a request that sets +/// `unrestricted_internet` without it turns it off: an agent that moves to +/// unrestricted and back to restricted does not come back logging. fn into_stored(req: Permissions, current: &AgentPermissions) -> AgentPermissions { + let internet_log = match req.internet_log { + Some(on) => on.then_some(true), + None if req.unrestricted_internet.is_some() => None, + None => current.internet_log, + }; AgentPermissions { permissions: req.permissions, unrestricted_internet: req.unrestricted_internet.or(current.unrestricted_internet), + internet_log, } } +/// Why `value` is refused: `internet_log` is only meaningful beside an explicit +/// `unrestricted_internet: false`, the value an older hive reads. +fn log_without_restriction(value: &AgentPermissions) -> Option<&'static str> { + (value.internet_log == Some(true) && value.unrestricted_internet != Some(false)).then_some( + "internet_log needs unrestricted_internet to be false, in this request or stored", + ) +} + /// Refuse a list that names a permission twice. pub(crate) fn unique(perms: &[Permission]) -> Result<(), String> { let mut seen = std::collections::HashSet::new(); @@ -62,6 +79,7 @@ fn view(v: AgentPermissions) -> Permissions { Permissions { permissions: v.permissions, unrestricted_internet: v.unrestricted_internet, + internet_log: v.internet_log, } } @@ -173,6 +191,9 @@ pub(crate) async fn put_agent_permissions( .await .map_err(|e| queue_problem(&e))?; let value = into_stored(req.clone(), ¤t); + if let Some(why) = log_without_restriction(&value) { + return Err(refuse(error_problem(StatusCode::BAD_REQUEST, why))); + } if (current == value && !undecodable) || write(&store, &agent, &value, revision) .await @@ -214,10 +235,12 @@ pub(crate) async fn delete_agent_permissions( let (current, revision, _) = read_or_default::(&store, &agent) .await .map_err(|e| queue_problem(&e))?; - // Only the flag outlives a clear: dropping it would turn an agent - // created with `false` on. + // Only the flags outlive a clear: dropping `unrestricted_internet` + // would turn an agent created with `false` on, and dropping + // `internet_log` would turn a logging agent into an enforced one. let kept = AgentPermissions { unrestricted_internet: current.unrestricted_internet, + internet_log: current.internet_log, ..AgentPermissions::default() }; if kept == AgentPermissions::default() { @@ -307,6 +330,7 @@ pub(crate) fn at_create( Ok(AgentPermissions { permissions: perms, unrestricted_internet, + internet_log: None, }) } @@ -329,6 +353,13 @@ fn merged_at_create( .unrestricted_internet .or(current.unrestricted_internet) .or_else(|| (!held).then_some(false)), + // A create is not a decision about logging: a request that sets + // `unrestricted_internet` ends it, as a `PUT` does. + internet_log: if value.unrestricted_internet.is_some() { + None + } else { + current.internet_log + }, } } @@ -596,6 +627,80 @@ mod tests { use hive_types::Permission; use swarm_wire::agent_permissions::{Import, ImportReply}; + fn stored(json: &str) -> swarm_wire::agent_permissions::AgentPermissions { + serde_json::from_str(json).expect("decodes") + } + + fn request(unrestricted_internet: Option, internet_log: Option) -> Permissions { + Permissions { + permissions: None, + unrestricted_internet, + internet_log, + } + } + + /// Logging is never switched by a request that does not name it, except + /// that setting the main flag ends it, so no agent comes back logging. + #[test] + fn a_request_changes_logging_only_by_naming_it_or_the_main_flag() { + let logging = stored(r#"{"unrestricted_internet":false,"internet_log":true}"#); + let after = |req| super::into_stored(req, &logging).internet_log; + assert_eq!(after(request(None, None)), Some(true), "left out: kept"); + assert_eq!(after(request(Some(false), Some(true))), Some(true)); + assert_eq!( + after(request(Some(false), None)), + None, + "main flag set: ended" + ); + assert_eq!( + after(request(Some(true), None)), + None, + "unrestricted: ended" + ); + assert_eq!(after(request(Some(false), Some(false))), None); + let plain = stored(r#"{"unrestricted_internet":false}"#); + assert_eq!( + super::into_stored(request(None, Some(true)), &plain).internet_log, + Some(true), + "control: the field turns it on beside a stored false" + ); + } + + /// `true` without a stored or requested `false` is refused; the pair and + /// the plain restricted record are not. + #[test] + fn logging_is_refused_unless_the_main_flag_is_false() { + let refused = |current: &str, req| { + super::log_without_restriction(&super::into_stored(req, &stored(current))).is_some() + }; + assert!(refused("{}", request(None, Some(true))), "flag absent"); + assert!( + refused("{}", request(Some(true), Some(true))), + "unrestricted" + ); + assert!(!refused("{}", request(Some(false), Some(true))), "the pair"); + assert!( + !refused( + r#"{"unrestricted_internet":false}"#, + request(None, Some(true)) + ), + "stored false" + ); + assert!(!refused("{}", request(Some(false), None)), "restricted"); + } + + /// A create keeps a stored logging flag unless it sets the main flag. + #[test] + fn a_create_keeps_logging_unless_it_sets_the_main_flag() { + let current = stored(r#"{"unrestricted_internet":false,"internet_log":true}"#); + let merged = |value: &str| super::merged_at_create(&stored(value), ¤t, true); + assert_eq!(merged("{}").internet_log, Some(true)); + assert_eq!( + merged(r#"{"unrestricted_internet":false}"#).internet_log, + None + ); + } + async fn published(client: &async_nats::Client, agent: &str) -> Option { let store = swarm_queue_client::agent_permissions::open_or_create(client) .await @@ -611,6 +716,7 @@ mod tests { Permissions { permissions: Some(vec![Permission::ManageRootAgent]), unrestricted_internet: None, + internet_log: None, } } @@ -655,6 +761,7 @@ mod tests { let none = Permissions { permissions: Some(Vec::new()), unrestricted_internet: None, + internet_log: None, }; let _ = put(&state, "atlas", none.clone()).await.expect("set"); assert_eq!( @@ -670,6 +777,53 @@ mod tests { assert_eq!(get(&state, "atlas").await, Permissions::default()); } + /// Through the handlers: `log` is stored beside the old flag at `false`, a + /// `DELETE` keeps both, and a `PUT` of the main flag alone ends it. + #[tokio::test] + async fn log_mode_is_stored_beside_the_old_flag_and_ends_with_the_main_flag() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = crate::agent_limits::tests::declared(&client).await; + let logging = r#"{"unrestricted_internet":false,"internet_log":true}"#; + + let log = Permissions { + permissions: None, + unrestricted_internet: Some(false), + internet_log: Some(true), + }; + let stored = put(&state, "atlas", log.clone()).await.expect("set").0; + assert_eq!(stored, log); + assert_eq!(published(&client, "atlas").await.as_deref(), Some(logging)); + + let _ = delete_agent_permissions(State(state.clone()), Path("atlas".to_owned())).await; + assert_eq!(published(&client, "atlas").await.as_deref(), Some(logging)); + + // Refused: `log` beside `true`, and nothing is written. + let both = Permissions { + unrestricted_internet: Some(true), + ..log.clone() + }; + let refused = put(&state, "atlas", both).await.expect_err("refused"); + assert_eq!(refused.status, Some(StatusCode::BAD_REQUEST)); + assert_eq!(published(&client, "atlas").await.as_deref(), Some(logging)); + + // The main flag alone ends it, and `false` again does not bring it back. + let on = Permissions { + permissions: None, + unrestricted_internet: Some(true), + internet_log: None, + }; + let _ = put(&state, "atlas", on.clone()).await.expect("set"); + let off = Permissions { + unrestricted_internet: Some(false), + ..on.clone() + }; + let _ = put(&state, "atlas", off).await.expect("set"); + assert_eq!( + published(&client, "atlas").await.as_deref(), + Some(r#"{"unrestricted_internet":false}"#) + ); + } /// An agent stored with `unrestricted_internet: false` keeps it through a /// `PUT` that leaves it out and through `DELETE`; a `PUT` that names it /// changes it. Without the flag, `DELETE` still removes the key. diff --git a/swarm-queue-client/src/agent_permissions.rs b/swarm-queue-client/src/agent_permissions.rs index 73aca48b..a9cad8fe 100644 --- a/swarm-queue-client/src/agent_permissions.rs +++ b/swarm-queue-client/src/agent_permissions.rs @@ -59,6 +59,34 @@ pub fn unrestricted(record: &AgentPermissions) -> bool { record.unrestricted_internet.unwrap_or(true) } +/// How squid treats an agent's requests to names its lists do not allow. +#[cfg(feature = "agent-permissions")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InternetMode { + /// Any destination the safety denies leave. + Unrestricted, + /// As `Restricted`, except that a request to an unlisted name goes through + /// and squid logs it as `would_deny`. + Log, + /// Only the listed names. + Restricted, +} + +/// The mode `record` puts the agent in. `Log` needs an explicit `false` on +/// `unrestricted_internet`: an absent flag is [`InternetMode::Unrestricted`] +/// whatever `internet_log` says. +#[cfg(feature = "agent-permissions")] +#[must_use] +pub fn internet_mode(record: &AgentPermissions) -> InternetMode { + if unrestricted(record) { + InternetMode::Unrestricted + } else if record.internet_log == Some(true) { + InternetMode::Log + } else { + InternetMode::Restricted + } +} + /// Read `agent`'s current permissions with one direct get: the one subject /// an agent's own grant covers, and one of those a hive's covers. /// @@ -150,6 +178,31 @@ mod tests { assert!(unrestricted(&decode(Some("DEL"), b"").unwrap())); } + /// `Log` takes `unrestricted_internet: false` and `internet_log: true` + /// together; either alone, or the pair on an unrestricted record, is not it. + #[test] + fn log_needs_the_old_flag_off_and_the_new_one_on() { + let mode = |json: &str| internet_mode(&serde_json::from_str(json).unwrap()); + assert_eq!( + mode(r#"{"unrestricted_internet":false,"internet_log":true}"#), + InternetMode::Log + ); + assert_eq!( + mode(r#"{"unrestricted_internet":false}"#), + InternetMode::Restricted + ); + assert_eq!( + mode(r#"{"unrestricted_internet":false,"internet_log":false}"#), + InternetMode::Restricted + ); + assert_eq!(mode(r#"{"internet_log":true}"#), InternetMode::Unrestricted); + assert_eq!( + mode(r#"{"unrestricted_internet":true,"internet_log":true}"#), + InternetMode::Unrestricted + ); + assert_eq!(mode("{}"), InternetMode::Unrestricted); + } + /// A decoder built before the flag existed has no such field and ignores /// unknown ones; it reads a record carrying the flag. #[test] diff --git a/swarm-wire/src/agent_permissions.rs b/swarm-wire/src/agent_permissions.rs index 7ba46fde..8a8eb8c7 100644 --- a/swarm-wire/src/agent_permissions.rs +++ b/swarm-wire/src/agent_permissions.rs @@ -77,6 +77,14 @@ pub struct AgentPermissions { /// it through `swarm_queue_client::agent_permissions::unrestricted`. #[serde(default, skip_serializing_if = "Option::is_none")] pub unrestricted_internet: Option, + /// With `unrestricted_internet` `false`: the agent's requests to names its + /// lists do not allow go through, and squid logs each as `would_deny` + /// instead of refusing it. Absent is off. It is only ever `true` beside an + /// explicit `false`, so a hive that does not read this field keeps + /// enforcing. Read it through + /// `swarm_queue_client::agent_permissions::internet_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub internet_log: Option, } /// What a hive sends once: the permissions its meta repo held, by agent. @@ -161,4 +169,30 @@ mod tests { let unset: AgentPermissions = serde_json::from_str("{}").unwrap(); assert_eq!(unset.unrestricted_internet, None); } + + /// The log flag rides beside the old flag, which stays `false`, so a + /// reader without the field takes the record as a restricted agent. + #[test] + fn the_log_flag_keeps_the_old_flag_off_and_old_readers_enforce() { + // A reader built before the field existed. + #[derive(serde::Deserialize)] + struct Old { + unrestricted_internet: Option, + } + let log = AgentPermissions { + unrestricted_internet: Some(false), + internet_log: Some(true), + ..AgentPermissions::default() + }; + let wire = r#"{"unrestricted_internet":false,"internet_log":true}"#; + assert_eq!(serde_json::to_string(&log).unwrap(), wire); + assert_eq!(serde_json::from_str::(wire).unwrap(), log); + + let old: Old = serde_json::from_str(wire).unwrap(); + assert_eq!(old.unrestricted_internet, Some(false)); + // A record from before the field decodes with it off. + let before: AgentPermissions = + serde_json::from_str(r#"{"unrestricted_internet":false}"#).unwrap(); + assert_eq!(before.internet_log, None); + } } diff --git a/swarmctl/src/permissions.rs b/swarmctl/src/permissions.rs index 28686f0c..3e7acc74 100644 --- a/swarmctl/src/permissions.rs +++ b/swarmctl/src/permissions.rs @@ -78,6 +78,7 @@ fn with_permissions( Ok(Permissions { permissions: Some(granted), unrestricted_internet, + internet_log: None, }) } @@ -121,10 +122,11 @@ fn render(p: &Permissions) -> String { .collect::>() .join(", "), }; - let internet = match p.unrestricted_internet { - None => "(not set; unrestricted)", - Some(true) => "unrestricted", - Some(false) => "restricted", + let internet = match (p.unrestricted_internet, p.internet_log) { + (None, _) => "(not set; unrestricted)", + (Some(true), _) => "unrestricted", + (Some(false), Some(true)) => "log (unlisted names are allowed and logged as would_deny)", + (Some(false), _) => "restricted", }; format!("permissions: {caps}\nunrestricted_internet: {internet}\n") } @@ -197,4 +199,17 @@ mod tests { r#"{"permissions":["web_tools"]}"# ); } + + /// A logging agent shows as such, not as plainly restricted. + #[test] + fn a_logging_agent_is_shown_as_logging() { + let log: Permissions = + serde_json::from_str(r#"{"unrestricted_internet":false,"internet_log":true}"#).unwrap(); + assert!(render(&log).contains("unrestricted_internet: log (")); + let off: Permissions = serde_json::from_str(r#"{"unrestricted_internet":false}"#).unwrap(); + assert!( + render(&off).ends_with("unrestricted_internet: restricted\n"), + "control" + ); + } }