From 0b99f17bc3bf85d4a7e5e5a0f7e1f9a9eb7cb91e Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 14:14:30 +0100 Subject: [PATCH 1/5] render squid's unrestricted-users list from each agent's flag --- hive-c0re/src/workers/squid_users.rs | 371 +++++++++++++++++++++++---- swarm-secret-client/src/proxy.rs | 17 +- 2 files changed, 339 insertions(+), 49 deletions(-) diff --git a/hive-c0re/src/workers/squid_users.rs b/hive-c0re/src/workers/squid_users.rs index 44737a8fb..c2fb0269b 100644 --- a/hive-c0re/src/workers/squid_users.rs +++ b/hive-c0re/src/workers/squid_users.rs @@ -1,5 +1,6 @@ -//! Squid's users file: the hashes of the proxy credentials this hive's agents -//! authenticate to its egress proxy with. +//! Squid's users file, and its list of unrestricted users: the hashes of the +//! proxy credentials this hive's agents authenticate to its egress proxy with, +//! and which of them may reach any destination. //! //! swarm-controller mints one credential per agent and one for this hive's own //! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this @@ -9,12 +10,19 @@ //! moves. The agents are the container list, not the running containers: a //! stopped agent stays in the file. //! -//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is -//! while it still verifies its credential, so the file changes only when a -//! user is added, removed or given a new credential. +//! The file named by [`ENV_UNRESTRICTED_FILE`] lists, one name per line, the +//! users whose agent holds `unrestricted_internet` in the permissions this +//! hive last read from the swarm (an agent with no record, or no flag in it, +//! holds it), plus the self-check's unrestricted user. Squid reads the list as +//! an ACL when it parses its configuration, so `nix/host-modules/hive-squid.nix` +//! reloads squid when the file changes. //! -//! A failed read leaves the file as it is: dropping a user on a store outage -//! would read exactly like the swarm revoking it. +//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is +//! while it still verifies its credential, so each file changes only when a +//! user is added, removed, given a new credential or has its flag flipped. +//! +//! A failed read leaves both files as they are: dropping a user on a store +//! outage would read exactly like the swarm revoking it. use std::future::Future; use std::path::{Path, PathBuf}; @@ -24,6 +32,7 @@ use anyhow::{Context, Result, bail}; use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt}; use swarm_secret_client::{SecretStore, client, proxy}; +use crate::permissions; use crate::stats::sweep_health::{self, SweepHealth}; /// How often the file is re-rendered. @@ -36,11 +45,15 @@ const FAILURES_BEFORE_WARNING: u32 = 3; /// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid. pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE"; +/// The environment variable naming the unrestricted-users list. Set by +/// `nix/host-modules/hive-c0re` from `services.hyperhive.squid.unrestrictedFile`. +pub const ENV_UNRESTRICTED_FILE: &str = "HIVE_SQUID_UNRESTRICTED_FILE"; + /// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()` /// accepts. const SALT_BYTES: usize = 12; -/// The file's mode. The group is squid's, from the setgid directory +/// Both files' mode. The group is squid's, from the setgid directory /// `nix/host-modules/hive-squid.nix` creates. const FILE_MODE: u32 = 0o640; @@ -49,14 +62,32 @@ const FILE_MODE: u32 = 0o640; struct User { name: String, secret: String, + /// Whether squid lets this user reach any destination the deny rules leave. + unrestricted: bool, } -/// Start the pass loop, when this host has a squid users file, a store and a -/// hive name. -pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { - let Some(path) = std::env::var_os(ENV_USERS_FILE) +/// The two files a pass writes. +struct Files<'a> { + users: &'a Path, + unrestricted: &'a Path, +} + +/// Whether `name` is one of the self-check's squid users rather than an agent's. +fn is_selfcheck(name: &str) -> bool { + name == proxy::SELFCHECK_USER || name == proxy::SELFCHECK_UNRESTRICTED_USER +} + +fn env_path(name: &str) -> Option { + std::env::var_os(name) .filter(|v| !v.is_empty()) .map(PathBuf::from) +} + +/// Start the pass loop, when this host has squid's two files, a store and a +/// hive name. +pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { + let (Some(users), Some(unrestricted)) = + (env_path(ENV_USERS_FILE), env_path(ENV_UNRESTRICTED_FILE)) else { return; }; @@ -73,7 +104,15 @@ pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { tokio::spawn(async move { let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING); loop { - match pass(&hive, &path).await { + match pass( + &hive, + &Files { + users: &users, + unrestricted: &unrestricted, + }, + ) + .await + { Ok(()) => health.record_ok(), Err(e) => { tracing::warn!(error = ?e, "squid users: the pass failed"); @@ -99,18 +138,21 @@ pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { }); } -/// Read this hive's credentials from the store and render them into `file`. -async fn pass(hive: &str, file: &Path) -> Result<()> { +/// Read this hive's credentials from the store and the swarm's permissions +/// from the last read of them, and render both files. +async fn pass(hive: &str, files: &Files<'_>) -> Result<()> { let agents: Vec = crate::lifecycle::agents_for_meta_listing() .await .context("enumerating this hive's agents")? .into_iter() .map(|spec| spec.name) .collect(); + let perms = permissions::read_at(&permissions::path()) + .context("reading the stored swarm permissions")?; let store = SecretStore::from_env(hive) .await .context("connecting to the store")?; - sync_with(hive, &agents, file, |path| { + sync_with(hive, &agents, &perms, files, |path| { let store = &store; async move { let held: Option = store @@ -123,49 +165,104 @@ async fn pass(hive: &str, file: &Path) -> Result<()> { .await } -/// [`pass`] with the store read and the file given. Nothing is written unless -/// every read succeeds and the rendered file differs from the current one. -async fn sync_with(hive: &str, agents: &[String], file: &Path, read: F) -> Result<()> +/// [`pass`] with the store read, the permissions and the files given. Nothing +/// is written unless every read succeeds, and a file is only written when its +/// rendering differs from what it holds. +/// +/// The unrestricted list goes first. A user that turns restricted is then +/// restricted before the users file stops listing it, and one that turns +/// unrestricted is not listed before it can log in. +async fn sync_with( + hive: &str, + agents: &[String], + perms: &permissions::Stored, + files: &Files<'_>, + read: F, +) -> Result<()> where F: Fn(String) -> Fut, Fut: Future>>, { let mut wanted = Vec::new(); for agent in agents { - wanted.push((agent.clone(), proxy::agent_credential_path(agent)?)); + let unrestricted = permissions::unrestricted_internet(perms.get(agent)); + wanted.push(( + agent.clone(), + proxy::agent_credential_path(agent)?, + unrestricted, + )); } + let selfcheck = proxy::selfcheck_path(hive)?; + wanted.push((proxy::SELFCHECK_USER.to_owned(), selfcheck.clone(), false)); wanted.push(( - proxy::SELFCHECK_USER.to_owned(), - proxy::selfcheck_path(hive)?, + proxy::SELFCHECK_UNRESTRICTED_USER.to_owned(), + selfcheck, + true, )); let mut users = Vec::new(); - for (name, path) in wanted { + for (name, path, unrestricted) in wanted { if let Some(secret) = read(path).await? { - users.push(User { name, secret }); + users.push(User { + name, + secret, + unrestricted, + }); } else { tracing::debug!(name, "squid users: no proxy credential minted yet"); } } - let current = match std::fs::read_to_string(file) { - Ok(text) => text, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(), - Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())), - }; + let list = render_unrestricted(&users); + if write_if_changed(files.unrestricted, &list)? { + tracing::info!( + unrestricted = list.lines().count(), + file = %files.unrestricted.display(), + "squid users: wrote the unrestricted list" + ); + } + let current = read_or_empty(files.users)?; let rendered = render(¤t, &users)?; - if rendered == current { - return Ok(()); + if write_if_changed(files.users, &rendered)? { + tracing::info!( + users = users.len(), + agents = users.iter().filter(|u| !is_selfcheck(&u.name)).count(), + file = %files.users.display(), + "squid users: wrote the users file" + ); } - crate::secret_file::write_secret_mode(file, rendered.as_bytes(), FILE_MODE) - .with_context(|| format!("writing {}", file.display()))?; - tracing::info!( - users = users.len(), - agents = users.iter().filter(|u| u.name != proxy::SELFCHECK_USER).count(), - file = %file.display(), - "squid users: wrote the users file" - ); Ok(()) } +/// The file's text, or the empty string when it does not exist. +fn read_or_empty(file: &Path) -> Result { + match std::fs::read_to_string(file) { + Ok(text) => Ok(text), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(String::new()), + Err(e) => Err(e).with_context(|| format!("reading {}", file.display())), + } +} + +/// Write `text` to `file` unless it already holds exactly that. Returns +/// whether it wrote. +fn write_if_changed(file: &Path, text: &str) -> Result { + if read_or_empty(file)? == text { + return Ok(false); + } + crate::secret_file::write_secret_mode(file, text.as_bytes(), FILE_MODE) + .with_context(|| format!("writing {}", file.display()))?; + Ok(true) +} + +/// The names of the unrestricted `users`, sorted, one per line. +fn render_unrestricted(users: &[User]) -> String { + let mut names: Vec<&str> = users + .iter() + .filter(|u| u.unrestricted) + .map(|u| u.name.as_str()) + .collect(); + names.sort_unstable(); + names.iter().flat_map(|n| [n, "\n"]).collect() +} + /// The users file for `users`, sorted by name, one `:` line each. /// /// A user whose line in `current` still verifies against its credential keeps @@ -215,6 +312,7 @@ mod tests { use std::os::unix::fs::PermissionsExt as _; use super::*; + use swarm_wire::agent_permissions::AgentPermissions; const HIVE: &str = "h1"; @@ -230,14 +328,54 @@ mod tests { names.iter().map(|n| (*n).to_owned()).collect() } - async fn sync(held: &HashMap, agents: &[&str], file: &Path) -> Result<()> { - sync_with(HIVE, &names(agents), file, |path| { + /// The unrestricted list that sits beside the users file `users`. + fn list_of(users: &Path) -> PathBuf { + users.with_file_name("unrestricted") + } + + /// The permissions record of an agent whose swarm flag is `flag`. + fn flagged(agent: &str, flag: Option) -> (String, AgentPermissions) { + let json = match flag { + Some(f) => format!(r#"{{"unrestricted_internet":{f}}}"#), + None => "{}".to_owned(), + }; + ( + agent.to_owned(), + serde_json::from_str(&json).expect("decodes"), + ) + } + + async fn sync_flags( + held: &HashMap, + agents: &[&str], + perms: &permissions::Stored, + file: &Path, + ) -> Result<()> { + let list = list_of(file); + let files = Files { + users: file, + unrestricted: &list, + }; + sync_with(HIVE, &names(agents), perms, &files, |path| { let found = held.get(&path).cloned(); async move { Ok(found) } }) .await } + /// [`sync_flags`] with no permissions read yet: every agent unrestricted. + async fn sync(held: &HashMap, agents: &[&str], file: &Path) -> Result<()> { + sync_flags(held, agents, &permissions::Stored::new(), file).await + } + + fn list(users: &Path) -> Vec { + std::fs::read_to_string(list_of(users)) + .expect("the list is there") + .lines() + .map(str::to_owned) + .collect() + } + fn lines(file: &Path) -> Vec<(String, String)> { std::fs::read_to_string(file) .expect("the file is there") @@ -267,7 +405,15 @@ mod tests { .await .expect("renders"); let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); - assert_eq!(users, ["argus", "atlas", "selfcheck.squid"]); + assert_eq!( + users, + [ + "argus", + "atlas", + "selfcheck-unrestricted.squid", + "selfcheck.squid" + ] + ); } #[tokio::test] @@ -298,17 +444,17 @@ mod tests { sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) .await .expect("renders"); - // Control: the search finds a name that is in the file. + // Control: the search finds a name that is in each file. let mut found = 0; for entry in std::fs::read_dir(dir.path()).expect("lists") { let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text"); - assert!(text.contains("atlas:"), "control: {text}"); + assert!(text.contains("atlas"), "control: {text}"); for secret in ["atlas-secret", "argus-secret", "check-secret"] { assert!(!text.contains(secret), "{secret} reached the disk"); } found += 1; } - assert_eq!(found, 1, "only the users file is left behind"); + assert_eq!(found, 2, "only the users file and the list are left behind"); } #[tokio::test] @@ -390,7 +536,10 @@ mod tests { std::fs::write(&file, text).expect("write"); sync(&held, &["atlas"], &file).await.expect("renders"); let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); - assert_eq!(users, ["atlas", "selfcheck.squid"]); + assert_eq!( + users, + ["atlas", "selfcheck-unrestricted.squid", "selfcheck.squid"] + ); } #[tokio::test] @@ -411,10 +560,17 @@ mod tests { .await .expect("renders"); let before = std::fs::read_to_string(&file).expect("read"); + let list_before = std::fs::read_to_string(list_of(&file)).expect("read"); + let list = list_of(&file); + let files = Files { + users: &file, + unrestricted: &list, + }; let failed = sync_with( HIVE, &names(&["atlas", "argus"]), - &file, + &permissions::Stored::from([flagged("atlas", Some(false))]), + &files, |path| async move { if path.contains("argus") { bail!("the store is down") @@ -425,6 +581,11 @@ mod tests { .await; assert!(failed.is_err()); assert_eq!(std::fs::read_to_string(&file).expect("read"), before); + assert_eq!( + std::fs::read_to_string(&list).expect("read"), + list_before, + "the flag flip waits for every read" + ); } #[test] @@ -433,8 +594,124 @@ mod tests { let user = User { name: bad.to_owned(), secret: "s".to_owned(), + unrestricted: false, }; assert!(render("", &[user]).is_err(), "{bad:?}"); } } + + #[tokio::test] + async fn an_agent_is_listed_unless_its_flag_is_off() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + let held = store(&[ + ("swarm/agents/old/proxy-credential", "s1"), + ("swarm/agents/unset/proxy-credential", "s2"), + ("swarm/agents/on/proxy-credential", "s3"), + ("swarm/agents/new/proxy-credential", "s4"), + ("swarm/hives/h1/squid-selfcheck", "check-secret"), + ]); + let perms = permissions::Stored::from([ + flagged("unset", None), + flagged("on", Some(true)), + flagged("new", Some(false)), + ]); + sync_flags(&held, &["old", "unset", "on", "new"], &perms, &file) + .await + .expect("renders"); + // `old` has no record at all, `unset` a record without the flag. + assert_eq!( + list(&file), + ["old", "on", "selfcheck-unrestricted.squid", "unset"] + ); + // Control: the restricted agent still has a login. + let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); + assert!(users.contains(&"new".to_owned()), "{users:?}"); + } + + #[tokio::test] + async fn the_selfchecks_log_in_with_one_secret_and_only_one_is_unrestricted() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas"], &file) + .await + .expect("renders"); + let by_name: HashMap<_, _> = lines(&file).into_iter().collect(); + assert!(verifies("check-secret", &by_name["selfcheck.squid"])); + assert!(verifies( + "check-secret", + &by_name["selfcheck-unrestricted.squid"] + )); + let listed = list(&file); + assert!(listed.contains(&"selfcheck-unrestricted.squid".to_owned())); + assert!( + !listed.contains(&"selfcheck.squid".to_owned()), + "{listed:?}" + ); + } + + #[tokio::test] + async fn an_agent_with_no_credential_is_not_listed() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas", "fresh"], &file) + .await + .expect("renders"); + let listed = list(&file); + assert!(listed.contains(&"atlas".to_owned()), "control: {listed:?}"); + assert!(!listed.contains(&"fresh".to_owned()), "{listed:?}"); + } + + #[tokio::test] + async fn a_flipped_flag_rewrites_the_list_and_leaves_the_users_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + let held = two_agents_and_a_stranger(); + let ino = + |p: &Path| std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(p).expect("stat")); + sync(&held, &["atlas", "argus"], &file) + .await + .expect("renders"); + assert!(list(&file).contains(&"argus".to_owned())); + let (users_ino, list_ino) = (ino(&file), ino(&list_of(&file))); + // An unchanged pass touches neither. + sync(&held, &["atlas", "argus"], &file) + .await + .expect("renders"); + assert_eq!((ino(&file), ino(&list_of(&file))), (users_ino, list_ino)); + // Turning argus off changes only the list. + let perms = permissions::Stored::from([flagged("argus", Some(false))]); + sync_flags(&held, &["atlas", "argus"], &perms, &file) + .await + .expect("renders"); + assert!(!list(&file).contains(&"argus".to_owned())); + assert!(list(&file).contains(&"atlas".to_owned())); + assert_eq!(ino(&file), users_ino, "the users file was rewritten"); + assert_ne!(ino(&list_of(&file)), list_ino, "the list was not"); + } + + #[tokio::test] + async fn the_list_is_group_readable() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas"], &file) + .await + .expect("renders"); + let meta = std::fs::metadata(list_of(&file)).expect("stat"); + assert_eq!(meta.permissions().mode() & 0o777, 0o640); + } + + /// A pass that cannot write the users file has still written the list: a + /// user turned restricted loses its access before the users file stops + /// listing it. + #[tokio::test] + async fn the_list_is_written_before_the_users_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + std::fs::create_dir(&file).expect("a directory where the users file belongs"); + sync(&two_agents_and_a_stranger(), &["atlas"], &file) + .await + .expect_err("the users file cannot be read"); + assert_eq!(list(&file), ["atlas", "selfcheck-unrestricted.squid"]); + } } diff --git a/swarm-secret-client/src/proxy.rs b/swarm-secret-client/src/proxy.rs index 319c6f342..81a0cad3c 100644 --- a/swarm-secret-client/src/proxy.rs +++ b/swarm-secret-client/src/proxy.rs @@ -28,6 +28,14 @@ pub const SELFCHECK_LEAF: &str = "squid-selfcheck"; /// `nix/host-modules/hive-squid.nix` spells the same string. pub const SELFCHECK_USER: &str = "selfcheck.squid"; +/// The squid user the self-check logs in as to prove an unrestricted agent +/// reaches an unlisted name. It authenticates with the same secret as +/// [`SELFCHECK_USER`] ([`selfcheck_path`]), and is always on squid's +/// unrestricted list. It cannot be an agent's name for the same reason as +/// [`SELFCHECK_USER`]. `nix/host-modules/hive-squid.nix` spells the same +/// string. +pub const SELFCHECK_UNRESTRICTED_USER: &str = "selfcheck-unrestricted.squid"; + /// The path holding the secret `agent` authenticates to the egress proxy with. /// /// A flat leaf under the agent's prefix, like @@ -127,8 +135,13 @@ mod tests { } #[test] - fn the_selfcheck_user_is_not_a_legal_agent_name() { - assert!(crate::path::checked_segment("agent", SELFCHECK_USER).is_err()); + fn the_selfcheck_users_are_not_legal_agent_names() { + for user in [SELFCHECK_USER, SELFCHECK_UNRESTRICTED_USER] { + assert!( + crate::path::checked_segment("agent", user).is_err(), + "{user}" + ); + } } #[test] From 501445ebc315828cbc0740c05c0daa4088ef1b88 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 14:14:30 +0100 Subject: [PATCH 2/5] apply each agent's internet flag to its squid credential --- docs/networking/network.md | 40 ++++-- docs/process/conventions.md | 5 +- docs/swarm/credentials-reference.md | 20 +-- docs/swarm/ui.md | 4 +- nix/host-modules/hive-c0re/environment.nix | 5 +- nix/host-modules/hive-squid.nix | 119 ++++++++++++---- nix/module-eval/hive-otel.nix | 68 +++++++-- nix/vm-tests/hive-squid.nix | 153 ++++++++++++++++----- 8 files changed, 323 insertions(+), 91 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index 3af915826..816155da4 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -230,17 +230,33 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. store and the next pass mints a new one. HTTP Basic crosses the bridge in clear between an agent and the proxy. See [the credentials reference](../swarm/credentials-reference.md). - A name that isn't on the allow-list gets the 403 whether or not the request - has a credential; a listed name without one gets a 407. + A request without a valid credential gets a 407, once the destination + checks below have passed it. + +- **Two lists.** The credential decides the policy. `hive-c0re` writes a + second file beside the users file, `services.hyperhive.squid.unrestrictedFile` + (`/var/lib/hive-squid/unrestricted`, `hive-core:squid`, `0640`): the names of + the users whose agent holds the `unrestricted_internet` flag in the + permissions the hive last read from the swarm, one per line, rewritten when + it changes. An agent with no record, or a record that never set the flag, + holds it, the same default as the + `hyperhive.agent.unrestricted_internet` metric. Squid reads the list when it + parses its configuration, so `squid-unrestricted-reload.path` reloads squid + when the file changes. A missing or empty file lists no one, and squid + starts either way. A flag change reaches squid after the hive's next read of + the permissions and its next 30-second pass. - **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each - entry matches that exact name. Everything else gets a 403. -- **Unlisted names.** Refused first, without a DNS lookup. + entry matches that exact name. A user on the unrestricted list reaches any + other name too, on ports 80 and 443. Everyone else gets a 403 for it. +- **Unlisted names.** For a user who isn't on the unrestricted list, refused + after the destination checks, which look the name up on the host. - **Denied before any allow.** Destinations that resolve to loopback, - link-local or the bridge subnet, and IP-literal hosts. Squid resolves names - on the host, so without these a listed name pointing at `127.0.0.1` would - walk around the bridge→loopback DROP rule. Only `CONNECT` to port 443 and + link-local or the bridge subnet, and IP-literal hosts, for every user + including the unrestricted ones. Squid resolves names on the host, so + without these a name pointing at `127.0.0.1` would walk around the + bridge→loopback DROP rule. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass. - **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through @@ -254,9 +270,13 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT` to `services.hyperhive.squid.selfcheck.canary` (default: the forge's name) must answer 200, to `deny-canary.invalid` 403, and to the canary without a - credential 407. It writes one `squid-selfcheck ok=… allow_status=… -deny_status=… anon_status=… credential=present|absent` line per run and - fails the unit when any is wrong. The password is the hive's + credential 407. It then logs in as `selfcheck-unrestricted.squid`, which has + the same secret and is always on the unrestricted list, and a `CONNECT` to + `services.hyperhive.squid.selfcheck.unrestrictedCanary` (default: + `example.com`, a name outside the allow-list) must answer 200. It writes one + `squid-selfcheck ok=… allow_status=… deny_status=… anon_status=… +credential=present|absent unrestricted_canary=… unrestricted_status=…` line + per run and fails the unit when any is wrong. The password is the hive's `swarm/hives//squid-selfcheck` secret, which `squid-selfcheck-credential` lands at `/run/credstore/hive-squid.selfcheck`. Before `swarm-controller` has minted it, or while `hive-c0re` hasn't diff --git a/docs/process/conventions.md b/docs/process/conventions.md index 9739928b4..61bd31410 100644 --- a/docs/process/conventions.md +++ b/docs/process/conventions.md @@ -320,7 +320,10 @@ an absent flag included, and a value in the request wins. A `PUT` that leaves the field out and a `DELETE` (`swarmctl agent permissions clear`) also keep the stored value; a `DELETE` on an agent that has one rewrites the record to just that value. Read it with -`swarm_queue_client::agent_permissions::unrestricted`. Nothing acts on it yet. +`swarm_queue_client::agent_permissions::unrestricted`. The hive reads it for +the agent's squid policy: [the egress proxy](../networking/network.md#egress-proxy-squid) +lets an unrestricted agent's credential reach any destination, and holds the +others to the allow-list. No agent sends a credential yet. **How a change applies** — - The agent's harness reads its own key every 10 seconds and builds every diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index ee3ecc572..77657ec67 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -148,16 +148,16 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. ### Hive-level — one of each per hive -| secret | generated by | lives at | -| ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | -| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | -| the cache's public key and pull token, for hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); the cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand, and a hive that never got them has `hive-priv` answer `not available`. | -| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | -| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | -| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity, and copies it (account on the first line, token on the second) to `/run/hyperhive/forge-hive-token` (tmpfs, `0600` `hive-core`) for nix's `git`, which `hive-priv` also runs. Each `internal/meta` read and each boot rewrites the copy when it differs; where they differ after a rotation, a build that starts before the next read fails to authenticate, and a retry reads the new token | -| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; no agent sends a credential, so no traffic changes | -| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | +| secret | generated by | lives at | +| ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | +| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | +| the cache's public key and pull token, for hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); the cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand, and a hive that never got them has `hive-priv` answer `not available`. | +| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | +| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | +| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity, and copies it (account on the first line, token on the second) to `/run/hyperhive/forge-hive-token` (tmpfs, `0600` `hive-core`) for nix's `git`, which `hive-priv` also runs. Each `internal/meta` read and each boot rewrites the copy when it differs; where they differ after a rotation, a build that starts before the next read fails to authenticate, and a retry reads the new token | +| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; no agent sends a credential, so no traffic changes | +| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | ### Store identities, one per principal diff --git a/docs/swarm/ui.md b/docs/swarm/ui.md index 15750fad2..d468c5b84 100644 --- a/docs/swarm/ui.md +++ b/docs/swarm/ui.md @@ -568,7 +568,9 @@ the swarm never set the flag, which counts as on. The **restrict** or **allow unrestricted** badge flips it with a `PUT` that carries the stored permission list along with the flag, because a `PUT` without the list resets a stored list to the default. An agent with no stored list sends the flag alone. -Nothing acts on the flag yet. +The hive's egress proxy applies the flag to each agent's proxy credential (see +[the egress proxy](../networking/network.md#egress-proxy-squid)); no agent sends +one yet, so the flag changes no traffic today. A saved change isn't instant. The hive reads the caps about every 30 seconds, and a running agent takes a changed cap at its next spawn or rebuild, so the diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index e2ef032ff..d342f9fb1 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -272,8 +272,11 @@ in BAO_CLIENT_CERT = "%d/bao-client.pem"; BAO_CLIENT_KEY = "%d/bao-client-key.pem"; # Where `workers::squid_users` renders the hashes of this hive's agents' proxy - # credentials for squid; it reads them from the store under the identity above. + # credentials for squid, and the list of the users whose agent holds the + # swarm's `unrestricted_internet` flag; it reads the credentials from the + # store under the identity above. HIVE_SQUID_USERS_FILE = config.services.hyperhive.squid.usersFile; + HIVE_SQUID_UNRESTRICTED_FILE = config.services.hyperhive.squid.unrestrictedFile; } // lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) { # Absent means the system trust store — right for a deployment with a real diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 465156629..a0c38dc08 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -1,9 +1,12 @@ # Egress proxy for this hive's agents: squid on the host, bound to the bridge # IP, default-deny on the destination hostname. It filters CONNECT targets and # plain-HTTP hosts and never decrypts, so it holds no signing key. A request -# for a listed name must carry a per-agent proxy credential, checked against the -# users file hive-c0re renders. No agent sends one yet; the exporter, journal -# forwarding and the self-check below are how a deploy shows it works. +# must carry a per-agent proxy credential, checked against the users file +# hive-c0re renders. A credential's agent either has the swarm's +# `unrestricted_internet` flag, and may then reach any destination the deny +# rules leave, or reaches only the listed names. No agent sends a credential +# yet; the exporter, journal forwarding and the self-check below are how a +# deploy shows it works. { pkgs, lib, @@ -24,18 +27,21 @@ let # The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`. selfcheckUser = "selfcheck.squid"; + # The same string in `swarm_secret_client::proxy::SELFCHECK_UNRESTRICTED_USER`. + selfcheckUnrestrictedUser = "selfcheck-unrestricted.squid"; usersDir = builtins.dirOf cfg.usersFile; # Evaluated top to bottom, first match wins. The three destination denies - # sit before the only allow: squid resolves names on the host, so a name - # that answers 127.0.0.1, 169.254.169.254 or a bridge address would - # otherwise reach services the bridge firewall keeps agents out of. The - # unlisted-name deny comes first because `dstdomain` needs no lookup while - # the `dst` ACLs do: an unlisted name is refused at once instead of waiting - # on the host resolver, and costs a client no DNS query. The credential - # check sits right before the allow, so a refused destination answers 403 - # to a client with or without a credential, and only a request that would - # otherwise be let through gets the 407 challenge. + # sit before every allow, `allow unrestricted` included: squid resolves + # names on the host, so a name that answers 127.0.0.1, 169.254.169.254 or a + # bridge address would otherwise reach services the bridge firewall keeps + # agents out of, and an unrestricted agent is still an agent. The IP-literal + # denies come first because `dstdomain` on an IP literal does a reverse + # lookup. The credential check sits right before the first allow, so only a + # request that would otherwise be let through gets the 407 challenge. An + # unrestricted user is then let through to any other destination on a safe + # port, and every other user to the listed names. The `dst` denies cost an + # unlisted name a lookup on the host's resolver before it is refused. squidConf = '' http_port ${netCfg.bridgeIp}:${toString port} pid_filename /run/squid/squid.pid @@ -59,6 +65,10 @@ let acl SSL_ports port 443 acl Safe_ports port 80 443 acl authed proxy_auth REQUIRED + # The users whose agent holds the flag. Squid reads the file when it parses + # its configuration, so `squid-unrestricted-reload` reloads squid when + # hive-c0re rewrites it. An empty or missing file lists no one. + acl unrestricted proxy_auth "${cfg.unrestrictedFile}" # Host-originated requests: the self-check and the exporter. A local # connect to the bridge IP takes the bridge IP as its source. @@ -82,11 +92,12 @@ let http_access deny CONNECT !SSL_ports http_access deny ip_v4_host http_access deny ip_v6_host - ${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"} http_access deny to_localhost http_access deny to_linklocal http_access deny to_bridge http_access deny !authed + http_access allow unrestricted + ${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"} ${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"} http_access deny all ''; @@ -128,30 +139,32 @@ let secret=$(< "$credential_file") fi - # : the status squid answers a CONNECT with; 000 when - # the proxy is down. + # : the status squid answers a CONNECT with; 000 when + # the proxy is down. Both users log in with the one secret. connect_status() { { - if [ "$2" = auth ] && [ -n "$secret" ]; then - printf 'proxy-user = "%s:%s"\n' ${selfcheckUser} "$secret" + if [ "$2" != anon ] && [ -n "$secret" ]; then + printf 'proxy-user = "%s:%s"\n' "$2" "$secret" fi } | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \ -o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true } - allow_status=$(connect_status ${cfg.selfcheck.canary} auth) - deny_status=$(connect_status ${denyCanary} auth) + allow_status=$(connect_status ${cfg.selfcheck.canary} ${selfcheckUser}) + deny_status=$(connect_status ${denyCanary} ${selfcheckUser}) anon_status=$(connect_status ${cfg.selfcheck.canary} anon) + unrestricted_status=$(connect_status ${cfg.selfcheck.unrestrictedCanary} ${selfcheckUnrestrictedUser}) ok=false - if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ]; then + if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ] \ + && [ "$unrestricted_status" = 200 ]; then ok=true fi credential=absent if [ -n "$secret" ]; then credential=present fi - echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential" + echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential unrestricted_canary=${cfg.selfcheck.unrestrictedCanary} unrestricted_status=$unrestricted_status" [ "$ok" = true ] ''; }; @@ -186,6 +199,23 @@ in ''; }; + unrestrictedFile = lib.mkOption { + type = lib.types.path; + default = "${builtins.dirOf cfg.usersFile}/unrestricted"; + readOnly = true; + defaultText = lib.literalMD "`unrestricted`, in the directory of `usersFile`"; + description = '' + The user names, one per line, that squid lets reach any destination the + deny rules leave open: the users of the agents that hold the swarm's + `unrestricted_internet` flag (set unless the swarm records it off), and + the self-check's unrestricted user. hive-c0re renders it beside + `usersFile`, and squid is reloaded when it changes. Every other user + reaches only `allowedDomains`. An empty file, which is what exists + until hive-c0re writes one, makes every user a restricted one. + Owned by `hive-core` and readable by the `squid` group. + ''; + }; + selfcheck.credentialName = lib.mkOption { type = lib.types.str; default = "hive-squid.selfcheck"; @@ -198,6 +228,17 @@ in ''; }; + selfcheck.unrestrictedCanary = lib.mkOption { + type = lib.types.str; + default = "example.com"; + description = '' + Hostname outside `allowedDomains` that the self-check CONNECTs to on + port 443, as its unrestricted user, and expects squid to let through. + Squid resolves it with this host's resolver and connects to it, so a + host with no route to it reports `ok=false`. + ''; + }; + selfcheck.canary = lib.mkOption { type = lib.types.str; default = hyperhiveCfg.swarm.forge.domain; @@ -230,11 +271,43 @@ in }; # The directory is setgid so the file hive-c0re renames into it is the - # squid group's. The file exists before squid starts, empty: the helper - # exits on a missing file and squid then rejects every login. + # squid group's. The users file exists before squid starts, empty: the + # helper exits on a missing file and squid then rejects every login. The + # unrestricted list is created the same way, though squid treats a missing + # one as empty. systemd.tmpfiles.rules = [ "d ${usersDir} 2750 hive-core squid - -" "f ${cfg.usersFile} 0640 hive-core squid - -" + "f ${cfg.unrestrictedFile} 0640 hive-core squid - -" + ]; + + # Squid reads the unrestricted list when it parses its configuration, not + # when it changes. hive-c0re writes the list only when its contents change, + # so each change is one reload. Nothing happens to a squid that is not + # running; it reads the list when it starts. + systemd.paths.squid-unrestricted-reload = { + description = "Watch squid's unrestricted-users list"; + wantedBy = [ "paths.target" ]; + pathConfig.PathChanged = cfg.unrestrictedFile; + }; + + systemd.services.squid-unrestricted-reload = { + description = "Reload squid to read its unrestricted-users list"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${config.systemd.package}/bin/systemctl try-reload-or-restart squid.service"; + }; + }; + + assertions = [ + { + assertion = !(builtins.elem cfg.selfcheck.unrestrictedCanary allowed); + message = '' + services.hyperhive.squid.selfcheck.unrestrictedCanary must be a name outside + allowedDomains, or the self-check cannot tell an unrestricted user from a + restricted one. + ''; + } ]; systemd.services.squid = { diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2cf4e19ea..2b03ba675 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -485,38 +485,80 @@ let && !(lib.hasInfix "localnet" conf); } { - # First match wins, so a destination deny after the allow is dead, and - # a credential check after it never challenges anyone. The two IP-literal - # denies stay ahead of the unlisted-name deny: `dstdomain` on an IP + # First match wins, so a destination deny after an allow is dead, and a + # credential check after one never challenges anyone. `allow unrestricted` + # is an allow: above the three destination denies an unrestricted user + # reaches loopback, link-local and the bridge through the proxy. The two + # IP-literal denies stay ahead of every allow: `dstdomain` on an IP # literal does a reverse lookup. - name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, then a missing credential, before any allow"; + name = "squid refuses loopback, link-local and bridge destinations and IP literals, then a missing credential, before any allow"; ok = let lines = lib.splitString "\n" hiveOtel.services.squid.configText; at = l: lib.lists.findFirstIndex (x: x == l) null lines; allow = at "http_access allow allowed_dst"; + unrestricted = at "http_access allow unrestricted"; authed = at "http_access deny !authed"; + firstAllow = if unrestricted < allow then unrestricted else allow; in allow != null + && unrestricted != null && authed != null && at "http_access deny to_localhost" != null - && at "http_access deny to_localhost" < allow && at "http_access deny to_linklocal" != null - && at "http_access deny to_linklocal" < allow && at "http_access deny to_bridge" != null - && at "http_access deny to_bridge" < allow + && at "http_access deny ip_v4_host" != null + && at "http_access deny ip_v6_host" != null + && lib.all (l: at l < firstAllow) [ + "http_access deny to_localhost" + "http_access deny to_linklocal" + "http_access deny to_bridge" + "http_access deny ip_v4_host" + "http_access deny ip_v6_host" + ] && at "http_access deny to_localhost" < authed && at "http_access deny to_linklocal" < authed && at "http_access deny to_bridge" < authed - && authed < allow - && at "http_access deny ip_v4_host" != null - && at "http_access deny ip_v4_host" < at "http_access deny !allowed_dst" - && at "http_access deny ip_v6_host" < at "http_access deny !allowed_dst" - && at "http_access deny !allowed_dst" != null - && at "http_access deny !allowed_dst" < at "http_access deny to_localhost" + && authed < firstAllow && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } + { + # An unrestricted user is let through before the unlisted-name deny and + # everyone else is held to the listed names after it. Control: both are + # there, so the ordering below is not a lookup of nothing. + name = "squid lets an unrestricted user through before the allow-list rules that bind everyone else"; + ok = + let + conf = hiveOtel.services.squid.configText; + lines = lib.splitString "\n" conf; + at = l: lib.lists.findFirstIndex (x: x == l) null lines; + list = hiveOtel.services.hyperhive.squid.unrestrictedFile; + in + at "http_access allow unrestricted" != null + && at "http_access deny !allowed_dst" != null + && at "http_access allow unrestricted" < at "http_access deny !allowed_dst" + && at "http_access deny !allowed_dst" < at "http_access allow allowed_dst" + && builtins.elem ''acl unrestricted proxy_auth "${list}"'' lines + && list == "/var/lib/hive-squid/unrestricted"; + } + { + # The list is read when squid parses its configuration, so a change to + # the file reloads squid. Control: the watched path is the list's own. + name = "squid is reloaded when its unrestricted-users list changes"; + ok = + let + list = hiveOtel.services.hyperhive.squid.unrestrictedFile; + rules = hiveOtel.systemd.tmpfiles.rules; + path = hiveOtel.systemd.paths.squid-unrestricted-reload; + svc = hiveOtel.systemd.services.squid-unrestricted-reload.serviceConfig; + in + path.pathConfig.PathChanged == list + && builtins.elem "paths.target" path.wantedBy + && lib.hasSuffix "systemctl try-reload-or-restart squid.service" svc.ExecStart + && builtins.elem "f ${list} 0640 hive-core squid - -" rules + && hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_UNRESTRICTED_FILE == list; + } { # The helper is the NCSA one, on the file hive-c0re renders. The file # exists, empty, before squid starts (the helper exits on a missing one), diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 801baa474..d08a86bc5 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -112,13 +112,18 @@ pkgs.testers.runNixOSTest { ]; # Names the proxy resolves, each a different destination class. networking.hosts = { - "192.168.2.2" = [ "forge.t.local" ]; + # `other.t.local` resolves, is reachable and is on no allow-list. + "192.168.2.2" = [ + "forge.t.local" + "other.t.local" + ]; "127.0.0.1" = [ "loop.t.local" ]; ${bridgeIp} = [ "bridge.t.local" ]; "169.254.169.254" = [ "meta.t.local" ]; }; # Renders the users file the way hive-c0re does. environment.systemPackages = [ pkgs.mkpasswd ]; + services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local"; services.hyperhive.squid.allowedDomains = [ "forge.t.local" "loop.t.local" @@ -157,14 +162,24 @@ pkgs.testers.runNixOSTest { testScript = '' import re import secrets + import time # Drawn fresh each run. The users file holds only their hashes, as - # hive-c0re renders it. - PW = {u: secrets.token_urlsafe(24) for u in ("atlas", "selfcheck.squid", "ruth")} + # hive-c0re renders it. `atlas` is a restricted agent and `uma` an + # unrestricted one; the two self-check users share one secret. + PW = { + u: secrets.token_urlsafe(24) + for u in ("atlas", "uma", "selfcheck.squid", "ruth") + } + PW["selfcheck-unrestricted.squid"] = PW["selfcheck.squid"] ATLAS = "atlas:" + PW["atlas"] + UMA = "uma:" + PW["uma"] SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"] + SELFCHECK_U = "selfcheck-unrestricted.squid:" + PW["selfcheck.squid"] RUTH = "ruth:" + PW["ruth"] USERS = "/var/lib/hive-squid/users" + LIST = "/var/lib/hive-squid/unrestricted" + UNRESTRICTED = ["uma", "selfcheck-unrestricted.squid"] start_all() proxy.wait_for_unit("squid.service") @@ -194,17 +209,42 @@ pkgs.testers.runNixOSTest { f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1" ) - with subtest("squid starts on the empty users file and refuses every login"): + def write_list(*names): + # What hive-c0re does for the list. Squid reads it on a reload, which + # squid-unrestricted-reload does when the file changes. + body = "printf '%s\\n' " + " ".join(names) if names else ":" + proxy.succeed( + f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new " + f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}" + ) + + def wait_status(host, cred, want): + # The reload that follows a list write is asynchronous. + got = None + for _ in range(30): + got = connect(host, cred) + if got == want: + return + time.sleep(1) + raise AssertionError(f"{host} as {cred.split(':')[0]}: wanted {want}, last got {got}") + + with subtest("squid starts on the empty users file and list and refuses every login"): assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640" + assert proxy.succeed(f"stat -c '%U:%G %a' {LIST}").strip() == "hive-core:squid 640" assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750" assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0" + assert proxy.succeed(f"stat -c %s {LIST}").strip() == "0" assert connect("forge.t.local", ATLAS) == "407", "no such user" - assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name needs no login" + assert connect("other.t.local", UMA) == "407", "no such user, unlisted name" with subtest("the users file holds hashes, never a password"): - write_users(ATLAS, SELFCHECK) + write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U) users = proxy.succeed(f"cat {USERS}") - assert re.fullmatch(r"(atlas|selfcheck\.squid):\$6\$\S+\n(atlas|selfcheck\.squid):\$6\$\S+\n", users), users + rows = [l.split(":", 1) for l in users.splitlines()] + assert sorted(r[0] for r in rows) == sorted( + ["atlas", "uma", "selfcheck.squid", "selfcheck-unrestricted.squid"] + ), users + assert all(re.fullmatch(r"\$6\$\S+", r[1]) for r in rows), users assert not any(p in users for p in PW.values()), users with subtest("an allowed name is tunnelled with a credential and challenged without one"): @@ -215,22 +255,31 @@ pkgs.testers.runNixOSTest { # DES would accept the first eight characters; the hash must not. assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password" - with subtest("an unlisted name is refused with or without a credential"): - assert connect("deny-canary.invalid", ATLAS) == "403", "deny canary" - assert connect("deny-canary.invalid") == "403", "deny canary, anonymous" + with subtest("with an empty list every user is restricted"): + assert connect("other.t.local", UMA) == "403", "uma, not listed yet" + assert connect("other.t.local", ATLAS) == "403" + assert connect("forge.t.local", UMA) == "200", "control: uma still reaches a listed name" - with subtest("a name resolving to a host-local destination is refused"): - assert connect("loop.t.local", ATLAS) == "403", "loopback" - assert connect("bridge.t.local", ATLAS) == "403", "bridge address" - assert connect("meta.t.local", ATLAS) == "403", "link-local" + with subtest("a listed user reaches an unlisted name and the others are refused"): + write_list(*UNRESTRICTED) + wait_status("other.t.local", UMA, "200") + assert connect("other.t.local", ATLAS) == "403", "restricted" + assert connect("forge.t.local", ATLAS) == "200", "restricted, listed name" + assert connect("forge.t.local", UMA) == "200", "unrestricted, listed name" + assert connect("other.t.local") == "407", "no credential" + assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password" + assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, a name that does not resolve" - with subtest("IP literals are refused even when the address is reachable"): - assert connect("127.0.0.1", ATLAS) == "403" - assert connect("192.168.2.2", ATLAS) == "403" - - with subtest("only a port-443 CONNECT is allowed"): - assert connect("forge.t.local:22", ATLAS) == "403" - assert connect("forge.t.local:80", ATLAS) == "403" + with subtest("neither kind of user reaches a host-local destination or another port"): + for cred in (ATLAS, UMA): + who = cred.split(":")[0] + assert connect("loop.t.local", cred) == "403", f"{who}: loopback" + assert connect("bridge.t.local", cred) == "403", f"{who}: bridge address" + assert connect("meta.t.local", cred) == "403", f"{who}: link-local" + assert connect("127.0.0.1", cred) == "403", f"{who}: loopback literal" + assert connect("192.168.2.2", cred) == "403", f"{who}: IP literal" + assert connect("other.t.local:22", cred) == "403", f"{who}: port 22" + assert connect("other.t.local:80", cred) == "403", f"{who}: CONNECT to port 80" with subtest("an agent cannot read cachemgr"): agent.succeed( @@ -247,32 +296,50 @@ pkgs.testers.runNixOSTest { "journalctl -u squid-selfcheck.service -o cat | " "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local " "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " - "anon_status=407 credential=present'" + "anon_status=407 credential=present " + "unrestricted_canary=other.t.local unrestricted_status=200'" ) + with subtest("the self-check fails, and says why, when the unrestricted user is not listed"): + write_list() + wait_status("other.t.local", UMA, "403") + proxy.fail("systemctl start squid-selfcheck.service") + proxy.succeed( + "journalctl -u squid-selfcheck.service -o cat | " + "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " + "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " + "anon_status=407 credential=present " + "unrestricted_canary=other.t.local unrestricted_status=403'" + ) + write_list(*UNRESTRICTED) + wait_status("other.t.local", UMA, "200") + with subtest("the self-check fails, and says why, without a credential or with the wrong one"): proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept") proxy.fail("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " - "allow_status=407 deny_canary=deny-canary.invalid deny_status=403 " - "anon_status=407 credential=absent'" + "allow_status=407 deny_canary=deny-canary.invalid deny_status=407 " + "anon_status=407 credential=absent " + "unrestricted_canary=other.t.local unrestricted_status=407'" ) proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck") proxy.fail("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " - "grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=403 " - "anon_status=407 credential=present'" + "grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=407 " + "anon_status=407 credential=present " + "unrestricted_canary=other.t.local unrestricted_status=407'" ) proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck") with subtest("squid's access log reaches the journal under squid.service, with the user"): journal = proxy.succeed("journalctl -u squid.service -o cat") assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT other\.t\.local:443 uma ", journal), journal assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal - assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443 ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal), journal assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal # Exporter scrapes are cachemgr requests and stay out of the log. assert "squid-internal-mgr" not in journal, journal @@ -287,15 +354,36 @@ pkgs.testers.runNixOSTest { assert m and float(m.group(1)) >= 1, metrics proxy.fail("curl -sf -m 5 http://${bridgeIp}:9301/metrics") - with subtest("squid survives a reload"): + with subtest("squid survives a reload and keeps the list"): proxy.succeed("systemctl reload squid.service") proxy.wait_for_unit("squid.service") assert connect("forge.t.local", ATLAS) == "200" + assert connect("other.t.local", UMA) == "200" + assert connect("other.t.local", ATLAS) == "403" - with subtest("a user added while squid runs can log in, with no reload"): - write_users(ATLAS, SELFCHECK, RUTH) + with subtest("a user added while squid runs can log in, with no reload, and is restricted"): + write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U, RUTH) assert connect("forge.t.local", RUTH) == "200" assert connect("forge.t.local", ATLAS) == "200" + assert connect("other.t.local", RUTH) == "403" + + with subtest("a user the list gains is unrestricted after the reload and loses it again"): + write_list(*UNRESTRICTED, "ruth") + wait_status("other.t.local", RUTH, "200") + assert connect("loop.t.local", RUTH) == "403", "still no loopback" + write_list(*UNRESTRICTED) + wait_status("other.t.local", RUTH, "403") + assert connect("other.t.local", UMA) == "200", "control: uma is still listed" + + with subtest("squid starts on a missing list, treats every user as restricted, and recovers when it appears"): + proxy.succeed(f"rm {LIST}") + proxy.succeed("systemctl restart squid.service") + proxy.wait_for_unit("squid.service") + proxy.wait_for_open_port(3128, "${bridgeIp}") + assert connect("forge.t.local", UMA) == "200", "logins work" + assert connect("other.t.local", UMA) == "403", "no list, no unrestricted user" + write_list(*UNRESTRICTED) + wait_status("other.t.local", UMA, "200") with subtest("squid starts on a missing users file, refuses logins, and recovers when it appears"): proxy.succeed(f"rm {USERS}") @@ -303,8 +391,9 @@ pkgs.testers.runNixOSTest { proxy.wait_for_unit("squid.service") proxy.wait_for_open_port(3128, "${bridgeIp}") assert connect("forge.t.local", ATLAS) == "407", "no file, no login" - assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name still answers" - write_users(ATLAS, SELFCHECK) + assert connect("other.t.local", UMA) == "407", "no file, no login, even when listed" + write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U) assert connect("forge.t.local", ATLAS) == "200", "the file appeared" + assert connect("other.t.local", UMA) == "200", "listed, and now logged in" ''; } From 3d9561a9a6257d5ad77800bc56d64521e130fa9c Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 14:23:21 +0100 Subject: [PATCH 3/5] deny private and reserved destinations for every squid user --- docs/networking/network.md | 28 +++++++--- nix/host-modules/hive-squid.nix | 46 +++++++++++---- nix/module-eval/hive-otel.nix | 31 ++++++++++- nix/vm-tests/hive-squid.nix | 99 +++++++++++++++++++++++++++------ 4 files changed, 167 insertions(+), 37 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index 816155da4..75a723f36 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -238,13 +238,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. (`/var/lib/hive-squid/unrestricted`, `hive-core:squid`, `0640`): the names of the users whose agent holds the `unrestricted_internet` flag in the permissions the hive last read from the swarm, one per line, rewritten when - it changes. An agent with no record, or a record that never set the flag, - holds it, the same default as the - `hyperhive.agent.unrestricted_internet` metric. Squid reads the list when it + it changes. The default is permissive: an agent with no record, or a record + that never set the flag, holds it, the same default as the + `hyperhive.agent.unrestricted_internet` metric. While the hive's read of the + permissions is stale or failing, a new agent that already holds a + credential counts as unrestricted. Squid reads the list when it parses its configuration, so `squid-unrestricted-reload.path` reloads squid when the file changes. A missing or empty file lists no one, and squid starts either way. A flag change reaches squid after the hive's next read of the permissions and its next 30-second pass. + An older `hive-c0re` writes no list, so every credential on that hive is + restricted. Agents must not start sending credentials until every hive runs + the `hive-c0re` that writes it, or a legacy agent loses general internet. - **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each @@ -253,11 +258,18 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. - **Unlisted names.** For a user who isn't on the unrestricted list, refused after the destination checks, which look the name up on the host. - **Denied before any allow.** Destinations that resolve to loopback, - link-local or the bridge subnet, and IP-literal hosts, for every user - including the unrestricted ones. Squid resolves names on the host, so - without these a name pointing at `127.0.0.1` would walk around the - bridge→loopback DROP rule. Only `CONNECT` to port 443 and - plain HTTP to ports 80 and 443 pass. + link-local, the bridge subnet, a private range (`10.0.0.0/8`, + `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`), shared address space + (`100.64.0.0/10`), `0.0.0.0/8`, multicast or reserved space, and IP-literal + hosts, for every user including the unrestricted ones and for listed names. + Squid resolves names on the host and connects from it, so without these a + name pointing at `127.0.0.1`, at the host's own LAN address or at a private + neighbour would walk around the bridge→loopback DROP rule and the swarm's + network. Squid checks the address it connects to, so a name that changes its + answer between a check and the connect gains nothing. A listed name must + therefore resolve to a public address on the host's resolver: a split-horizon + name that answers a LAN address gets a 403, and so does the self-check + canary. Only `CONNECT` to port 443 and plain HTTP to ports 80 and 443 pass. - **Metrics.** `squid-exporter` serves `prometheus-squid-exporter` on `127.0.0.1:9301`. The hive collector scrapes it as job `squid` through `services.hyperhive.otel.scrapeTargets`. `squid_up` and diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index a0c38dc08..fa53d1bb3 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -31,11 +31,30 @@ let selfcheckUnrestrictedUser = "selfcheck-unrestricted.squid"; usersDir = builtins.dirOf cfg.usersFile; - # Evaluated top to bottom, first match wins. The three destination denies - # sit before every allow, `allow unrestricted` included: squid resolves - # names on the host, so a name that answers 127.0.0.1, 169.254.169.254 or a - # bridge address would otherwise reach services the bridge firewall keeps - # agents out of, and an unrestricted agent is still an agent. The IP-literal + # Destinations that are not the public internet: loopback, link-local, the + # bridge, and the private, shared (CGNAT), unspecified, multicast and + # reserved ranges. Squid checks the address it connects to, so a name an + # agent controls cannot point past them. + privateRanges = [ + "10.0.0.0/8" + "172.16.0.0/12" + "192.168.0.0/16" + "100.64.0.0/10" + "0.0.0.0/8" + "224.0.0.0/4" + "240.0.0.0/4" + "fc00::/7" + "ff00::/8" + ]; + + # Evaluated top to bottom, first match wins. The destination denies sit + # before every allow, `allow unrestricted` included: squid resolves names on + # the host and connects from it, so a name that answers 127.0.0.1, + # 169.254.169.254, the host's own LAN address or a private neighbour would + # otherwise reach services the bridge firewall keeps agents out of, and an + # unrestricted agent is still an agent. The same denies bind a listed name, + # so a listed name must resolve to a public address on the host's resolver. + # The IP-literal # denies come first because `dstdomain` on an IP literal does a reverse # lookup. The credential check sits right before the first allow, so only a # request that would otherwise be let through gets the 407 challenge. An @@ -75,6 +94,7 @@ let acl selfcheck src ${netCfg.bridgeIp}/32 acl to_bridge dst ${netCfg.networkCidr} + acl to_private dst ${lib.concatStringsSep " " privateRanges} # `dstdomain` on an IP-literal host does a reverse lookup, so a PTR record # an agent controls could match an allowed name. acl ip_v4_host url_regex ^([a-z]+://)?[0-9.]+(:[0-9]+)?(/|$) @@ -95,6 +115,7 @@ let http_access deny to_localhost http_access deny to_linklocal http_access deny to_bridge + http_access deny to_private http_access deny !authed http_access allow unrestricted ${lib.optionalString (allowed != [ ]) "http_access deny !allowed_dst"} @@ -181,8 +202,12 @@ in ]; description = '' Hostnames squid lets a client reach, by CONNECT or plain HTTP. Each - entry matches that exact name. Everything else gets a 403. The - self-check canary is allowed on top of this list. + entry matches that exact name. Everything else gets a 403, except + for an unrestricted user. The self-check canary is allowed on top of + this list. A name must resolve, on this host's resolver, to a public + address: one in a loopback, link-local, private, shared (100.64.0.0/10), + multicast or reserved range, or in the bridge subnet, gets a 403 for + every user. ''; }; @@ -246,9 +271,10 @@ in description = '' Hostname the self-check CONNECTs to on port 443 and expects squid to let through. Squid resolves it with this host's resolver, not the - hive's: a host whose resolver answers a loopback or bridge address - for it (a single-host swarm with the gateway's `localHostsEntry`) gets - a 403, and the self-check reports `ok=false`. + hive's: a host whose resolver answers a loopback, bridge or other + private address for it (a single-host swarm with the gateway's + `localHostsEntry`, or a split-horizon LAN) gets a 403, and the + self-check reports `ok=false`. ''; }; }; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2b03ba675..0e94ae039 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -491,7 +491,7 @@ let # reaches loopback, link-local and the bridge through the proxy. The two # IP-literal denies stay ahead of every allow: `dstdomain` on an IP # literal does a reverse lookup. - name = "squid refuses loopback, link-local and bridge destinations and IP literals, then a missing credential, before any allow"; + name = "squid refuses loopback, link-local, bridge and private destinations and IP literals, then a missing credential, before any allow"; ok = let lines = lib.splitString "\n" hiveOtel.services.squid.configText; @@ -507,22 +507,51 @@ let && at "http_access deny to_localhost" != null && at "http_access deny to_linklocal" != null && at "http_access deny to_bridge" != null + && at "http_access deny to_private" != null && at "http_access deny ip_v4_host" != null && at "http_access deny ip_v6_host" != null && lib.all (l: at l < firstAllow) [ "http_access deny to_localhost" "http_access deny to_linklocal" "http_access deny to_bridge" + "http_access deny to_private" "http_access deny ip_v4_host" "http_access deny ip_v6_host" ] && at "http_access deny to_localhost" < authed && at "http_access deny to_linklocal" < authed && at "http_access deny to_bridge" < authed + && at "http_access deny to_private" < authed && authed < firstAllow && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } + { + # The ranges are what keeps an unrestricted user on the public internet. + # Control: the bridge deny and the ACL line are found, so a range missing + # below is not a lookup of nothing. + name = "squid's private-destination ACL names every private, shared, multicast and reserved range"; + ok = + let + lines = lib.splitString "\n" hiveOtel.services.squid.configText; + acl = lib.findFirst (l: lib.hasPrefix "acl to_private dst " l) null lines; + ranges = lib.splitString " " (lib.removePrefix "acl to_private dst " acl); + in + acl != null + && builtins.elem "acl to_bridge dst ${hiveOtel.services.hyperhive.network.networkCidr}" lines + && + lib.sort lib.lessThan ranges == lib.sort lib.lessThan [ + "10.0.0.0/8" + "172.16.0.0/12" + "192.168.0.0/16" + "100.64.0.0/10" + "0.0.0.0/8" + "224.0.0.0/4" + "240.0.0.0/4" + "fc00::/7" + "ff00::/8" + ]; + } { # An unrestricted user is let through before the unlisted-name deny and # everyone else is held to the listed names after it. Control: both are diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index d08a86bc5..5e1fcb03c 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -66,6 +66,38 @@ let config.services.hyperhive.network.exposeHostPorts; }; + # The uplink is a public range. The private ranges squid must refuse are + # extra subnets on the same link, so the proxy can reach a listener in each: + # a name that resolves there is refused only by squid's own rules. + privateNets = [ + { + name = "priv10.t.local"; + net = "10.99.0"; + } + { + name = "priv172.t.local"; + net = "172.16.99"; + } + { + name = "priv192.t.local"; + net = "192.168.99"; + } + { + name = "cgnat.t.local"; + net = "100.64.99"; + } + ]; + privateAddrs = + host: + map (p: { + address = "${p.net}.${toString host}"; + prefixLength = 24; + }) privateNets; + ulaAddr = host: { + address = "fd00:99::${toString host}"; + prefixLength = 64; + }; + # Plain TCP on 443: a CONNECT is answered before any TLS, which is all the # test reads. listen443 = addrs: { @@ -104,23 +136,34 @@ pkgs.testers.runNixOSTest { prefixLength = 24; } ]; - networking.interfaces.eth2.ipv4.addresses = lib.mkForce [ - { - address = "192.168.2.1"; - prefixLength = 24; - } - ]; + networking.interfaces.eth2.ipv4.addresses = lib.mkForce ( + [ + { + address = "203.0.113.1"; + prefixLength = 24; + } + ] + ++ privateAddrs 1 + ); + networking.interfaces.eth2.ipv6.addresses = [ (ulaAddr 1) ]; # Names the proxy resolves, each a different destination class. networking.hosts = { # `other.t.local` resolves, is reachable and is on no allow-list. - "192.168.2.2" = [ + "203.0.113.2" = [ "forge.t.local" "other.t.local" ]; "127.0.0.1" = [ "loop.t.local" ]; ${bridgeIp} = [ "bridge.t.local" ]; "169.254.169.254" = [ "meta.t.local" ]; - }; + "fd00:99::2" = [ "ula.t.local" ]; + } + // lib.listToAttrs ( + map (p: { + name = "${p.net}.2"; + value = [ p.name ]; + }) privateNets + ); # Renders the users file the way hive-c0re does. environment.systemPackages = [ pkgs.mkpasswd ]; services.hyperhive.squid.selfcheck.unrestrictedCanary = "other.t.local"; @@ -129,20 +172,31 @@ pkgs.testers.runNixOSTest { "loop.t.local" "bridge.t.local" "meta.t.local" - ]; + "ula.t.local" + ] + ++ map (p: p.name) privateNets; }; web = { lib, ... }: { - imports = [ (listen443 [ "0.0.0.0" ]) ]; - virtualisation.vlans = [ 2 ]; - networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ - { - address = "192.168.2.2"; - prefixLength = 24; - } + imports = [ + (listen443 [ + "0.0.0.0" + "[::]" + ]) ]; + virtualisation.vlans = [ 2 ]; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce ( + [ + { + address = "203.0.113.2"; + prefixLength = 24; + } + ] + ++ privateAddrs 2 + ); + networking.interfaces.eth1.ipv6.addresses = [ (ulaAddr 2) ]; networking.firewall.allowedTCPPorts = [ 443 ]; }; @@ -188,6 +242,8 @@ pkgs.testers.runNixOSTest { web.wait_for_unit("nginx.service") proxy.wait_for_open_port(3128, "${bridgeIp}") + PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local"] + def connect(host, cred=None): # The status squid answered the CONNECT with; 000 when none came back. auth = f"-U '{cred}' " if cred else "" @@ -228,6 +284,11 @@ pkgs.testers.runNixOSTest { time.sleep(1) raise AssertionError(f"{host} as {cred.split(':')[0]}: wanted {want}, last got {got}") + with subtest("the private listeners answer the proxy itself, so a refusal below is squid's"): + for addr in ["10.99.0.2", "172.16.99.2", "192.168.99.2", "100.64.99.2", "fd00:99::2"]: + proxy.wait_until_succeeds(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'") + assert connect("forge.t.local", None) == "407", "control: the public name is challenged, not refused" + with subtest("squid starts on the empty users file and list and refuses every login"): assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640" assert proxy.succeed(f"stat -c '%U:%G %a' {LIST}").strip() == "hive-core:squid 640" @@ -270,14 +331,16 @@ pkgs.testers.runNixOSTest { assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password" assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, a name that does not resolve" - with subtest("neither kind of user reaches a host-local destination or another port"): + with subtest("neither kind of user reaches a host-local or private destination, or another port"): for cred in (ATLAS, UMA): who = cred.split(":")[0] assert connect("loop.t.local", cred) == "403", f"{who}: loopback" assert connect("bridge.t.local", cred) == "403", f"{who}: bridge address" assert connect("meta.t.local", cred) == "403", f"{who}: link-local" assert connect("127.0.0.1", cred) == "403", f"{who}: loopback literal" - assert connect("192.168.2.2", cred) == "403", f"{who}: IP literal" + assert connect("203.0.113.2", cred) == "403", f"{who}: IP literal" + for name in PRIVATE: + assert connect(name, cred) == "403", f"{who}: {name}, a listed name that resolves to a private address" assert connect("other.t.local:22", cred) == "403", f"{who}: port 22" assert connect("other.t.local:80", cred) == "403", f"{who}: CONNECT to port 80" From e8868d52a8b42ad1997db7b10e8755c08babef5d Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 14:27:43 +0100 Subject: [PATCH 4/5] deny the IPv6 unspecified address at squid --- docs/networking/network.md | 2 +- nix/host-modules/hive-squid.nix | 7 ++++++- nix/module-eval/hive-otel.nix | 1 + nix/vm-tests/hive-squid.nix | 7 +++++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index 75a723f36..74ff49a98 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -260,7 +260,7 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. - **Denied before any allow.** Destinations that resolve to loopback, link-local, the bridge subnet, a private range (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`), shared address space - (`100.64.0.0/10`), `0.0.0.0/8`, multicast or reserved space, and IP-literal + (`100.64.0.0/10`), `0.0.0.0/8`, `::`, multicast or reserved space, and IP-literal hosts, for every user including the unrestricted ones and for listed names. Squid resolves names on the host and connects from it, so without these a name pointing at `127.0.0.1`, at the host's own LAN address or at a private diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index fa53d1bb3..7f1e107ed 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -35,6 +35,10 @@ let # bridge, and the private, shared (CGNAT), unspecified, multicast and # reserved ranges. Squid checks the address it connects to, so a name an # agent controls cannot point past them. + # `::` is listed because a connect to it lands on loopback; squid's + # built-in `to_localhost` has `0.0.0.0/32` and `::1` but not `::`. NAT64 + # (`64:ff9b::/96`) and 6to4 (`2002::/16`) embed IPv4 addresses and are left + # out: they matter only on a host with such a route. privateRanges = [ "10.0.0.0/8" "172.16.0.0/12" @@ -45,6 +49,7 @@ let "240.0.0.0/4" "fc00::/7" "ff00::/8" + "::/128" ]; # Evaluated top to bottom, first match wins. The destination denies sit @@ -206,7 +211,7 @@ in for an unrestricted user. The self-check canary is allowed on top of this list. A name must resolve, on this host's resolver, to a public address: one in a loopback, link-local, private, shared (100.64.0.0/10), - multicast or reserved range, or in the bridge subnet, gets a 403 for + multicast or reserved range (or is `::`), or in the bridge subnet, gets a 403 for every user. ''; }; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 0e94ae039..942945944 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -550,6 +550,7 @@ let "240.0.0.0/4" "fc00::/7" "ff00::/8" + "::/128" ]; } { diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 5e1fcb03c..36f0283e9 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -120,9 +120,10 @@ pkgs.testers.runNixOSTest { imports = [ hiveStubs ../host-modules/hive-squid.nix - # Listeners a leaked destination would reach: loopback and the bridge. + # Listeners a leaked destination would reach: loopback (`::` lands on `::1`) and the bridge. (listen443 [ "127.0.0.1" + "[::1]" bridgeIp ]) ]; @@ -157,6 +158,7 @@ pkgs.testers.runNixOSTest { ${bridgeIp} = [ "bridge.t.local" ]; "169.254.169.254" = [ "meta.t.local" ]; "fd00:99::2" = [ "ula.t.local" ]; + "::" = [ "any6.t.local" ]; } // lib.listToAttrs ( map (p: { @@ -173,6 +175,7 @@ pkgs.testers.runNixOSTest { "bridge.t.local" "meta.t.local" "ula.t.local" + "any6.t.local" ] ++ map (p: p.name) privateNets; }; @@ -242,7 +245,7 @@ pkgs.testers.runNixOSTest { web.wait_for_unit("nginx.service") proxy.wait_for_open_port(3128, "${bridgeIp}") - PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local"] + PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local", "any6.t.local"] def connect(host, cred=None): # The status squid answered the CONNECT with; 000 when none came back. From 86e9f49fe684ebabf54d3eb46dda04de84434688 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 14:56:10 +0100 Subject: [PATCH 5/5] resolve the self-check's deny canary in the squid VM test --- nix/vm-tests/hive-squid.nix | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 36f0283e9..735bf3d02 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -149,10 +149,14 @@ pkgs.testers.runNixOSTest { networking.interfaces.eth2.ipv6.addresses = [ (ulaAddr 1) ]; # Names the proxy resolves, each a different destination class. networking.hosts = { - # `other.t.local` resolves, is reachable and is on no allow-list. + # `other.t.local` and the self-check's deny canary resolve, are + # reachable and are on no allow-list. The canary is in the hosts file + # because the destination denies look every name up first, and the VM + # has no resolver to answer for an unknown one before the client gives up. "203.0.113.2" = [ "forge.t.local" "other.t.local" + "deny-canary.invalid" ]; "127.0.0.1" = [ "loop.t.local" ]; ${bridgeIp} = [ "bridge.t.local" ]; @@ -332,7 +336,7 @@ pkgs.testers.runNixOSTest { assert connect("forge.t.local", UMA) == "200", "unrestricted, listed name" assert connect("other.t.local") == "407", "no credential" assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password" - assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, a name that does not resolve" + assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, an unlisted name" with subtest("neither kind of user reaches a host-local or private destination, or another port"): for cred in (ATLAS, UMA):