From ec06a91debc6687562d123a31761421a03d44190 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 13:02:25 +0100 Subject: [PATCH 1/5] mint per-agent proxy credentials and a per-hive squid self-check credential --- nix/host-modules/swarm-bao.nix | 4 + nix/module-eval/bao-grants.nix | 7 +- swarm-controller/src/main.rs | 1 + swarm-controller/src/proxy_credential.rs | 320 +++++++++++++++++++++++ swarm-controller/src/startup.rs | 3 +- swarm-secret-client/src/lib.rs | 3 +- swarm-secret-client/src/proxy.rs | 153 +++++++++++ 7 files changed, 487 insertions(+), 4 deletions(-) create mode 100644 swarm-controller/src/proxy_credential.rs create mode 100644 swarm-secret-client/src/proxy.rs diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index ecd6b3959..4f66b668f 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -486,6 +486,10 @@ let capabilities = ["create", "read", "update"] } + path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" { + capabilities = ["create", "read"] + } + ${lib.concatMapStrings readStanza ( map (leaf: "${credentialMountPath}/data/${leaf}") atticFactLeaves )} diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 2f7951d64..022a51c1d 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -1724,17 +1724,20 @@ let { # `matrix_account::hive_sender` writes every hive's sender token and # `forge::hive_token` every hive's forge token, and nothing else under - # `hives/`, where each hive's queue credential also lives. `+` is one + # `hives/`, where each hive's queue credential also lives. `proxy_credential` + # creates each hive's squid self-check credential and reads it back, and + # has no `update`, so it cannot replace one. `+` is one # segment, which keeps each stanza to its one leaf; a `*` is a glob only # at the end of a path. Pinned as whole stanzas, so an added capability # fails. - name = "the controller writes each hive's sender and forge tokens and nothing else under hives"; + name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, and nothing else under hives"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/+/matrix/sender-token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s && lib.hasInfix "path \"secret/data/swarm/hives/+/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s + && lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s && !(lib.hasInfix "secret/data/swarm/hives/*" s) && !(lib.hasInfix "secret/metadata/swarm/hives" s); } diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 9d882782d..f90980dd5 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -60,6 +60,7 @@ mod meta_flake; mod notify_flows; mod notify_router_api; mod otel_http_client; +mod proxy_credential; mod pulls; mod queue_identity; mod read_policy; diff --git a/swarm-controller/src/proxy_credential.rs b/swarm-controller/src/proxy_credential.rs new file mode 100644 index 000000000..3b242ef16 --- /dev/null +++ b/swarm-controller/src/proxy_credential.rs @@ -0,0 +1,320 @@ +//! The credentials squid authenticates its clients with: one per agent, at +//! `swarm_secret_client::proxy::agent_credential_path`, and one per hive for +//! that hive's squid self-check, at `proxy::selfcheck_path`. +//! +//! Minted here because the agent's store policy is read-only and a hive's +//! policy cannot create under another principal's prefix. [`spawn`]'s pass runs +//! at start and every [`RECONCILE_INTERVAL`] over every agent that holds a store +//! identity, so an agent that existed before this pass was added gets one on the +//! next tick, with no restart. +//! +//! ⚠️ **Never replaced.** A path that holds anything is left alone, and the write +//! is a `cas=0` create. Replacing a credential is a rotation: squid's users file +//! is rendered from whatever the store holds, and a rotation has to reach every +//! client that carries the old one in the same step. + +use anyhow::{Context, Result}; +use serde::de::IgnoredAny; +use swarm_secret_client::{Error, SecretStore, client::DEFAULT_CERT_MOUNT, policy, proxy}; + +/// How often [`spawn`] re-checks every agent and hive. +const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5); + +/// What [`ensure`] did at one path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// The path already held a credential. + Present, + /// This pass wrote one. + Minted, + /// Another writer's create landed between this pass's read and its own. + LostRace, +} + +/// The two store calls [`ensure`] makes, so a test can stand in for the store. +pub(crate) trait CredentialStore { + /// Whether anything is stored at `path`, as [`SecretStore::read_optional`] + /// answers it. The value is decoded as [`IgnoredAny`] and never held. + fn exists(&self, path: &str) -> impl Future> + Send; + + /// As [`SecretStore::create`]. + fn create( + &self, + path: &str, + value: &proxy::Credential, + ) -> impl Future> + Send; +} + +impl CredentialStore for SecretStore { + async fn exists(&self, path: &str) -> Result { + Ok(self.read_optional::(path).await?.is_some()) + } + + async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> { + SecretStore::create(self, path, value).await + } +} + +/// Make sure `path` holds a credential, writing one only if it holds nothing. +/// +/// A create that fails is read back: if the path now holds something, a +/// concurrent writer won and that is the credential. +/// +/// # Errors +/// When the store refuses the read, or the create fails and the path is still +/// empty. +pub(crate) async fn ensure(store: &impl CredentialStore, path: &str) -> Result { + if store + .exists(path) + .await + .with_context(|| format!("reading {path}"))? + { + return Ok(Outcome::Present); + } + let credential = proxy::Credential { + value: crate::agent_identity::generate_queue_secret()?, + }; + let Err(e) = store.create(path, &credential).await else { + return Ok(Outcome::Minted); + }; + if store + .exists(path) + .await + .with_context(|| format!("reading {path} back after a failed create"))? + { + return Ok(Outcome::LostRace); + } + Err(anyhow::Error::new(e).context(format!("creating {path}"))) +} + +/// [`ensure`] at each of `paths`, which `what` names in the log. Returns how +/// many it minted. A failure at one path is logged and the rest still run. +async fn ensure_all( + store: &impl CredentialStore, + what: &str, + paths: Vec<(String, Result)>, +) -> usize { + let mut minted = 0; + for (name, path) in paths { + let path = match path { + Ok(path) => path, + Err(e) => { + tracing::warn!(name, error = %e, "{what}: unusable name"); + continue; + } + }; + match ensure(store, &path).await { + Ok(Outcome::Present) => {} + Ok(Outcome::Minted) => { + minted += 1; + tracing::info!(name, %path, "{what} minted"); + } + Ok(Outcome::LostRace) => { + tracing::info!(name, %path, "{what}: another writer created it first"); + } + Err(e) => tracing::warn!(name, error = %format!("{e:#}"), "{what}: mint failed"), + } + } + minted +} + +/// One pass: a credential for every agent in `agents` and a self-check +/// credential for every hive in `hives`. Returns how many were minted. +async fn mint_missing(store: &impl CredentialStore, agents: &[String], hives: &[String]) -> usize { + let for_agents = agents + .iter() + .map(|a| (a.clone(), proxy::agent_credential_path(a))) + .collect(); + let for_hives = hives + .iter() + .map(|h| (h.clone(), proxy::selfcheck_path(h))) + .collect(); + ensure_all(store, "agent proxy credential", for_agents).await + + ensure_all(store, "hive squid self-check credential", for_hives).await +} + +/// The agents that hold a store identity: exactly the ones whose hive reads +/// what a pass writes. +async fn roster(store: &SecretStore) -> Result> { + let roles = store + .list_cert_roles(DEFAULT_CERT_MOUNT) + .await + .context("listing the store's cert-auth roles")?; + Ok(policy::agents_from_role_names(&roles)) +} + +/// Run a pass now and every [`RECONCILE_INTERVAL`] after. A pass that fails is +/// logged and retried on the next tick; it never stops the daemon. +pub fn spawn(hives: Vec) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); + loop { + ticker.tick().await; + let pass = async { + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + let agents = roster(&store).await?; + anyhow::Ok((agents.len(), mint_missing(&store, &agents, &hives).await)) + }; + match pass.await { + Ok((agents, minted)) => { + tracing::debug!(agents, minted, "proxy credentials: pass done"); + } + Err(e) => tracing::warn!( + error = %format!("{e:#}"), + retry_in_s = RECONCILE_INTERVAL.as_secs(), + "proxy credentials: pass failed; retrying next tick" + ), + } + } + }); +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Mutex; + + use super::*; + + /// An in-memory store that refuses a create over a held path the way a + /// `cas=0` write does, and counts the creates it was asked for. + #[derive(Default)] + struct Fake { + held: Mutex>, + creates: Mutex, + /// A path whose create fails and leaves nothing behind. + broken: Option, + /// Makes the next create find the path taken by someone else. + race: Option, + } + + impl Fake { + fn holding(path: &str, value: &str) -> Self { + let fake = Self::default(); + fake.held + .lock() + .unwrap() + .insert(path.to_owned(), value.to_owned()); + fake + } + + fn value(&self, path: &str) -> Option { + self.held.lock().unwrap().get(path).cloned() + } + } + + fn refused() -> Error { + Error::MissingEnv("BAO_ADDR") + } + + impl CredentialStore for Fake { + async fn exists(&self, path: &str) -> Result { + Ok(self.held.lock().unwrap().contains_key(path)) + } + + async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> { + *self.creates.lock().unwrap() += 1; + let mut held = self.held.lock().unwrap(); + if self.broken.as_deref() == Some(path) { + return Err(refused()); + } + if let Some(winner) = &self.race { + held.insert(path.to_owned(), winner.clone()); + return Err(refused()); + } + if held.contains_key(path) { + return Err(refused()); + } + held.insert(path.to_owned(), value.value.clone()); + Ok(()) + } + } + + const ATLAS: &str = "swarm/agents/atlas/proxy-credential"; + const ARGUS: &str = "swarm/agents/argus/proxy-credential"; + const H1: &str = "swarm/hives/h1/squid-selfcheck"; + + fn names(names: &[&str]) -> Vec { + names.iter().map(|n| (*n).to_owned()).collect() + } + + #[tokio::test] + async fn an_absent_credential_is_created() { + let store = Fake::default(); + assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Minted); + let value = store.value(ATLAS).expect("written"); + // 32 bytes, base64url without padding. + assert_eq!(value.len(), 43, "{value}"); + assert!( + value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + ); + } + + #[tokio::test] + async fn a_present_credential_is_untouched_and_never_updated() { + let store = Fake::holding(ATLAS, "existing"); + assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Present); + assert_eq!(store.value(ATLAS).as_deref(), Some("existing")); + assert_eq!(*store.creates.lock().unwrap(), 0, "no write was attempted"); + } + + #[tokio::test] + async fn a_lost_race_keeps_the_winners_value() { + let store = Fake { + race: Some("winner".to_owned()), + ..Fake::default() + }; + assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::LostRace); + assert_eq!(store.value(ATLAS).as_deref(), Some("winner")); + } + + #[tokio::test] + async fn a_failed_create_with_nothing_behind_it_is_an_error() { + let store = Fake { + broken: Some(ATLAS.to_owned()), + ..Fake::default() + }; + assert!(ensure(&store, ATLAS).await.is_err()); + assert_eq!(store.value(ATLAS), None); + } + + #[tokio::test] + async fn a_pass_mints_for_every_agent_and_hive_that_lacks_one() { + let store = Fake::holding(ARGUS, "existing"); + let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await; + assert_eq!(minted, 2); + assert!(store.value(ATLAS).is_some()); + assert!(store.value(H1).is_some()); + assert_eq!(store.value(ARGUS).as_deref(), Some("existing")); + // Control: the second pass has nothing left to mint, and writes nothing. + let creates = *store.creates.lock().unwrap(); + assert_eq!( + mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await, + 0 + ); + assert_eq!(*store.creates.lock().unwrap(), creates); + } + + #[tokio::test] + async fn one_agent_failing_does_not_stop_the_rest() { + let store = Fake { + broken: Some(ATLAS.to_owned()), + ..Fake::default() + }; + let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await; + assert_eq!(minted, 2); + assert!(store.value(ARGUS).is_some()); + assert!(store.value(H1).is_some()); + } + + #[tokio::test] + async fn an_unusable_agent_name_mints_nothing() { + let store = Fake::default(); + assert_eq!(mint_missing(&store, &names(&["../argus"]), &[]).await, 0); + assert!(store.held.lock().unwrap().is_empty()); + } +} diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index b30325acd..815319482 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -368,7 +368,7 @@ fn spawn_subagent_streams(status: Option<&Arc>, hives: &[H /// The per-agent background passes: subagent and terminal streams, and /// schedule keys, and the permissions bucket and its import, and the /// archetypes agents are created from. Beside them, the -/// copy of the cache facts to each hive. +/// copy of the cache facts to each hive, and the proxy credentials squid checks. /// Lifted out of `main` for `clippy::too_many_lines`. pub(crate) fn spawn_per_agent_upkeep( status: Option<&Arc>, @@ -378,6 +378,7 @@ pub(crate) fn spawn_per_agent_upkeep( spawn_schedule_reconcile(status, hives); spawn_agent_permissions(status, hives); crate::attic_copy::spawn(hives.iter().map(|h| h.name.clone()).collect()); + crate::proxy_credential::spawn(hives.iter().map(|h| h.name.clone()).collect()); if let Some(status) = status { archetypes::spawn(status.queue_client()); } diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index a21ea5b0d..502f2b847 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -7,7 +7,7 @@ //! of secret, the path it lives at together with the fields it holds //! ([`matrix`], [`queue`], [`mtls`], [`forge`], [`github`], [`grafana`], //! [`acp`], [`attic`], [`backend_env`], [`ci`], [`review_classifier`], [`builder`], [`ci_metrics`], -//! [`claude_login`]). +//! [`claude_login`], [`proxy`]). //! Each of those is a thing the controller and a hive must say identically, so //! it is said once here. //! @@ -47,6 +47,7 @@ pub mod matrix; pub mod mtls; pub mod path; pub mod policy; +pub mod proxy; pub mod queue; pub mod review_classifier; pub mod startup_read; diff --git a/swarm-secret-client/src/proxy.rs b/swarm-secret-client/src/proxy.rs new file mode 100644 index 000000000..319c6f342 --- /dev/null +++ b/swarm-secret-client/src/proxy.rs @@ -0,0 +1,153 @@ +//! The egress-proxy agreement: the credentials squid authenticates its clients +//! with, where they live in the store, and what the objects at those paths hold. +//! +//! Each agent has one ([`agent_credential_path`]), minted by `swarm-controller` +//! and read by the agent's hive, which turns it into the hash squid's users file +//! holds. Each hive has one more ([`selfcheck_path`]) for its own squid +//! self-check, so the self-check can prove an authenticated request is let +//! through. Neither end is senior, so both halves are stated once, here, beside +//! [`crate::forge`]. + +use serde::{Deserialize, Serialize}; + +use crate::{ + Error, + path::{Kind, principal_prefix}, +}; + +/// The name of an agent's leaf. +pub const AGENT_LEAF: &str = "proxy-credential"; + +/// The name of a hive's self-check leaf. +pub const SELFCHECK_LEAF: &str = "squid-selfcheck"; + +/// The squid user the self-check logs in as. +/// +/// An agent's squid user is its name, and agent names cannot hold a `.` +/// ([`crate::path::checked_segment`]), so this one cannot be an agent's. +/// `nix/host-modules/hive-squid.nix` spells the same string. +pub const SELFCHECK_USER: &str = "selfcheck.squid"; + +/// The path holding the secret `agent` authenticates to the egress proxy with. +/// +/// A flat leaf under the agent's prefix, like +/// [`crate::forge::agent_token_path`], so the agent's own read stanza +/// ([`crate::policy::render_shared_agent`]) already covers it, and so does the +/// hive's. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`, +/// which is what keeps one agent's name from addressing another agent's secret. +pub fn agent_credential_path(agent: &str) -> Result { + let prefix = principal_prefix(Kind::Agent, agent)?; + Ok(format!("{prefix}/{AGENT_LEAF}")) +} + +/// The path holding the secret `hive`'s own squid self-check authenticates with. +/// +/// Under [`Kind::Hive`], so the grant that reaches it is the hive's own +/// `swarm/hives//*` stanza: hive `alpha` reads its own and gets a 403 +/// on `beta`'s. +/// +/// # Errors +/// [`Error::PathSegment`] when `hive` contains anything but `[A-Za-z0-9_-]`. +pub fn selfcheck_path(hive: &str) -> Result { + let prefix = principal_prefix(Kind::Hive, hive)?; + Ok(format!("{prefix}/{SELFCHECK_LEAF}")) +} + +/// What [`agent_credential_path`] and [`selfcheck_path`] hold. +/// +/// No `Debug` derive: `value` is a live credential, and a derived `Debug` is +/// one `{:?}` away from a log line. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Credential { + /// The secret: 32 random bytes, base64url without padding, so it can sit in + /// a proxy URL's userinfo unescaped. + pub value: String, +} + +impl std::fmt::Debug for Credential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credential") + .field("value", &"") + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn covered_by(policy: &str, path: &str) -> bool { + policy.lines().any(|line| { + line.strip_prefix("path \"") + .and_then(|rest| rest.split_once("\" {")) + .and_then(|(p, _)| p.strip_suffix('*')) + .is_some_and(|prefix| format!("secret/data/{path}").starts_with(prefix)) + }) + } + + #[test] + fn the_paths_are_the_leaves_the_nix_side_spells() { + assert_eq!( + agent_credential_path("atlas").expect("legal"), + "swarm/agents/atlas/proxy-credential" + ); + assert_eq!( + selfcheck_path("h1").expect("legal"), + "swarm/hives/h1/squid-selfcheck" + ); + } + + #[test] + fn a_traversal_in_either_name_is_refused() { + for bad in ["../argus", "a/b", "a.b", ""] { + assert!(agent_credential_path(bad).is_err(), "{bad:?}"); + assert!(selfcheck_path(bad).is_err(), "{bad:?}"); + } + assert!(agent_credential_path("a-b_C9").is_ok()); + assert!(selfcheck_path("a-b_C9").is_ok()); + } + + #[test] + fn the_agent_reads_its_own_credential_and_a_hive_reads_every_one() { + let path = agent_credential_path("atlas").expect("legal"); + assert!(covered_by(&crate::policy::effective_agent("atlas"), &path)); + assert!(!covered_by(&crate::policy::effective_agent("argus"), &path)); + assert!(covered_by(&crate::policy::effective_hive("h1"), &path)); + } + + #[test] + fn a_hive_reads_its_own_selfcheck_credential_and_no_other_hives() { + let path = selfcheck_path("h1").expect("legal"); + assert!(covered_by(&crate::policy::effective_hive("h1"), &path)); + assert!(!covered_by(&crate::policy::effective_hive("h2"), &path)); + assert!(!covered_by(&crate::policy::effective_agent("atlas"), &path)); + } + + #[test] + fn the_selfcheck_user_is_not_a_legal_agent_name() { + assert!(crate::path::checked_segment("agent", SELFCHECK_USER).is_err()); + } + + #[test] + fn debug_hides_the_value() { + let shown = format!( + "{:?}", + Credential { + value: "s3cret".to_owned() + } + ); + assert!(!shown.contains("s3cret"), "{shown}"); + } + + #[test] + fn the_value_field_is_named_value() { + let json = serde_json::to_string(&Credential { + value: "t".to_owned(), + }) + .expect("a String serialises"); + assert_eq!(json, r#"{"value":"t"}"#); + } +} From 52d532ff5acddd5c8bc1f3d6a0daf38d6b47489c Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 13:02:25 +0100 Subject: [PATCH 2/5] render squid's users file from this hive's proxy credentials --- Cargo.lock | 30 ++ Cargo.toml | 3 + docs/swarm/credentials-reference.md | 3 + hive-c0re/Cargo.toml | 2 + hive-c0re/src/main.rs | 2 + hive-c0re/src/secret_file.rs | 41 +- hive-c0re/src/workers/mod.rs | 3 +- hive-c0re/src/workers/squid_users.rs | 440 +++++++++++++++++++++ nix/host-modules/hive-c0re/default.nix | 6 +- nix/host-modules/hive-c0re/environment.nix | 3 + nix/host-modules/hive-squid.nix | 20 + nix/module-eval/hive-otel.nix | 15 + 12 files changed, 562 insertions(+), 6 deletions(-) create mode 100644 hive-c0re/src/workers/squid_users.rs diff --git a/Cargo.lock b/Cargo.lock index e7d202fed..d2363cf51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1777,6 +1777,7 @@ dependencies = [ "ed25519-dalek", "forgejo-api", "futures-util", + "getrandom 0.3.4", "hive-core-agent-sock", "hive-host-sock", "hive-jobq", @@ -1800,6 +1801,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha-crypt", "sha2 0.11.0", "strum", "swarm-matrix-client", @@ -3082,6 +3084,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "mcf" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2044bdbade272ded4a1529324f4f30ca7b0c5b496541f2a9095bbd1c9d03cb06" +dependencies = [ + "base64ct", +] + [[package]] name = "memchr" version = "2.8.3" @@ -3357,6 +3368,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "password-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" + [[package]] name = "paste" version = "1.0.15" @@ -4567,6 +4584,19 @@ dependencies = [ "serde", ] +[[package]] +name = "sha-crypt" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32ee31e0190f9dc366d26b032ac36a99328532f40a237a79385a20d2d7f29810" +dependencies = [ + "base64ct", + "ctutils", + "mcf", + "password-hash", + "sha2 0.11.0", +] + [[package]] name = "sha1" version = "0.10.7" diff --git a/Cargo.toml b/Cargo.toml index 5ef450f61..3157d61b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -248,6 +248,9 @@ matrix-sdk = { version = "0.18", default-features = false, features = [ futures-util = "0.3" hmac = "0.13" sha2 = "0.11" +# SHA-512 crypt (`$6$`) for the users file hive-c0re renders; squid's `basic_ncsa_auth` +# hands a `$` hash to the system `crypt()`. +sha-crypt = "0.6" # Verifies the cache signature over a narinfo (hive-c0re). Already in the lock # through nkeys. ed25519-dalek = "2" diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index c516f69f8..e87bbcdb8 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -36,6 +36,8 @@ the rest of the cell says how. | `swarm/agents//claude-credentials` | `swarm-claude-login`, when you log the agent in from its card; one version per login | the agent, while it waits for a login | ❌ log in again | ✅ every 30 seconds while waiting | | `swarm/hives//matrix/sender-token` | `swarm-controller`, every 5 minutes | `swarm-controller` and the hive | ✅ when missing or rejected by the homeserver | ✅ the hive re-reads on every sweep | | `swarm/hives//forge/token` | `swarm-controller`, every 5 minutes | `swarm-controller` and the hive | ✅ when missing or no longer matching the forge | ✅ read on every forge call | +| `swarm/agents//proxy-credential` | `swarm-controller`: 32 random bytes, at start and every 5 minutes for every agent that holds a store identity | the hive (`hive-c0re`); the agent can read it, nothing sends it yet | ❌ never replaced: written only while the path holds nothing | ✅ `hive-c0re` re-reads it every 30 seconds | +| `swarm/hives//squid-selfcheck` | `swarm-controller`, in the same pass as the agents' credentials | the hive: `hive-c0re` and `squid-selfcheck-credential` (root) | ❌ never replaced: written only while the path holds nothing | ✅ `hive-c0re` every 30 seconds, the root unit every 10 minutes | | `swarm/hives//oidc/client` | authelia, first boot; `swarm-secret-publish` copies it in | the hive | ❌ minted once; `swarm-secret-publish` republishes a re-mint | ✅ re-read when authelia refuses it | | `swarm/services//oidc/client` | authelia | that service | must be stated | must be stated | | `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge, at start and every 5 minutes | `swarm-bao-ci-runner` on the runner's host | ❌ the forge keeps one token until an admin resets it; the controller then writes the new one within 5 minutes | ✅ every 10 minutes; restarts the runner on a change | @@ -154,6 +156,7 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. | the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | | the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | | this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity, and copies it (account on the first line, token on the second) to `/run/hyperhive/forge-hive-token` (tmpfs, `0600` `hive-core`) for nix's `git`, which `hive-priv` also runs. Each `internal/meta` read and each boot rewrites the copy when it differs; where they differ after a rotation, a build that starts before the next read fails to authenticate, and a retry reads the new token | +| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and no agent sends a credential and squid doesn't read the file yet, so no traffic changes | ### Store identities, one per principal diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index b1cb8c29e..fa1713dfb 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -19,6 +19,8 @@ axum.workspace = true backon.workspace = true chrono.workspace = true base64.workspace = true +getrandom.workspace = true +sha-crypt.workspace = true reqwest.workspace = true forgejo-api.workspace = true url.workspace = true diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 11b0b6152..13f037ae8 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -363,6 +363,8 @@ async fn cmd_serve( // not a part of the above; see swarm_agent_status's module doc for why // the two buckets stay separate. swarm_agent_status::spawn(coord.shutdown_rx()); + // Squid's users file: the hashes of this hive's agents' proxy credentials. + workers::squid_users::spawn(coord.shutdown_rx()); // Per-agent events.sqlite + bash-tasks file cleanup now runs // agent-side in the harness (`hive_agent::vacuum`): the files are // agent-owned, so host-side deletes hit PermissionDenied / readonly-db diff --git a/hive-c0re/src/secret_file.rs b/hive-c0re/src/secret_file.rs index f13f51e0c..6f3f8d316 100644 --- a/hive-c0re/src/secret_file.rs +++ b/hive-c0re/src/secret_file.rs @@ -1,7 +1,8 @@ -//! [`write_secret_0600`]: the one way hive-c0re puts a credential on disk. +//! [`write_secret_0600`] and [`write_secret_mode`]: the ways hive-c0re puts a +//! credential, or a hash of one, on disk. use std::io::Write as _; -use std::os::unix::fs::OpenOptionsExt as _; +use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; use std::path::Path; use anyhow::{Context as _, Result}; @@ -13,6 +14,12 @@ use anyhow::{Context as _, Result}; /// reader (or a crash) never sees half of it. Errors carry paths, never /// `contents`. pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> { + write_secret_mode(path, contents, 0o600) +} + +/// [`write_secret_0600`] with the file's mode given, for a file another user +/// has to read (the group comes from the directory). +pub fn write_secret_mode(path: &Path, contents: &[u8], mode: u32) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; } @@ -27,9 +34,13 @@ pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> { let mut f = std::fs::OpenOptions::new() .write(true) .create_new(true) - .mode(0o600) + .mode(mode) .open(&tmp) .with_context(|| format!("create {}", tmp.display()))?; + // `open` masks the mode with the process umask; the mode is a property of + // the writer, not of the unit's umask. + f.set_permissions(std::fs::Permissions::from_mode(mode)) + .with_context(|| format!("chmod {}", tmp.display()))?; f.write_all(contents) .and_then(|()| f.sync_all()) .with_context(|| format!("write {}", tmp.display()))?; @@ -38,7 +49,7 @@ pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> { #[cfg(test)] mod tests { - use super::write_secret_0600; + use super::{write_secret_0600, write_secret_mode}; use std::os::unix::fs::PermissionsExt as _; fn mode(path: &std::path::Path) -> u32 { @@ -76,4 +87,26 @@ mod tests { write_secret_0600(&path, b"new\n").expect("writes"); assert_eq!(std::fs::read(&path).expect("read"), b"new\n"); } + + /// A restrictive umask doesn't change the mode asked for. The umask is + /// process-wide, so the write happens in a child run of this test. + #[test] + fn the_mode_survives_a_restrictive_umask() { + const DIR: &str = "SECRET_FILE_UMASK_TEST_DIR"; + if let Some(dir) = std::env::var_os(DIR) { + write_secret_mode(&std::path::Path::new(&dir).join("shared"), b"x\n", 0o640) + .expect("writes"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + let status = std::process::Command::new("sh") + .arg("-c") + .arg(r#"umask 077 && exec "$0" --exact secret_file::tests::the_mode_survives_a_restrictive_umask"#) + .arg(std::env::current_exe().expect("test binary")) + .env(DIR, dir.path()) + .status() + .expect("spawns the child"); + assert!(status.success()); + assert_eq!(mode(&dir.path().join("shared")), 0o640); + } } diff --git a/hive-c0re/src/workers/mod.rs b/hive-c0re/src/workers/mod.rs index 118f923bd..87da7e4eb 100644 --- a/hive-c0re/src/workers/mod.rs +++ b/hive-c0re/src/workers/mod.rs @@ -1,7 +1,7 @@ //! Background tasks and periodic sweeps: crash/login watcher, boot-time //! auto-update reconcile, the agent-sockets.json writer loop, the MCP socket listener //! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the -//! swarm resource-cap and permissions reads. +//! swarm resource-cap and permissions reads, and the squid users file. //! Each submodule is re-exported //! at the crate root, so `crate::crash_watch::…` etc. keep working unchanged. @@ -11,6 +11,7 @@ pub mod crash_watch; pub mod knowledge; pub mod mcp_sockets; pub mod permissions_import; +pub mod squid_users; pub mod swarm_limits; pub mod swarm_permissions; pub mod wanted; diff --git a/hive-c0re/src/workers/squid_users.rs b/hive-c0re/src/workers/squid_users.rs new file mode 100644 index 000000000..44737a8fb --- /dev/null +++ b/hive-c0re/src/workers/squid_users.rs @@ -0,0 +1,440 @@ +//! Squid's users file: the hashes of the proxy credentials this hive's agents +//! authenticate to its egress proxy with. +//! +//! swarm-controller mints one credential per agent and one for this hive's own +//! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this +//! reads the credentials of the agents this hive has, plus the self-check's, +//! and renders `:` lines into the file named by +//! [`ENV_USERS_FILE`], which squid's `basic_ncsa_auth` re-reads when its mtime +//! moves. The agents are the container list, not the running containers: a +//! stopped agent stays in the file. +//! +//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is +//! while it still verifies its credential, so the file changes only when a +//! user is added, removed or given a new credential. +//! +//! A failed read leaves the file as it is: dropping a user on a store outage +//! would read exactly like the swarm revoking it. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt}; +use swarm_secret_client::{SecretStore, client, proxy}; + +use crate::stats::sweep_health::{self, SweepHealth}; + +/// How often the file is re-rendered. +pub const INTERVAL: Duration = Duration::from_secs(30); + +/// At [`INTERVAL`] this is ~90 seconds of failing passes. +const FAILURES_BEFORE_WARNING: u32 = 3; + +/// The environment variable naming the file. Set by `nix/host-modules/hive-c0re` +/// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid. +pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE"; + +/// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()` +/// accepts. +const SALT_BYTES: usize = 12; + +/// The file's mode. The group is squid's, from the setgid directory +/// `nix/host-modules/hive-squid.nix` creates. +const FILE_MODE: u32 = 0o640; + +/// One squid user and the credential it logs in with. No `Debug`: `secret` is +/// live. +struct User { + name: String, + secret: String, +} + +/// Start the pass loop, when this host has a squid users file, a store and a +/// hive name. +pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { + let Some(path) = std::env::var_os(ENV_USERS_FILE) + .filter(|v| !v.is_empty()) + .map(PathBuf::from) + else { + return; + }; + let Some(hive) = crate::container_view::hive_swarm_names().0 else { + tracing::info!("squid users: HYPERHIVE_HIVE_NAME is unset; no users file is written"); + return; + }; + if std::env::var_os(client::ENV_ADDR).is_none_or(|v| v.is_empty()) { + tracing::info!( + "squid users: no swarm secret store is configured; no users file is written" + ); + return; + } + tokio::spawn(async move { + let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING); + loop { + match pass(&hive, &path).await { + Ok(()) => health.record_ok(), + Err(e) => { + tracing::warn!(error = ?e, "squid users: the pass failed"); + let err = format!("{e:#}"); + health.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "rendering squid's users file is failing ({} consecutive, {age}) \ + — squid keeps the users last written: {err}", + ctx.consecutive + ) + }); + } + } + tokio::select! { + () = tokio::time::sleep(INTERVAL) => {} + _ = shutdown.changed() => return, + } + } + }); +} + +/// Read this hive's credentials from the store and render them into `file`. +async fn pass(hive: &str, file: &Path) -> Result<()> { + let agents: Vec = crate::lifecycle::agents_for_meta_listing() + .await + .context("enumerating this hive's agents")? + .into_iter() + .map(|spec| spec.name) + .collect(); + let store = SecretStore::from_env(hive) + .await + .context("connecting to the store")?; + sync_with(hive, &agents, file, |path| { + let store = &store; + async move { + let held: Option = store + .read_optional(&path) + .await + .with_context(|| format!("reading {path}"))?; + Ok(held.map(|c| c.value)) + } + }) + .await +} + +/// [`pass`] with the store read and the file given. Nothing is written unless +/// every read succeeds and the rendered file differs from the current one. +async fn sync_with(hive: &str, agents: &[String], file: &Path, read: F) -> Result<()> +where + F: Fn(String) -> Fut, + Fut: Future>>, +{ + let mut wanted = Vec::new(); + for agent in agents { + wanted.push((agent.clone(), proxy::agent_credential_path(agent)?)); + } + wanted.push(( + proxy::SELFCHECK_USER.to_owned(), + proxy::selfcheck_path(hive)?, + )); + let mut users = Vec::new(); + for (name, path) in wanted { + if let Some(secret) = read(path).await? { + users.push(User { name, secret }); + } else { + tracing::debug!(name, "squid users: no proxy credential minted yet"); + } + } + let current = match std::fs::read_to_string(file) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())), + }; + let rendered = render(¤t, &users)?; + if rendered == current { + return Ok(()); + } + crate::secret_file::write_secret_mode(file, rendered.as_bytes(), FILE_MODE) + .with_context(|| format!("writing {}", file.display()))?; + tracing::info!( + users = users.len(), + agents = users.iter().filter(|u| u.name != proxy::SELFCHECK_USER).count(), + file = %file.display(), + "squid users: wrote the users file" + ); + Ok(()) +} + +/// The users file for `users`, sorted by name, one `:` line each. +/// +/// A user whose line in `current` still verifies against its credential keeps +/// that line. Every other line of `current` is dropped, so a destroyed agent, +/// or one that belongs to another hive, does not outlive the pass. +fn render(current: &str, users: &[User]) -> Result { + let mut lines: Vec = Vec::with_capacity(users.len()); + for user in users { + if user.name.contains([':', '\n']) { + bail!("{:?} cannot be a squid user name", user.name); + } + let kept = current + .lines() + .filter_map(|l| l.split_once(':')) + .find(|(name, hash)| *name == user.name && verifies(&user.secret, hash)) + .map(|(_, hash)| hash.to_owned()); + let hash = match kept { + Some(hash) => hash, + None => hash_secret(&user.secret)?, + }; + lines.push(format!("{}:{hash}\n", user.name)); + } + lines.sort(); + Ok(lines.concat()) +} + +/// A fresh salted SHA-512 crypt hash of `secret`: `$6$rounds=5000$$`. +fn hash_secret(secret: &str) -> Result { + let mut salt = [0u8; SALT_BYTES]; + getrandom::fill(&mut salt).context("drawing a salt from the kernel's CSPRNG")?; + let hash = ShaCrypt::SHA512 + .hash_password_with_salt(secret.as_bytes(), &salt) + .map_err(|e| anyhow::anyhow!("hashing a proxy credential: {e}"))?; + Ok(hash.as_str().to_owned()) +} + +/// Whether `hash` is `secret`'s SHA-crypt hash. +fn verifies(secret: &str, hash: &str) -> bool { + ShaCrypt::SHA512 + .verify_password(secret.as_bytes(), hash) + .is_ok() +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::os::unix::fs::PermissionsExt as _; + + use super::*; + + const HIVE: &str = "h1"; + + /// What the store holds, by path. + fn store(entries: &[(&str, &str)]) -> HashMap { + entries + .iter() + .map(|(p, v)| ((*p).to_owned(), (*v).to_owned())) + .collect() + } + + fn names(names: &[&str]) -> Vec { + names.iter().map(|n| (*n).to_owned()).collect() + } + + async fn sync(held: &HashMap, agents: &[&str], file: &Path) -> Result<()> { + sync_with(HIVE, &names(agents), file, |path| { + let found = held.get(&path).cloned(); + async move { Ok(found) } + }) + .await + } + + fn lines(file: &Path) -> Vec<(String, String)> { + std::fs::read_to_string(file) + .expect("the file is there") + .lines() + .map(|l| { + let (name, hash) = l.split_once(':').expect("a name and a hash"); + (name.to_owned(), hash.to_owned()) + }) + .collect() + } + + fn two_agents_and_a_stranger() -> HashMap { + store(&[ + ("swarm/agents/atlas/proxy-credential", "atlas-secret"), + ("swarm/agents/argus/proxy-credential", "argus-secret"), + ("swarm/agents/other-hive-agent/proxy-credential", "stranger"), + ("swarm/hives/h1/squid-selfcheck", "check-secret"), + ("swarm/hives/h2/squid-selfcheck", "other-check"), + ]) + } + + #[tokio::test] + async fn only_this_hives_agents_and_its_selfcheck_are_rendered() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) + .await + .expect("renders"); + let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); + assert_eq!(users, ["argus", "atlas", "selfcheck.squid"]); + } + + #[tokio::test] + async fn each_line_is_a_salted_sha512_crypt_hash_of_its_users_credential() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) + .await + .expect("renders"); + let by_name: HashMap<_, _> = lines(&file).into_iter().collect(); + assert!(by_name["atlas"].starts_with("$6$"), "{}", by_name["atlas"]); + assert!(verifies("atlas-secret", &by_name["atlas"])); + assert!(verifies("argus-secret", &by_name["argus"])); + assert!(verifies("check-secret", &by_name["selfcheck.squid"])); + // Control: a hash is of its own user's credential only. + assert!(!verifies("argus-secret", &by_name["atlas"])); + // Salted: the same credential hashes differently twice. + assert_ne!( + hash_secret("atlas-secret").expect("hashes"), + hash_secret("atlas-secret").expect("hashes") + ); + } + + #[tokio::test] + async fn no_credential_is_written_to_the_file_or_its_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) + .await + .expect("renders"); + // Control: the search finds a name that is in the file. + let mut found = 0; + for entry in std::fs::read_dir(dir.path()).expect("lists") { + let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text"); + assert!(text.contains("atlas:"), "control: {text}"); + for secret in ["atlas-secret", "argus-secret", "check-secret"] { + assert!(!text.contains(secret), "{secret} reached the disk"); + } + found += 1; + } + assert_eq!(found, 1, "only the users file is left behind"); + } + + #[tokio::test] + async fn the_file_is_group_readable_and_replaced_atomically() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas"], &file) + .await + .expect("renders"); + let mode = std::fs::metadata(&file).expect("stat").permissions().mode() & 0o777; + assert_eq!(mode, 0o640); + let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); + sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) + .await + .expect("renders"); + let after = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); + assert_ne!( + inode, after, + "a rename onto the path, not a rewrite in place" + ); + assert!(!dir.path().join("users.tmp").exists()); + } + + #[tokio::test] + async fn an_unchanged_pass_leaves_the_file_alone() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + let held = two_agents_and_a_stranger(); + sync(&held, &["atlas", "argus"], &file) + .await + .expect("renders"); + let first = std::fs::read_to_string(&file).expect("read"); + let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); + sync(&held, &["atlas", "argus"], &file) + .await + .expect("renders"); + assert_eq!(std::fs::read_to_string(&file).expect("read"), first); + let again = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); + assert_eq!(inode, again, "nothing was written"); + } + + #[tokio::test] + async fn a_changed_credential_gets_a_new_hash_and_the_others_keep_theirs() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + let mut held = two_agents_and_a_stranger(); + sync(&held, &["atlas", "argus"], &file) + .await + .expect("renders"); + let before: HashMap<_, _> = lines(&file).into_iter().collect(); + held.insert( + "swarm/agents/atlas/proxy-credential".to_owned(), + "rotated".to_owned(), + ); + sync(&held, &["atlas", "argus"], &file) + .await + .expect("renders"); + let after: HashMap<_, _> = lines(&file).into_iter().collect(); + assert!(verifies("rotated", &after["atlas"])); + assert_ne!(before["atlas"], after["atlas"]); + assert_eq!(before["argus"], after["argus"]); + } + + #[tokio::test] + async fn a_stopped_agent_stays_and_a_destroyed_one_goes() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + let held = two_agents_and_a_stranger(); + // The agent list is the container list, so an agent whose container + // is down is in it: nothing here looks at whether anything runs. + sync(&held, &["atlas", "argus"], &file) + .await + .expect("renders"); + assert!(lines(&file).iter().any(|(n, _)| n == "argus")); + // Control: once the container is gone the user is dropped, and a line + // for an agent this hive never had does not survive a pass either. + let mut text = std::fs::read_to_string(&file).expect("read"); + text.push_str("stray:$6$rounds=5000$saltsaltsaltsalt$x\n"); + std::fs::write(&file, text).expect("write"); + sync(&held, &["atlas"], &file).await.expect("renders"); + let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); + assert_eq!(users, ["atlas", "selfcheck.squid"]); + } + + #[tokio::test] + async fn an_agent_without_a_credential_yet_is_left_out() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + // The empty file squid's unit creates before its helper starts. + std::fs::write(&file, "").expect("write"); + sync(&store(&[]), &["atlas"], &file).await.expect("renders"); + assert_eq!(std::fs::read_to_string(&file).expect("read"), ""); + } + + #[tokio::test] + async fn a_failed_read_leaves_the_file_as_it_was() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("users"); + sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) + .await + .expect("renders"); + let before = std::fs::read_to_string(&file).expect("read"); + let failed = sync_with( + HIVE, + &names(&["atlas", "argus"]), + &file, + |path| async move { + if path.contains("argus") { + bail!("the store is down") + } + Ok(Some("atlas-secret".to_owned())) + }, + ) + .await; + assert!(failed.is_err()); + assert_eq!(std::fs::read_to_string(&file).expect("read"), before); + } + + #[test] + fn a_name_that_would_break_the_line_is_refused() { + for bad in ["a:b", "a\nb"] { + let user = User { + name: bad.to_owned(), + secret: "s".to_owned(), + }; + assert!(render("", &[user]).is_err(), "{bad:?}"); + } + } +} diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index a1d0f02e6..6a26ae02c 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -347,7 +347,11 @@ in # needs access to the rest of c0re's state — that separation # means c0re itself needs an explicit carve-out to write there. ProtectSystem = "strict"; - ReadWritePaths = [ "/var/lib/hive-gateway/conf" ]; + # `-` so a host without the squid users directory still starts the daemon. + ReadWritePaths = [ + "/var/lib/hive-gateway/conf" + "-${builtins.dirOf config.services.hyperhive.squid.usersFile}" + ]; ProtectKernelTunables = true; # no sysctl writes ProtectKernelLogs = true; # no /dev/kmsg access needed ProtectControlGroups = true; # cgroup writes go through hive-priv, not c0re directly diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 288e52865..e2ef032ff 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -271,6 +271,9 @@ in # See the LoadCredential in ./default.nix. BAO_CLIENT_CERT = "%d/bao-client.pem"; BAO_CLIENT_KEY = "%d/bao-client-key.pem"; + # Where `workers::squid_users` renders the hashes of this hive's agents' proxy + # credentials for squid; it reads them from the store under the identity above. + HIVE_SQUID_USERS_FILE = config.services.hyperhive.squid.usersFile; } // lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) { # Absent means the system trust store — right for a deployment with a real diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 41f2556b5..c1b926453 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -21,6 +21,8 @@ let allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]); + usersDir = builtins.dirOf cfg.usersFile; + # Evaluated top to bottom, first match wins. The three destination denies # sit before the only allow: squid resolves names on the host, so a name # that answers 127.0.0.1, 169.254.169.254 or a bridge address would @@ -136,6 +138,17 @@ in ''; }; + usersFile = lib.mkOption { + type = lib.types.path; + default = "/var/lib/hive-squid/users"; + description = '' + The `:` file hive-c0re renders from the credentials in the + swarm secret store: one line per agent of this hive and one for the + self-check, as SHA-512 crypt hashes. Created empty, owned by `hive-core` + and readable by the `squid` group. Squid doesn't read it yet. + ''; + }; + selfcheck.canary = lib.mkOption { type = lib.types.str; default = hyperhiveCfg.swarm.forge.domain; @@ -167,6 +180,13 @@ in configText = squidConf; }; + # The directory is setgid so the file hive-c0re renames into it is the + # squid group's. + systemd.tmpfiles.rules = [ + "d ${usersDir} 2750 hive-core squid - -" + "f ${cfg.usersFile} 0640 hive-core squid - -" + ]; + systemd.services.squid = { serviceConfig = sandbox // { User = "squid"; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2e4090336..3db591080 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -505,6 +505,21 @@ let && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } + { + # The file hive-c0re renders exists, owned by it and readable by squid's + # group, from boot; hive-c0re gets its path only on a host with a store + # identity. + name = "squid's users file exists for hive-c0re to write, and hive-c0re is told where it is"; + ok = + let + users = hiveOtel.services.hyperhive.squid.usersFile; + rules = hiveOtel.systemd.tmpfiles.rules; + in + users == "/var/lib/hive-squid/users" + && builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules + && builtins.elem "f ${users} 0640 hive-core squid - -" rules + && hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users; + } { name = "squid's port is open on the bridge interface and on no other"; ok = From b7adb90f537852b79b7a46a5e9592b87e6703b2d Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 13:07:15 +0100 Subject: [PATCH 3/5] require a proxy credential at squid and prove it in the self-check --- nix/host-modules/hive-squid.nix | 85 +++++++++++++++++---- nix/module-eval/hive-otel.nix | 35 +++++++-- nix/vm-tests/hive-squid.nix | 130 +++++++++++++++++++++++++++----- 3 files changed, 207 insertions(+), 43 deletions(-) diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index c1b926453..465156629 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -1,8 +1,9 @@ # Egress proxy for this hive's agents: squid on the host, bound to the bridge # IP, default-deny on the destination hostname. It filters CONNECT targets and -# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is -# pointed at it yet; the exporter, journal forwarding and the self-check below -# are how a deploy shows it works. +# plain-HTTP hosts and never decrypts, so it holds no signing key. A request +# for a listed name must carry a per-agent proxy credential, checked against the +# users file hive-c0re renders. No agent sends one yet; the exporter, journal +# forwarding and the self-check below are how a deploy shows it works. { pkgs, lib, @@ -21,6 +22,8 @@ let allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]); + # The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`. + selfcheckUser = "selfcheck.squid"; usersDir = builtins.dirOf cfg.usersFile; # Evaluated top to bottom, first match wins. The three destination denies @@ -29,7 +32,10 @@ let # otherwise reach services the bridge firewall keeps agents out of. The # unlisted-name deny comes first because `dstdomain` needs no lookup while # the `dst` ACLs do: an unlisted name is refused at once instead of waiting - # on the host resolver, and costs a client no DNS query. + # on the host resolver, and costs a client no DNS query. The credential + # check sits right before the allow, so a refused destination answers 403 + # to a client with or without a credential, and only a request that would + # otherwise be let through gets the 407 challenge. squidConf = '' http_port ${netCfg.bridgeIp}:${toString port} pid_filename /run/squid/squid.pid @@ -42,8 +48,17 @@ let # Cachemgr scrapes are left out. access_log syslog:daemon.info squid !manager + # The helper reads the users file on every request and reloads it when its + # mtime moves. It exits when the file is missing, which is why the file is + # created before squid starts. A positive answer is cached for a minute, so + # that is also how long a removed user keeps working. + auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile} + auth_param basic realm hyperhive egress + auth_param basic credentialsttl 1 minute + acl SSL_ports port 443 acl Safe_ports port 80 443 + acl authed proxy_auth REQUIRED # Host-originated requests: the self-check and the exporter. A local # connect to the bridge IP takes the bridge IP as its source. @@ -71,6 +86,7 @@ let http_access deny to_localhost http_access deny to_linklocal http_access deny to_bridge + http_access deny !authed ${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"} http_access deny all ''; @@ -103,20 +119,39 @@ let name = "squid-selfcheck"; runtimeInputs = [ pkgs.curl ]; text = '' - # The status squid answers a CONNECT with; 000 when the proxy is down. + # The credential comes from the unit's imported credential, and goes to + # curl on its stdin so it never appears in a command line. Without one + # the requests are anonymous and the self-check reports it. + secret="" + credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}" + if [ -s "$credential_file" ]; then + secret=$(< "$credential_file") + fi + + # : the status squid answers a CONNECT with; 000 when + # the proxy is down. connect_status() { - curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \ + { + if [ "$2" = auth ] && [ -n "$secret" ]; then + printf 'proxy-user = "%s:%s"\n' ${selfcheckUser} "$secret" + fi + } | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \ -o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true } - allow_status=$(connect_status ${cfg.selfcheck.canary}) - deny_status=$(connect_status ${denyCanary}) + allow_status=$(connect_status ${cfg.selfcheck.canary} auth) + deny_status=$(connect_status ${denyCanary} auth) + anon_status=$(connect_status ${cfg.selfcheck.canary} anon) ok=false - if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then + if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ]; then ok=true fi - echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status" + credential=absent + if [ -n "$secret" ]; then + credential=present + fi + echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential" [ "$ok" = true ] ''; }; @@ -142,10 +177,24 @@ in type = lib.types.path; default = "/var/lib/hive-squid/users"; description = '' - The `:` file hive-c0re renders from the credentials in the - swarm secret store: one line per agent of this hive and one for the - self-check, as SHA-512 crypt hashes. Created empty, owned by `hive-core` - and readable by the `squid` group. Squid doesn't read it yet. + The `:` file squid checks proxy credentials against. hive-c0re + renders it from the credentials in the swarm secret store, one line per + agent of this hive and one for the self-check, as SHA-512 crypt hashes. + An empty file, which is what exists until hive-c0re writes one, makes + every login fail, and every request to a listed name answers 407. + Owned by `hive-core` and readable by the `squid` group. + ''; + }; + + selfcheck.credentialName = lib.mkOption { + type = lib.types.str; + default = "hive-squid.selfcheck"; + readOnly = true; + description = '' + Name of the systemd credential the self-check imports its squid + password from. `squid-selfcheck-credential` writes it to + `/run/credstore/`. Without one the self-check sends no + credential and reports `ok=false`. ''; }; @@ -181,7 +230,8 @@ in }; # The directory is setgid so the file hive-c0re renames into it is the - # squid group's. + # squid group's. The file exists before squid starts, empty: the helper + # exits on a missing file and squid then rejects every login. systemd.tmpfiles.rules = [ "d ${usersDir} 2750 hive-core squid - -" "f ${cfg.usersFile} 0640 hive-core squid - -" @@ -224,10 +274,13 @@ in }; systemd.services.squid-selfcheck = { - description = "Squid allow and deny canaries"; + description = "Squid allow, deny and credential canaries"; after = [ "squid.service" ]; serviceConfig = sandbox // { Type = "oneshot"; + # A glob that matches nothing is not an error: a host with no + # credential yet still runs the self-check, and it reports that. + ImportCredential = cfg.selfcheck.credentialName; ExecStart = lib.getExe selfcheck; DynamicUser = true; IPAddressAllow = netCfg.bridgeIp; diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 3db591080..2cf4e19ea 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -485,39 +485,60 @@ let && !(lib.hasInfix "localnet" conf); } { - # First match wins, so a destination deny after the allow is dead. - name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, before any allow"; + # First match wins, so a destination deny after the allow is dead, and + # a credential check after it never challenges anyone. The two IP-literal + # denies stay ahead of the unlisted-name deny: `dstdomain` on an IP + # literal does a reverse lookup. + name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, then a missing credential, before any allow"; ok = let lines = lib.splitString "\n" hiveOtel.services.squid.configText; at = l: lib.lists.findFirstIndex (x: x == l) null lines; allow = at "http_access allow allowed_dst"; + authed = at "http_access deny !authed"; in allow != null + && authed != null && at "http_access deny to_localhost" != null && at "http_access deny to_localhost" < allow && at "http_access deny to_linklocal" != null && at "http_access deny to_linklocal" < allow && at "http_access deny to_bridge" != null && at "http_access deny to_bridge" < allow + && at "http_access deny to_localhost" < authed + && at "http_access deny to_linklocal" < authed + && at "http_access deny to_bridge" < authed + && authed < allow + && at "http_access deny ip_v4_host" != null + && at "http_access deny ip_v4_host" < at "http_access deny !allowed_dst" + && at "http_access deny ip_v6_host" < at "http_access deny !allowed_dst" && at "http_access deny !allowed_dst" != null && at "http_access deny !allowed_dst" < at "http_access deny to_localhost" && builtins.elem "cache deny all" lines && lib.last (lib.filter (l: l != "") lines) == "http_access deny all"; } { - # The file hive-c0re renders exists, owned by it and readable by squid's - # group, from boot; hive-c0re gets its path only on a host with a store - # identity. - name = "squid's users file exists for hive-c0re to write, and hive-c0re is told where it is"; + # The helper is the NCSA one, on the file hive-c0re renders. The file + # exists, empty, before squid starts (the helper exits on a missing one), + # and the self-check imports its credential without failing when there is + # none. + name = "squid authenticates with the NCSA helper on a users file that exists before it starts"; ok = let + conf = hiveOtel.services.squid.configText; + lines = lib.splitString "\n" conf; users = hiveOtel.services.hyperhive.squid.usersFile; rules = hiveOtel.systemd.tmpfiles.rules; + sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig; in - users == "/var/lib/hive-squid/users" + lib.any ( + l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l + ) lines + && builtins.elem "acl authed proxy_auth REQUIRED" lines + && users == "/var/lib/hive-squid/users" && builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules && builtins.elem "f ${users} 0640 hive-core squid - -" rules + && sc.ImportCredential == hiveOtel.services.hyperhive.squid.selfcheck.credentialName && hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users; } { diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 382fb971e..801baa474 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -55,6 +55,12 @@ let }; }; }; + # The user hive-c0re runs as, which owns the users file squid reads. + config.users.users.hive-core = { + isSystemUser = true; + group = "hive-core"; + }; + config.users.groups.hive-core = { }; # The firewall hole hive-network.nix opens for `exposeHostPorts`. config.networking.firewall.interfaces.eth1.allowedTCPPorts = config.services.hyperhive.network.exposeHostPorts; @@ -77,7 +83,7 @@ pkgs.testers.runNixOSTest { nodes = { proxy = - { lib, ... }: + { lib, pkgs, ... }: { imports = [ hiveStubs @@ -111,6 +117,8 @@ pkgs.testers.runNixOSTest { ${bridgeIp} = [ "bridge.t.local" ]; "169.254.169.254" = [ "meta.t.local" ]; }; + # Renders the users file the way hive-c0re does. + environment.systemPackages = [ pkgs.mkpasswd ]; services.hyperhive.squid.allowedDomains = [ "forge.t.local" "loop.t.local" @@ -148,6 +156,15 @@ pkgs.testers.runNixOSTest { testScript = '' import re + import secrets + + # Drawn fresh each run. The users file holds only their hashes, as + # hive-c0re renders it. + PW = {u: secrets.token_urlsafe(24) for u in ("atlas", "selfcheck.squid", "ruth")} + ATLAS = "atlas:" + PW["atlas"] + SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"] + RUTH = "ruth:" + PW["ruth"] + USERS = "/var/lib/hive-squid/users" start_all() proxy.wait_for_unit("squid.service") @@ -156,29 +173,64 @@ pkgs.testers.runNixOSTest { web.wait_for_unit("nginx.service") proxy.wait_for_open_port(3128, "${bridgeIp}") - def connect(host): + def connect(host, cred=None): # The status squid answered the CONNECT with; 000 when none came back. + auth = f"-U '{cred}' " if cred else "" return agent.succeed( - "curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null " + "curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null " f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true" ).strip() - with subtest("an allowed name is tunnelled and an unlisted one is refused"): - assert connect("forge.t.local") == "200", "allow canary" - assert connect("deny-canary.invalid") == "403", "deny canary" + def write_users(*creds): + # What hive-c0re does: hashes only, written aside and renamed into + # place, in squid's group. The sleep keeps two writes in different + # seconds, which is how the helper tells the file changed. + lines = " ".join( + f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";" + for u, p in (c.split(":", 1) for c in creds) + ) + proxy.succeed( + f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new " + f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1" + ) + + with subtest("squid starts on the empty users file and refuses every login"): + assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640" + assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750" + assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0" + assert connect("forge.t.local", ATLAS) == "407", "no such user" + assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name needs no login" + + with subtest("the users file holds hashes, never a password"): + write_users(ATLAS, SELFCHECK) + users = proxy.succeed(f"cat {USERS}") + assert re.fullmatch(r"(atlas|selfcheck\.squid):\$6\$\S+\n(atlas|selfcheck\.squid):\$6\$\S+\n", users), users + assert not any(p in users for p in PW.values()), users + + with subtest("an allowed name is tunnelled with a credential and challenged without one"): + assert connect("forge.t.local", ATLAS) == "200", "allow canary" + assert connect("forge.t.local") == "407", "no credential" + assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password" + assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user" + # DES would accept the first eight characters; the hash must not. + assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password" + + with subtest("an unlisted name is refused with or without a credential"): + assert connect("deny-canary.invalid", ATLAS) == "403", "deny canary" + assert connect("deny-canary.invalid") == "403", "deny canary, anonymous" with subtest("a name resolving to a host-local destination is refused"): - assert connect("loop.t.local") == "403", "loopback" - assert connect("bridge.t.local") == "403", "bridge address" - assert connect("meta.t.local") == "403", "link-local" + assert connect("loop.t.local", ATLAS) == "403", "loopback" + assert connect("bridge.t.local", ATLAS) == "403", "bridge address" + assert connect("meta.t.local", ATLAS) == "403", "link-local" with subtest("IP literals are refused even when the address is reachable"): - assert connect("127.0.0.1") == "403" - assert connect("192.168.2.2") == "403" + assert connect("127.0.0.1", ATLAS) == "403" + assert connect("192.168.2.2", ATLAS) == "403" with subtest("only a port-443 CONNECT is allowed"): - assert connect("forge.t.local:22") == "403" - assert connect("forge.t.local:80") == "403" + assert connect("forge.t.local:22", ATLAS) == "403" + assert connect("forge.t.local:80", ATLAS) == "403" with subtest("an agent cannot read cachemgr"): agent.succeed( @@ -186,19 +238,42 @@ pkgs.testers.runNixOSTest { "http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403" ) - with subtest("the self-check passes against the same squid"): + with subtest("the self-check passes with its credential"): + proxy.succeed("install -d -m 0700 /run/credstore") + proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck") + proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck") proxy.succeed("systemctl start squid-selfcheck.service") proxy.succeed( "journalctl -u squid-selfcheck.service -o cat | " "grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local " - "allow_status=200 deny_canary=deny-canary.invalid deny_status=403'" + "allow_status=200 deny_canary=deny-canary.invalid deny_status=403 " + "anon_status=407 credential=present'" ) - with subtest("squid's access log reaches the journal under squid.service"): + with subtest("the self-check fails, and says why, without a credential or with the wrong one"): + proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept") + proxy.fail("systemctl start squid-selfcheck.service") + proxy.succeed( + "journalctl -u squid-selfcheck.service -o cat | " + "grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local " + "allow_status=407 deny_canary=deny-canary.invalid deny_status=403 " + "anon_status=407 credential=absent'" + ) + proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck") + proxy.fail("systemctl start squid-selfcheck.service") + proxy.succeed( + "journalctl -u squid-selfcheck.service -o cat | " + "grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=403 " + "anon_status=407 credential=present'" + ) + proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck") + + with subtest("squid's access log reaches the journal under squid.service, with the user"): journal = proxy.succeed("journalctl -u squid.service -o cat") - assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", journal), journal - assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", journal), journal - assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443 ", journal), journal + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal # Exporter scrapes are cachemgr requests and stay out of the log. assert "squid-internal-mgr" not in journal, journal @@ -215,6 +290,21 @@ pkgs.testers.runNixOSTest { with subtest("squid survives a reload"): proxy.succeed("systemctl reload squid.service") proxy.wait_for_unit("squid.service") - assert connect("forge.t.local") == "200" + assert connect("forge.t.local", ATLAS) == "200" + + with subtest("a user added while squid runs can log in, with no reload"): + write_users(ATLAS, SELFCHECK, RUTH) + assert connect("forge.t.local", RUTH) == "200" + assert connect("forge.t.local", ATLAS) == "200" + + with subtest("squid starts on a missing users file, refuses logins, and recovers when it appears"): + proxy.succeed(f"rm {USERS}") + proxy.succeed("systemctl restart squid.service") + proxy.wait_for_unit("squid.service") + proxy.wait_for_open_port(3128, "${bridgeIp}") + assert connect("forge.t.local", ATLAS) == "407", "no file, no login" + assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name still answers" + write_users(ATLAS, SELFCHECK) + assert connect("forge.t.local", ATLAS) == "200", "the file appeared" ''; } From fd63b402e1d51ea84bffd67fcd2e1af413aa0acb Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 13:07:15 +0100 Subject: [PATCH 4/5] fetch the self-check's password as a systemd credential --- nix/host-modules/default.nix | 1 + nix/host-modules/hive-squid-credential.nix | 103 +++++++++++++++++++++ nix/host-modules/swarm-bao.nix | 16 ++-- nix/module-eval/bao-grants.nix | 13 ++- nix/module-eval/hive-tls.nix | 22 +++++ 5 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 nix/host-modules/hive-squid-credential.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index e720cfe83..17039af3f 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -25,6 +25,7 @@ ./hive-priv.nix ./hive-priv-cache.nix ./hive-squid.nix + ./hive-squid-credential.nix ./hive-tls.nix ./otel.nix ./glue-bao-readers-policy-order.nix diff --git a/nix/host-modules/hive-squid-credential.nix b/nix/host-modules/hive-squid-credential.nix new file mode 100644 index 000000000..47be3e2a6 --- /dev/null +++ b/nix/host-modules/hive-squid-credential.nix @@ -0,0 +1,103 @@ +# The password this hive's squid self-check logs in to squid with, kept where +# only root can read it. +# +# swarm-controller mints it at `swarm/hives//squid-selfcheck` +# (`swarm_secret_client::proxy::selfcheck_path`). hive-c0re reads the same path +# to put its hash in squid's users file; this root oneshot reads it under the +# hive's own store identity and lands it as the systemd credential +# `squid-selfcheck` imports (`/run/credstore/`, root, `0400`, tmpfs). +# +# A hive with no minted credential yet leaves the file alone and the self-check +# reports `credential=absent`. +{ + pkgs, + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + hiveController = hyperhiveCfg.deploy.hive-controller; + tls = hiveController.tls; + baoCfg = hyperhiveCfg.swarm.bao; + baoDeploy = hyperhiveCfg.deploy.bao; + + enabled = + hiveController.enable + && hiveController.hiveName != null + && tls.baoHiveIdentityCertFile != null + && tls.baoHiveIdentityKeyFile != null; + + credentialFile = "/run/credstore/${hyperhiveCfg.squid.selfcheck.credentialName}"; + + storeRetry = import ./lib/store-retry.nix { }; + storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; +in +{ + config = lib.mkIf enabled { + systemd.services.squid-selfcheck-credential = { + description = "Read the squid self-check's password from the swarm secret store"; + wantedBy = [ "multi-user.target" ]; + before = [ "squid-selfcheck.service" ]; + wants = [ "network-online.target" ]; + after = [ + "network-online.target" + "systemd-tmpfiles-setup.service" + ]; + path = [ + baoDeploy.package + pkgs.coreutils + pkgs.gnugrep + ]; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile; + BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + # ./lib/store-retry.nix: the login fails for as long as the store is + # sealed or down. + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "squid-selfcheck-credential"; + TimeoutStartSec = "5min"; + }; + script = '' + set -euo pipefail + ${import ./lib/store-seeded-secret.nix { }} + ${import ./lib/atomic-write-secret.nix { }} + ${import ./lib/refresh-consumer.nix { }} + + install -d -m 0700 /run/credstore + store_init + store_login + if [ "$STORE_UP" != 1 ]; then + echo "could not read the squid self-check credential; ${credentialFile} stays as it is." >&2 + exit 1 + fi + if ! value="$(store_bao kv get -field=value ${lib.escapeShellArg "secret/swarm/hives/${hiveController.hiveName}/squid-selfcheck"} 2>"$STORE_ERR")"; then + if grep -qi 'no value found' "$STORE_ERR"; then + echo "the swarm has not minted this hive's squid self-check credential yet; ${credentialFile} stays as it is." + exit 0 + fi + cat "$STORE_ERR" >&2 + exit 1 + fi + if [[ ! "$value" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "the stored squid self-check credential is not one base64url word; ${credentialFile} stays as it is." >&2 + exit 1 + fi + # Written only when the value changed, so the file keeps its mtime. + if secret_differs ${credentialFile} "$value"; then + atomic_write_secret 0400 root:root ${credentialFile} "$value" + fi + ''; + }; + + systemd.timers.squid-selfcheck-credential = storeRefreshTimer "re-read the squid self-check's password"; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 4f66b668f..b7f617e07 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -942,12 +942,13 @@ let inherit hiveName; }) hyperhiveCfg.swarm.hives; - # What a hive's gateway-leaf identity may do: issue from its own role, and - # read two of its own cache facts. The root unit on the hive host - # (`hive-priv-cache-facts`, ./hive-priv-cache.nix) reads them under this - # identity: the key hive-priv's downloads must be signed by, and the token it - # pulls with. The cache's URL and name come from the host's configuration, - # not from the store. + # What a hive's gateway-leaf identity may do: issue from its own role, read + # two of its own cache facts, and read its own squid self-check credential. + # The root unit on the hive host (`hive-priv-cache-facts`, + # ./hive-priv-cache.nix) reads the facts under this identity: the key + # hive-priv's downloads must be signed by, and the token it pulls with. The + # cache's URL and name come from the host's configuration, not from the + # store. `squid-selfcheck-credential` (./hive-squid.nix) reads the credential. hiveLeafPolicyText = obj: '' @@ -961,7 +962,8 @@ let [ "public-key" "pull-token" - ]; + ] + + readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/squid-selfcheck"; # The PKI mount agent client certificates are issued from. Its root is # generated inside the store, so the agent CA's key never exists outside it. diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 022a51c1d..b6e48f1e6 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -1496,9 +1496,10 @@ let { # One role, policy and login role per hive in the directory, each # naming only its own hive: the policy text in the unit is exactly the - # `update` on its own role and `read` on its own two cache facts, and each + # `update` on its own role and `read` on its own two cache facts and its own + # squid self-check credential, and each # hive's block names neither the other's role nor its subject. - name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts, and nothing else"; + name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts and squid self-check credential, and nothing else"; ok = let s = baoGrantTwoHives.systemd.services.swarm-bao-hive-leaf-policy.script; @@ -1536,6 +1537,10 @@ let path = "secret/data/swarm/hives/h1/attic/pull-token"; caps = [ "read" ]; } + { + path = "secret/data/swarm/hives/h1/squid-selfcheck"; + caps = [ "read" ]; + } { path = "pki/issue/swarm-hive-leaf-h2"; caps = [ "update" ]; @@ -1548,6 +1553,10 @@ let path = "secret/data/swarm/hives/h2/attic/pull-token"; caps = [ "read" ]; } + { + path = "secret/data/swarm/hives/h2/squid-selfcheck"; + caps = [ "read" ]; + } ] && issues "h1" "h1.t.local" h1 && issues "h2" "h2.example.org" h2 diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 45f265c7e..c59cbdbe7 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -800,6 +800,28 @@ let noService.systemd.services.hive-priv-cache-facts.environment.BAO_CLIENT_CERT == "/etc/pki/bao-hive.pem"; } + { + # Control: the lookup finds the sibling root unit of the same shape. The + # squid self-check's password is read under this hive's own identity and + # lands where only root reads it, as the credential `squid-selfcheck` + # imports. + name = "the squid self-check credential is read by a root oneshot under this hive's own identity into the credstore"; + ok = + let + u = units.squid-selfcheck-credential; + in + lib.hasInfix "secret/swarm/hives/h1/squid-selfcheck" u.script + && !(lib.hasInfix "swarm/hives/h2" u.script) + && lib.hasInfix "/run/credstore/hive-squid.selfcheck" u.script + && lib.hasInfix "atomic_write_secret 0400 root:root" u.script + && u.environment.BAO_CLIENT_CERT == units.hive-priv-cache-facts.environment.BAO_CLIENT_CERT + && u.environment.BAO_CLIENT_KEY == units.hive-priv-cache-facts.environment.BAO_CLIENT_KEY + && u.serviceConfig.Type == "oneshot" + && !(remainsAfterExit u) + && allLocal.systemd.timers ? squid-selfcheck-credential + && !(noIdentity.systemd.services ? squid-selfcheck-credential) + && !(noIdentity.systemd.timers ? squid-selfcheck-credential); + } { name = "hive-priv is told the cache's address by the host's configuration"; ok = From 8328c38a0738292a395266627c6b9bfa625c3832 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 13:07:15 +0100 Subject: [PATCH 5/5] document squid's proxy credentials --- docs/networking/network.md | 35 +++++++++++++++++++++++------ docs/swarm/ca.md | 6 +++-- docs/swarm/credentials-reference.md | 3 ++- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index e12748dc4..3af915826 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -212,13 +212,27 @@ DNS/gateway), so only expose services safe for any agent to reach. Every hive host runs squid in the host netns, listening on `:3128` (`nix/host-modules/hive-squid.nix`). The port is open on the bridge interface -only. **No agent uses it yet**: agent traffic still goes straight out through -the NAT. The proxy goes first so you can check its metrics and logs before -anything reroutes. +only. **No agent uses it yet**: none sends a credential, and agent traffic +still goes straight out through the NAT. The proxy goes first so you can check +its metrics and logs before anything reroutes. Squid filters on the destination hostname: a `CONNECT` target or a plain-HTTP host. It never decrypts TLS, so it holds no signing key. It caches nothing. +- **Credentials.** A request for a listed name must carry a per-agent proxy + credential (HTTP Basic, the agent's name and its secret). `swarm-controller` + mints one per agent at `swarm/agents//proxy-credential`, and never + replaces it. `hive-c0re` hashes the credentials of this hive's agents, and + one more for the self-check, into `services.hyperhive.squid.usersFile` + every 30 seconds. Squid's `basic_ncsa_auth` helper re-reads the file when it + changes. A stopped agent stays in the file. Squid caches a good login for a + minute. Nothing rotates a credential: to replace one, delete its leaf in the + store and the next pass mints a new one. HTTP Basic crosses the bridge in + clear between an agent and the proxy. See + [the credentials reference](../swarm/credentials-reference.md). + A name that isn't on the allow-list gets the 403 whether or not the request + has a credential; a listed name without one gets a 407. + - **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each entry matches that exact name. Everything else gets a 403. @@ -237,10 +251,17 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. unit and the self-check's to the swarm's log store. A refusal shows as `TCP_DENIED/403` with the client's source address. - **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service` - every minute from the bridge IP: `CONNECT` to `services.hyperhive.squid.selfcheck.canary` - (default: the forge's name) must answer 200, and to `deny-canary.invalid` - must answer 403. It writes one `squid-selfcheck ok=… allow_status=… -deny_status=…` line per run and fails the unit when either is wrong. The + every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT` + to `services.hyperhive.squid.selfcheck.canary` (default: the forge's name) + must answer 200, to `deny-canary.invalid` 403, and to the canary without a + credential 407. It writes one `squid-selfcheck ok=… allow_status=… +deny_status=… anon_status=… credential=present|absent` line per run and + fails the unit when any is wrong. The password is the hive's + `swarm/hives//squid-selfcheck` secret, which + `squid-selfcheck-credential` lands at `/run/credstore/hive-squid.selfcheck`. + Before `swarm-controller` has minted it, or while `hive-c0re` hasn't + written the users file, the line reads `credential=absent` or + `allow_status=407` and `ok=false`, and nothing else breaks. The host's resolver looks up the canary, so a host whose resolver maps that name to a loopback or bridge address (a single-host swarm with `gateway.localHostsEntry`) fails the check. diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 0b55be954..696094f07 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -22,8 +22,10 @@ hive's domain and `*.`, read from `services.hyperhive.swarm.hives..domain` on the store's host, with no IP SANs and no `localhost`. A request for any other name is a refusal from the store naming that name. The hive's policy grants `update` on that one -issue path and `read` on its own `attic/public-key` and `attic/pull-token` -(see [the cache facts for hive-priv](credentials-reference.md)), and nothing +issue path and `read` on its own `attic/public-key`, `attic/pull-token` and +`squid-selfcheck` +(see [the cache facts for hive-priv](credentials-reference.md) and the +squid self-check password), and nothing else. `swarm-bao-hive-leaf-policy.service`, on the store's host, writes the role, the policy and the cert-auth role for every hive in `swarm.hives`. diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index e87bbcdb8..7c75ab53d 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -156,7 +156,8 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. | the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | | the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | | this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity, and copies it (account on the first line, token on the second) to `/run/hyperhive/forge-hive-token` (tmpfs, `0600` `hive-core`) for nix's `git`, which `hive-priv` also runs. Each `internal/meta` read and each boot rewrites the copy when it differs; where they differ after a rotation, a build that starts before the next read fails to authenticate, and a retry reads the new token | -| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and no agent sends a credential and squid doesn't read the file yet, so no traffic changes | +| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; no agent sends a credential, so no traffic changes | +| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | ### Store identities, one per principal