//! Squid's users file, and its list of unrestricted users: the hashes of the //! proxy credentials this hive's agents authenticate to its egress proxy with, //! and which of them may reach any destination. //! //! swarm-controller mints one credential per agent and one for this hive's own //! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this //! reads the credentials of the agents this hive has, plus the self-check's, //! and renders `:` lines into the file named by //! [`ENV_USERS_FILE`], which squid's `basic_ncsa_auth` re-reads when its mtime //! moves. The agents are the container list, not the running containers: a //! stopped agent stays in the file. //! //! The file named by [`ENV_UNRESTRICTED_FILE`] lists, one name per line, the //! users whose agent holds `unrestricted_internet` in the permissions this //! hive last read from the swarm (an agent with no record, or no flag in it, //! holds it), plus the self-check's unrestricted user. Squid reads the list as //! an ACL when it parses its configuration, so `nix/host-modules/hive-squid.nix` //! reloads squid when the file changes. //! //! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is //! while it still verifies its credential, so each file changes only when a //! user is added, removed, given a new credential or has its flag flipped. //! //! A failed read leaves both files as they are: dropping a user on a store //! outage would read exactly like the swarm revoking it. use std::future::Future; use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::{Context, Result, bail}; use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt}; use swarm_secret_client::{SecretStore, client, proxy}; use crate::permissions; use crate::stats::sweep_health::{self, SweepHealth}; /// How often the file is re-rendered. pub const INTERVAL: Duration = Duration::from_secs(30); /// At [`INTERVAL`] this is ~90 seconds of failing passes. const FAILURES_BEFORE_WARNING: u32 = 3; /// The environment variable naming the file. Set by `nix/host-modules/hive-c0re` /// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid. pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE"; /// The environment variable naming the unrestricted-users list. Set by /// `nix/host-modules/hive-c0re` from `services.hyperhive.squid.unrestrictedFile`. pub const ENV_UNRESTRICTED_FILE: &str = "HIVE_SQUID_UNRESTRICTED_FILE"; /// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()` /// accepts. const SALT_BYTES: usize = 12; /// Both files' mode. The group is squid's, from the setgid directory /// `nix/host-modules/hive-squid.nix` creates. const FILE_MODE: u32 = 0o640; /// One squid user and the credential it logs in with. No `Debug`: `secret` is /// live. struct User { name: String, secret: String, /// Whether squid lets this user reach any destination the deny rules leave. unrestricted: bool, } /// The two files a pass writes. struct Files<'a> { users: &'a Path, unrestricted: &'a Path, } /// Whether `name` is one of the self-check's squid users rather than an agent's. fn is_selfcheck(name: &str) -> bool { name == proxy::SELFCHECK_USER || name == proxy::SELFCHECK_UNRESTRICTED_USER } fn env_path(name: &str) -> Option { std::env::var_os(name) .filter(|v| !v.is_empty()) .map(PathBuf::from) } /// Start the pass loop, when this host has squid's two files, a store and a /// hive name. pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { let (Some(users), Some(unrestricted)) = (env_path(ENV_USERS_FILE), env_path(ENV_UNRESTRICTED_FILE)) else { return; }; let Some(hive) = crate::container_view::hive_swarm_names().0 else { tracing::info!("squid users: HYPERHIVE_HIVE_NAME is unset; no users file is written"); return; }; if std::env::var_os(client::ENV_ADDR).is_none_or(|v| v.is_empty()) { tracing::info!( "squid users: no swarm secret store is configured; no users file is written" ); return; } tokio::spawn(async move { let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING); loop { match pass( &hive, &Files { users: &users, unrestricted: &unrestricted, }, ) .await { Ok(()) => health.record_ok(), Err(e) => { tracing::warn!(error = ?e, "squid users: the pass failed"); let err = format!("{e:#}"); health.record_err(|ctx| { let age = ctx.since_last_ok.map_or_else( || "no success this session".to_owned(), |d| format!("last ok {} ago", sweep_health::fmt_age(d)), ); format!( "rendering squid's users file is failing ({} consecutive, {age}) \ — squid keeps the users last written: {err}", ctx.consecutive ) }); } } tokio::select! { () = tokio::time::sleep(INTERVAL) => {} _ = shutdown.changed() => return, } } }); } /// Read this hive's credentials from the store and the swarm's permissions /// from the last read of them, and render both files. async fn pass(hive: &str, files: &Files<'_>) -> Result<()> { let agents: Vec = crate::lifecycle::agents_for_meta_listing() .await .context("enumerating this hive's agents")? .into_iter() .map(|spec| spec.name) .collect(); let perms = permissions::read_at(&permissions::path()) .context("reading the stored swarm permissions")?; let store = SecretStore::from_env(hive) .await .context("connecting to the store")?; sync_with(hive, &agents, &perms, files, |path| { let store = &store; async move { let held: Option = store .read_optional(&path) .await .with_context(|| format!("reading {path}"))?; Ok(held.map(|c| c.value)) } }) .await } /// [`pass`] with the store read, the permissions and the files given. Nothing /// is written unless every read succeeds, and a file is only written when its /// rendering differs from what it holds. /// /// The unrestricted list goes first. A user that turns restricted is then /// restricted before the users file stops listing it, and one that turns /// unrestricted is not listed before it can log in. async fn sync_with( hive: &str, agents: &[String], perms: &permissions::Stored, files: &Files<'_>, read: F, ) -> Result<()> where F: Fn(String) -> Fut, Fut: Future>>, { let mut wanted = Vec::new(); for agent in agents { let unrestricted = permissions::unrestricted_internet(perms.get(agent)); wanted.push(( agent.clone(), proxy::agent_credential_path(agent)?, unrestricted, )); } let selfcheck = proxy::selfcheck_path(hive)?; wanted.push((proxy::SELFCHECK_USER.to_owned(), selfcheck.clone(), false)); wanted.push(( proxy::SELFCHECK_UNRESTRICTED_USER.to_owned(), selfcheck, true, )); let mut users = Vec::new(); for (name, path, unrestricted) in wanted { if let Some(secret) = read(path).await? { users.push(User { name, secret, unrestricted, }); } else { tracing::debug!(name, "squid users: no proxy credential minted yet"); } } let list = render_unrestricted(&users); if write_if_changed(files.unrestricted, &list)? { tracing::info!( file = %files.unrestricted.display(), "squid users: wrote the unrestricted list, {} unrestricted", list.lines().count() ); } let current = read_or_empty(files.users)?; let rendered = render(¤t, &users)?; if write_if_changed(files.users, &rendered)? { tracing::info!( file = %files.users.display(), "squid users: wrote the users file, {} users for {} agents", users.len(), users.iter().filter(|u| !is_selfcheck(&u.name)).count() ); } Ok(()) } /// The file's text, or the empty string when it does not exist. fn read_or_empty(file: &Path) -> Result { match std::fs::read_to_string(file) { Ok(text) => Ok(text), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(String::new()), Err(e) => Err(e).with_context(|| format!("reading {}", file.display())), } } /// Write `text` to `file` unless it already holds exactly that. Returns /// whether it wrote. fn write_if_changed(file: &Path, text: &str) -> Result { if read_or_empty(file)? == text { return Ok(false); } crate::secret_file::write_secret_mode(file, text.as_bytes(), FILE_MODE) .with_context(|| format!("writing {}", file.display()))?; Ok(true) } /// The names of the unrestricted `users`, sorted, one per line. fn render_unrestricted(users: &[User]) -> String { let mut names: Vec<&str> = users .iter() .filter(|u| u.unrestricted) .map(|u| u.name.as_str()) .collect(); names.sort_unstable(); names.iter().flat_map(|n| [n, "\n"]).collect() } /// The users file for `users`, sorted by name, one `:` line each. /// /// A user whose line in `current` still verifies against its credential keeps /// that line. Every other line of `current` is dropped, so a destroyed agent, /// or one that belongs to another hive, does not outlive the pass. fn render(current: &str, users: &[User]) -> Result { let mut lines: Vec = Vec::with_capacity(users.len()); for user in users { if user.name.contains([':', '\n']) { bail!("{:?} cannot be a squid user name", user.name); } let kept = current .lines() .filter_map(|l| l.split_once(':')) .find(|(name, hash)| *name == user.name && verifies(&user.secret, hash)) .map(|(_, hash)| hash.to_owned()); let hash = match kept { Some(hash) => hash, None => hash_secret(&user.secret)?, }; lines.push(format!("{}:{hash}\n", user.name)); } lines.sort(); Ok(lines.concat()) } /// A fresh salted SHA-512 crypt hash of `secret`: `$6$rounds=5000$$`. fn hash_secret(secret: &str) -> Result { let mut salt = [0u8; SALT_BYTES]; getrandom::fill(&mut salt).context("drawing a salt from the kernel's CSPRNG")?; let hash = ShaCrypt::SHA512 .hash_password_with_salt(secret.as_bytes(), &salt) .map_err(|e| anyhow::anyhow!("hashing a proxy credential: {e}"))?; Ok(hash.as_str().to_owned()) } /// Whether `hash` is `secret`'s SHA-crypt hash. fn verifies(secret: &str, hash: &str) -> bool { ShaCrypt::SHA512 .verify_password(secret.as_bytes(), hash) .is_ok() } #[cfg(test)] mod tests { use std::collections::HashMap; use std::os::unix::fs::PermissionsExt as _; use super::*; use swarm_wire::agent_permissions::AgentPermissions; const HIVE: &str = "h1"; /// What the store holds, by path. fn store(entries: &[(&str, &str)]) -> HashMap { entries .iter() .map(|(p, v)| ((*p).to_owned(), (*v).to_owned())) .collect() } fn names(names: &[&str]) -> Vec { names.iter().map(|n| (*n).to_owned()).collect() } /// The unrestricted list that sits beside the users file `users`. fn list_of(users: &Path) -> PathBuf { users.with_file_name("unrestricted") } /// The permissions record of an agent whose swarm flag is `flag`. fn flagged(agent: &str, flag: Option) -> (String, AgentPermissions) { let json = match flag { Some(f) => format!(r#"{{"unrestricted_internet":{f}}}"#), None => "{}".to_owned(), }; ( agent.to_owned(), serde_json::from_str(&json).expect("decodes"), ) } async fn sync_flags( held: &HashMap, agents: &[&str], perms: &permissions::Stored, file: &Path, ) -> Result<()> { let list = list_of(file); let files = Files { users: file, unrestricted: &list, }; sync_with(HIVE, &names(agents), perms, &files, |path| { let found = held.get(&path).cloned(); async move { Ok(found) } }) .await } /// [`sync_flags`] with no permissions read yet: every agent unrestricted. async fn sync(held: &HashMap, agents: &[&str], file: &Path) -> Result<()> { sync_flags(held, agents, &permissions::Stored::new(), file).await } fn list(users: &Path) -> Vec { std::fs::read_to_string(list_of(users)) .expect("the list is there") .lines() .map(str::to_owned) .collect() } fn lines(file: &Path) -> Vec<(String, String)> { std::fs::read_to_string(file) .expect("the file is there") .lines() .map(|l| { let (name, hash) = l.split_once(':').expect("a name and a hash"); (name.to_owned(), hash.to_owned()) }) .collect() } fn two_agents_and_a_stranger() -> HashMap { store(&[ ("swarm/agents/atlas/proxy-credential", "atlas-secret"), ("swarm/agents/argus/proxy-credential", "argus-secret"), ("swarm/agents/other-hive-agent/proxy-credential", "stranger"), ("swarm/hives/h1/squid-selfcheck", "check-secret"), ("swarm/hives/h2/squid-selfcheck", "other-check"), ]) } #[tokio::test] async fn only_this_hives_agents_and_its_selfcheck_are_rendered() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) .await .expect("renders"); let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); assert_eq!( users, [ "argus", "atlas", "selfcheck-unrestricted.squid", "selfcheck.squid" ] ); } #[tokio::test] async fn each_line_is_a_salted_sha512_crypt_hash_of_its_users_credential() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) .await .expect("renders"); let by_name: HashMap<_, _> = lines(&file).into_iter().collect(); assert!(by_name["atlas"].starts_with("$6$"), "{}", by_name["atlas"]); assert!(verifies("atlas-secret", &by_name["atlas"])); assert!(verifies("argus-secret", &by_name["argus"])); assert!(verifies("check-secret", &by_name["selfcheck.squid"])); // Control: a hash is of its own user's credential only. assert!(!verifies("argus-secret", &by_name["atlas"])); // Salted: the same credential hashes differently twice. assert_ne!( hash_secret("atlas-secret").expect("hashes"), hash_secret("atlas-secret").expect("hashes") ); } #[tokio::test] async fn no_credential_is_written_to_the_file_or_its_directory() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) .await .expect("renders"); // Control: the search finds a name that is in each file. let mut found = 0; for entry in std::fs::read_dir(dir.path()).expect("lists") { let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text"); assert!(text.contains("atlas"), "control: {text}"); for secret in ["atlas-secret", "argus-secret", "check-secret"] { assert!(!text.contains(secret), "{secret} reached the disk"); } found += 1; } assert_eq!(found, 2, "only the users file and the list are left behind"); } #[tokio::test] async fn the_file_is_group_readable_and_replaced_atomically() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas"], &file) .await .expect("renders"); let mode = std::fs::metadata(&file).expect("stat").permissions().mode() & 0o777; assert_eq!(mode, 0o640); let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) .await .expect("renders"); let after = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); assert_ne!( inode, after, "a rename onto the path, not a rewrite in place" ); assert!(!dir.path().join("users.tmp").exists()); } #[tokio::test] async fn an_unchanged_pass_leaves_the_file_alone() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); let held = two_agents_and_a_stranger(); sync(&held, &["atlas", "argus"], &file) .await .expect("renders"); let first = std::fs::read_to_string(&file).expect("read"); let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); sync(&held, &["atlas", "argus"], &file) .await .expect("renders"); assert_eq!(std::fs::read_to_string(&file).expect("read"), first); let again = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat")); assert_eq!(inode, again, "nothing was written"); } #[tokio::test] async fn a_changed_credential_gets_a_new_hash_and_the_others_keep_theirs() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); let mut held = two_agents_and_a_stranger(); sync(&held, &["atlas", "argus"], &file) .await .expect("renders"); let before: HashMap<_, _> = lines(&file).into_iter().collect(); held.insert( "swarm/agents/atlas/proxy-credential".to_owned(), "rotated".to_owned(), ); sync(&held, &["atlas", "argus"], &file) .await .expect("renders"); let after: HashMap<_, _> = lines(&file).into_iter().collect(); assert!(verifies("rotated", &after["atlas"])); assert_ne!(before["atlas"], after["atlas"]); assert_eq!(before["argus"], after["argus"]); } #[tokio::test] async fn a_stopped_agent_stays_and_a_destroyed_one_goes() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); let held = two_agents_and_a_stranger(); // The agent list is the container list, so an agent whose container // is down is in it: nothing here looks at whether anything runs. sync(&held, &["atlas", "argus"], &file) .await .expect("renders"); assert!(lines(&file).iter().any(|(n, _)| n == "argus")); // Control: once the container is gone the user is dropped, and a line // for an agent this hive never had does not survive a pass either. let mut text = std::fs::read_to_string(&file).expect("read"); text.push_str("stray:$6$rounds=5000$saltsaltsaltsalt$x\n"); std::fs::write(&file, text).expect("write"); sync(&held, &["atlas"], &file).await.expect("renders"); let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); assert_eq!( users, ["atlas", "selfcheck-unrestricted.squid", "selfcheck.squid"] ); } #[tokio::test] async fn an_agent_without_a_credential_yet_is_left_out() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); // The empty file squid's unit creates before its helper starts. std::fs::write(&file, "").expect("write"); sync(&store(&[]), &["atlas"], &file).await.expect("renders"); assert_eq!(std::fs::read_to_string(&file).expect("read"), ""); } #[tokio::test] async fn a_failed_read_leaves_the_file_as_it_was() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file) .await .expect("renders"); let before = std::fs::read_to_string(&file).expect("read"); let list_before = std::fs::read_to_string(list_of(&file)).expect("read"); let list = list_of(&file); let files = Files { users: &file, unrestricted: &list, }; let failed = sync_with( HIVE, &names(&["atlas", "argus"]), &permissions::Stored::from([flagged("atlas", Some(false))]), &files, |path| async move { if path.contains("argus") { bail!("the store is down") } Ok(Some("atlas-secret".to_owned())) }, ) .await; assert!(failed.is_err()); assert_eq!(std::fs::read_to_string(&file).expect("read"), before); assert_eq!( std::fs::read_to_string(&list).expect("read"), list_before, "the flag flip waits for every read" ); } #[test] fn a_name_that_would_break_the_line_is_refused() { for bad in ["a:b", "a\nb"] { let user = User { name: bad.to_owned(), secret: "s".to_owned(), unrestricted: false, }; assert!(render("", &[user]).is_err(), "{bad:?}"); } } #[tokio::test] async fn an_agent_is_listed_unless_its_flag_is_off() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); let held = store(&[ ("swarm/agents/old/proxy-credential", "s1"), ("swarm/agents/unset/proxy-credential", "s2"), ("swarm/agents/on/proxy-credential", "s3"), ("swarm/agents/new/proxy-credential", "s4"), ("swarm/hives/h1/squid-selfcheck", "check-secret"), ]); let perms = permissions::Stored::from([ flagged("unset", None), flagged("on", Some(true)), flagged("new", Some(false)), ]); sync_flags(&held, &["old", "unset", "on", "new"], &perms, &file) .await .expect("renders"); // `old` has no record at all, `unset` a record without the flag. assert_eq!( list(&file), ["old", "on", "selfcheck-unrestricted.squid", "unset"] ); // Control: the restricted agent still has a login. let users: Vec = lines(&file).into_iter().map(|(n, _)| n).collect(); assert!(users.contains(&"new".to_owned()), "{users:?}"); } #[tokio::test] async fn the_selfchecks_log_in_with_one_secret_and_only_one_is_unrestricted() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas"], &file) .await .expect("renders"); let by_name: HashMap<_, _> = lines(&file).into_iter().collect(); assert!(verifies("check-secret", &by_name["selfcheck.squid"])); assert!(verifies( "check-secret", &by_name["selfcheck-unrestricted.squid"] )); let listed = list(&file); assert!(listed.contains(&"selfcheck-unrestricted.squid".to_owned())); assert!( !listed.contains(&"selfcheck.squid".to_owned()), "{listed:?}" ); } #[tokio::test] async fn an_agent_with_no_credential_is_not_listed() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas", "fresh"], &file) .await .expect("renders"); let listed = list(&file); assert!(listed.contains(&"atlas".to_owned()), "control: {listed:?}"); assert!(!listed.contains(&"fresh".to_owned()), "{listed:?}"); } #[tokio::test] async fn a_flipped_flag_rewrites_the_list_and_leaves_the_users_file() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); let held = two_agents_and_a_stranger(); let ino = |p: &Path| std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(p).expect("stat")); sync(&held, &["atlas", "argus"], &file) .await .expect("renders"); assert!(list(&file).contains(&"argus".to_owned())); let (users_ino, list_ino) = (ino(&file), ino(&list_of(&file))); // An unchanged pass touches neither. sync(&held, &["atlas", "argus"], &file) .await .expect("renders"); assert_eq!((ino(&file), ino(&list_of(&file))), (users_ino, list_ino)); // Turning argus off changes only the list. let perms = permissions::Stored::from([flagged("argus", Some(false))]); sync_flags(&held, &["atlas", "argus"], &perms, &file) .await .expect("renders"); assert!(!list(&file).contains(&"argus".to_owned())); assert!(list(&file).contains(&"atlas".to_owned())); assert_eq!(ino(&file), users_ino, "the users file was rewritten"); assert_ne!(ino(&list_of(&file)), list_ino, "the list was not"); } #[tokio::test] async fn the_list_is_group_readable() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); sync(&two_agents_and_a_stranger(), &["atlas"], &file) .await .expect("renders"); let meta = std::fs::metadata(list_of(&file)).expect("stat"); assert_eq!(meta.permissions().mode() & 0o777, 0o640); } /// A pass that cannot write the users file has still written the list: a /// user turned restricted loses its access before the users file stops /// listing it. #[tokio::test] async fn the_list_is_written_before_the_users_file() { let dir = tempfile::tempdir().expect("tempdir"); let file = dir.path().join("users"); std::fs::create_dir(&file).expect("a directory where the users file belongs"); sync(&two_agents_and_a_stranger(), &["atlas"], &file) .await .expect_err("the users file cannot be read"); assert_eq!(list(&file), ["atlas", "selfcheck-unrestricted.squid"]); } }