vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

render squid's unrestricted-users list from each agent's flag

Browse source
This commit is contained in:
atlas 2026-10-28 14:14:30 +01:00 • committed by mara
commit 0b99f17bc3
2 changed files with 338 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

371
hive-c0re/src/workers/squid_users.rs
View file

@ -1,5 +1,6 @@
//! Squid's users file: the hashes of the proxy credentials this hive's agents
//! authenticate to its egress proxy with.
//! Squid's users file, and its list of unrestricted users: the hashes of the
//! proxy credentials this hive's agents authenticate to its egress proxy with,
//! and which of them may reach any destination.
//!
//! swarm-controller mints one credential per agent and one for this hive's own
//! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this
@ -9,12 +10,19 @@
//! moves. The agents are the container list, not the running containers: a
//! stopped agent stays in the file.
//!
//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is
//! while it still verifies its credential, so the file changes only when a
//! user is added, removed or given a new credential.
//! The file named by [`ENV_UNRESTRICTED_FILE`] lists, one name per line, the
//! users whose agent holds `unrestricted_internet` in the permissions this
//! hive last read from the swarm (an agent with no record, or no flag in it,
//! holds it), plus the self-check's unrestricted user. Squid reads the list as
//! an ACL when it parses its configuration, so `nix/host-modules/hive-squid.nix`
//! reloads squid when the file changes.
//!
//! A failed read leaves the file as it is: dropping a user on a store outage
//! would read exactly like the swarm revoking it.
//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is
//! while it still verifies its credential, so each file changes only when a
//! user is added, removed, given a new credential or has its flag flipped.
//!
//! A failed read leaves both files as they are: dropping a user on a store
//! outage would read exactly like the swarm revoking it.
use std::future::Future;
use std::path::{Path, PathBuf};
@ -24,6 +32,7 @@ use anyhow::{Context, Result, bail};
use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt};
use swarm_secret_client::{SecretStore, client, proxy};
use crate::permissions;
use crate::stats::sweep_health::{self, SweepHealth};
/// How often the file is re-rendered.
@ -36,11 +45,15 @@ const FAILURES_BEFORE_WARNING: u32 = 3;
/// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid.
pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE";
/// The environment variable naming the unrestricted-users list. Set by
/// `nix/host-modules/hive-c0re` from `services.hyperhive.squid.unrestrictedFile`.
pub const ENV_UNRESTRICTED_FILE: &str = "HIVE_SQUID_UNRESTRICTED_FILE";
/// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()`
/// accepts.
const SALT_BYTES: usize = 12;
/// The file's mode. The group is squid's, from the setgid directory
/// Both files' mode. The group is squid's, from the setgid directory
/// `nix/host-modules/hive-squid.nix` creates.
const FILE_MODE: u32 = 0o640;
@ -49,14 +62,32 @@ const FILE_MODE: u32 = 0o640;
struct User {
name: String,
secret: String,
/// Whether squid lets this user reach any destination the deny rules leave.
unrestricted: bool,
}
/// Start the pass loop, when this host has a squid users file, a store and a
/// hive name.
pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
let Some(path) = std::env::var_os(ENV_USERS_FILE)
/// The two files a pass writes.
struct Files<'a> {
users: &'a Path,
unrestricted: &'a Path,
}
/// Whether `name` is one of the self-check's squid users rather than an agent's.
fn is_selfcheck(name: &str) -> bool {
name == proxy::SELFCHECK_USER || name == proxy::SELFCHECK_UNRESTRICTED_USER
}
fn env_path(name: &str) -> Option<PathBuf> {
std::env::var_os(name)
.filter(|v| !v.is_empty())
.map(PathBuf::from)
}
/// Start the pass loop, when this host has squid's two files, a store and a
/// hive name.
pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
let (Some(users), Some(unrestricted)) =
(env_path(ENV_USERS_FILE), env_path(ENV_UNRESTRICTED_FILE))
else {
return;
};
@ -73,7 +104,15 @@ pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
tokio::spawn(async move {
let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING);
loop {
match pass(&hive, &path).await {
match pass(
&hive,
&Files {
users: &users,
unrestricted: &unrestricted,
},
)
.await
{
Ok(()) => health.record_ok(),
Err(e) => {
tracing::warn!(error = ?e, "squid users: the pass failed");
@ -99,18 +138,21 @@ pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
});
}
/// Read this hive's credentials from the store and render them into `file`.
async fn pass(hive: &str, file: &Path) -> Result<()> {
/// Read this hive's credentials from the store and the swarm's permissions
/// from the last read of them, and render both files.
async fn pass(hive: &str, files: &Files<'_>) -> Result<()> {
let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
.await
.context("enumerating this hive's agents")?
.into_iter()
.map(|spec| spec.name)
.collect();
let perms = permissions::read_at(&permissions::path())
.context("reading the stored swarm permissions")?;
let store = SecretStore::from_env(hive)
.await
.context("connecting to the store")?;
sync_with(hive, &agents, file, |path| {
sync_with(hive, &agents, &perms, files, |path| {
let store = &store;
async move {
let held: Option<proxy::Credential> = store
@ -123,49 +165,104 @@ async fn pass(hive: &str, file: &Path) -> Result<()> {
.await
}
/// [`pass`] with the store read and the file given. Nothing is written unless
/// every read succeeds and the rendered file differs from the current one.
async fn sync_with<F, Fut>(hive: &str, agents: &[String], file: &Path, read: F) -> Result<()>
/// [`pass`] with the store read, the permissions and the files given. Nothing
/// is written unless every read succeeds, and a file is only written when its
/// rendering differs from what it holds.
///
/// The unrestricted list goes first. A user that turns restricted is then
/// restricted before the users file stops listing it, and one that turns
/// unrestricted is not listed before it can log in.
async fn sync_with<F, Fut>(
hive: &str,
agents: &[String],
perms: &permissions::Stored,
files: &Files<'_>,
read: F,
) -> Result<()>
where
F: Fn(String) -> Fut,
Fut: Future<Output = Result<Option<String>>>,
{
let mut wanted = Vec::new();
for agent in agents {
wanted.push((agent.clone(), proxy::agent_credential_path(agent)?));
let unrestricted = permissions::unrestricted_internet(perms.get(agent));
wanted.push((
agent.clone(),
proxy::agent_credential_path(agent)?,
unrestricted,
));
}
let selfcheck = proxy::selfcheck_path(hive)?;
wanted.push((proxy::SELFCHECK_USER.to_owned(), selfcheck.clone(), false));
wanted.push((
proxy::SELFCHECK_USER.to_owned(),
proxy::selfcheck_path(hive)?,
proxy::SELFCHECK_UNRESTRICTED_USER.to_owned(),
selfcheck,
true,
));
let mut users = Vec::new();
for (name, path) in wanted {
for (name, path, unrestricted) in wanted {
if let Some(secret) = read(path).await? {
users.push(User { name, secret });
users.push(User {
name,
secret,
unrestricted,
});
} else {
tracing::debug!(name, "squid users: no proxy credential minted yet");
}
}
let current = match std::fs::read_to_string(file) {
Ok(text) => text,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())),
};
let list = render_unrestricted(&users);
if write_if_changed(files.unrestricted, &list)? {
tracing::info!(
unrestricted = list.lines().count(),
file = %files.unrestricted.display(),
"squid users: wrote the unrestricted list"
);
}
let current = read_or_empty(files.users)?;
let rendered = render(&current, &users)?;
if rendered == current {
return Ok(());
if write_if_changed(files.users, &rendered)? {
tracing::info!(
users = users.len(),
agents = users.iter().filter(|u| !is_selfcheck(&u.name)).count(),
file = %files.users.display(),
"squid users: wrote the users file"
);
}
crate::secret_file::write_secret_mode(file, rendered.as_bytes(), FILE_MODE)
.with_context(|| format!("writing {}", file.display()))?;
tracing::info!(
users = users.len(),
agents = users.iter().filter(|u| u.name != proxy::SELFCHECK_USER).count(),
file = %file.display(),
"squid users: wrote the users file"
);
Ok(())
}
/// The file's text, or the empty string when it does not exist.
fn read_or_empty(file: &Path) -> Result<String> {
match std::fs::read_to_string(file) {
Ok(text) => Ok(text),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(String::new()),
Err(e) => Err(e).with_context(|| format!("reading {}", file.display())),
}
}
/// Write `text` to `file` unless it already holds exactly that. Returns
/// whether it wrote.
fn write_if_changed(file: &Path, text: &str) -> Result<bool> {
if read_or_empty(file)? == text {
return Ok(false);
}
crate::secret_file::write_secret_mode(file, text.as_bytes(), FILE_MODE)
.with_context(|| format!("writing {}", file.display()))?;
Ok(true)
}
/// The names of the unrestricted `users`, sorted, one per line.
fn render_unrestricted(users: &[User]) -> String {
let mut names: Vec<&str> = users
.iter()
.filter(|u| u.unrestricted)
.map(|u| u.name.as_str())
.collect();
names.sort_unstable();
names.iter().flat_map(|n| [n, "\n"]).collect()
}
/// The users file for `users`, sorted by name, one `<user>:<hash>` line each.
///
/// A user whose line in `current` still verifies against its credential keeps
@ -215,6 +312,7 @@ mod tests {
use std::os::unix::fs::PermissionsExt as _;
use super::*;
use swarm_wire::agent_permissions::AgentPermissions;
const HIVE: &str = "h1";
@ -230,14 +328,54 @@ mod tests {
names.iter().map(|n| (*n).to_owned()).collect()
}
async fn sync(held: &HashMap<String, String>, agents: &[&str], file: &Path) -> Result<()> {
sync_with(HIVE, &names(agents), file, |path| {
/// The unrestricted list that sits beside the users file `users`.
fn list_of(users: &Path) -> PathBuf {
users.with_file_name("unrestricted")
}
/// The permissions record of an agent whose swarm flag is `flag`.
fn flagged(agent: &str, flag: Option<bool>) -> (String, AgentPermissions) {
let json = match flag {
Some(f) => format!(r#"{{"unrestricted_internet":{f}}}"#),
None => "{}".to_owned(),
};
(
agent.to_owned(),
serde_json::from_str(&json).expect("decodes"),
)
}
async fn sync_flags(
held: &HashMap<String, String>,
agents: &[&str],
perms: &permissions::Stored,
file: &Path,
) -> Result<()> {
let list = list_of(file);
let files = Files {
users: file,
unrestricted: &list,
};
sync_with(HIVE, &names(agents), perms, &files, |path| {
let found = held.get(&path).cloned();
async move { Ok(found) }
})
.await
}
/// [`sync_flags`] with no permissions read yet: every agent unrestricted.
async fn sync(held: &HashMap<String, String>, agents: &[&str], file: &Path) -> Result<()> {
sync_flags(held, agents, &permissions::Stored::new(), file).await
}
fn list(users: &Path) -> Vec<String> {
std::fs::read_to_string(list_of(users))
.expect("the list is there")
.lines()
.map(str::to_owned)
.collect()
}
fn lines(file: &Path) -> Vec<(String, String)> {
std::fs::read_to_string(file)
.expect("the file is there")
@ -267,7 +405,15 @@ mod tests {
.await
.expect("renders");
let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
assert_eq!(users, ["argus", "atlas", "selfcheck.squid"]);
assert_eq!(
users,
[
"argus",
"atlas",
"selfcheck-unrestricted.squid",
"selfcheck.squid"
]
);
}
#[tokio::test]
@ -298,17 +444,17 @@ mod tests {
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
// Control: the search finds a name that is in the file.
// Control: the search finds a name that is in each file.
let mut found = 0;
for entry in std::fs::read_dir(dir.path()).expect("lists") {
let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text");
assert!(text.contains("atlas:"), "control: {text}");
assert!(text.contains("atlas"), "control: {text}");
for secret in ["atlas-secret", "argus-secret", "check-secret"] {
assert!(!text.contains(secret), "{secret} reached the disk");
}
found += 1;
}
assert_eq!(found, 1, "only the users file is left behind");
assert_eq!(found, 2, "only the users file and the list are left behind");
}
#[tokio::test]
@ -390,7 +536,10 @@ mod tests {
std::fs::write(&file, text).expect("write");
sync(&held, &["atlas"], &file).await.expect("renders");
let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
assert_eq!(users, ["atlas", "selfcheck.squid"]);
assert_eq!(
users,
["atlas", "selfcheck-unrestricted.squid", "selfcheck.squid"]
);
}
#[tokio::test]
@ -411,10 +560,17 @@ mod tests {
.await
.expect("renders");
let before = std::fs::read_to_string(&file).expect("read");
let list_before = std::fs::read_to_string(list_of(&file)).expect("read");
let list = list_of(&file);
let files = Files {
users: &file,
unrestricted: &list,
};
let failed = sync_with(
HIVE,
&names(&["atlas", "argus"]),
&file,
&permissions::Stored::from([flagged("atlas", Some(false))]),
&files,
|path| async move {
if path.contains("argus") {
bail!("the store is down")
@ -425,6 +581,11 @@ mod tests {
.await;
assert!(failed.is_err());
assert_eq!(std::fs::read_to_string(&file).expect("read"), before);
assert_eq!(
std::fs::read_to_string(&list).expect("read"),
list_before,
"the flag flip waits for every read"
);
}
#[test]
@ -433,8 +594,124 @@ mod tests {
let user = User {
name: bad.to_owned(),
secret: "s".to_owned(),
unrestricted: false,
};
assert!(render("", &[user]).is_err(), "{bad:?}");
}
}
#[tokio::test]
async fn an_agent_is_listed_unless_its_flag_is_off() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let held = store(&[
("swarm/agents/old/proxy-credential", "s1"),
("swarm/agents/unset/proxy-credential", "s2"),
("swarm/agents/on/proxy-credential", "s3"),
("swarm/agents/new/proxy-credential", "s4"),
("swarm/hives/h1/squid-selfcheck", "check-secret"),
]);
let perms = permissions::Stored::from([
flagged("unset", None),
flagged("on", Some(true)),
flagged("new", Some(false)),
]);
sync_flags(&held, &["old", "unset", "on", "new"], &perms, &file)
.await
.expect("renders");
// `old` has no record at all, `unset` a record without the flag.
assert_eq!(
list(&file),
["old", "on", "selfcheck-unrestricted.squid", "unset"]
);
// Control: the restricted agent still has a login.
let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
assert!(users.contains(&"new".to_owned()), "{users:?}");
}
#[tokio::test]
async fn the_selfchecks_log_in_with_one_secret_and_only_one_is_unrestricted() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas"], &file)
.await
.expect("renders");
let by_name: HashMap<_, _> = lines(&file).into_iter().collect();
assert!(verifies("check-secret", &by_name["selfcheck.squid"]));
assert!(verifies(
"check-secret",
&by_name["selfcheck-unrestricted.squid"]
));
let listed = list(&file);
assert!(listed.contains(&"selfcheck-unrestricted.squid".to_owned()));
assert!(
!listed.contains(&"selfcheck.squid".to_owned()),
"{listed:?}"
);
}
#[tokio::test]
async fn an_agent_with_no_credential_is_not_listed() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "fresh"], &file)
.await
.expect("renders");
let listed = list(&file);
assert!(listed.contains(&"atlas".to_owned()), "control: {listed:?}");
assert!(!listed.contains(&"fresh".to_owned()), "{listed:?}");
}
#[tokio::test]
async fn a_flipped_flag_rewrites_the_list_and_leaves_the_users_file() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let held = two_agents_and_a_stranger();
let ino =
|p: &Path| std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(p).expect("stat"));
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
assert!(list(&file).contains(&"argus".to_owned()));
let (users_ino, list_ino) = (ino(&file), ino(&list_of(&file)));
// An unchanged pass touches neither.
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
assert_eq!((ino(&file), ino(&list_of(&file))), (users_ino, list_ino));
// Turning argus off changes only the list.
let perms = permissions::Stored::from([flagged("argus", Some(false))]);
sync_flags(&held, &["atlas", "argus"], &perms, &file)
.await
.expect("renders");
assert!(!list(&file).contains(&"argus".to_owned()));
assert!(list(&file).contains(&"atlas".to_owned()));
assert_eq!(ino(&file), users_ino, "the users file was rewritten");
assert_ne!(ino(&list_of(&file)), list_ino, "the list was not");
}
#[tokio::test]
async fn the_list_is_group_readable() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas"], &file)
.await
.expect("renders");
let meta = std::fs::metadata(list_of(&file)).expect("stat");
assert_eq!(meta.permissions().mode() & 0o777, 0o640);
}
/// A pass that cannot write the users file has still written the list: a
/// user turned restricted loses its access before the users file stops
/// listing it.
#[tokio::test]
async fn the_list_is_written_before_the_users_file() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
std::fs::create_dir(&file).expect("a directory where the users file belongs");
sync(&two_agents_and_a_stranger(), &["atlas"], &file)
.await
.expect_err("the users file cannot be read");
assert_eq!(list(&file), ["atlas", "selfcheck-unrestricted.squid"]);
}
}

17
swarm-secret-client/src/proxy.rs
View file

@ -28,6 +28,14 @@ pub const SELFCHECK_LEAF: &str = "squid-selfcheck";
/// `nix/host-modules/hive-squid.nix` spells the same string.
pub const SELFCHECK_USER: &str = "selfcheck.squid";
/// The squid user the self-check logs in as to prove an unrestricted agent
/// reaches an unlisted name. It authenticates with the same secret as
/// [`SELFCHECK_USER`] ([`selfcheck_path`]), and is always on squid's
/// unrestricted list. It cannot be an agent's name for the same reason as
/// [`SELFCHECK_USER`]. `nix/host-modules/hive-squid.nix` spells the same
/// string.
pub const SELFCHECK_UNRESTRICTED_USER: &str = "selfcheck-unrestricted.squid";
/// The path holding the secret `agent` authenticates to the egress proxy with.
///
/// A flat leaf under the agent's prefix, like
@ -127,8 +135,13 @@ mod tests {
}
#[test]
fn the_selfcheck_user_is_not_a_legal_agent_name() {
assert!(crate::path::checked_segment("agent", SELFCHECK_USER).is_err());
fn the_selfcheck_users_are_not_legal_agent_names() {
for user in [SELFCHECK_USER, SELFCHECK_UNRESTRICTED_USER] {
assert!(
crate::path::checked_segment("agent", user).is_err(),
"{user}"
);
}
}
#[test]