render squid's unrestricted-users list from each agent's flag

hive-c0re/src/workers/squid_users.rs

@@ -1,5 +1,6 @@
-//! Squid's users file: the hashes of the proxy credentials this hive's agents
+//! Squid's users file, and its list of unrestricted users: the hashes of the
-//! authenticate to its egress proxy with.
+//! proxy credentials this hive's agents authenticate to its egress proxy with,
+//! and which of them may reach any destination.
 //!
 //! swarm-controller mints one credential per agent and one for this hive's own
 //! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this
@@ -9,12 +10,19 @@
 //! moves. The agents are the container list, not the running containers: a
 //! stopped agent stays in the file.
 //!
-//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is
+//! The file named by [`ENV_UNRESTRICTED_FILE`] lists, one name per line, the
-//! while it still verifies its credential, so the file changes only when a
+//! users whose agent holds `unrestricted_internet` in the permissions this
-//! user is added, removed or given a new credential.
+//! hive last read from the swarm (an agent with no record, or no flag in it,
+//! holds it), plus the self-check's unrestricted user. Squid reads the list as
+//! an ACL when it parses its configuration, so `nix/host-modules/hive-squid.nix`
+//! reloads squid when the file changes.
 //!
-//! A failed read leaves the file as it is: dropping a user on a store outage
+//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is
-//! would read exactly like the swarm revoking it.
+//! while it still verifies its credential, so each file changes only when a
+//! user is added, removed, given a new credential or has its flag flipped.
+//!
+//! A failed read leaves both files as they are: dropping a user on a store
+//! outage would read exactly like the swarm revoking it.
 
 use std::future::Future;
 use std::path::{Path, PathBuf};
@@ -24,6 +32,7 @@ use anyhow::{Context, Result, bail};
 use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt};
 use swarm_secret_client::{SecretStore, client, proxy};
 
+use crate::permissions;
 use crate::stats::sweep_health::{self, SweepHealth};
 
 /// How often the file is re-rendered.
@@ -36,11 +45,15 @@ const FAILURES_BEFORE_WARNING: u32 = 3;
 /// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid.
 pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE";
 
+/// The environment variable naming the unrestricted-users list. Set by
+/// `nix/host-modules/hive-c0re` from `services.hyperhive.squid.unrestrictedFile`.
+pub const ENV_UNRESTRICTED_FILE: &str = "HIVE_SQUID_UNRESTRICTED_FILE";
+
 /// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()`
 /// accepts.
 const SALT_BYTES: usize = 12;
 
-/// The file's mode. The group is squid's, from the setgid directory
+/// Both files' mode. The group is squid's, from the setgid directory
 /// `nix/host-modules/hive-squid.nix` creates.
 const FILE_MODE: u32 = 0o640;
 
@@ -49,14 +62,32 @@ const FILE_MODE: u32 = 0o640;
 struct User {
     name: String,
     secret: String,
+    /// Whether squid lets this user reach any destination the deny rules leave.
+    unrestricted: bool,
 }
 
-/// Start the pass loop, when this host has a squid users file, a store and a
+/// The two files a pass writes.
-/// hive name.
+struct Files<'a> {
-pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
+    users: &'a Path,
-    let Some(path) = std::env::var_os(ENV_USERS_FILE)
+    unrestricted: &'a Path,
+}
+
+/// Whether `name` is one of the self-check's squid users rather than an agent's.
+fn is_selfcheck(name: &str) -> bool {
+    name == proxy::SELFCHECK_USER || name == proxy::SELFCHECK_UNRESTRICTED_USER
+}
+
+fn env_path(name: &str) -> Option<PathBuf> {
+    std::env::var_os(name)
         .filter(|v| !v.is_empty())
         .map(PathBuf::from)
+}
+
+/// Start the pass loop, when this host has squid's two files, a store and a
+/// hive name.
+pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
+    let (Some(users), Some(unrestricted)) =
+        (env_path(ENV_USERS_FILE), env_path(ENV_UNRESTRICTED_FILE))
     else {
         return;
     };
@@ -73,7 +104,15 @@ pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
     tokio::spawn(async move {
         let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING);
         loop {
-            match pass(&hive, &path).await {
+            match pass(
+                &hive,
+                &Files {
+                    users: &users,
+                    unrestricted: &unrestricted,
+                },
+            )
+            .await
+            {
                 Ok(()) => health.record_ok(),
                 Err(e) => {
                     tracing::warn!(error = ?e, "squid users: the pass failed");
@@ -99,18 +138,21 @@ pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
     });
 }
 
-/// Read this hive's credentials from the store and render them into `file`.
+/// Read this hive's credentials from the store and the swarm's permissions
-async fn pass(hive: &str, file: &Path) -> Result<()> {
+/// from the last read of them, and render both files.
+async fn pass(hive: &str, files: &Files<'_>) -> Result<()> {
     let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
         .await
         .context("enumerating this hive's agents")?
         .into_iter()
         .map(|spec| spec.name)
         .collect();
+    let perms = permissions::read_at(&permissions::path())
+        .context("reading the stored swarm permissions")?;
     let store = SecretStore::from_env(hive)
         .await
         .context("connecting to the store")?;
-    sync_with(hive, &agents, file, |path| {
+    sync_with(hive, &agents, &perms, files, |path| {
         let store = &store;
         async move {
             let held: Option<proxy::Credential> = store
@@ -123,49 +165,104 @@ async fn pass(hive: &str, file: &Path) -> Result<()> {
     .await
 }
 
-/// [`pass`] with the store read and the file given. Nothing is written unless
+/// [`pass`] with the store read, the permissions and the files given. Nothing
-/// every read succeeds and the rendered file differs from the current one.
+/// is written unless every read succeeds, and a file is only written when its
-async fn sync_with<F, Fut>(hive: &str, agents: &[String], file: &Path, read: F) -> Result<()>
+/// rendering differs from what it holds.
+///
+/// The unrestricted list goes first. A user that turns restricted is then
+/// restricted before the users file stops listing it, and one that turns
+/// unrestricted is not listed before it can log in.
+async fn sync_with<F, Fut>(
+    hive: &str,
+    agents: &[String],
+    perms: &permissions::Stored,
+    files: &Files<'_>,
+    read: F,
+) -> Result<()>
 where
     F: Fn(String) -> Fut,
     Fut: Future<Output = Result<Option<String>>>,
 {
     let mut wanted = Vec::new();
     for agent in agents {
-        wanted.push((agent.clone(), proxy::agent_credential_path(agent)?));
+        let unrestricted = permissions::unrestricted_internet(perms.get(agent));
-    }
         wanted.push((
-        proxy::SELFCHECK_USER.to_owned(),
+            agent.clone(),
-        proxy::selfcheck_path(hive)?,
+            proxy::agent_credential_path(agent)?,
+            unrestricted,
+        ));
+    }
+    let selfcheck = proxy::selfcheck_path(hive)?;
+    wanted.push((proxy::SELFCHECK_USER.to_owned(), selfcheck.clone(), false));
+    wanted.push((
+        proxy::SELFCHECK_UNRESTRICTED_USER.to_owned(),
+        selfcheck,
+        true,
     ));
     let mut users = Vec::new();
-    for (name, path) in wanted {
+    for (name, path, unrestricted) in wanted {
         if let Some(secret) = read(path).await? {
-            users.push(User { name, secret });
+            users.push(User {
+                name,
+                secret,
+                unrestricted,
+            });
         } else {
             tracing::debug!(name, "squid users: no proxy credential minted yet");
         }
     }
-    let current = match std::fs::read_to_string(file) {
+    let list = render_unrestricted(&users);
-        Ok(text) => text,
+    if write_if_changed(files.unrestricted, &list)? {
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
+        tracing::info!(
-        Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())),
+            unrestricted = list.lines().count(),
-    };
+            file = %files.unrestricted.display(),
-    let rendered = render(&current, &users)?;
+            "squid users: wrote the unrestricted list"
-    if rendered == current {
+        );
-        return Ok(());
     }
-    crate::secret_file::write_secret_mode(file, rendered.as_bytes(), FILE_MODE)
+    let current = read_or_empty(files.users)?;
-        .with_context(|| format!("writing {}", file.display()))?;
+    let rendered = render(&current, &users)?;
+    if write_if_changed(files.users, &rendered)? {
         tracing::info!(
             users = users.len(),
-        agents = users.iter().filter(|u| u.name != proxy::SELFCHECK_USER).count(),
+            agents = users.iter().filter(|u| !is_selfcheck(&u.name)).count(),
-        file = %file.display(),
+            file = %files.users.display(),
             "squid users: wrote the users file"
         );
+    }
     Ok(())
 }
 
+/// The file's text, or the empty string when it does not exist.
+fn read_or_empty(file: &Path) -> Result<String> {
+    match std::fs::read_to_string(file) {
+        Ok(text) => Ok(text),
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(String::new()),
+        Err(e) => Err(e).with_context(|| format!("reading {}", file.display())),
+    }
+}
+
+/// Write `text` to `file` unless it already holds exactly that. Returns
+/// whether it wrote.
+fn write_if_changed(file: &Path, text: &str) -> Result<bool> {
+    if read_or_empty(file)? == text {
+        return Ok(false);
+    }
+    crate::secret_file::write_secret_mode(file, text.as_bytes(), FILE_MODE)
+        .with_context(|| format!("writing {}", file.display()))?;
+    Ok(true)
+}
+
+/// The names of the unrestricted `users`, sorted, one per line.
+fn render_unrestricted(users: &[User]) -> String {
+    let mut names: Vec<&str> = users
+        .iter()
+        .filter(|u| u.unrestricted)
+        .map(|u| u.name.as_str())
+        .collect();
+    names.sort_unstable();
+    names.iter().flat_map(|n| [n, "\n"]).collect()
+}
+
 /// The users file for `users`, sorted by name, one `<user>:<hash>` line each.
 ///
 /// A user whose line in `current` still verifies against its credential keeps
@@ -215,6 +312,7 @@ mod tests {
     use std::os::unix::fs::PermissionsExt as _;
 
     use super::*;
+    use swarm_wire::agent_permissions::AgentPermissions;
 
     const HIVE: &str = "h1";
 
@@ -230,14 +328,54 @@ mod tests {
         names.iter().map(|n| (*n).to_owned()).collect()
     }
 
-    async fn sync(held: &HashMap<String, String>, agents: &[&str], file: &Path) -> Result<()> {
+    /// The unrestricted list that sits beside the users file `users`.
-        sync_with(HIVE, &names(agents), file, |path| {
+    fn list_of(users: &Path) -> PathBuf {
+        users.with_file_name("unrestricted")
+    }
+
+    /// The permissions record of an agent whose swarm flag is `flag`.
+    fn flagged(agent: &str, flag: Option<bool>) -> (String, AgentPermissions) {
+        let json = match flag {
+            Some(f) => format!(r#"{{"unrestricted_internet":{f}}}"#),
+            None => "{}".to_owned(),
+        };
+        (
+            agent.to_owned(),
+            serde_json::from_str(&json).expect("decodes"),
+        )
+    }
+
+    async fn sync_flags(
+        held: &HashMap<String, String>,
+        agents: &[&str],
+        perms: &permissions::Stored,
+        file: &Path,
+    ) -> Result<()> {
+        let list = list_of(file);
+        let files = Files {
+            users: file,
+            unrestricted: &list,
+        };
+        sync_with(HIVE, &names(agents), perms, &files, |path| {
             let found = held.get(&path).cloned();
             async move { Ok(found) }
         })
         .await
     }
 
+    /// [`sync_flags`] with no permissions read yet: every agent unrestricted.
+    async fn sync(held: &HashMap<String, String>, agents: &[&str], file: &Path) -> Result<()> {
+        sync_flags(held, agents, &permissions::Stored::new(), file).await
+    }
+
+    fn list(users: &Path) -> Vec<String> {
+        std::fs::read_to_string(list_of(users))
+            .expect("the list is there")
+            .lines()
+            .map(str::to_owned)
+            .collect()
+    }
+
     fn lines(file: &Path) -> Vec<(String, String)> {
         std::fs::read_to_string(file)
             .expect("the file is there")
@@ -267,7 +405,15 @@ mod tests {
             .await
             .expect("renders");
         let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
-        assert_eq!(users, ["argus", "atlas", "selfcheck.squid"]);
+        assert_eq!(
+            users,
+            [
+                "argus",
+                "atlas",
+                "selfcheck-unrestricted.squid",
+                "selfcheck.squid"
+            ]
+        );
     }
 
     #[tokio::test]
@@ -298,17 +444,17 @@ mod tests {
         sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
             .await
             .expect("renders");
-        // Control: the search finds a name that is in the file.
+        // Control: the search finds a name that is in each file.
         let mut found = 0;
         for entry in std::fs::read_dir(dir.path()).expect("lists") {
             let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text");
-            assert!(text.contains("atlas:"), "control: {text}");
+            assert!(text.contains("atlas"), "control: {text}");
             for secret in ["atlas-secret", "argus-secret", "check-secret"] {
                 assert!(!text.contains(secret), "{secret} reached the disk");
             }
             found += 1;
         }
-        assert_eq!(found, 1, "only the users file is left behind");
+        assert_eq!(found, 2, "only the users file and the list are left behind");
     }
 
     #[tokio::test]
@@ -390,7 +536,10 @@ mod tests {
         std::fs::write(&file, text).expect("write");
         sync(&held, &["atlas"], &file).await.expect("renders");
         let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
-        assert_eq!(users, ["atlas", "selfcheck.squid"]);
+        assert_eq!(
+            users,
+            ["atlas", "selfcheck-unrestricted.squid", "selfcheck.squid"]
+        );
     }
 
     #[tokio::test]
@@ -411,10 +560,17 @@ mod tests {
             .await
             .expect("renders");
         let before = std::fs::read_to_string(&file).expect("read");
+        let list_before = std::fs::read_to_string(list_of(&file)).expect("read");
+        let list = list_of(&file);
+        let files = Files {
+            users: &file,
+            unrestricted: &list,
+        };
         let failed = sync_with(
             HIVE,
             &names(&["atlas", "argus"]),
-            &file,
+            &permissions::Stored::from([flagged("atlas", Some(false))]),
+            &files,
             |path| async move {
                 if path.contains("argus") {
                     bail!("the store is down")
@@ -425,6 +581,11 @@ mod tests {
         .await;
         assert!(failed.is_err());
         assert_eq!(std::fs::read_to_string(&file).expect("read"), before);
+        assert_eq!(
+            std::fs::read_to_string(&list).expect("read"),
+            list_before,
+            "the flag flip waits for every read"
+        );
     }
 
     #[test]
@@ -433,8 +594,124 @@ mod tests {
             let user = User {
                 name: bad.to_owned(),
                 secret: "s".to_owned(),
+                unrestricted: false,
             };
             assert!(render("", &[user]).is_err(), "{bad:?}");
         }
     }
+
+    #[tokio::test]
+    async fn an_agent_is_listed_unless_its_flag_is_off() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        let held = store(&[
+            ("swarm/agents/old/proxy-credential", "s1"),
+            ("swarm/agents/unset/proxy-credential", "s2"),
+            ("swarm/agents/on/proxy-credential", "s3"),
+            ("swarm/agents/new/proxy-credential", "s4"),
+            ("swarm/hives/h1/squid-selfcheck", "check-secret"),
+        ]);
+        let perms = permissions::Stored::from([
+            flagged("unset", None),
+            flagged("on", Some(true)),
+            flagged("new", Some(false)),
+        ]);
+        sync_flags(&held, &["old", "unset", "on", "new"], &perms, &file)
+            .await
+            .expect("renders");
+        // `old` has no record at all, `unset` a record without the flag.
+        assert_eq!(
+            list(&file),
+            ["old", "on", "selfcheck-unrestricted.squid", "unset"]
+        );
+        // Control: the restricted agent still has a login.
+        let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
+        assert!(users.contains(&"new".to_owned()), "{users:?}");
+    }
+
+    #[tokio::test]
+    async fn the_selfchecks_log_in_with_one_secret_and_only_one_is_unrestricted() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas"], &file)
+            .await
+            .expect("renders");
+        let by_name: HashMap<_, _> = lines(&file).into_iter().collect();
+        assert!(verifies("check-secret", &by_name["selfcheck.squid"]));
+        assert!(verifies(
+            "check-secret",
+            &by_name["selfcheck-unrestricted.squid"]
+        ));
+        let listed = list(&file);
+        assert!(listed.contains(&"selfcheck-unrestricted.squid".to_owned()));
+        assert!(
+            !listed.contains(&"selfcheck.squid".to_owned()),
+            "{listed:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn an_agent_with_no_credential_is_not_listed() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas", "fresh"], &file)
+            .await
+            .expect("renders");
+        let listed = list(&file);
+        assert!(listed.contains(&"atlas".to_owned()), "control: {listed:?}");
+        assert!(!listed.contains(&"fresh".to_owned()), "{listed:?}");
+    }
+
+    #[tokio::test]
+    async fn a_flipped_flag_rewrites_the_list_and_leaves_the_users_file() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        let held = two_agents_and_a_stranger();
+        let ino =
+            |p: &Path| std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(p).expect("stat"));
+        sync(&held, &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        assert!(list(&file).contains(&"argus".to_owned()));
+        let (users_ino, list_ino) = (ino(&file), ino(&list_of(&file)));
+        // An unchanged pass touches neither.
+        sync(&held, &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        assert_eq!((ino(&file), ino(&list_of(&file))), (users_ino, list_ino));
+        // Turning argus off changes only the list.
+        let perms = permissions::Stored::from([flagged("argus", Some(false))]);
+        sync_flags(&held, &["atlas", "argus"], &perms, &file)
+            .await
+            .expect("renders");
+        assert!(!list(&file).contains(&"argus".to_owned()));
+        assert!(list(&file).contains(&"atlas".to_owned()));
+        assert_eq!(ino(&file), users_ino, "the users file was rewritten");
+        assert_ne!(ino(&list_of(&file)), list_ino, "the list was not");
+    }
+
+    #[tokio::test]
+    async fn the_list_is_group_readable() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas"], &file)
+            .await
+            .expect("renders");
+        let meta = std::fs::metadata(list_of(&file)).expect("stat");
+        assert_eq!(meta.permissions().mode() & 0o777, 0o640);
+    }
+
+    /// A pass that cannot write the users file has still written the list: a
+    /// user turned restricted loses its access before the users file stops
+    /// listing it.
+    #[tokio::test]
+    async fn the_list_is_written_before_the_users_file() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        std::fs::create_dir(&file).expect("a directory where the users file belongs");
+        sync(&two_agents_and_a_stranger(), &["atlas"], &file)
+            .await
+            .expect_err("the users file cannot be read");
+        assert_eq!(list(&file), ["atlas", "selfcheck-unrestricted.squid"]);
+    }
 }

swarm-secret-client/src/proxy.rs

@@ -28,6 +28,14 @@ pub const SELFCHECK_LEAF: &str = "squid-selfcheck";
 /// `nix/host-modules/hive-squid.nix` spells the same string.
 pub const SELFCHECK_USER: &str = "selfcheck.squid";
 
+/// The squid user the self-check logs in as to prove an unrestricted agent
+/// reaches an unlisted name. It authenticates with the same secret as
+/// [`SELFCHECK_USER`] ([`selfcheck_path`]), and is always on squid's
+/// unrestricted list. It cannot be an agent's name for the same reason as
+/// [`SELFCHECK_USER`]. `nix/host-modules/hive-squid.nix` spells the same
+/// string.
+pub const SELFCHECK_UNRESTRICTED_USER: &str = "selfcheck-unrestricted.squid";
+
 /// The path holding the secret `agent` authenticates to the egress proxy with.
 ///
 /// A flat leaf under the agent's prefix, like
@@ -127,8 +135,13 @@ mod tests {
     }
 
     #[test]
-    fn the_selfcheck_user_is_not_a_legal_agent_name() {
+    fn the_selfcheck_users_are_not_legal_agent_names() {
-        assert!(crate::path::checked_segment("agent", SELFCHECK_USER).is_err());
+        for user in [SELFCHECK_USER, SELFCHECK_UNRESTRICTED_USER] {
+            assert!(
+                crate::path::checked_segment("agent", user).is_err(),
+                "{user}"
+            );
+        }
     }
 
     #[test]