render squid's users file from this hive's proxy credentials

hive-c0re/Cargo.toml

@@ -19,6 +19,8 @@ axum.workspace = true
 backon.workspace = true
 chrono.workspace = true
 base64.workspace = true
+getrandom.workspace = true
+sha-crypt.workspace = true
 reqwest.workspace = true
 forgejo-api.workspace = true
 url.workspace = true

hive-c0re/src/main.rs

@@ -363,6 +363,8 @@ async fn cmd_serve(
     // not a part of the above; see swarm_agent_status's module doc for why
     // the two buckets stay separate.
     swarm_agent_status::spawn(coord.shutdown_rx());
+    // Squid's users file: the hashes of this hive's agents' proxy credentials.
+    workers::squid_users::spawn(coord.shutdown_rx());
     // Per-agent events.sqlite + bash-tasks file cleanup now runs
     // agent-side in the harness (`hive_agent::vacuum`): the files are
     // agent-owned, so host-side deletes hit PermissionDenied / readonly-db

hive-c0re/src/secret_file.rs

@@ -1,7 +1,8 @@
-//! [`write_secret_0600`]: the one way hive-c0re puts a credential on disk.
+//! [`write_secret_0600`] and [`write_secret_mode`]: the ways hive-c0re puts a
+//! credential, or a hash of one, on disk.
 
 use std::io::Write as _;
-use std::os::unix::fs::OpenOptionsExt as _;
+use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _};
 use std::path::Path;
 
 use anyhow::{Context as _, Result};
@@ -13,6 +14,12 @@ use anyhow::{Context as _, Result};
 /// reader (or a crash) never sees half of it. Errors carry paths, never
 /// `contents`.
 pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> {
+    write_secret_mode(path, contents, 0o600)
+}
+
+/// [`write_secret_0600`] with the file's mode given, for a file another user
+/// has to read (the group comes from the directory).
+pub fn write_secret_mode(path: &Path, contents: &[u8], mode: u32) -> Result<()> {
     if let Some(parent) = path.parent() {
         std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
     }
@@ -27,9 +34,13 @@ pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> {
     let mut f = std::fs::OpenOptions::new()
         .write(true)
         .create_new(true)
-        .mode(0o600)
+        .mode(mode)
         .open(&tmp)
         .with_context(|| format!("create {}", tmp.display()))?;
+    // `open` masks the mode with the process umask; the mode is a property of
+    // the writer, not of the unit's umask.
+    f.set_permissions(std::fs::Permissions::from_mode(mode))
+        .with_context(|| format!("chmod {}", tmp.display()))?;
     f.write_all(contents)
         .and_then(|()| f.sync_all())
         .with_context(|| format!("write {}", tmp.display()))?;
@@ -38,7 +49,7 @@ pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> {
 
 #[cfg(test)]
 mod tests {
-    use super::write_secret_0600;
+    use super::{write_secret_0600, write_secret_mode};
     use std::os::unix::fs::PermissionsExt as _;
 
     fn mode(path: &std::path::Path) -> u32 {
@@ -76,4 +87,26 @@ mod tests {
         write_secret_0600(&path, b"new\n").expect("writes");
         assert_eq!(std::fs::read(&path).expect("read"), b"new\n");
     }
+
+    /// A restrictive umask doesn't change the mode asked for. The umask is
+    /// process-wide, so the write happens in a child run of this test.
+    #[test]
+    fn the_mode_survives_a_restrictive_umask() {
+        const DIR: &str = "SECRET_FILE_UMASK_TEST_DIR";
+        if let Some(dir) = std::env::var_os(DIR) {
+            write_secret_mode(&std::path::Path::new(&dir).join("shared"), b"x\n", 0o640)
+                .expect("writes");
+            return;
+        }
+        let dir = tempfile::tempdir().expect("tempdir");
+        let status = std::process::Command::new("sh")
+            .arg("-c")
+            .arg(r#"umask 077 && exec "$0" --exact secret_file::tests::the_mode_survives_a_restrictive_umask"#)
+            .arg(std::env::current_exe().expect("test binary"))
+            .env(DIR, dir.path())
+            .status()
+            .expect("spawns the child");
+        assert!(status.success());
+        assert_eq!(mode(&dir.path().join("shared")), 0o640);
+    }
 }

hive-c0re/src/workers/mod.rs

@@ -1,7 +1,7 @@
 //! Background tasks and periodic sweeps: crash/login watcher, boot-time
 //! auto-update reconcile, the agent-sockets.json writer loop, the MCP socket listener
 //! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the
-//! swarm resource-cap and permissions reads.
+//! swarm resource-cap and permissions reads, and the squid users file.
 //! Each submodule is re-exported
 //! at the crate root, so `crate::crash_watch::…` etc. keep working unchanged.
 
@@ -11,6 +11,7 @@ pub mod crash_watch;
 pub mod knowledge;
 pub mod mcp_sockets;
 pub mod permissions_import;
+pub mod squid_users;
 pub mod swarm_limits;
 pub mod swarm_permissions;
 pub mod wanted;

hive-c0re/src/workers/squid_users.rs

@@ -0,0 +1,440 @@
+//! Squid's users file: the hashes of the proxy credentials this hive's agents
+//! authenticate to its egress proxy with.
+//!
+//! swarm-controller mints one credential per agent and one for this hive's own
+//! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this
+//! reads the credentials of the agents this hive has, plus the self-check's,
+//! and renders `<user>:<hash>` lines into the file named by
+//! [`ENV_USERS_FILE`], which squid's `basic_ncsa_auth` re-reads when its mtime
+//! moves. The agents are the container list, not the running containers: a
+//! stopped agent stays in the file.
+//!
+//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is
+//! while it still verifies its credential, so the file changes only when a
+//! user is added, removed or given a new credential.
+//!
+//! A failed read leaves the file as it is: dropping a user on a store outage
+//! would read exactly like the swarm revoking it.
+
+use std::future::Future;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use anyhow::{Context, Result, bail};
+use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt};
+use swarm_secret_client::{SecretStore, client, proxy};
+
+use crate::stats::sweep_health::{self, SweepHealth};
+
+/// How often the file is re-rendered.
+pub const INTERVAL: Duration = Duration::from_secs(30);
+
+/// At [`INTERVAL`] this is ~90 seconds of failing passes.
+const FAILURES_BEFORE_WARNING: u32 = 3;
+
+/// The environment variable naming the file. Set by `nix/host-modules/hive-c0re`
+/// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid.
+pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE";
+
+/// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()`
+/// accepts.
+const SALT_BYTES: usize = 12;
+
+/// The file's mode. The group is squid's, from the setgid directory
+/// `nix/host-modules/hive-squid.nix` creates.
+const FILE_MODE: u32 = 0o640;
+
+/// One squid user and the credential it logs in with. No `Debug`: `secret` is
+/// live.
+struct User {
+    name: String,
+    secret: String,
+}
+
+/// Start the pass loop, when this host has a squid users file, a store and a
+/// hive name.
+pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
+    let Some(path) = std::env::var_os(ENV_USERS_FILE)
+        .filter(|v| !v.is_empty())
+        .map(PathBuf::from)
+    else {
+        return;
+    };
+    let Some(hive) = crate::container_view::hive_swarm_names().0 else {
+        tracing::info!("squid users: HYPERHIVE_HIVE_NAME is unset; no users file is written");
+        return;
+    };
+    if std::env::var_os(client::ENV_ADDR).is_none_or(|v| v.is_empty()) {
+        tracing::info!(
+            "squid users: no swarm secret store is configured; no users file is written"
+        );
+        return;
+    }
+    tokio::spawn(async move {
+        let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING);
+        loop {
+            match pass(&hive, &path).await {
+                Ok(()) => health.record_ok(),
+                Err(e) => {
+                    tracing::warn!(error = ?e, "squid users: the pass failed");
+                    let err = format!("{e:#}");
+                    health.record_err(|ctx| {
+                        let age = ctx.since_last_ok.map_or_else(
+                            || "no success this session".to_owned(),
+                            |d| format!("last ok {} ago", sweep_health::fmt_age(d)),
+                        );
+                        format!(
+                            "rendering squid's users file is failing ({} consecutive, {age}) \
+                             — squid keeps the users last written: {err}",
+                            ctx.consecutive
+                        )
+                    });
+                }
+            }
+            tokio::select! {
+                () = tokio::time::sleep(INTERVAL) => {}
+                _ = shutdown.changed() => return,
+            }
+        }
+    });
+}
+
+/// Read this hive's credentials from the store and render them into `file`.
+async fn pass(hive: &str, file: &Path) -> Result<()> {
+    let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
+        .await
+        .context("enumerating this hive's agents")?
+        .into_iter()
+        .map(|spec| spec.name)
+        .collect();
+    let store = SecretStore::from_env(hive)
+        .await
+        .context("connecting to the store")?;
+    sync_with(hive, &agents, file, |path| {
+        let store = &store;
+        async move {
+            let held: Option<proxy::Credential> = store
+                .read_optional(&path)
+                .await
+                .with_context(|| format!("reading {path}"))?;
+            Ok(held.map(|c| c.value))
+        }
+    })
+    .await
+}
+
+/// [`pass`] with the store read and the file given. Nothing is written unless
+/// every read succeeds and the rendered file differs from the current one.
+async fn sync_with<F, Fut>(hive: &str, agents: &[String], file: &Path, read: F) -> Result<()>
+where
+    F: Fn(String) -> Fut,
+    Fut: Future<Output = Result<Option<String>>>,
+{
+    let mut wanted = Vec::new();
+    for agent in agents {
+        wanted.push((agent.clone(), proxy::agent_credential_path(agent)?));
+    }
+    wanted.push((
+        proxy::SELFCHECK_USER.to_owned(),
+        proxy::selfcheck_path(hive)?,
+    ));
+    let mut users = Vec::new();
+    for (name, path) in wanted {
+        if let Some(secret) = read(path).await? {
+            users.push(User { name, secret });
+        } else {
+            tracing::debug!(name, "squid users: no proxy credential minted yet");
+        }
+    }
+    let current = match std::fs::read_to_string(file) {
+        Ok(text) => text,
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
+        Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())),
+    };
+    let rendered = render(&current, &users)?;
+    if rendered == current {
+        return Ok(());
+    }
+    crate::secret_file::write_secret_mode(file, rendered.as_bytes(), FILE_MODE)
+        .with_context(|| format!("writing {}", file.display()))?;
+    tracing::info!(
+        users = users.len(),
+        agents = users.iter().filter(|u| u.name != proxy::SELFCHECK_USER).count(),
+        file = %file.display(),
+        "squid users: wrote the users file"
+    );
+    Ok(())
+}
+
+/// The users file for `users`, sorted by name, one `<user>:<hash>` line each.
+///
+/// A user whose line in `current` still verifies against its credential keeps
+/// that line. Every other line of `current` is dropped, so a destroyed agent,
+/// or one that belongs to another hive, does not outlive the pass.
+fn render(current: &str, users: &[User]) -> Result<String> {
+    let mut lines: Vec<String> = Vec::with_capacity(users.len());
+    for user in users {
+        if user.name.contains([':', '\n']) {
+            bail!("{:?} cannot be a squid user name", user.name);
+        }
+        let kept = current
+            .lines()
+            .filter_map(|l| l.split_once(':'))
+            .find(|(name, hash)| *name == user.name && verifies(&user.secret, hash))
+            .map(|(_, hash)| hash.to_owned());
+        let hash = match kept {
+            Some(hash) => hash,
+            None => hash_secret(&user.secret)?,
+        };
+        lines.push(format!("{}:{hash}\n", user.name));
+    }
+    lines.sort();
+    Ok(lines.concat())
+}
+
+/// A fresh salted SHA-512 crypt hash of `secret`: `$6$rounds=5000$<salt>$<hash>`.
+fn hash_secret(secret: &str) -> Result<String> {
+    let mut salt = [0u8; SALT_BYTES];
+    getrandom::fill(&mut salt).context("drawing a salt from the kernel's CSPRNG")?;
+    let hash = ShaCrypt::SHA512
+        .hash_password_with_salt(secret.as_bytes(), &salt)
+        .map_err(|e| anyhow::anyhow!("hashing a proxy credential: {e}"))?;
+    Ok(hash.as_str().to_owned())
+}
+
+/// Whether `hash` is `secret`'s SHA-crypt hash.
+fn verifies(secret: &str, hash: &str) -> bool {
+    ShaCrypt::SHA512
+        .verify_password(secret.as_bytes(), hash)
+        .is_ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::os::unix::fs::PermissionsExt as _;
+
+    use super::*;
+
+    const HIVE: &str = "h1";
+
+    /// What the store holds, by path.
+    fn store(entries: &[(&str, &str)]) -> HashMap<String, String> {
+        entries
+            .iter()
+            .map(|(p, v)| ((*p).to_owned(), (*v).to_owned()))
+            .collect()
+    }
+
+    fn names(names: &[&str]) -> Vec<String> {
+        names.iter().map(|n| (*n).to_owned()).collect()
+    }
+
+    async fn sync(held: &HashMap<String, String>, agents: &[&str], file: &Path) -> Result<()> {
+        sync_with(HIVE, &names(agents), file, |path| {
+            let found = held.get(&path).cloned();
+            async move { Ok(found) }
+        })
+        .await
+    }
+
+    fn lines(file: &Path) -> Vec<(String, String)> {
+        std::fs::read_to_string(file)
+            .expect("the file is there")
+            .lines()
+            .map(|l| {
+                let (name, hash) = l.split_once(':').expect("a name and a hash");
+                (name.to_owned(), hash.to_owned())
+            })
+            .collect()
+    }
+
+    fn two_agents_and_a_stranger() -> HashMap<String, String> {
+        store(&[
+            ("swarm/agents/atlas/proxy-credential", "atlas-secret"),
+            ("swarm/agents/argus/proxy-credential", "argus-secret"),
+            ("swarm/agents/other-hive-agent/proxy-credential", "stranger"),
+            ("swarm/hives/h1/squid-selfcheck", "check-secret"),
+            ("swarm/hives/h2/squid-selfcheck", "other-check"),
+        ])
+    }
+
+    #[tokio::test]
+    async fn only_this_hives_agents_and_its_selfcheck_are_rendered() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
+        assert_eq!(users, ["argus", "atlas", "selfcheck.squid"]);
+    }
+
+    #[tokio::test]
+    async fn each_line_is_a_salted_sha512_crypt_hash_of_its_users_credential() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        let by_name: HashMap<_, _> = lines(&file).into_iter().collect();
+        assert!(by_name["atlas"].starts_with("$6$"), "{}", by_name["atlas"]);
+        assert!(verifies("atlas-secret", &by_name["atlas"]));
+        assert!(verifies("argus-secret", &by_name["argus"]));
+        assert!(verifies("check-secret", &by_name["selfcheck.squid"]));
+        // Control: a hash is of its own user's credential only.
+        assert!(!verifies("argus-secret", &by_name["atlas"]));
+        // Salted: the same credential hashes differently twice.
+        assert_ne!(
+            hash_secret("atlas-secret").expect("hashes"),
+            hash_secret("atlas-secret").expect("hashes")
+        );
+    }
+
+    #[tokio::test]
+    async fn no_credential_is_written_to_the_file_or_its_directory() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        // Control: the search finds a name that is in the file.
+        let mut found = 0;
+        for entry in std::fs::read_dir(dir.path()).expect("lists") {
+            let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text");
+            assert!(text.contains("atlas:"), "control: {text}");
+            for secret in ["atlas-secret", "argus-secret", "check-secret"] {
+                assert!(!text.contains(secret), "{secret} reached the disk");
+            }
+            found += 1;
+        }
+        assert_eq!(found, 1, "only the users file is left behind");
+    }
+
+    #[tokio::test]
+    async fn the_file_is_group_readable_and_replaced_atomically() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas"], &file)
+            .await
+            .expect("renders");
+        let mode = std::fs::metadata(&file).expect("stat").permissions().mode() & 0o777;
+        assert_eq!(mode, 0o640);
+        let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
+        sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        let after = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
+        assert_ne!(
+            inode, after,
+            "a rename onto the path, not a rewrite in place"
+        );
+        assert!(!dir.path().join("users.tmp").exists());
+    }
+
+    #[tokio::test]
+    async fn an_unchanged_pass_leaves_the_file_alone() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        let held = two_agents_and_a_stranger();
+        sync(&held, &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        let first = std::fs::read_to_string(&file).expect("read");
+        let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
+        sync(&held, &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        assert_eq!(std::fs::read_to_string(&file).expect("read"), first);
+        let again = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
+        assert_eq!(inode, again, "nothing was written");
+    }
+
+    #[tokio::test]
+    async fn a_changed_credential_gets_a_new_hash_and_the_others_keep_theirs() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        let mut held = two_agents_and_a_stranger();
+        sync(&held, &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        let before: HashMap<_, _> = lines(&file).into_iter().collect();
+        held.insert(
+            "swarm/agents/atlas/proxy-credential".to_owned(),
+            "rotated".to_owned(),
+        );
+        sync(&held, &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        let after: HashMap<_, _> = lines(&file).into_iter().collect();
+        assert!(verifies("rotated", &after["atlas"]));
+        assert_ne!(before["atlas"], after["atlas"]);
+        assert_eq!(before["argus"], after["argus"]);
+    }
+
+    #[tokio::test]
+    async fn a_stopped_agent_stays_and_a_destroyed_one_goes() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        let held = two_agents_and_a_stranger();
+        // The agent list is the container list, so an agent whose container
+        // is down is in it: nothing here looks at whether anything runs.
+        sync(&held, &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        assert!(lines(&file).iter().any(|(n, _)| n == "argus"));
+        // Control: once the container is gone the user is dropped, and a line
+        // for an agent this hive never had does not survive a pass either.
+        let mut text = std::fs::read_to_string(&file).expect("read");
+        text.push_str("stray:$6$rounds=5000$saltsaltsaltsalt$x\n");
+        std::fs::write(&file, text).expect("write");
+        sync(&held, &["atlas"], &file).await.expect("renders");
+        let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
+        assert_eq!(users, ["atlas", "selfcheck.squid"]);
+    }
+
+    #[tokio::test]
+    async fn an_agent_without_a_credential_yet_is_left_out() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        // The empty file squid's unit creates before its helper starts.
+        std::fs::write(&file, "").expect("write");
+        sync(&store(&[]), &["atlas"], &file).await.expect("renders");
+        assert_eq!(std::fs::read_to_string(&file).expect("read"), "");
+    }
+
+    #[tokio::test]
+    async fn a_failed_read_leaves_the_file_as_it_was() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let file = dir.path().join("users");
+        sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
+            .await
+            .expect("renders");
+        let before = std::fs::read_to_string(&file).expect("read");
+        let failed = sync_with(
+            HIVE,
+            &names(&["atlas", "argus"]),
+            &file,
+            |path| async move {
+                if path.contains("argus") {
+                    bail!("the store is down")
+                }
+                Ok(Some("atlas-secret".to_owned()))
+            },
+        )
+        .await;
+        assert!(failed.is_err());
+        assert_eq!(std::fs::read_to_string(&file).expect("read"), before);
+    }
+
+    #[test]
+    fn a_name_that_would_break_the_line_is_refused() {
+        for bad in ["a:b", "a\nb"] {
+            let user = User {
+                name: bad.to_owned(),
+                secret: "s".to_owned(),
+            };
+            assert!(render("", &[user]).is_err(), "{bad:?}");
+        }
+    }
+}