hyperhive/swarm-builder/src/push.rs

//! The push of a built closure to the binary cache, with `attic push`.
//!
//! The cache is an attic server: the swarm's own or an external instance. Its push token is read
//! from the secret store per push and written to a file in a
//! directory only the builder's unit can enter, beside a config that names it with
//! `token-file`. `attic` finds the config through `XDG_CONFIG_HOME`.
//! `attic login <name> <url> <token>` is not used: it puts the token in the
//! process list. `attic` verifies the server's certificate
//! against the system roots.
//!
//! A push that does not happen or does not finish leaves the build green with
//! `cached` false: the closure is still in the host's store. The controller's
//! cache-miss pass sends the same build again, which pushes it.

use std::future::Future;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::Duration;

use anyhow::{Result, bail};
use swarm_wire::builder::FromMetaReply;

/// The longest one push may run, when the reply deadline leaves that much.
pub const PUSH_DEADLINE: Duration = Duration::from_mins(10);

/// The server name in the config file, and in the cache argument of the push.
const SERVER: &str = "swarm";

/// Where the builder pushes: one cache on one attic server.
pub struct Remote {
    /// The `attic` binary: a name found in the unit's path in production.
    pub attic: PathBuf,
    /// The server's name, without scheme: `host` or `host:port`.
    pub domain: String,
    /// The cache's name on that server.
    pub cache: String,
    /// A directory only the builder can enter, which the config files go in.
    pub state_dir: PathBuf,
}

impl Remote {
    /// The cache the environment names: `None` when neither value is set, so
    /// the builder pushes nothing.
    ///
    /// # Errors
    /// When only one of the two is set, or a value would bend the config file
    /// or the command.
    pub fn from_settings(
        domain: Option<String>,
        cache: Option<String>,
        state_dir: Option<String>,
    ) -> Result<Option<Self>> {
        let (domain, cache) = match (domain, cache) {
            (None, None) => return Ok(None),
            (Some(domain), Some(cache)) => (domain, cache),
            _ => bail!("the cache domain and the cache name must be set together"),
        };
        let plain_domain = !domain.is_empty()
            && domain
                .bytes()
                .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b':'));
        if !plain_domain {
            bail!("the cache domain {domain:?} is not a host name with an optional port");
        }
        if !cache_name_ok(&cache) {
            bail!("the cache name {cache:?} is not one attic accepts");
        }
        let Some(state_dir) = state_dir else {
            bail!("a cache is set, but the unit has no runtime directory to hold its config");
        };
        Ok(Some(Self {
            attic: PathBuf::from("attic"),
            domain,
            cache,
            state_dir: PathBuf::from(state_dir),
        }))
    }

    /// Push `out_path` with `token`. The result is what `attic` printed.
    ///
    /// # Errors
    /// Why the push did not finish: the config could not be written, `attic`
    /// could not start, or it exited with an error.
    pub async fn push(&self, token: &str, out_path: &str) -> Result<String, String> {
        if !out_path.starts_with("/nix/store/") || out_path.contains(char::is_whitespace) {
            return Err(format!("{out_path:?} is not a store path"));
        }
        let home = ConfigHome::write(&self.state_dir, &self.domain, token)
            .map_err(|e| format!("writing the attic config: {e}"))?;
        let output = self
            .command(&home.0, out_path)
            .output()
            .await
            .map_err(|e| format!("starting attic: {e}"))?;
        let said = |bytes: &[u8]| {
            crate::nix_build::last_lines(&String::from_utf8_lossy(bytes))
                .unwrap_or_default()
                .replace(token, "<redacted>")
        };
        if output.status.success() {
            Ok(said(&output.stdout))
        } else {
            Err(format!(
                "attic exited with {}: {}",
                output.status,
                said(&output.stderr)
            ))
        }
    }

    /// `attic push` of `out_path`, reading its config from `config_home`. The
    /// token is in a file beside it and in no argument or variable.
    fn command(&self, config_home: &Path, out_path: &str) -> tokio::process::Command {
        let mut cmd = tokio::process::Command::new(&self.attic);
        cmd.arg("push")
            .arg(format!("{SERVER}:{}", self.cache))
            .arg(out_path)
            .env("XDG_CONFIG_HOME", config_home)
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true);
        cmd
    }
}

/// Whether `name` is a cache name attic accepts, which is
/// `^[A-Za-z0-9][A-Za-z0-9-_+]{0,49}$`. `services.hyperhive.swarm.attic.cache`
/// states the same rule in nix.
fn cache_name_ok(name: &str) -> bool {
    let mut bytes = name.bytes();
    bytes.next().is_some_and(|b| b.is_ascii_alphanumeric())
        && name.len() <= 50
        && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'+'))
}

/// Numbers the config directories of one process's pushes.
static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);

/// The config directory of one push, removed with the token in it on drop.
struct ConfigHome(PathBuf);

impl ConfigHome {
    /// `<parent>/push-<n>/attic/config.toml`, which names `token` beside it, both
    /// readable by this user alone.
    fn write(parent: &Path, domain: &str, token: &str) -> std::io::Result<Self> {
        use std::io::Write as _;
        use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _};
        if token.is_empty() || token.bytes().any(|b| b < b' ') {
            return Err(std::io::Error::other(
                "the push token is empty or holds a control character",
            ));
        }
        let home = Self(parent.join(format!(
            "push-{}-{}",
            std::process::id(),
            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
        )));
        let dir = home.0.join("attic");
        let token_file = dir.join("token");
        let token_path = token_file.to_string_lossy();
        // A quote or backslash in the path would end the TOML string early.
        if token_path.contains(['"', '\\']) {
            return Err(std::io::Error::other(
                "the runtime directory's path holds a quote or a backslash",
            ));
        }
        std::fs::DirBuilder::new()
            .recursive(true)
            .mode(0o700)
            .create(&dir)?;
        let private = |name: &str| {
            std::fs::OpenOptions::new()
                .write(true)
                .create_new(true)
                .mode(0o600)
                .open(dir.join(name))
        };
        private("token")?.write_all(token.as_bytes())?;
        write!(
            private("config.toml")?,
            "default-server = \"{SERVER}\"\n\n[servers.{SERVER}]\n\
             endpoint = \"https://{domain}\"\ntoken-file = \"{token_path}\"\n"
        )?;
        Ok(home)
    }
}

impl Drop for ConfigHome {
    fn drop(&mut self) {
        let _ = std::fs::remove_dir_all(&self.0);
    }
}

/// `reply` with `cached` set by pushing its closure, when it is a built one and
/// a cache is set. `read_token` is not polled without a cache. The token read
/// and the push together get `budget`.
///
/// Whatever happens to the push, a built reply stays built: a failure is logged
/// and leaves `cached` false.
pub async fn cache_built(
    reply: FromMetaReply,
    cache: Option<&Remote>,
    read_token: impl Future<Output = Result<String>>,
    budget: Duration,
    agent: &str,
) -> FromMetaReply {
    let (
        Some(cache),
        FromMetaReply::Built {
            out_path,
            system,
            private_users,
            nix_builds,
            ..
        },
    ) = (cache, &reply)
    else {
        return reply;
    };
    let pushed = tokio::time::timeout(budget, async {
        let token = read_token
            .await
            .map_err(|e| format!("reading the push token: {e:#}"))?;
        cache.push(&token, out_path).await
    })
    .await
    .unwrap_or_else(|_| {
        Err(format!(
            "the push did not finish within {} s",
            budget.as_secs()
        ))
    });
    match pushed {
        Ok(_) => {
            tracing::info!(%agent, %out_path, cache = %cache.cache, "closure pushed to the cache");
            FromMetaReply::Built {
                out_path: out_path.clone(),
                system: system.clone(),
                cached: true,
                private_users: *private_users,
                nix_builds: *nix_builds,
            }
        }
        Err(reason) => {
            tracing::warn!(%agent, %out_path, cache = %cache.cache, %reason, "closure not pushed to the cache");
            reply
        }
    }
}

#[cfg(test)]
mod tests {
    use std::os::unix::fs::PermissionsExt as _;
    use std::sync::atomic::{AtomicBool, Ordering};

    use hive_types::NixBuilds;

    use super::*;

    const TOKEN: &str = "eyJ0b2tlbiI6InNlY3JldCJ9.c2lnbmVk";
    const OUT: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas";

    fn built() -> FromMetaReply {
        FromMetaReply::Built {
            out_path: OUT.to_owned(),
            system: "x86_64-linux".to_owned(),
            cached: false,
            private_users: None,
            nix_builds: Some(NixBuilds::Disabled),
        }
    }

    fn cached(reply: &FromMetaReply) -> bool {
        match reply {
            FromMetaReply::Built { cached, .. } => *cached,
            other => panic!("not built: {other:?}"),
        }
    }

    /// A cache whose `attic` is the program `attic`: `true` and `false` stand
    /// in for a push that works and one that fails, and `echo` for one that
    /// says what it was run with.
    fn cache_with(attic: &str) -> Remote {
        Remote {
            attic: PathBuf::from(attic),
            domain: "cache.example.org".to_owned(),
            cache: "grid".to_owned(),
            state_dir: std::env::temp_dir(),
        }
    }

    fn token() -> impl Future<Output = Result<String>> {
        std::future::ready(Ok(TOKEN.to_owned()))
    }

    #[tokio::test]
    async fn a_push_that_works_marks_the_build_cached() {
        let cache = cache_with("true");
        let reply = cache_built(built(), Some(&cache), token(), PUSH_DEADLINE, "atlas").await;
        assert!(cached(&reply), "{reply:?}");
        // The rewritten reply keeps what the evaluated config said.
        assert!(
            matches!(
                reply,
                FromMetaReply::Built {
                    private_users: None,
                    nix_builds: Some(NixBuilds::Disabled),
                    ..
                }
            ),
            "{reply:?}"
        );
    }

    #[tokio::test]
    async fn a_push_that_fails_leaves_the_build_green_and_not_cached() {
        let cache = cache_with("false");
        let reply = cache_built(built(), Some(&cache), token(), PUSH_DEADLINE, "atlas").await;
        assert_eq!(reply, built());
    }

    #[tokio::test]
    async fn a_push_that_cannot_start_leaves_the_build_green_and_not_cached() {
        let cache = cache_with("/nonexistent/attic");
        let reply = cache_built(built(), Some(&cache), token(), PUSH_DEADLINE, "atlas").await;
        assert_eq!(reply, built());
    }

    #[tokio::test]
    async fn an_unreadable_token_leaves_the_build_green_and_not_cached() {
        let cache = cache_with("true");
        let reply = cache_built(
            built(),
            Some(&cache),
            async { anyhow::bail!("nothing at the path") },
            PUSH_DEADLINE,
            "atlas",
        )
        .await;
        assert_eq!(reply, built());
    }

    #[tokio::test(start_paused = true)]
    async fn a_token_read_that_outlasts_the_budget_leaves_the_build_green() {
        let cache = cache_with("true");
        let started = tokio::time::Instant::now();
        let reply = cache_built(
            built(),
            Some(&cache),
            std::future::pending(),
            Duration::from_secs(5),
            "atlas",
        )
        .await;
        assert_eq!(reply, built());
        assert_eq!(started.elapsed(), Duration::from_secs(5));
    }

    #[tokio::test]
    async fn without_a_cache_nothing_is_read_or_pushed() {
        let polled = std::sync::Arc::new(AtomicBool::new(false));
        let tracked = |polled: &std::sync::Arc<AtomicBool>| {
            let polled = std::sync::Arc::clone(polled);
            async move {
                polled.store(true, Ordering::SeqCst);
                token().await
            }
        };
        let reply = cache_built(built(), None, tracked(&polled), PUSH_DEADLINE, "atlas").await;
        assert_eq!(reply, built());
        assert!(!polled.load(Ordering::SeqCst), "the token was read");

        // Control: with a cache the same future runs.
        let cache = cache_with("true");
        cache_built(
            built(),
            Some(&cache),
            tracked(&polled),
            PUSH_DEADLINE,
            "atlas",
        )
        .await;
        assert!(polled.load(Ordering::SeqCst));
    }

    #[tokio::test]
    async fn a_build_that_is_not_built_is_not_pushed() {
        let cache = cache_with("true");
        let failed = FromMetaReply::Failed {
            error_tail: "e".to_owned(),
        };
        let reply = cache_built(
            failed.clone(),
            Some(&cache),
            token(),
            PUSH_DEADLINE,
            "atlas",
        )
        .await;
        assert_eq!(reply, failed);
    }

    #[tokio::test]
    async fn the_push_command_has_no_token_in_it() {
        // `echo` prints the arguments it is given.
        let cache = cache_with("echo");
        let said = cache.push(TOKEN, OUT).await.expect("echo succeeds");
        assert_eq!(said, format!("push swarm:grid {OUT}"));
        assert!(!said.contains(TOKEN));

        let cmd = cache.command(Path::new("/run/home"), OUT);
        let cmd = cmd.as_std();
        let args: Vec<_> = cmd.get_args().collect();
        assert_eq!(args, ["push", "swarm:grid", OUT]);
        let env: Vec<_> = cmd.get_envs().collect();
        assert_eq!(
            env,
            [(
                std::ffi::OsStr::new("XDG_CONFIG_HOME"),
                Some(std::ffi::OsStr::new("/run/home"))
            )]
        );
    }

    #[test]
    fn the_token_is_in_a_private_file_the_config_names_and_both_are_removed_after() {
        let parent = std::env::temp_dir().join(format!("push-test-{}", std::process::id()));
        std::fs::create_dir_all(&parent).expect("parent");
        let home = ConfigHome::write(&parent, "cache.example.org", TOKEN).expect("written");
        let file = home.0.join("attic/config.toml");
        let text = std::fs::read_to_string(&file).expect("reads");
        let token_file = home.0.join("attic/token");
        assert!(
            text.contains(&format!("token-file = \"{}\"", token_file.display())),
            "{text}"
        );
        assert!(!text.contains(TOKEN), "{text}");
        assert_eq!(std::fs::read_to_string(&token_file).expect("reads"), TOKEN);
        assert!(
            text.contains("endpoint = \"https://cache.example.org\""),
            "{text}"
        );
        let mode = |p: &Path| std::fs::metadata(p).expect("stat").permissions().mode() & 0o777;
        assert_eq!(mode(&file), 0o600);
        assert_eq!(mode(&token_file), 0o600);
        assert_eq!(mode(&home.0), 0o700);
        let root = home.0.clone();
        drop(home);
        assert!(!root.exists());
        std::fs::remove_dir_all(&parent).expect("cleanup");
    }

    #[test]
    fn an_empty_token_or_one_with_a_control_character_is_refused() {
        let parent = std::env::temp_dir();
        for bad in ["", "a\nb", "a\0b"] {
            assert!(ConfigHome::write(&parent, "h", bad).is_err(), "{bad:?}");
        }
    }

    #[test]
    fn a_path_that_would_end_the_config_string_is_refused() {
        let parent = std::env::temp_dir().join("a\"b");
        assert!(ConfigHome::write(&parent, "h", TOKEN).is_err());
        assert!(!parent.exists());
    }

    #[test]
    fn the_cache_name_rule_is_attics() {
        let fifty = "a".repeat(50);
        for ok in ["grid", "a", "A-b_c+d", "0", fifty.as_str()] {
            assert!(cache_name_ok(ok), "{ok:?}");
        }
        let fifty_one = "a".repeat(51);
        for bad in [
            "",
            "grid.v2",
            "-x",
            "_x",
            "+x",
            "a:b",
            "a b",
            "a/b",
            fifty_one.as_str(),
        ] {
            assert!(!cache_name_ok(bad), "{bad:?}");
        }
    }

    #[tokio::test]
    async fn a_path_that_is_not_a_store_path_is_not_pushed() {
        let cache = cache_with("true");
        assert!(cache.push(TOKEN, "--help").await.is_err());
        assert!(cache.push(TOKEN, "/nix/store/x y").await.is_err());
        assert!(cache.push(TOKEN, OUT).await.is_ok());
    }

    #[test]
    fn the_settings_name_a_cache_only_when_both_are_set() {
        let state = || Some("/run/swarm-builder".to_owned());
        let some = |s: &str| Some(s.to_owned());
        assert!(
            Remote::from_settings(None, None, None)
                .expect("ok")
                .is_none()
        );
        let cache = Remote::from_settings(some("cache.example.org"), some("grid"), state())
            .expect("ok")
            .expect("set");
        assert_eq!(cache.domain, "cache.example.org");
        assert_eq!(cache.cache, "grid");
        assert!(Remote::from_settings(some("cache.example.org"), None, state()).is_err());
        assert!(Remote::from_settings(None, some("grid"), state()).is_err());
        assert!(Remote::from_settings(some("https://cache"), some("grid"), state()).is_err());
        assert!(Remote::from_settings(some("cache"), some("a\"b"), state()).is_err());
        assert!(Remote::from_settings(some("cache"), some("grid.v2"), state()).is_err());
        assert!(Remote::from_settings(some("cache"), some("-grid"), state()).is_err());
        assert!(Remote::from_settings(some("cache"), some("g_r+i-d"), state()).is_ok());
        assert!(Remote::from_settings(some("cache"), some("grid"), None).is_err());
    }
}