//! The check on a closure before it is applied, and the two small verbs around //! the swarm cache. //! //! hive-c0re (unprivileged) downloads the swarm's closure through the host's //! nix daemon. `CreateContainer` and `UpdateContainer` then apply a toplevel //! only when it is in the store and carries a signature by the swarm cache's //! key, which `nix store verify` checks against the key in [`ROOT_FACTS_DIR`]. //! The key is a file a root unit keeps; nothing in the request, and nothing //! hive-c0re writes, chooses it. //! //! A path already in the store from a local build has no signature in the //! store database, and the daemon's pull leaves it so. Before the check, //! [`fetch_signature`] imports the signature the swarm cache holds for it; the //! check still decides. use std::os::unix::fs::MetadataExt as _; use std::path::Path; use anyhow::{Context as _, Result, bail}; use hive_priv_sock::valid_closure_path; use crate::container::{BoundedRun, run_bounded}; use crate::shared::{RealSpawn, Spawn, validate_agent_name}; /// Where the root unit `hive-priv-cache-facts` keeps `public-key`. /// Not `/run/hyperhive/attic`, which hive-c0re writes. const ROOT_FACTS_DIR: &str = "/run/hyperhive-priv/attic"; /// The root unit that reads this hive's cache facts from the store. const FACTS_UNIT: &str = "hive-priv-cache-facts.service"; /// Where the facts unit writes `cache.conf` (the cache's address) and `netrc` /// (the pull token), both root-only to write. const CONF_DIR: &str = "/run/hyperhive-nix"; /// Wall-clock bound on one `nix store copy-sigs`, which fetches one narinfo. const COPY_SIGS_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(1); /// Wall-clock bound on one `nix store verify`, which reads a path's metadata. const VERIFY_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(1); /// Wall-clock bound on the facts unit: a store login and three small writes. const REFRESH_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(1); /// Whether `path` is a regular file or directory owned by `owner` that nobody /// else can read or write. fn private_to(path: &Path, owner: u32, directory: bool) -> Result<()> { let meta = std::fs::symlink_metadata(path) .with_context(|| format!("{} is missing", path.display()))?; let kind_ok = if directory { meta.is_dir() } else { meta.is_file() }; if !kind_ok { bail!( "{} is not a plain {}", path.display(), if directory { "directory" } else { "file" } ); } if meta.uid() != owner { bail!("{} is not owned by uid {owner}", path.display()); } if meta.mode() & 0o077 != 0 { bail!("{} is readable or writable by others", path.display()); } Ok(()) } /// The swarm cache's public key in `dir`, as `:`. /// /// # Errors /// A `dir` or key file that is missing, not owned by `owner` or open to /// others, or a key that is not `:`. fn swarm_key(dir: &Path, owner: u32) -> Result { private_to(dir, owner, true)?; let key_file = dir.join("public-key"); private_to(&key_file, owner, false)?; let key = std::fs::read_to_string(&key_file) .with_context(|| format!("reading {}", key_file.display()))?; let key = key.trim(); if key.contains(char::is_whitespace) || key .split_once(':') .is_none_or(|(name, key)| name.is_empty() || key.is_empty()) { bail!("the cache's public key is not :"); } Ok(key.to_owned()) } /// The `nix` arguments that check `toplevel` is in the store and signed by /// `key`, and by no other key. /// /// - `--no-contents` checks signatures and validity, not the NAR hash. /// - `--sigs-needed 1` with `trusted-public-keys` replaced by `key` means one /// signature by the swarm cache's key; the host's other keys do not count. fn verify_args(toplevel: &str, key: &str) -> Vec { [ "--extra-experimental-features", "nix-command", "store", "verify", "--no-contents", "--sigs-needed", "1", "--option", "trusted-public-keys", key, toplevel, ] .map(str::to_owned) .into() } /// What `nix store verify` reports through its exit status, as a bit set: 2 is /// a path no trusted key signed, 4 a path that is not in the store. fn verdict(code: Option, toplevel: &str, stderr: &str) -> Result<()> { match code { Some(0) => Ok(()), Some(c) if c & 4 != 0 => bail!("{toplevel} is not in the store"), Some(c) if c & 2 != 0 => bail!("{toplevel} has no signature by the swarm cache's key"), _ => bail!( "nix store verify failed: {}", stderr.lines().last().unwrap_or("").trim() ), } } /// The swarm cache's address in a `cache.conf` fragment: the value of its /// `extra-substituters` line, when that is one word. fn cache_url(fragment: &str) -> Option<&str> { fragment .lines() .filter_map(|line| line.split_once('=')) .find(|(name, _)| name.trim() == "extra-substituters") .map(|(_, value)| value.trim()) .filter(|url| !url.is_empty() && !url.contains(char::is_whitespace)) } /// The `nix` arguments that import the signatures the cache at `url` holds for /// `toplevel` into the daemon's store database, authenticating with `netrc`. /// /// - `--store daemon` is the store the verify reads, whatever the environment. /// - No `--recursive`: only the path that is verified is touched. fn copy_sigs_args(toplevel: &str, url: &str, netrc: &Path) -> Vec { [ "--extra-experimental-features", "nix-command", "store", "copy-sigs", "--store", "daemon", "--substituter", url, "--option", "netrc-file", &netrc.to_string_lossy(), toplevel, ] .map(str::to_owned) .into() } /// Import the signature the swarm cache holds for `toplevel`, so that a path /// already in the store without it can pass the verify. Nothing is trusted /// here: the verify still needs the swarm key's signature. /// /// A failure (no `cache.conf`, the cache unreachable, no narinfo for the path) /// is logged and the verify runs anyway: the path may already carry the /// signature. async fn fetch_signature(spawn: &impl Spawn, conf_dir: &Path, toplevel: &str) { let conf = conf_dir.join("cache.conf"); let fragment = match std::fs::read_to_string(&conf) { Ok(text) => text, Err(e) => { tracing::warn!(path = %conf.display(), error = %e, "not fetching the swarm signature: cannot read the cache address"); return; } }; let Some(url) = cache_url(&fragment) else { tracing::warn!(path = %conf.display(), "not fetching the swarm signature: no single extra-substituters address"); return; }; let mut cmd = spawn.command("nix"); cmd.args(copy_sigs_args(toplevel, url, &conf_dir.join("netrc"))); match run_bounded(cmd, COPY_SIGS_TIMEOUT).await { Ok(BoundedRun::Exited { status, .. }) if status.success() => {} Ok(BoundedRun::Exited { status, stderr }) => tracing::warn!( %toplevel, %status, stderr = stderr.lines().last().unwrap_or("").trim(), "nix store copy-sigs failed; verifying without it" ), Ok(BoundedRun::TimedOut) => tracing::warn!( %toplevel, "nix store copy-sigs timed out after {}s; verifying without it", COPY_SIGS_TIMEOUT.as_secs() ), Err(e) => tracing::warn!( %toplevel, error = %format!("{e:#}"), "could not run nix store copy-sigs; verifying without it" ), } } /// Whether `toplevel` is in the store with a signature by the key in `dir`, /// after importing the cache's signature for it ([`fetch_signature`]). async fn check_signed( spawn: &impl Spawn, dir: &Path, conf_dir: &Path, owner: u32, toplevel: &str, ) -> Result<()> { if !valid_closure_path(toplevel) { bail!("{toplevel:?} is not a store path"); } let key = swarm_key(dir, owner)?; fetch_signature(spawn, conf_dir, toplevel).await; let mut cmd = spawn.command("nix"); cmd.args(verify_args(toplevel, &key)); match run_bounded(cmd, VERIFY_TIMEOUT) .await .context("running nix store verify")? { BoundedRun::Exited { status, stderr } => verdict(status.code(), toplevel, &stderr), BoundedRun::TimedOut => bail!( "nix store verify timed out after {}s; killed it", VERIFY_TIMEOUT.as_secs() ), } } /// `toplevel` when it is a store path that is in the store with a signature by /// the swarm cache's key; else the refusal, which is also logged at error /// level with the agent and the path. /// /// The request comes from hive-c0re, which is not root and can ask for any /// path of that shape. A path in the store is not enough: a build of the host's /// own, or a path from another cache, carries no signature by this key. /// /// # Errors /// `toplevel` is not a store path, is not in the store, or carries no /// signature by the swarm cache's key, or the key cannot be read. pub(crate) async fn applicable_for<'a>(agent: &str, toplevel: &'a str) -> Result<&'a str> { applicable_with( &RealSpawn, Path::new(ROOT_FACTS_DIR), Path::new(CONF_DIR), 0, agent, toplevel, ) .await } async fn applicable_with<'a>( spawn: &impl Spawn, dir: &Path, conf_dir: &Path, owner: u32, agent: &str, toplevel: &'a str, ) -> Result<&'a str> { match check_signed(spawn, dir, conf_dir, owner, toplevel).await { Ok(()) => Ok(toplevel), Err(e) => { tracing::error!( %agent, requested = %toplevel, error = %format!("{e:#}"), "refusing to apply a closure the swarm cache did not sign" ); bail!("{agent}: refusing {toplevel}: {e:#}") } } } /// `RealiseClosure`, from an older hive-c0re: nothing is downloaded. `Ok` when /// the path is already what an apply requires, else `not available: `. /// /// # Errors /// Bad arguments, or `not available: `. pub(crate) async fn realise_closure(agent: &str, store_path: &str) -> Result<(String, String)> { validate_agent_name(agent)?; if !valid_closure_path(store_path) { bail!("{store_path:?} is not a store path"); } match check_signed( &RealSpawn, Path::new(ROOT_FACTS_DIR), Path::new(CONF_DIR), 0, store_path, ) .await { Ok(()) => Ok(("present".to_owned(), String::new())), Err(e) => bail!( "not available: {e:#}; hive-priv no longer downloads, hive-c0re pulls through the nix daemon" ), } } /// `RefreshCacheFacts`: start the root unit that reads this hive's cache facts /// from the store and writes the nix.conf fragment, and wait for it. The unit /// is fixed here, not named by the request. /// /// # Errors /// The unit failed to start or did not finish within [`REFRESH_TIMEOUT`]. pub(crate) async fn refresh_cache_facts() -> Result<(String, String)> { refresh_with(&RealSpawn, REFRESH_TIMEOUT).await } async fn refresh_with(spawn: &impl Spawn, limit: std::time::Duration) -> Result<(String, String)> { let mut cmd = spawn.command("systemctl"); cmd.args(["start", FACTS_UNIT]); match run_bounded(cmd, limit) .await .with_context(|| format!("running systemctl start {FACTS_UNIT}"))? { BoundedRun::Exited { status, .. } if status.success() => { tracing::info!("refreshed the cache facts"); Ok((String::new(), String::new())) } BoundedRun::Exited { status, stderr } => bail!( "systemctl start {FACTS_UNIT} exited {status}: {}", stderr.lines().last().unwrap_or("").trim() ), BoundedRun::TimedOut => bail!( "{FACTS_UNIT} did not finish within {}s; killed the wait", limit.as_secs() ), } } #[cfg(test)] mod tests { use std::os::unix::fs::PermissionsExt as _; use std::path::PathBuf; use super::*; use crate::shared::test_support::{ScriptSpawn, scratch}; const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; const KEY: &str = "swarm:AAAA"; /// The uid the test runs as, which stands in for root. fn me() -> u32 { std::fs::metadata(scratch()).expect("stat").uid() } fn put(path: &Path, text: &str, mode: u32) { std::fs::write(path, text).expect("write"); std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).expect("chmod"); } /// A facts directory as the root unit leaves it. fn facts(key: &str) -> PathBuf { let dir = scratch(); std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("chmod"); put(&dir.join("public-key"), &format!("{key}\n"), 0o600); put( &dir.join("netrc"), "machine cache.example password t\n", 0o600, ); dir } const URL: &str = "https://cache.example/swarm"; /// A `cache.conf` directory as the facts unit leaves it. fn conf() -> PathBuf { let dir = scratch(); put( &dir.join("cache.conf"), &format!( "extra-substituters = {URL}\nextra-trusted-public-keys = {KEY}\nnetrc-file = {}/netrc\n", dir.display() ), 0o644, ); dir } /// A `nix` over a store holding `PATH`, which `nix store verify /// --sigs-needed 1` finds signed by `KEY` only: exit 0 for that path under /// that key, 2 for that path under any other key, 4 for any other path. It /// exits 99 for anything but the verify and copy-sigs invocations, and logs /// its arguments. /// /// `signed` is whether `PATH` carries the signature before anything runs; /// otherwise the verify exits 2 until a copy-sigs has exited 0, which /// imports it. `copy_exit` is that copy-sigs' exit status. fn nix_with(log: &Path, signed: bool, copy_exit: i32) -> ScriptSpawn { let marker = log.with_extension("signed"); if signed { std::fs::write(&marker, "").expect("marker"); } ScriptSpawn(format!( r#"echo "$@" >>{log} if [ "$3 $4" = "store copy-sigs" ]; then [ "$5 $6 $7" = "--store daemon --substituter" ] || exit 99 [ "$8" = {URL} ] || exit 99 [ "$9 ${{10}}" = "--option netrc-file" ] || exit 99 [ "${{12}}" = {PATH} ] || exit 99 [ {copy_exit} = 0 ] && touch {marker} exit {copy_exit} fi [ "$1 $2 $3 $4 $5" = "--extra-experimental-features nix-command store verify --no-contents" ] || exit 99 [ "$6 $7 $8 $9" = "--sigs-needed 1 --option trusted-public-keys" ] || exit 99 [ "${{11}}" = {PATH} ] || exit 4 [ "${{10}}" = {KEY} ] || exit 2 [ -e {marker} ] || exit 2 exit 0"#, log = log.display(), marker = marker.display() )) } /// A `nix` whose path is signed already and whose copy-sigs succeeds. fn nix(log: &Path) -> ScriptSpawn { nix_with(log, true, 0) } #[tokio::test] async fn a_toplevel_signed_by_the_swarm_key_is_applied() { let log = scratch().join("calls"); let got = applicable_with(&nix(&log), &facts(KEY), &conf(), me(), "atlas", PATH) .await .expect("signed"); assert_eq!(got, PATH); // The key checked against is the one in the root file. let calls = std::fs::read_to_string(&log).expect("log"); assert!( calls.contains(&format!("trusted-public-keys {KEY} {PATH}")), "{calls}" ); } /// A path no swarm key signed, one the store lacks, and a key file that /// names another key are each refused, naming the agent and the path. #[tokio::test] async fn an_unsigned_or_absent_toplevel_is_refused() { let log = scratch().join("calls"); let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-iris"; for (case, dir, path, want) in [ ("absent", facts(KEY), other, "is not in the store"), ( "unsigned by the key on file", facts("other:BBBB"), PATH, "has no signature by the swarm cache's key", ), ] { let err = applicable_with(&nix(&log), &dir, &conf(), me(), "atlas", path) .await .expect_err(case); let msg = format!("{err:#}"); assert!( msg.contains("atlas") && msg.contains(path) && msg.contains(want), "{case}: {msg}" ); } } /// A tool failure other than the two verdicts is a refusal too, with the /// tool's last line. #[tokio::test] async fn a_failing_verify_refuses() { let spawn = ScriptSpawn("echo 'error: cannot connect' >&2; exit 1".to_owned()); let err = applicable_with(&spawn, &facts(KEY), &conf(), me(), "atlas", PATH) .await .expect_err("refused"); assert!(format!("{err:#}").contains("cannot connect"), "{err:#}"); } #[tokio::test] async fn only_a_store_path_is_verified() { // A stand-in that would accept anything, so only the shape check refuses. let yes = ScriptSpawn("exit 0".to_owned()); for bad in [ "", "main", "/etc/passwd", "/nix/store/x", &format!("{PATH}/bin"), ] { let err = applicable_with(&yes, &facts(KEY), &conf(), me(), "atlas", bad) .await .expect_err(bad); assert!(format!("{err:#}").contains("not a store path"), "{bad}"); } // Control: the same stand-in accepts a store path. assert!( applicable_with(&yes, &facts(KEY), &conf(), me(), "atlas", PATH) .await .is_ok() ); } /// Without the root key there is nothing to verify against: refused, and /// nix is not asked. #[tokio::test] async fn without_the_root_key_nothing_is_applied() { let log = scratch().join("calls"); let nowhere = scratch().join("nowhere"); let err = applicable_with(&nix(&log), &nowhere, &conf(), 0, "atlas", PATH) .await .expect_err("no key"); assert!(format!("{err:#}").contains("missing"), "{err:#}"); assert!(!log.exists(), "nix ran without a key"); } #[test] fn the_key_is_the_root_files_and_only_a_private_one() { let me = me(); assert_eq!(swarm_key(&facts(KEY), me).unwrap(), KEY); // Another owner than the one trusted. assert!(swarm_key(&facts(KEY), me + 1).is_err()); // The key open to others, or the directory open. let dir = facts(KEY); std::fs::set_permissions( dir.join("public-key"), std::fs::Permissions::from_mode(0o644), ) .expect("chmod"); assert!(swarm_key(&dir, me).is_err()); let dir = facts(KEY); std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).expect("chmod"); assert!(swarm_key(&dir, me).is_err()); // A symlink in place of the key. let dir = facts(KEY); std::fs::remove_file(dir.join("public-key")).expect("rm"); std::os::unix::fs::symlink(dir.join("netrc"), dir.join("public-key")).expect("link"); assert!(swarm_key(&dir, me).is_err()); for key in ["nokeyname", "swarm: AAAA", ":AAAA", "swarm:"] { assert!(swarm_key(&facts(key), me).is_err(), "{key:?}"); } // The directory hive-c0re writes is not the one this reads. assert_eq!(ROOT_FACTS_DIR, "/run/hyperhive-priv/attic"); } #[test] fn the_command_counts_only_the_swarm_key() { let args = verify_args(PATH, KEY); assert_eq!(args.last().map(String::as_str), Some(PATH)); let at = args .iter() .position(|a| a == "--sigs-needed") .expect("flag"); assert_eq!(args[at + 1], "1"); let at = args .iter() .position(|a| a == "trusted-public-keys") .expect("option"); assert_eq!(args[at - 1], "--option"); assert_eq!(args[at + 1], KEY); // The setting that would add to the host's keys is not used. assert!( !args.iter().any(|a| a.starts_with("extra-trusted")), "{args:?}" ); } /// An older hive-c0re's download request downloads nothing: it succeeds /// only for a path an apply would accept. #[tokio::test] async fn the_older_download_request_answers_not_available_for_a_path_not_in_the_store() { // The real `nix` is not on the path of the test, so this is refused for // want of the root key before anything runs; the shape and name checks // come first. for (agent, path) in [ ("Bad Agent", PATH), ("", PATH), ("atlas", "/nix/store/x"), ("atlas", "/etc/passwd"), ("atlas", &format!("{PATH}/bin/sh")), ] { let err = realise_closure(agent, path).await.expect_err("refused"); assert!(!format!("{err:#}").contains("not available"), "{err:#}"); } let err = realise_closure("atlas", PATH).await.expect_err("no key"); let msg = format!("{err:#}"); assert!( msg.starts_with("not available: ") && msg.contains("no longer downloads"), "{msg}" ); } /// The refresh starts the fixed unit and nothing else, and says why it /// failed or timed out. #[tokio::test] async fn the_refresh_starts_the_facts_unit_and_reports_a_failure() { let log = scratch().join("calls"); let ok = ScriptSpawn(format!("echo \"$0 $@\" >>{}", log.display())); refresh_with(&ok, REFRESH_TIMEOUT).await.expect("started"); assert_eq!( std::fs::read_to_string(&log).expect("log").trim(), "systemctl start hive-priv-cache-facts.service" ); let failing = ScriptSpawn("echo 'Unit not found.' >&2; exit 5".to_owned()); let err = refresh_with(&failing, REFRESH_TIMEOUT) .await .expect_err("failed"); assert!(format!("{err:#}").contains("Unit not found."), "{err:#}"); let slow = ScriptSpawn("sleep 30".to_owned()); let err = refresh_with(&slow, std::time::Duration::from_millis(100)) .await .expect_err("timed out"); assert!(format!("{err:#}").contains("did not finish"), "{err:#}"); } fn calls(log: &Path) -> Vec { std::fs::read_to_string(log) .unwrap_or_default() .lines() .map(str::to_owned) .collect() } /// A path in the store without the swarm signature passes once the cache's /// signature is imported, and the import runs before the verify, with the /// cache's address and netrc from the facts unit's directory. #[tokio::test] async fn the_cache_signature_is_imported_before_the_verify() { let log = scratch().join("calls"); let conf = conf(); applicable_with( &nix_with(&log, false, 0), &facts(KEY), &conf, me(), "atlas", PATH, ) .await .expect("signed after the import"); let calls = calls(&log); assert_eq!(calls.len(), 2, "{calls:?}"); assert_eq!( calls[0], format!( "--extra-experimental-features nix-command store copy-sigs --store daemon --substituter {URL} --option netrc-file {}/netrc {PATH}", conf.display() ) ); assert!(calls[1].contains("store verify"), "{calls:?}"); } /// A failed import is not the verdict: the verify runs and decides, so an /// unsigned path is refused for its missing signature and a path that /// already carries it is applied. #[tokio::test] async fn a_failed_import_still_runs_the_verify() { let log = scratch().join("calls"); let err = applicable_with( &nix_with(&log, false, 1), &facts(KEY), &conf(), me(), "atlas", PATH, ) .await .expect_err("still unsigned"); assert!( format!("{err:#}").contains("has no signature by the swarm cache's key"), "{err:#}" ); let calls = calls(&log); assert!( calls.len() == 2 && calls[0].contains("copy-sigs") && calls[1].contains("store verify"), "{calls:?}" ); // Control: the same failing import leaves a signed path applicable. let log = scratch().join("calls"); applicable_with( &nix_with(&log, true, 1), &facts(KEY), &conf(), me(), "atlas", PATH, ) .await .expect("signed already"); } /// Without a readable `cache.conf` naming one address there is no import, /// and the verify decides as before. #[tokio::test] async fn without_a_cache_address_the_verify_runs_alone() { for (case, text) in [ ("no file", None), ("no setting", Some("netrc-file = /x\n")), ("empty", Some("extra-substituters =\n")), ( "two addresses", Some("extra-substituters = https://a https://b\n"), ), ] { let conf = scratch(); if let Some(text) = text { put(&conf.join("cache.conf"), text, 0o644); } let log = scratch().join("calls"); let err = applicable_with( &nix_with(&log, false, 0), &facts(KEY), &conf, me(), "atlas", PATH, ) .await .expect_err(case); assert!(format!("{err:#}").contains("has no signature"), "{case}"); let calls = calls(&log); assert!( calls.len() == 1 && calls[0].contains("store verify"), "{case}: {calls:?}" ); } } #[test] fn the_cache_address_is_the_extra_substituters_value() { assert_eq!( cache_url("extra-substituters = https://c.example/swarm\nnetrc-file = /n\n"), Some("https://c.example/swarm") ); // `substituters` is the host's own list, not the swarm cache. assert_eq!(cache_url("substituters = https://c.example\n"), None); assert_eq!(cache_url(""), None); } #[test] fn the_import_is_for_the_one_path_and_never_trusts_a_key() { let args = copy_sigs_args(PATH, URL, Path::new("/run/hyperhive-nix/netrc")); assert_eq!(args.last().map(String::as_str), Some(PATH)); assert!(!args.iter().any(|a| a == "--recursive" || a == "-r")); assert!( !args.iter().any(|a| a.contains("trusted-public-keys")), "{args:?}" ); } }