diff --git a/docs/4-implementation/swarm/builder.md b/docs/4-implementation/swarm/builder.md index 57596c812..c8eee16bb 100644 --- a/docs/4-implementation/swarm/builder.md +++ b/docs/4-implementation/swarm/builder.md @@ -315,6 +315,22 @@ a `nix eval` in its sandbox, and an eval has three outcomes: and hive-priv keeps what the container's last write chose, as it does for an agent with no record. +The built reply carries a third option, the agent's icon +(`services.hyperhive.agent.icon`), but the controller doesn't keep it in the +closure record. The same `nix eval` reads the SVG file the option names (the eval +applies `builtins.readFile`, so the builder needs no store path), and the reply's +`icon` is one of: + +- `set` with the SVG's text, for an option that names a file of at most 256 KiB; +- `unset`, for a null option or a module that doesn't declare it; +- left out, for a failed eval, a timeout, an empty file or an oversize one. + +The controller writes the key `` of the `agent-icons` KV bucket from a +`set`, deletes it on `unset` and leaves it on a missing field. A write that +fails is logged and doesn't fail the build or the deploy. So an agent's icon +changes when its config is built and deployed, not when its container restarts, +and an agent that has never run has one as soon as its first build succeeds. + Mixed versions: | builder | controller and hive-c0re | what happens | diff --git a/docs/4-implementation/swarm/controller.md b/docs/4-implementation/swarm/controller.md index 34789be28..94abd7c3b 100644 --- a/docs/4-implementation/swarm/controller.md +++ b/docs/4-implementation/swarm/controller.md @@ -266,13 +266,15 @@ arrives, so for up to a minute after the controller starts, and for an agent whose harness doesn't send it. The row's `paused_by_hive` comes from the same headers and is `null` in the same cases. -The third thing an agent publishes is its **icon**, the same SVG its own -`GET /icon` serves. It goes into the `agent-icons` KV bucket under the key +The third thing a hive's agent has is its **icon**, the same SVG its own +`GET /icon` serves. It's in the `agent-icons` KV bucket under the key ``, with no hive in it, so the swarm can show the icon of an agent that's -stopped or has moved hives. The queue grants the agent's own credential -`$KV.agent-icons.` and no other key of that bucket. The harness writes -once per start, because a config change reaches an agent by restarting its -container. An agent with no icon deletes its key. Swarm-side, +stopped or has moved hives. The [builder](builder.md) reads it from the agent's +evaluated config when it builds the agent for a deploy, and the controller writes +the key from the reply: an agent with no icon has its key deleted. The harness +still publishes the same file once per start, with its own credential's grant of +`$KV.agent-icons.`; the controller's write is the one that matters, since it +doesn't need the agent running. Swarm-side, `GET /api/agents//icon` serves the stored bytes, and 404 means the agent has no icon. swarm-ui's agent cards load it as an `` and show the dimmed default agent icon (a robot head in a diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index 0d69268a5..b93cf0af4 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -623,6 +623,7 @@ async fn handle_meta( cached, private_users, nix_builds, + .. } => tracing::info!( agent = %job.agent, meta_rev = %job.meta_rev, @@ -864,6 +865,7 @@ mod tests { cached: false, private_users: Some(false), nix_builds: Some(hive_types::NixBuilds::Host), + icon: None, } }, ) @@ -887,6 +889,7 @@ mod tests { cached: false, private_users: Some(false), nix_builds: Some(hive_types::NixBuilds::Host), + icon: None, } }, ) @@ -1473,9 +1476,9 @@ mod tests { "nix_builds": "host", }) ); - assert_eq!(fake.lines("tokens"), ["forge-token"; 3]); + assert_eq!(fake.lines("tokens"), ["forge-token"; 4]); let calls = fake.calls(); - assert_eq!(calls.len(), 3, "{calls:?}"); + assert_eq!(calls.len(), 4, "{calls:?}"); assert!( calls[0].contains(&format!( "git+https://forge.example/internal/meta.git?rev={REV}#nixosConfigurations.atlas.config.system.build.toplevel" @@ -1490,6 +1493,11 @@ mod tests { calls[2].contains("services.hyperhive.agent.nixBuilds"), "{calls:?}" ); + // The icon is read as the text of the file the option names. + assert!( + calls[3].contains("services.hyperhive.agent.icon") && calls[3].contains("--apply"), + "{calls:?}" + ); } /// An option the module does not declare is false; one whose eval failed, or @@ -1605,7 +1613,7 @@ mod tests { ]; serve_meta(fake.builder(), env, msgs).await; assert_eq!(next(&mut sent).await.0, "_INBOX.2"); - assert_eq!(fake.calls().len(), 3, "one build and its two option reads"); + assert_eq!(fake.calls().len(), 4, "one build and its three option reads"); } #[tokio::test(start_paused = true)] diff --git a/swarm-builder/src/nix_build.rs b/swarm-builder/src/nix_build.rs index 6ed4e70e7..aba96310a 100644 --- a/swarm-builder/src/nix_build.rs +++ b/swarm-builder/src/nix_build.rs @@ -21,7 +21,7 @@ use std::time::Duration; use crate::hyperhive_flake::HyperhiveFlake; use hive_types::NixBuilds; use swarm_wire::builder::{ - BuildAgent, BuildFromMeta, FromMetaReply, LockMeta, LockReply, META_REPO_PATH, Reply, + BuildAgent, BuildFromMeta, FromMetaReply, IconRead, LockMeta, LockReply, META_REPO_PATH, Reply, }; use tokio::io::AsyncReadExt as _; @@ -96,6 +96,7 @@ impl Builder { cached: false, private_users: self.private_users(job, token).await, nix_builds: self.nix_builds(job, token).await, + icon: self.icon(job, token).await, }, Reply::Failed { error_tail } => FromMetaReply::Failed { error_tail }, }, @@ -189,9 +190,23 @@ impl Builder { /// sandbox against the flake the build just evaluated. Logs why when the /// answer is not a value. async fn eval_option(&self, job: &BuildFromMeta, token: &str, option: &str) -> Eval { + self.eval_option_applied(job, token, option, None).await + } + + /// [`Self::eval_option`] with `apply`, a nix function, applied to the value. + async fn eval_option_applied( + &self, + job: &BuildFromMeta, + token: &str, + option: &str, + apply: Option<&str>, + ) -> Eval { let mut cmd = self.nix_command(token); - cmd.args(EVAL_FLAGS) - .arg(format!("{}.{option}", self.meta_config(job))); + cmd.args(EVAL_FLAGS); + if let Some(apply) = apply { + cmd.args(["--apply", apply]); + } + cmd.arg(format!("{}.{option}", self.meta_config(job))); match run_raw(cmd, self.deadline).await { Ok((status, out, _)) if status.success() => Eval::Value(out), Ok((_, _, err)) if attribute_missing(&String::from_utf8_lossy(&err)) => { @@ -232,6 +247,22 @@ impl Builder { } } + /// The agent's `services.hyperhive.agent.icon`: the text of the SVG file the + /// option names, read by the eval itself so the builder needs no store + /// path. An option that is null, or that the agent's module does not declare, + /// is [`IconRead::Unset`]; a failed eval, a timeout, an output that is no + /// string and a file over [`ICON_MAX_BYTES`] are `None`. + async fn icon(&self, job: &BuildFromMeta, token: &str) -> Option { + match self + .eval_option_applied(job, token, ICON_OPTION, Some(ICON_APPLY)) + .await + { + Eval::Value(out) => parse_icon(&out), + Eval::Missing => Some(IconRead::Unset), + Eval::Unknown => None, + } + } + /// The config commit as a flake reference, fetched through the PR's head /// ref. fn agent_url(&self, job: &BuildAgent) -> String { @@ -273,6 +304,16 @@ const PRIVATE_USERS_OPTION: &str = "services.hyperhive.agent.user.privateUsers"; /// The agent option holding where its nix builds run. const NIX_BUILDS_OPTION: &str = "services.hyperhive.agent.nixBuilds"; +/// The agent option naming its icon, an SVG file. +const ICON_OPTION: &str = "services.hyperhive.agent.icon"; + +/// Applied to the icon option: the file's text, or null when none is set. +const ICON_APPLY: &str = "p: if p == null then null else builtins.readFile p"; + +/// The largest icon the reply carries, in bytes: a reply is one queue message, +/// and an icon is a small drawing. A larger file is left out as unread. +const ICON_MAX_BYTES: usize = 256 * 1024; + /// What `nix eval --json` of an agent option came to. enum Eval { /// It printed a value. @@ -298,6 +339,24 @@ fn parse_flag(out: &[u8]) -> Option { } } +/// `nix eval --json` of the icon option after [`ICON_APPLY`]: null is unset, a +/// string is the SVG's text, and anything else, an empty or an oversize file +/// included, is `None`. +fn parse_icon(out: &[u8]) -> Option { + match serde_json::from_slice::>(out).ok()? { + None => Some(IconRead::Unset), + Some(svg) if svg.is_empty() => None, + Some(svg) if svg.len() > ICON_MAX_BYTES => { + tracing::warn!( + bytes = svg.len(), + "the agent's icon is too large; not sending it" + ); + None + } + Some(svg) => Some(IconRead::Set { svg }), + } +} + /// `nix eval --json` of a [`NixBuilds`]: the spelling, or `None` when the /// output is not a string. fn parse_nix_builds(out: &[u8]) -> Option { @@ -660,6 +719,56 @@ mod tests { } } + /// The reply carries the agent's icon: the file's text for a set option, `Unset` + /// for a null one or a module that predates it, and nothing for an eval that + /// failed or printed what no icon is. The eval is asked to read the file, which + /// is what the `--apply` check pins. + #[tokio::test] + async fn a_meta_build_reports_the_agents_icon() { + for (i, (answer, want)) in [ + ( + "echo '\"\"'", + Some(IconRead::Set { + svg: "".to_owned(), + }), + ), + ("echo null", Some(IconRead::Unset)), + (ATTRIBUTE_MISSING, Some(IconRead::Unset)), + ("echo '\"\"'", None), + ("echo 5", None), + ("echo error: boom >&2; exit 1", None), + ] + .into_iter() + .enumerate() + { + let nix = fake_nix( + &format!("icon-{i}"), + &format!( + "for a in \"$@\"; do last=$a; done; case \"$last\" in *agent.icon) case \"$*\" in *--apply*) {answer};; *) echo 'no apply' >&2; exit 1;; esac;; *) if [ \"$1\" = eval ]; then echo false; else echo /nix/store/x; fi;; esac\n" + ), + ); + let b = Builder { nix, ..builder() }; + let reply = b.build_from_meta(&meta_job(), "t").await; + assert!( + matches!(&reply, FromMetaReply::Built { icon, .. } if *icon == want), + "{answer}: {reply:?}" + ); + } + } + + /// An icon file over the cap is left out of the reply rather than sent: the + /// reply is one queue message. + #[test] + fn an_icon_over_the_cap_is_not_sent() { + let big = format!("\"{}\"", "a".repeat(ICON_MAX_BYTES + 1)); + assert_eq!(parse_icon(big.as_bytes()), None); + let fits = format!("\"{}\"", "a".repeat(ICON_MAX_BYTES)); + assert!(matches!( + parse_icon(fits.as_bytes()), + Some(IconRead::Set { .. }) + )); + } + /// An eval that does not finish in time is unknown for both options, not /// their defaults. The build itself finishes, which is the control. #[tokio::test] @@ -741,6 +850,7 @@ mod tests { cached: false, private_users: None, nix_builds: None, + icon: None, } ); } diff --git a/swarm-builder/src/push.rs b/swarm-builder/src/push.rs index e0a88a196..911bc0e4f 100644 --- a/swarm-builder/src/push.rs +++ b/swarm-builder/src/push.rs @@ -212,6 +212,7 @@ pub async fn cache_built( system, private_users, nix_builds, + icon, .. }, ) = (cache, &reply) @@ -240,6 +241,7 @@ pub async fn cache_built( cached: true, private_users: *private_users, nix_builds: *nix_builds, + icon: icon.clone(), } } Err(reason) => { @@ -268,6 +270,7 @@ mod tests { cached: false, private_users: None, nix_builds: Some(NixBuilds::Disabled), + icon: Some(swarm_wire::builder::IconRead::Unset), } } @@ -306,6 +309,7 @@ mod tests { FromMetaReply::Built { private_users: None, nix_builds: Some(NixBuilds::Disabled), + icon: Some(swarm_wire::builder::IconRead::Unset), .. } ), diff --git a/swarm-controller/src/cache_miss.rs b/swarm-controller/src/cache_miss.rs index ce050741e..e2dbe8d0c 100644 --- a/swarm-controller/src/cache_miss.rs +++ b/swarm-controller/src/cache_miss.rs @@ -317,6 +317,7 @@ async fn rebuild(world: &impl World, agent: &str, stored: Stored) -> Outcome { cached, private_users, nix_builds, + .. } => (out_path, system, cached, private_users, nix_builds), FromMetaReply::Failed { error_tail } => { tracing::warn!(%agent, %meta_rev, %error_tail, "cache miss: the rebuild failed"); @@ -684,6 +685,7 @@ mod tests { cached, private_users: Some(false), nix_builds: Some(NixBuilds::Host), + icon: None, } } @@ -1028,6 +1030,7 @@ mod tests { cached, private_users: Some(true), nix_builds: Some(NixBuilds::Disabled), + icon: None, }; let world = Fake::missing(&["iris"]).answering("iris", answer); pass_at(&world, &mut Backoff::default(), Instant::now()).await; diff --git a/swarm-controller/src/meta_build.rs b/swarm-controller/src/meta_build.rs index 8d5c1a53e..a28631854 100644 --- a/swarm-controller/src/meta_build.rs +++ b/swarm-controller/src/meta_build.rs @@ -34,7 +34,7 @@ use std::future::Future; use anyhow::{Context as _, Result}; use hive_jobq::scheduler::Outcome; use swarm_wire::agent_closures::AgentClosure; -use swarm_wire::builder::{BuildFromMeta, FromMetaReply, META_SUBJECT, REPLY_DEADLINE}; +use swarm_wire::builder::{BuildFromMeta, FromMetaReply, IconRead, META_SUBJECT, REPLY_DEADLINE}; use crate::agent_renewal::unix_now; use crate::builder::runs::{BuildOutcome, BuildRecord, Builds}; @@ -140,6 +140,8 @@ pub(crate) trait MetaBuilds { agent: &str, closure: &AgentClosure, ) -> impl Future> + Send; + /// Sets `agent`'s icon in the swarm, or removes it. + fn record_icon(&self, agent: &str, icon: &IconRead) -> impl Future> + Send; } impl MetaBuilds for async_nats::Client { @@ -177,6 +179,23 @@ impl MetaBuilds for async_nats::Client { .map(|_| ()) .with_context(|| format!("writing agent-closures/{agent}")) } + + async fn record_icon(&self, agent: &str, icon: &IconRead) -> Result<()> { + let store = swarm_queue_client::agent_icon::open_or_create(self) + .await + .context("opening the agent-icons bucket")?; + match icon { + IconRead::Set { svg } => store + .put(agent, svg.clone().into_bytes().into()) + .await + .map(|_| ()) + .with_context(|| format!("writing agent-icons/{agent}")), + IconRead::Unset => store + .delete(agent) + .await + .with_context(|| format!("deleting agent-icons/{agent}")), + } + } } /// The most of `nix`'s tail a PR comment carries, in bytes: the builder's own cap @@ -374,7 +393,16 @@ pub(crate) async fn run( cached, private_users, nix_builds, + icon, }) => { + // The icon is the agent's config, so it follows the build, not the + // cache push or the record below. A write that fails is logged and + // leaves the deploy alone: the next build writes it again. + if let Some(icon) = &icon + && let Err(e) = queue.record_icon(&agent, icon).await + { + tracing::warn!(%agent, error = %format!("{e:#}"), "build_meta: the agent's icon was not recorded"); + } let built_rev = meta_rev.clone(); let closure = AgentClosure { meta_rev, @@ -480,6 +508,8 @@ mod tests { record_fails: bool, sent: Mutex>, recorded: Mutex>, + icon_fails: bool, + icons: Mutex>, } impl FakeQueue { @@ -489,6 +519,8 @@ mod tests { record_fails: false, sent: Mutex::new(Vec::new()), recorded: Mutex::new(Vec::new()), + icon_fails: false, + icons: Mutex::new(Vec::new()), } } @@ -511,6 +543,18 @@ mod tests { cached, private_users, nix_builds, + icon: None, + })) + } + + fn built_with_icon(icon: Option) -> Self { + Self::answering(Ok(FromMetaReply::Built { + out_path: "/nix/store/x-toplevel".to_owned(), + system: "x86_64-linux".to_owned(), + cached: true, + private_users: Some(false), + nix_builds: Some(NixBuilds::Host), + icon, })) } } @@ -532,6 +576,15 @@ mod tests { .push((agent.to_owned(), closure.clone())); Ok(()) } + + async fn record_icon(&self, agent: &str, icon: &IconRead) -> Result<()> { + anyhow::ensure!(!self.icon_fails, "the bucket refused the icon"); + self.icons + .lock() + .unwrap() + .push((agent.to_owned(), icon.clone())); + Ok(()) + } } async fn run_with(head: &FakeHead, queue: &FakeQueue) -> Outcome { @@ -668,6 +721,55 @@ mod tests { } } + /// The agent's icon, as the builder read it from the config, is written to the + /// swarm: a file sets it, an unset option removes it, and an option the builder + /// could not read leaves it as it is. + #[tokio::test] + async fn the_icon_the_builder_read_is_set_removed_or_left() { + let svg = IconRead::Set { + svg: "".to_owned(), + }; + for (read, written) in [ + (Some(svg.clone()), vec![("atlas".to_owned(), svg)]), + ( + Some(IconRead::Unset), + vec![("atlas".to_owned(), IconRead::Unset)], + ), + (None, vec![]), + ] { + let queue = FakeQueue::built_with_icon(read); + assert_eq!( + run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await, + Outcome::Done + ); + assert_eq!(*queue.icons.lock().unwrap(), written); + } + } + + /// The icon is not what the deploy waits on: a write the queue refuses + /// leaves the build green. + #[tokio::test] + async fn an_icon_that_cannot_be_written_does_not_fail_the_build() { + let queue = FakeQueue { + icon_fails: true, + ..FakeQueue::built_with_icon(Some(IconRead::Unset)) + }; + assert_eq!( + run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await, + Outcome::Done + ); + assert!(queue.icons.lock().unwrap().is_empty()); + assert_eq!(queue.recorded.lock().unwrap().len(), 1, "still recorded"); + } + + /// A build that fails writes no icon. + #[tokio::test] + async fn a_failed_build_writes_no_icon() { + let queue = nix_failed("error: x"); + let _ = run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await; + assert!(queue.icons.lock().unwrap().is_empty()); + } + /// Any `nix` failure, a timeout included, fails the node at once with what /// the builder said: not a no-verdict failure, and nothing is recorded. The /// green test above is the control that a record is written when the build diff --git a/swarm-wire/src/builder.rs b/swarm-wire/src/builder.rs index 0c2c7acc5..5bdb8498c 100644 --- a/swarm-wire/src/builder.rs +++ b/swarm-wire/src/builder.rs @@ -136,6 +136,18 @@ pub struct BuildFromMeta { pub meta_rev: String, } +/// What an agent's `services.hyperhive.agent.icon` came to in its evaluated +/// config. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "kebab-case")] +pub enum IconRead { + /// The option names an SVG file; `svg` is its text. + Set { svg: String }, + /// The option is not set: the agent has no icon, and an icon it had is + /// removed. + Unset, +} + /// The builder's answer to one [`BuildFromMeta`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "state", rename_all = "kebab-case")] @@ -160,6 +172,11 @@ pub enum FromMetaReply { /// when the builder could not read it, as for `private_users`. #[serde(default, skip_serializing_if = "Option::is_none")] nix_builds: Option, + /// The agent's `services.hyperhive.agent.icon`. Absent (`None`) when + /// the builder could not read it, as for `private_users`: the + /// controller then leaves the agent's icon as it is. + #[serde(default, skip_serializing_if = "Option::is_none")] + icon: Option, }, /// The build did not succeed: the last lines `nix` printed, or why it /// stopped (it timed out, or could not start). The controller holds the @@ -275,6 +292,9 @@ mod tests { cached: true, private_users: Some(true), nix_builds: Some(NixBuilds::Disabled), + icon: Some(IconRead::Set { + svg: "".to_owned(), + }), }, json!({ "state": "built", @@ -283,6 +303,24 @@ mod tests { "cached": true, "private_users": true, "nix_builds": "none", + "icon": {"state": "set", "svg": ""}, + }), + ), + ( + FromMetaReply::Built { + out_path: "/nix/store/x".to_owned(), + system: "x86_64-linux".to_owned(), + cached: false, + private_users: None, + nix_builds: None, + icon: Some(IconRead::Unset), + }, + json!({ + "state": "built", + "out_path": "/nix/store/x", + "system": "x86_64-linux", + "cached": false, + "icon": {"state": "unset"}, }), ), ( @@ -317,6 +355,7 @@ mod tests { cached: false, private_users: None, nix_builds: None, + icon: None, } ); } @@ -337,11 +376,13 @@ mod tests { cached: false, private_users: None, nix_builds: Some(NixBuilds::Host), + icon: None, }; let wire = serde_json::to_value(&reply).expect("encodes"); let fields = wire.as_object().expect("an object"); assert!(!fields.contains_key("private_users"), "{wire}"); assert_eq!(fields["nix_builds"], "host"); + assert!(!fields.contains_key("icon"), "{wire}"); assert!( !serde_json::from_value::(wire.clone()) .expect("decodes")