From 51035eac1d597b921c66d81324e2364291fa11f7 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 17 Oct 2026 00:24:08 +0200 Subject: [PATCH 1/3] archetypes: prefill the create form from flake-declared agent archetypes A flake declares agent archetypes in its `hyperhiveArchetypes` output, ` = { module; meta; }`. The hyperhive flake ships `reviewer` (default permissions without `execution`). swarm-builder answers `$SWARM.builder.archetypes` with the `meta` of every archetype its own hyperhive source declares. swarm-controller asks once the queue connects and every 10 minutes, checks each archetype as `POST /api/agents` checks its settings, and stores the result in the `agent-archetypes` KV bucket when it changed. `GET /api/archetypes` serves it, and the create form's archetype select prefills permissions, resource caps and inference from it. The agent created is an ordinary agent with its own config repo; the archetype's module is not applied yet. Refs #5386 --- CLAUDE.md | 6 +- docs/README.md | 2 + docs/swarm/archetypes.md | 87 ++++ docs/swarm/builder.md | 8 + docs/swarm/ui.md | 2 +- flake.nix | 4 + .../src/pages/CreateAgentArchetype.ts | 63 +++ .../swarm-ui/src/pages/CreateAgentForm.css | 5 + .../swarm-ui/src/pages/CreateAgentForm.tsx | 70 +++ .../test/create-agent-archetype.test.mjs | 67 +++ nix/archetypes/default.nix | 20 + nix/module-eval/builder.nix | 27 ++ swarm-builder/README.md | 8 + swarm-builder/src/archetypes.rs | 189 ++++++++ swarm-builder/src/main.rs | 55 ++- swarm-builder/src/nix_build.rs | 2 +- swarm-controller/src/archetypes.rs | 432 ++++++++++++++++++ swarm-controller/src/main.rs | 8 +- swarm-nats-auth/src/policy.rs | 45 +- swarm-queue-client/src/archetypes.rs | 117 +++++ swarm-queue-client/src/builder.rs | 52 ++- swarm-queue-client/src/lib.rs | 5 + 22 files changed, 1244 insertions(+), 30 deletions(-) create mode 100644 docs/swarm/archetypes.md create mode 100644 frontend/packages/swarm-ui/src/pages/CreateAgentArchetype.ts create mode 100644 frontend/packages/swarm-ui/test/create-agent-archetype.test.mjs create mode 100644 nix/archetypes/default.nix create mode 100644 swarm-builder/src/archetypes.rs create mode 100644 swarm-controller/src/archetypes.rs create mode 100644 swarm-queue-client/src/archetypes.rs diff --git a/CLAUDE.md b/CLAUDE.md index e7b57d43..f19648bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,7 +148,8 @@ Swarm-level: - **`swarm-matrix-ctl/`** — the one binary run inside `containers.hive-matrix`; new in-container jobs are verbs here, not crates. -- **`swarm-builder/`** — builds agent config PR heads for commit status. +- **`swarm-builder/`** — builds agent config PR heads for commit status, + and evaluates the agent archetypes the create form lists. Design: [`docs/swarm/builder.md`](docs/swarm/builder.md). - **`swarm-ci-metrics/`** — polls the forge's Actions runs and pushes each finished run's duration and queue wait to the metrics store. Design: @@ -176,7 +177,8 @@ Swarm-level: - **`nix/`** — `host-modules/` (the host stack), `agent-modules/` (per-agent harness features), `container-modules/` (guest side of service containers), `templates/agent.nix` (the container entry - point), `packages/`, `docs/` (options doc), `module-eval/` (NixOS + point), `archetypes/` (the built-in agent archetypes), `packages/`, + `docs/` (options doc), `module-eval/` (NixOS module-eval tests), `builder/` (`check-flake`), `script-tests/`, plus `sources.nix` / `rust.nix` / `checks.nix` / `devshell.nix` / `treefmt.nix` behind the thin `flake.nix`. diff --git a/docs/README.md b/docs/README.md index 47015297..6fc5894d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -91,6 +91,8 @@ declarations. (planned, not implemented yet). - **Does an agent config PR build before it's merged?** → [`swarm/builder.md`](swarm/builder.md). +- **How do I create an agent from a ready-made set of permissions and + settings?** → [`swarm/archetypes.md`](swarm/archetypes.md). - **How long do CI runs take, per repo and PR, and where does the board get it?** → [`swarm/ci-metrics.md`](swarm/ci-metrics.md). - **How do I log an agent in to claude from the swarm UI, and why doesn't a diff --git a/docs/swarm/archetypes.md b/docs/swarm/archetypes.md new file mode 100644 index 00000000..3b58317f --- /dev/null +++ b/docs/swarm/archetypes.md @@ -0,0 +1,87 @@ +# Agent archetypes + +An archetype is a named set of settings to create an agent from. The swarm +UI's create form lists the archetypes the swarm knows. Picking one fills the +form's permissions, resource caps and inference fields from it, and you can +still change each field before you create the agent. + +The agent you create this way is an ordinary agent: it gets its own +`agent-configs/` repo like any other, and the request carries only the +fields as the form shows them. Nothing records which archetype you picked. + +## The built-in archetypes + +The hyperhive flake ships these: + +| archetype | what it sets | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `reviewer` | permissions `messaging`, `meta`, `inbox`: the default set without `execution`. It has no shell, so it can't run `hive-forge`, `git` or any command. | + +## Where an archetype comes from + +A flake declares archetypes in its `hyperhiveArchetypes` output: + +```nix +hyperhiveArchetypes.reviewer = { + module = { }; + meta = { + description = "Reviews what it is sent."; + permissions = [ "messaging" "meta" "inbox" ]; + limits = { memory_max = "4G"; }; + inference = { effort = "high"; }; + }; +}; +``` + +- `meta` is plain data: no derivation, no path, no function. The swarm + evaluates `meta` alone. +- Every archetype needs a `description`. `permissions`, `limits` and + `inference` are optional and take the shapes `POST /api/agents` takes. The swarm ignores + other keys in `meta`. +- `module` is a NixOS module for the agent's container. The swarm doesn't + apply it yet. + +Today the swarm reads one flake: the hyperhive source the +[builder](builder.md)'s host runs, so only the built-ins appear. + +## How the list stays current + +1. swarm-controller asks the builder for the list on the queue subject + `$SWARM.builder.archetypes` once the queue connects, and every 10 minutes + after that. +2. The builder evaluates the `meta` of each `hyperhiveArchetypes` entry on + its own hyperhive and nixpkgs sources, and answers with the JSON or the + last lines `nix` printed: + + ```sh + nix eval --json #hyperhiveArchetypes \ + --apply 'builtins.mapAttrs (_: a: a.meta)' + ``` + +3. The controller checks each archetype the way `POST /api/agents` checks + its settings and stores the result in the `agent-archetypes` KV bucket, + under the key `hyperhive`, when it differs from the stored one. The list + leaves out an archetype that fails the check, and its name and reason go + into the record's `refused` map and the controller's journal. +4. `GET /api/archetypes` answers from that record, and the create form lists + from it. + +A builder host redeploy can change the list. The next pass, within 10 +minutes, picks it up. When no builder answers, the controller keeps the +stored list and logs a warning on each pass. A swarm with no builder enabled +shows no archetype select. + +The stored value: + +```json +{ + "source": "/nix/store/…-hyperhive-flake-source", + "archetypes": { + "reviewer": { + "description": "…", + "permissions": ["messaging", "meta", "inbox"] + } + }, + "refused": { "broken": "limits: …" } +} +``` diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index 6f6273f7..a70b90de 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -61,6 +61,14 @@ Where it runs and how it's enabled: [`services.md`](services.md#builder). push. The jobs graph is in memory, so a controller restart drops a build in flight and leaves its status `pending`. +## Archetype evaluation + +The builder also answers `$SWARM.builder.archetypes`: it evaluates the +`meta` of every agent archetype its own hyperhive source declares, with no +forge token, and answers with the JSON. An evaluation doesn't wait for a +build in progress and has two minutes. What the controller does with the +answer: [`archetypes.md`](archetypes.md). + ## Credentials | What | Where | Who reads it | diff --git a/docs/swarm/ui.md b/docs/swarm/ui.md index 45ea7975..d0a58a76 100644 --- a/docs/swarm/ui.md +++ b/docs/swarm/ui.md @@ -11,7 +11,7 @@ domain, covers host-level detail for one hive. | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `/` | the start page: the swarm's name over a digital-rain backdrop, plus a paperclip offering to set up an agent while no `ruth` exists (it opens the create form with `ruth` filled in); the swarm name in the header links here | | `/hives` | the hive directory, each hive with its last reported status | -| `/agents` | every agent: status, config PR, wanted state, inference settings, resource caps; create agents (with their permissions), link forge, matrix and GitHub accounts | +| `/agents` | every agent: status, config PR, wanted state, inference settings, resource caps; create agents (with their permissions, optionally from an [archetype](archetypes.md)), link forge, matrix and GitHub accounts | | `/agents//terminal` | one agent's live terminal | | `/agents//subagents//terminal` | one of that agent's subagents' live terminal | | `/alerts` | the alerts the swarm's Grafana holds now (firing, pending, no data, error), read through the controller with a Viewer service-account token | diff --git a/flake.nix b/flake.nix index a726e2a5..4a85bc31 100644 --- a/flake.nix +++ b/flake.nix @@ -206,6 +206,10 @@ agent-base = mkContainer self.nixosModules.agent-base; }; + # Agent archetypes, ` = { module; meta; }`. Not per system: `meta` + # is plain data, and `module` takes its `pkgs` from the agent's system. + hyperhiveArchetypes = import ./nix/archetypes; + devShells = forAllSystems ({ pkgs, rust, ... }: import ./nix/devshell.nix { inherit pkgs rust; }); formatter = forAllSystems ({ treefmt-eval, ... }: treefmt-eval.config.build.wrapper); diff --git a/frontend/packages/swarm-ui/src/pages/CreateAgentArchetype.ts b/frontend/packages/swarm-ui/src/pages/CreateAgentArchetype.ts new file mode 100644 index 00000000..6b1edc73 --- /dev/null +++ b/frontend/packages/swarm-ui/src/pages/CreateAgentArchetype.ts @@ -0,0 +1,63 @@ +// The create-agent form's archetype picker: what picking one puts in the +// form. Kept out of the component so `npm test` can run it. +import type { InferenceDraft } from "../agent/detail/AgentInference.js"; + +// Mirrors swarm-controller's `ArchetypeView` (`GET /api/archetypes`). +export interface Archetype { + source: string; + name: string; + description: string; + permissions?: string[]; + limits?: { cpu_quota?: string; memory_max?: string }; + inference?: { + model?: string; + effort?: string; + subagent_models?: string[]; + max_subagents?: number; + }; +} + +// Mirrors swarm-controller's `Archetypes`. +export interface ArchetypeList { + archetypes: Archetype[]; + refused: { source: string; name: string; reason: string }[]; +} + +/** The settings fields a pick fills in. */ +export interface Prefill { + permissions: string[]; + cpu: string; + memory: string; + inference: InferenceDraft; +} + +/** The select's value for `a`; unique across sources. */ +export function archetypeKey(a: Archetype): string { + return `${a.source}/${a.name}`; +} + +/** + * The form's settings for `archetype`, or for no archetype when it is + * `undefined`. A setting the archetype leaves out is the form's empty value, + * and permissions it leaves out are `defaults`. + */ +export function prefill( + archetype: Archetype | undefined, + defaults: string[], +): Prefill { + const inference = archetype?.inference ?? {}; + return { + permissions: archetype?.permissions ?? defaults, + cpu: archetype?.limits?.cpu_quota ?? "", + memory: archetype?.limits?.memory_max ?? "", + inference: { + model: inference.model ?? "", + effort: inference.effort ?? "", + subagents: (inference.subagent_models ?? []).join(", "), + maxSubagents: + inference.max_subagents === undefined + ? "" + : String(inference.max_subagents), + }, + }; +} diff --git a/frontend/packages/swarm-ui/src/pages/CreateAgentForm.css b/frontend/packages/swarm-ui/src/pages/CreateAgentForm.css index 18755318..e6e2ac3a 100644 --- a/frontend/packages/swarm-ui/src/pages/CreateAgentForm.css +++ b/frontend/packages/swarm-ui/src/pages/CreateAgentForm.css @@ -69,3 +69,8 @@ .create-agent-result-ok { color: var(--green); } +/* The picked archetype's description, under its select. */ +.create-agent-archetype-description { + margin: 0; + color: var(--muted); +} diff --git a/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx b/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx index d47cfd73..8eaf846d 100644 --- a/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx +++ b/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx @@ -54,6 +54,11 @@ import { requestPermissions, togglePermission, } from "./CreateAgentPermissions.js"; +import { + archetypeKey, + prefill, + type ArchetypeList, +} from "./CreateAgentArchetype.js"; import "./CreateAgentForm.css"; // Mirrors swarm-controller's `HiveEntry` — same shape `HivesPage` @@ -131,6 +136,16 @@ export function CreateAgentForm({ null, ); const [selected, setSelected] = useState([]); + // `null` until `/api/archetypes` answers; `archetype` is the picked one's + // `archetypeKey`, `""` for none. + const [archetypes, setArchetypes] = useState(null); + const [archetypesError, setArchetypesError] = useState( + null, + ); + const [archetype, setArchetype] = useState(""); + const picked = archetypes?.archetypes.find( + (a) => archetypeKey(a) === archetype, + ); // Loaded once, not re-fetched on submit: the roster changes rarely // enough (a nix-level swarm config change) that staleness within one @@ -165,6 +180,31 @@ export function CreateAgentForm({ })().catch((e: unknown) => setCatalogueError({ detail: String(e) })); }, []); + useEffect(() => { + (async () => { + const r = await fetch("/api/archetypes"); + if (!r.ok) { + setArchetypesError(await readApiError(r)); + return; + } + setArchetypes((await r.json()) as ArchetypeList); + })().catch((e: unknown) => setArchetypesError({ detail: String(e) })); + }, []); + + // Fills permissions, caps and inference from the picked archetype; each + // stays editable, and the request carries only those fields. + function pickArchetype(key: string) { + setArchetype(key); + const p = prefill( + archetypes?.archetypes.find((a) => archetypeKey(a) === key), + catalogue?.default ?? [], + ); + setSelected(p.permissions); + setCpu(p.cpu); + setMemory(p.memory); + setInference(p.inference); + } + const hiveOptions: SelectOption[] = (hives ?? []).map((h) => ({ value: h.name, label: h.name, @@ -199,6 +239,7 @@ export function CreateAgentForm({ const data = (await r.json()) as CreateAgentResponse; setResult({ status: "done", nodeId: data.node_id }); setName(""); + setArchetype(""); if (catalogue) setSelected( initialSelection(catalogue.all, catalogue.default, extraPermissions), @@ -230,6 +271,12 @@ export function CreateAgentForm({ problem={catalogueError} /> )} + {archetypesError && ( + + )}
+ {archetypes && archetypes.archetypes.length > 0 && ( + <> + ({ + value: archetypeKey(a), + label: a.name, + })), + ]} + /> + {picked && ( +

+ {picked.description} +

+ )} + + )}
{catalogue ? ( diff --git a/frontend/packages/swarm-ui/test/create-agent-archetype.test.mjs b/frontend/packages/swarm-ui/test/create-agent-archetype.test.mjs new file mode 100644 index 00000000..fb884822 --- /dev/null +++ b/frontend/packages/swarm-ui/test/create-agent-archetype.test.mjs @@ -0,0 +1,67 @@ +// The create-agent form's archetype picker: what a pick puts in the form. +// Run by `npm test` with Node's type stripping. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { archetypeKey, prefill } from "../src/pages/CreateAgentArchetype.ts"; + +const defaults = ["messaging", "meta", "inbox", "execution"]; + +const empty = { + permissions: defaults, + cpu: "", + memory: "", + inference: { model: "", effort: "", subagents: "", maxSubagents: "" }, +}; + +test("no archetype is the default permissions and empty settings", () => { + assert.deepEqual(prefill(undefined, defaults), empty); +}); + +test("an archetype's settings fill the form", () => { + const reviewer = { + source: "hyperhive", + name: "reviewer", + description: "reviews", + permissions: ["messaging", "meta", "inbox"], + limits: { cpu_quota: "200%", memory_max: "4G" }, + inference: { + model: "opus", + effort: "high", + subagent_models: ["haiku", "sonnet"], + max_subagents: 0, + }, + }; + assert.deepEqual(prefill(reviewer, defaults), { + permissions: ["messaging", "meta", "inbox"], + cpu: "200%", + memory: "4G", + inference: { + model: "opus", + effort: "high", + subagents: "haiku, sonnet", + maxSubagents: "0", + }, + }); +}); + +test("an archetype that sets nothing leaves the form as no archetype does", () => { + const bare = { source: "hyperhive", name: "bare", description: "d" }; + assert.deepEqual(prefill(bare, defaults), empty); +}); + +test("an empty permission list is kept, not replaced by the default", () => { + const none = { + source: "hyperhive", + name: "none", + description: "d", + permissions: [], + }; + assert.deepEqual(prefill(none, defaults).permissions, []); +}); + +test("the key tells sources apart", () => { + assert.notEqual( + archetypeKey({ source: "a", name: "x", description: "" }), + archetypeKey({ source: "b", name: "x", description: "" }), + ); +}); diff --git a/nix/archetypes/default.nix b/nix/archetypes/default.nix new file mode 100644 index 00000000..718b42b6 --- /dev/null +++ b/nix/archetypes/default.nix @@ -0,0 +1,20 @@ +# The archetypes this flake ships, exposed as `hyperhiveArchetypes`. Each is +# `{ module; meta; }`: `module` is a NixOS module for the agent's container, +# `meta` is plain data (no derivations, no `pkgs`), because the swarm +# builder evaluates `meta` alone and swarm-controller stores it as JSON. +# `meta` fields: `description` (required), `permissions`, `limits`, +# `inference`, in the shapes `POST /api/agents` takes them; see +# docs/swarm/archetypes.md. +{ + reviewer = { + module = { }; + meta = { + description = "Reviews what it is sent. No shell: it holds the default permissions minus execution, so it cannot run hive-forge, git or any other command."; + permissions = [ + "messaging" + "meta" + "inbox" + ]; + }; + }; +} diff --git a/nix/module-eval/builder.nix b/nix/module-eval/builder.nix index f26cf4cf..4249d087 100644 --- a/nix/module-eval/builder.nix +++ b/nix/module-eval/builder.nix @@ -121,6 +121,33 @@ let && env.SWARM_BUILDER_CONFIG_ORG == "agent-configs" && env.SWARM_BUILDER_FORGE_URL == "https://${withBuilder.services.hyperhive.swarm.forge.domain}"; } + { + # The builder evaluates `builtins.mapAttrs (_: a: a.meta)` over this + # output and prints it as JSON. A derivation or a path in `meta` would + # make that evaluation build or copy something; a function would not + # print at all. + name = "the built-in archetypes carry a module and a meta of plain data, and reviewer has no shell"; + ok = + let + plain = + v: + if lib.isDerivation v || lib.isFunction v || builtins.isPath v then + false + else if lib.isAttrs v then + lib.all plain (lib.attrValues v) + else if lib.isList v then + lib.all plain v + else + true; + archetypes = self.hyperhiveArchetypes; + metas = builtins.mapAttrs (_: a: a.meta) archetypes; + reviewer = metas.reviewer.permissions; + in + lib.all (a: a ? module && a.meta.description != "") (lib.attrValues archetypes) + && plain metas + && lib.elem "messaging" reviewer + && !(lib.elem "execution" reviewer); + } { # The responder denies a client id it does not know, and a NATS denial # arrives as a timeout. diff --git a/swarm-builder/README.md b/swarm-builder/README.md index 78e9cc38..6892a178 100644 --- a/swarm-builder/README.md +++ b/swarm-builder/README.md @@ -31,3 +31,11 @@ read-only forge token at every build. environment, never in a URL or an argument, and the helper answers only an `https` request for the forge's host. Its store role reads exactly those two paths. + +## Archetypes + +The service also answers archetype evaluations: `nix eval` of the `meta` of +every `hyperhiveArchetypes` entry in this host's hyperhive source, with +nixpkgs pointed at this host's. It fetches nothing and uses no token, and it +doesn't wait for a running build. See +[`docs/swarm/archetypes.md`](../docs/swarm/archetypes.md). diff --git a/swarm-builder/src/archetypes.rs b/swarm-builder/src/archetypes.rs new file mode 100644 index 00000000..76222ade --- /dev/null +++ b/swarm-builder/src/archetypes.rs @@ -0,0 +1,189 @@ +//! One archetype evaluation: `nix eval` of the `meta` of every entry in this +//! host's hyperhive source's `hyperhiveArchetypes` output, as JSON. +//! +//! Only `meta` is evaluated, never `module`: `meta` is plain data, so the +//! evaluation builds nothing. Both inputs are store paths, so it fetches +//! nothing and needs no forge token. + +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use swarm_queue_client::builder::ArchetypesReply; + +/// What every evaluation is run with, fixed at start. +pub struct Evaluator { + /// The `nix` binary. + pub nix: PathBuf, + /// The hyperhive source this host was deployed from. + pub hyperhive: String, + /// The nixpkgs source this host was deployed from, which the hyperhive + /// source's own `nixpkgs` input is pointed at, as a hive's meta flake does. + pub nixpkgs: String, + /// How long one evaluation may run. + pub deadline: Duration, +} + +impl Evaluator { + /// Evaluate the archetypes' `meta`. + pub async fn eval(&self) -> ArchetypesReply { + let mut cmd = tokio::process::Command::new(&self.nix); + cmd.args(self.args()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + run(cmd, &self.hyperhive, self.deadline).await + } + + /// The `nix eval` arguments. + fn args(&self) -> Vec { + [ + "eval", + "--json", + "--no-write-lock-file", + "--option", + "accept-flake-config", + "false", + ] + .into_iter() + .map(str::to_owned) + .chain([ + format!("{}#hyperhiveArchetypes", self.hyperhive), + "--override-input".to_owned(), + "nixpkgs".to_owned(), + self.nixpkgs.clone(), + "--apply".to_owned(), + "builtins.mapAttrs (_: a: a.meta)".to_owned(), + ]) + .collect() + } +} + +/// Run `cmd` to its end or `deadline`, and read the JSON object it printed or +/// the tail of what it said. +async fn run( + mut cmd: tokio::process::Command, + source: &str, + deadline: Duration, +) -> ArchetypesReply { + let failed = |error_tail: String| ArchetypesReply::Failed { error_tail }; + let output = match tokio::time::timeout(deadline, cmd.output()).await { + Err(_) => { + return failed(format!( + "the evaluation did not finish within {} seconds", + deadline.as_secs() + )); + } + Ok(Err(e)) => return failed(format!("starting nix: {e}")), + Ok(Ok(output)) => output, + }; + if !output.status.success() { + return failed(crate::nix_build::tail( + &String::from_utf8_lossy(&output.stderr), + output.status, + )); + } + match serde_json::from_slice::(&output.stdout) { + Ok(serde_json::Value::Object(archetypes)) => ArchetypesReply::Evaluated { + source: source.to_owned(), + archetypes, + }, + Ok(other) => failed(format!( + "hyperhiveArchetypes is not an attribute set of archetypes: {other}" + )), + Err(e) => failed(format!("nix printed no JSON: {e}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn evaluator() -> Evaluator { + Evaluator { + nix: PathBuf::from("nix"), + hyperhive: "/nix/store/h-source".to_owned(), + nixpkgs: "/nix/store/n-source".to_owned(), + deadline: swarm_queue_client::builder::ARCHETYPES_DEADLINE, + } + } + + #[test] + fn only_meta_is_evaluated_on_this_hosts_sources() { + let args = evaluator().args(); + assert!( + args.iter() + .any(|a| a == "/nix/store/h-source#hyperhiveArchetypes") + ); + let overridden: Vec<&str> = args + .windows(2) + .filter(|w| w[0] == "--override-input") + .map(|w| w[1].as_str()) + .collect(); + assert_eq!(overridden, ["nixpkgs"]); + let apply = args + .windows(2) + .find(|w| w[0] == "--apply") + .map(|w| w[1].as_str()); + assert_eq!(apply, Some("builtins.mapAttrs (_: a: a.meta)")); + assert!(args.iter().any(|a| a == "--no-write-lock-file")); + } + + fn sh(script: &str, deadline: Duration) -> impl Future { + let mut cmd = tokio::process::Command::new("sh"); + cmd.args(["-c", script]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + run(cmd, "/nix/store/h-source", deadline) + } + + #[tokio::test] + async fn an_evaluation_replies_with_each_archetypes_meta() { + let reply = sh( + r#"echo '{"reviewer":{"description":"reads"}}'"#, + Duration::from_secs(30), + ) + .await; + let ArchetypesReply::Evaluated { source, archetypes } = reply else { + panic!("{reply:?}") + }; + assert_eq!(source, "/nix/store/h-source"); + assert_eq!(archetypes["reviewer"]["description"], "reads"); + } + + #[tokio::test] + async fn a_failure_replies_with_what_nix_said_last() { + let reply = sh( + "echo 'error: attribute hyperhiveArchetypes missing' >&2; exit 1", + Duration::from_secs(30), + ) + .await; + assert!( + matches!(&reply, ArchetypesReply::Failed { error_tail } if error_tail.ends_with("missing")), + "{reply:?}" + ); + } + + #[tokio::test] + async fn output_that_is_not_an_attribute_set_is_refused() { + for script in ["echo '[1]'", "echo 'not json'"] { + let reply = sh(script, Duration::from_secs(30)).await; + assert!( + matches!(reply, ArchetypesReply::Failed { .. }), + "{script}: {reply:?}" + ); + } + } + + #[tokio::test] + async fn an_evaluation_past_its_deadline_is_stopped() { + let reply = sh("sleep 30", Duration::from_millis(200)).await; + assert!( + matches!(&reply, ArchetypesReply::Failed { error_tail } if error_tail.contains("did not finish")), + "{reply:?}" + ); + } +} diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index 7983b21b..7809e900 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -4,7 +4,9 @@ //! from swarm-controller on [`swarm_queue_client::builder::SUBJECT`], builds //! each agent config at its commit, answers on the request's reply subject, //! and logs one line per build. It writes nothing to the forge: the -//! controller turns the reply into a commit status. +//! controller turns the reply into a commit status. It also answers +//! [`swarm_queue_client::builder::ARCHETYPES_SUBJECT`] with the agent +//! archetypes its hyperhive source declares, which the controller stores. //! //! The store identity comes from the `BAO_*` environment. The queue client //! secret is read at start and again when authelia refuses it, the forge token @@ -15,11 +17,15 @@ use std::sync::Arc; use anyhow::{Context, Result, bail}; use futures_util::StreamExt as _; -use swarm_queue_client::builder::{BUILD_DEADLINE, BuildAgent, QUEUE_GROUP, Reply, SUBJECT}; +use swarm_queue_client::builder::{ + ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, BUILD_DEADLINE, BuildAgent, QUEUE_GROUP, Reply, + SUBJECT, +}; use swarm_queue_client::stored_secret::StoredSecret; use swarm_queue_client::{ClientSecret, QueueConfig}; use swarm_secret_client::forge::Credential; +mod archetypes; mod nix_build; /// The cert-auth role this service logs in to the store as. `swarm-bao.nix` @@ -52,6 +58,12 @@ async fn main() -> Result<()> { client_secret(&client_id) }) .await?; + let evaluator = archetypes::Evaluator { + nix: builder.nix.clone(), + hyperhive: builder.hyperhive.clone(), + nixpkgs: builder.nixpkgs.clone(), + deadline: ARCHETYPES_DEADLINE, + }; let reread_id = client_id.clone(); let secret = StoredSecret::new(client_id.clone(), secret, move || { let client_id = reread_id.clone(); @@ -72,6 +84,12 @@ async fn main() -> Result<()> { .await .with_context(|| format!("subscribing to {SUBJECT}"))?; tracing::info!(subject = SUBJECT, "builder: awaiting builds"); + let evaluations = queue + .queue_subscribe(ARCHETYPES_SUBJECT, QUEUE_GROUP.to_owned()) + .await + .with_context(|| format!("subscribing to {ARCHETYPES_SUBJECT}"))?; + // Outside the build slot below: a list refresh does not wait on a build. + tokio::spawn(answer_evaluations(evaluations, queue.clone(), evaluator)); // One build at a time. swarm-controller already sends one at a time; this // holds the bound for any request it did not serialise. @@ -103,6 +121,39 @@ async fn main() -> Result<()> { bail!("the subscription to {SUBJECT} ended") } +/// Answer every archetype evaluation request, one at a time, with the +/// evaluation's result. +async fn answer_evaluations( + mut requests: async_nats::Subscriber, + queue: async_nats::Client, + evaluator: archetypes::Evaluator, +) { + while let Some(msg) = requests.next().await { + let Some(reply_to) = msg.reply.clone() else { + tracing::warn!("an archetype evaluation arrived with no reply subject; dropping it"); + continue; + }; + let reply = evaluator.eval().await; + match &reply { + swarm_queue_client::builder::ArchetypesReply::Evaluated { archetypes, .. } => { + tracing::info!(count = archetypes.len(), "archetypes evaluated"); + } + swarm_queue_client::builder::ArchetypesReply::Failed { error_tail } => { + tracing::warn!(%error_tail, "archetype evaluation failed"); + } + } + match serde_json::to_vec(&reply) { + Ok(body) => { + if let Err(e) = queue.publish(reply_to, body.into()).await { + tracing::warn!(error = %e, "publishing an evaluation's reply failed"); + } + } + Err(e) => tracing::warn!(error = %e, "encoding an evaluation's reply failed"), + } + } + tracing::error!("the subscription to {ARCHETYPES_SUBJECT} ended"); +} + /// Decode one request, build it and log the build. async fn handle(payload: &[u8], builder: &nix_build::Builder) -> Reply { let job: BuildAgent = match serde_json::from_slice(payload) { diff --git a/swarm-builder/src/nix_build.rs b/swarm-builder/src/nix_build.rs index 23c22507..1fd17015 100644 --- a/swarm-builder/src/nix_build.rs +++ b/swarm-builder/src/nix_build.rs @@ -271,7 +271,7 @@ async fn read_tail( /// The last [`TAIL_LINES`] lines of `stderr`, at most [`TAIL_BYTES`] bytes, /// or the exit status when it said nothing. -fn tail(stderr: &str, status: std::process::ExitStatus) -> String { +pub(crate) fn tail(stderr: &str, status: std::process::ExitStatus) -> String { let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect(); if lines.is_empty() { return format!("nix exited with {status} and printed nothing"); diff --git a/swarm-controller/src/archetypes.rs b/swarm-controller/src/archetypes.rs new file mode 100644 index 00000000..1507e57e --- /dev/null +++ b/swarm-controller/src/archetypes.rs @@ -0,0 +1,432 @@ +//! Agent archetypes: the `agent-archetypes` record this daemon keeps from the +//! swarm builder's evaluation, and `GET /api/archetypes`, which the create +//! screen prefills from. +//! +//! Every [`REFRESH_INTERVAL`], starting once the queue is connected, the pass +//! asks the builder for the `meta` of its hyperhive source's archetypes, checks +//! each one the way `POST /api/agents` checks its settings, and writes the +//! result under [`BUILTIN_SOURCE`] when it differs from what is stored. A +//! failed or unanswered evaluation keeps the stored record. + +use std::time::Duration; + +use axum::Json; +use axum::extract::State; +use hive_sh4re::permissions::Permission; +use problem_details::ProblemDetails; +use serde::{Deserialize, Serialize}; +use swarm_queue_client::archetypes::{ArchetypeMeta, BUILTIN_SOURCE, SourceRecord}; +use swarm_queue_client::builder::{ + ARCHETYPES_DEADLINE, ARCHETYPES_SUBJECT, ArchetypesReply, EvalArchetypes, +}; +use utoipa::ToSchema; + +use crate::schedules::{queue, read_or_default, refused, write}; +use crate::{AppState, agent_limits, agent_permissions, error_problem, inference}; + +/// How often the builder is asked. Its source only changes when its host is +/// redeployed, so this bounds how long a redeploy takes to reach the create +/// screen. +const REFRESH_INTERVAL: Duration = Duration::from_mins(10); + +/// One archetype's `meta` as a source flake declares it. Keys this daemon +/// does not know, `ui` among them, are ignored, so an archetype written for a +/// newer swarm still lists. +#[derive(Deserialize)] +struct DeclaredMeta { + description: String, + #[serde(default)] + permissions: Option>, + #[serde(default)] + limits: Option, + #[serde(default)] + inference: Option, +} + +/// `meta` as stored, or why a create would refuse it. +fn check(name: &str, meta: serde_json::Value) -> Result { + hive_types::Ident::parse(name).map_err(|reason| format!("name: {reason}"))?; + let declared: DeclaredMeta = serde_json::from_value(meta).map_err(|e| e.to_string())?; + if declared.description.trim().is_empty() { + return Err("description is empty".to_owned()); + } + let permissions = match declared.permissions { + Some(perms) => { + agent_permissions::unique(&perms)?; + Some( + perms + .into_iter() + .map(|p| <&str>::from(p).to_owned()) + .collect(), + ) + } + None => None, + }; + Ok(ArchetypeMeta { + description: declared.description, + permissions, + limits: declared + .limits + .map(agent_limits::Caps::into_stored) + .transpose() + .map_err(|e| format!("limits: {e}"))?, + inference: declared + .inference + .map(inference::InferenceSettings::into_stored) + .transpose() + .map_err(|e| format!("inference: {e}"))?, + }) +} + +/// The record to store for one evaluation: every archetype that passes +/// [`check`], and every one that does not with its reason. +fn record(source: String, archetypes: serde_json::Map) -> SourceRecord { + let mut out = SourceRecord { + source, + ..SourceRecord::default() + }; + for (name, meta) in archetypes { + match check(&name, meta) { + Ok(meta) => { + out.archetypes.insert(name, meta); + } + Err(why) => { + out.refused.insert(name, why); + } + } + } + out +} + +/// Ask the builder once and answer what to store, or why there is nothing to. +async fn evaluate(client: &async_nats::Client) -> Result { + use async_nats::RequestErrorKind; + + let body = serde_json::to_vec(&EvalArchetypes {}).map_err(|e| e.to_string())?; + let sent = async_nats::Request::new() + .payload(body.into()) + .timeout(Some(ARCHETYPES_DEADLINE)); + let msg = client + .send_request(ARCHETYPES_SUBJECT, sent) + .await + .map_err(|e| match e.kind() { + RequestErrorKind::NoResponders => "no swarm builder is listening: \ + deploy.builder.enable is set on no host, or its service is not connected to \ + the swarm queue" + .to_owned(), + _ => format!("asking the swarm builder failed: {e}"), + })?; + match serde_json::from_slice(&msg.payload) { + Ok(ArchetypesReply::Evaluated { source, archetypes }) => Ok(record(source, archetypes)), + Ok(ArchetypesReply::Failed { error_tail }) => { + Err(format!("the evaluation failed: {error_tail}")) + } + Err(e) => Err(format!("the builder's reply did not decode: {e}")), + } +} + +/// Store `value` under [`BUILTIN_SOURCE`] unless it is already there. +/// Answers whether it wrote. +async fn store(client: &async_nats::Client, value: &SourceRecord) -> anyhow::Result { + let store = swarm_queue_client::archetypes::open_or_create(client).await?; + let (current, revision, undecodable) = + read_or_default::(&store, BUILTIN_SOURCE).await?; + if current == *value && !undecodable { + return Ok(false); + } + // A lost race is another write of the same evaluation; the next pass + // compares again. + write(&store, BUILTIN_SOURCE, value, revision).await +} + +/// Refresh the record once the queue is connected, then every +/// [`REFRESH_INTERVAL`]. +pub(crate) fn spawn(client: async_nats::Client) { + tokio::spawn(async move { + const CONNECT_POLL: Duration = Duration::from_secs(5); + loop { + while swarm_queue_client::ensure_connected(&client).is_err() { + tokio::time::sleep(CONNECT_POLL).await; + } + refresh(&client).await; + tokio::time::sleep(REFRESH_INTERVAL).await; + } + }); +} + +/// One evaluation, stored when it changed anything. +async fn refresh(client: &async_nats::Client) { + let value = match evaluate(client).await { + Ok(value) => value, + Err(error) => { + tracing::warn!(%error, "archetypes: kept the stored list"); + return; + } + }; + for (name, why) in &value.refused { + tracing::warn!(archetype = %name, %why, "archetypes: refused"); + } + match store(client, &value).await { + Ok(true) => tracing::info!( + count = value.archetypes.len(), + source = %value.source, + "archetypes: stored" + ), + Ok(false) => {} + Err(e) => tracing::warn!(error = %format!("{e:#}"), "archetypes: storing failed"), + } +} + +/// One archetype the create screen can offer, with the settings it prefills. +#[derive(Debug, PartialEq, Eq, Serialize, ToSchema)] +pub(crate) struct ArchetypeView { + /// The source flake's key, `hyperhive` for the built-ins. + source: String, + name: String, + description: String, + /// Permission names; absent leaves the default set. + #[serde(skip_serializing_if = "Option::is_none")] + permissions: Option>, + /// As `PUT /api/agents/{name}/limits` takes them. + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(value_type = Option)] + limits: Option, + /// As `PUT /api/agents/{name}/inference` takes them. + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(value_type = Option)] + inference: Option, +} + +/// One archetype a source declared and this daemon refused. +#[derive(Debug, PartialEq, Eq, Serialize, ToSchema)] +pub(crate) struct RefusedView { + source: String, + name: String, + reason: String, +} + +#[derive(Debug, Default, PartialEq, Eq, Serialize, ToSchema)] +pub(crate) struct Archetypes { + archetypes: Vec, + refused: Vec, +} + +fn view(key: &str, record: SourceRecord) -> Archetypes { + let mut out = Archetypes::default(); + for (name, meta) in record.archetypes { + out.archetypes.push(ArchetypeView { + source: key.to_owned(), + name, + description: meta.description, + permissions: meta.permissions, + limits: meta.limits, + inference: meta.inference, + }); + } + for (name, reason) in record.refused { + out.refused.push(RefusedView { + source: key.to_owned(), + name, + reason, + }); + } + out +} + +#[utoipa::path( + get, + path = "/api/archetypes", + responses( + (status = 200, description = "every archetype the swarm builder's evaluation passed, \ + and every one it refused; both empty before the first evaluation", body = Archetypes), + (status = 503, description = "no swarm queue is configured on this host, or it is \ + not connected (problem+json)", body = String), + (status = 500, description = "the archetypes could not be read (problem+json)", + body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_archetypes( + State(state): State, +) -> Result, ProblemDetails> { + let client = queue(&state).map_err(refused)?; + let failed = |e: anyhow::Error| { + tracing::warn!(error = %format!("{e:#}"), "archetypes: read failed"); + error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &format!("{e:#}"), + ) + }; + let Some(store) = swarm_queue_client::archetypes::open(&client) + .await + .map_err(|e| failed(e.into()))? + else { + return Ok(Json(Archetypes::default())); + }; + let (record, _, _) = read_or_default::(&store, BUILTIN_SOURCE) + .await + .map_err(failed)?; + Ok(Json(view(BUILTIN_SOURCE, record))) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn evaluated(archetypes: serde_json::Value) -> SourceRecord { + let serde_json::Value::Object(map) = archetypes else { + panic!("an object") + }; + record("/nix/store/h".to_owned(), map) + } + + #[test] + fn a_valid_archetype_is_stored_with_its_settings() { + let r = evaluated(json!({ + "reviewer": { + "description": "reviews", + "permissions": ["messaging", "meta", "inbox"], + "limits": { "memory_max": "4G" }, + "inference": { "effort": "high" }, + "ui": { "anything": true } + } + })); + assert!(r.refused.is_empty(), "{:?}", r.refused); + let meta = &r.archetypes["reviewer"]; + assert_eq!(meta.description, "reviews"); + assert_eq!( + meta.permissions.as_deref(), + Some( + &[ + "messaging".to_owned(), + "meta".to_owned(), + "inbox".to_owned() + ][..] + ) + ); + assert_eq!( + meta.limits.as_ref().and_then(|l| l.memory_max.as_deref()), + Some("4G") + ); + assert!(meta.inference.as_ref().is_some_and(|i| i.effort.is_some())); + } + + #[test] + fn a_bad_archetype_is_refused_by_name_and_the_rest_stay() { + let r = evaluated(json!({ + "ok": { "description": "fine" }, + "no-description": { "description": " " }, + "bad-permission": { "description": "x", "permissions": ["root"] }, + "twice": { "description": "x", "permissions": ["meta", "meta"] }, + "bad-limits": { "description": "x", "limits": { "cpu_quota": "lots" } }, + "typo-limits": { "description": "x", "limits": { "memory": "4G" } }, + "bad-inference": { "description": "x", "inference": { "effort": "huge" } }, + "Bad_Name": { "description": "x" } + })); + assert_eq!(r.archetypes.keys().collect::>(), ["ok"]); + let refused: Vec<&str> = r.refused.keys().map(String::as_str).collect(); + assert_eq!( + refused, + [ + "Bad_Name", + "bad-inference", + "bad-limits", + "bad-permission", + "no-description", + "twice", + "typo-limits" + ] + ); + assert!(r.refused["bad-limits"].starts_with("limits: ")); + assert!(r.refused["twice"].contains("listed twice")); + } + + /// Against a real server: no builder listening names the option; a + /// listener's evaluation is checked and stored, and storing the same + /// record again writes nothing. + #[tokio::test] + async fn an_evaluation_is_stored_and_rewritten_only_when_it_changes() { + use futures_util::StreamExt as _; + + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + let unanswered = evaluate(&client).await; + assert!( + matches!(&unanswered, Err(e) if e.contains("deploy.builder.enable")), + "{unanswered:?}" + ); + + let builder = nats.client().await; + let mut requests = builder + .queue_subscribe( + ARCHETYPES_SUBJECT, + swarm_queue_client::builder::QUEUE_GROUP.to_owned(), + ) + .await + .expect("subscribe"); + // A message the builder sends itself and receives back proves the + // server processed the subscription before it. + let mut probe = builder + .subscribe("archetypes-test.probe") + .await + .expect("subscribe"); + builder + .publish("archetypes-test.probe", "".into()) + .await + .expect("publish"); + probe.next().await.expect("the probe comes back"); + tokio::spawn(async move { + let msg = requests.next().await.expect("a request"); + let serde_json::Value::Object(archetypes) = json!({ + "reviewer": { "description": "reviews", "permissions": ["meta"] }, + "broken": { "description": "" } + }) else { + unreachable!() + }; + let reply = serde_json::to_vec(&ArchetypesReply::Evaluated { + source: "/nix/store/h".to_owned(), + archetypes, + }) + .expect("encodes"); + builder + .publish(msg.reply.expect("a reply subject"), reply.into()) + .await + .expect("publish"); + }); + let value = evaluate(&client).await.expect("evaluated"); + assert_eq!(value.archetypes.keys().collect::>(), ["reviewer"]); + assert_eq!(value.refused.keys().collect::>(), ["broken"]); + + assert!(store(&client, &value).await.expect("stored")); + assert!(!store(&client, &value).await.expect("compared")); + let changed = SourceRecord { + source: "/nix/store/h2".to_owned(), + ..value + }; + assert!(store(&client, &changed).await.expect("stored")); + let bucket = swarm_queue_client::archetypes::open(&client) + .await + .expect("opens") + .expect("exists"); + let (stored, _, _) = read_or_default::(&bucket, BUILTIN_SOURCE) + .await + .expect("reads"); + assert_eq!(stored, changed); + } + + #[test] + fn the_view_lists_each_archetype_under_its_source() { + let r = evaluated(json!({ + "reviewer": { "description": "reviews" }, + "broken": { "description": "" } + })); + let v = view(BUILTIN_SOURCE, r); + assert_eq!(v.archetypes.len(), 1); + assert_eq!(v.archetypes[0].source, "hyperhive"); + assert_eq!(v.archetypes[0].name, "reviewer"); + assert_eq!(v.archetypes[0].permissions, None); + assert_eq!(v.refused.len(), 1); + assert_eq!(v.refused[0].name, "broken"); + } +} diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 1a2d1a14..987b6422 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -53,6 +53,7 @@ mod agent_renewal; mod agent_state_stream; mod agent_status; mod alerts; +mod archetypes; mod auth; mod builder; mod claude_login; @@ -2988,12 +2989,16 @@ fn spawn_subagent_streams(status: Option<&Arc>, hives: &[H } /// The per-agent background passes: subagent and terminal streams, and -/// schedule keys, and the permissions bucket and its import. +/// schedule keys, and the permissions bucket and its import, and the +/// archetypes agents are created from. /// Lifted out of `main` for `clippy::too_many_lines`. fn spawn_per_agent_upkeep(status: Option<&Arc>, hives: &[HiveEntry]) { spawn_subagent_streams(status, hives); spawn_schedule_reconcile(status, hives); spawn_agent_permissions(status, hives); + if let Some(status) = status { + archetypes::spawn(status.queue_client()); + } } /// Create the agent-permissions bucket and answer hives' imports when a swarm @@ -3624,6 +3629,7 @@ fn build_app(state: AppState) -> axum::Router { agent_permissions::delete_agent_permissions )) .routes(routes!(agent_permissions::get_permission_catalogue)) + .routes(routes!(archetypes::get_archetypes)) .routes(routes!(get_config_prs)) .routes(routes!(create_agent)) .routes(routes!(mint_agent_forge_token)) diff --git a/swarm-nats-auth/src/policy.rs b/swarm-nats-auth/src/policy.rs index 8c73f618..41962bc6 100644 --- a/swarm-nats-auth/src/policy.rs +++ b/swarm-nats-auth/src/policy.rs @@ -263,14 +263,16 @@ impl Policy { } } - /// What the builder may do: hear build requests and answer each on its - /// reply subject, for as long as a build may take. Its publish list is - /// its own inbox, for [`Self::review_classifier_permissions`]' reason. + /// What the builder may do: hear build and archetype evaluation requests + /// and answer each on its reply subject, for as long as a build may take. + /// Its publish list is its own inbox, for + /// [`Self::review_classifier_permissions`]' reason. fn builder_permissions(client_id: &str) -> Permissions { Permissions { publish: vec![Self::inbox(client_id)], subscribe: Some(vec![ swarm_queue_client::builder::SUBJECT.to_owned(), + swarm_queue_client::builder::ARCHETYPES_SUBJECT.to_owned(), Self::inbox(client_id), ]), responses: Some(swarm_queue_client::builder::REPLY_DEADLINE), @@ -643,6 +645,8 @@ impl Policy { swarm_queue_client::review_classifier::SUBJECT.to_owned(), // A build of an agent config, sent as a request the same way. swarm_queue_client::builder::SUBJECT.to_owned(), + // An archetype evaluation, sent as a request the same way. + swarm_queue_client::builder::ARCHETYPES_SUBJECT.to_owned(), // An operator's claude login step, sent as a request the same way. swarm_queue_client::claude_login::SUBJECT.to_owned(), // The wanted-state buckets — one per hive, all created and written @@ -1463,7 +1467,7 @@ mod tests { #[test] fn the_builder_hears_its_subject_and_inbox_only_and_answers_by_reply() { - use swarm_queue_client::builder::{REPLY_DEADLINE, SUBJECT}; + use swarm_queue_client::builder::{ARCHETYPES_SUBJECT, REPLY_DEADLINE, SUBJECT}; use swarm_queue_client::client_inbox_prefix; let p = policy() @@ -1472,6 +1476,7 @@ mod tests { .expect("a configured builder is admitted"); let inbox = client_inbox_prefix(BUILDER); assert!(may_subscribe(&p, SUBJECT)); + assert!(may_subscribe(&p, ARCHETYPES_SUBJECT)); assert!(may_subscribe(&p, &reply(&inbox))); for refused in [ ">".to_owned(), @@ -1492,24 +1497,26 @@ mod tests { #[test] fn only_the_reader_may_send_a_build() { - use swarm_queue_client::builder::SUBJECT; + use swarm_queue_client::builder::{ARCHETYPES_SUBJECT, SUBJECT}; let p = with_classifier().with_builders(vec![BUILDER.to_owned()]); let reader = p.permissions("swarm-controller").expect("the reader"); - assert!(reader.publish.iter().any(|g| covers(g, SUBJECT))); - for (who, grant) in [ - ("a hive", p.permissions("hive-alpha").expect("a hive")), - ("the builder", p.permissions(BUILDER).expect("admitted")), - ( - "the classifier", - p.permissions(CLASSIFIER).expect("admitted"), - ), - ] { - assert!( - !grant.publish.iter().any(|g| covers(g, SUBJECT)), - "{who} may send a build: {:?}", - grant.publish - ); + for subject in [SUBJECT, ARCHETYPES_SUBJECT] { + assert!(reader.publish.iter().any(|g| covers(g, subject))); + for (who, grant) in [ + ("a hive", p.permissions("hive-alpha").expect("a hive")), + ("the builder", p.permissions(BUILDER).expect("admitted")), + ( + "the classifier", + p.permissions(CLASSIFIER).expect("admitted"), + ), + ] { + assert!( + !grant.publish.iter().any(|g| covers(g, subject)), + "{who} may send to {subject}: {:?}", + grant.publish + ); + } } } diff --git a/swarm-queue-client/src/archetypes.rs b/swarm-queue-client/src/archetypes.rs new file mode 100644 index 00000000..519a9a4e --- /dev/null +++ b/swarm-queue-client/src/archetypes.rs @@ -0,0 +1,117 @@ +//! The agent archetype KV bucket: one key per archetype source flake, holding +//! the `meta` of every archetype that flake's `hyperhiveArchetypes` output +//! declares, as swarm-controller last checked it. +//! +//! **swarm-controller is the only writer and the only reader.** The swarm +//! builder evaluates the source and answers on +//! [`crate::builder::ARCHETYPES_SUBJECT`]; the controller checks each +//! archetype's settings the way `POST /api/agents` checks them and writes the +//! result here, so the create screen lists archetypes from a value that a +//! create would accept. + +use std::collections::BTreeMap; + +use crate::agent_inference::AgentInference; +use crate::agent_limits::AgentCaps; + +/// The KV bucket archetype sources are published into, keyed by source name. +pub const BUCKET: &str = "agent-archetypes"; + +/// The key of the archetypes the hyperhive flake itself ships, the one source +/// there is so far. +pub const BUILTIN_SOURCE: &str = "hyperhive"; + +/// One source's archetypes: the value under `BUCKET`/``. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SourceRecord { + /// The flake reference the builder evaluated. + pub source: String, + /// Every archetype whose `meta` passed the controller's checks, by name. + #[serde(default)] + pub archetypes: BTreeMap, + /// Every archetype whose `meta` did not, by name, with the reason. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub refused: BTreeMap, +} + +/// One archetype's `meta`, as checked: the settings a create takes, in the +/// shapes the per-agent buckets store them in. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ArchetypeMeta { + pub description: String, + /// Permission names, `snake_case`. Absent leaves the default set. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub permissions: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limits: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inference: Option, +} + +/// Open the archetype bucket for writing, creating it if nothing has yet. +/// +/// **Controller-side only.** `history: 1`: a reader wants the current value. +#[cfg(feature = "kv")] +pub async fn open_or_create( + client: &async_nats::Client, +) -> Result { + let js = async_nats::jetstream::new(client.clone()); + crate::kv::open_or_create( + &js, + BUCKET, + "agent-archetypes", + "Agent archetypes the swarm builder evaluated, one key per source flake".to_owned(), + ) + .await +} + +/// Open the archetype bucket for reading, or `None` when nothing has created +/// it yet, which reads the same as no archetypes. +/// +/// # Errors +/// +/// [`crate::Error::OpenBucket`] for every failure other than the bucket not +/// existing. +#[cfg(feature = "kv")] +pub async fn open( + client: &async_nats::Client, +) -> Result, crate::Error> { + let js = async_nats::jetstream::new(client.clone()); + match js.get_key_value(BUCKET).await { + Ok(store) => Ok(Some(store)), + Err(e) if crate::wanted::is_bucket_not_found(&e) => Ok(None), + Err(source) => Err(crate::Error::OpenBucket { + bucket: BUCKET.to_owned(), + source, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An unset setting is left out on the wire, and so is an empty refusal + /// list. + #[test] + fn unset_fields_are_absent() { + let mut archetypes = BTreeMap::new(); + archetypes.insert( + "reviewer".to_owned(), + ArchetypeMeta { + description: "reads".to_owned(), + permissions: Some(vec!["messaging".to_owned()]), + ..ArchetypeMeta::default() + }, + ); + let record = SourceRecord { + source: "/nix/store/h".to_owned(), + archetypes, + refused: BTreeMap::new(), + }; + assert_eq!( + serde_json::to_string(&record).unwrap(), + r#"{"source":"/nix/store/h","archetypes":{"reviewer":{"description":"reads","permissions":["messaging"]}}}"# + ); + } +} diff --git a/swarm-queue-client/src/builder.rs b/swarm-queue-client/src/builder.rs index 3a35d402..801b4971 100644 --- a/swarm-queue-client/src/builder.rs +++ b/swarm-queue-client/src/builder.rs @@ -1,8 +1,8 @@ -//! The swarm builder's builds: the subject swarm-controller sends a build on, -//! and what travels each way. +//! The swarm builder's requests: the subjects swarm-controller sends a build +//! and an archetype evaluation on, and what travels each way. //! -//! Three crates name these: the controller sends the request, the builder -//! answers it, and the auth-callout responder grants the one subject to each. +//! Three crates name these: the controller sends the requests, the builder +//! answers them, and the auth-callout responder grants the subjects to each. use std::time::Duration; @@ -49,6 +49,35 @@ pub enum Reply { Failed { error_tail: String }, } +/// The subject an archetype evaluation goes to: the controller asks, the +/// builder evaluates the `meta` of every `hyperhiveArchetypes` entry of its own +/// hyperhive source and answers on the request's reply subject. +pub const ARCHETYPES_SUBJECT: &str = "$SWARM.builder.archetypes"; + +/// The longest one archetype evaluation may take, and how long the controller +/// waits for its reply. +pub const ARCHETYPES_DEADLINE: Duration = Duration::from_mins(2); + +/// One archetype evaluation. No field yet: the builder evaluates the hyperhive +/// source it was deployed with. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct EvalArchetypes {} + +/// The builder's answer to one [`EvalArchetypes`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "kebab-case")] +pub enum ArchetypesReply { + /// `source` is the flake evaluated; `archetypes` maps each archetype's + /// name to its `meta`, as `nix eval --json` printed it and unchecked. + Evaluated { + source: String, + archetypes: serde_json::Map, + }, + /// The evaluation did not finish: the last lines `nix` printed, or why the + /// builder could not start it. + Failed { error_tail: String }, +} + #[cfg(test)] mod tests { use super::*; @@ -72,4 +101,19 @@ mod tests { fn the_reply_deadline_outlasts_the_build() { assert!(REPLY_DEADLINE > BUILD_DEADLINE); } + + #[test] + fn the_archetypes_reply_is_tagged_by_state() { + let json = serde_json::to_value(ArchetypesReply::Evaluated { + source: "/nix/store/h".to_owned(), + archetypes: serde_json::Map::new(), + }) + .expect("encodes"); + assert_eq!(json["state"], "evaluated"); + let json = serde_json::to_value(ArchetypesReply::Failed { + error_tail: "e".to_owned(), + }) + .expect("encodes"); + assert_eq!(json["state"], "failed"); + } } diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index 00d8979c..3e31d601 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -231,6 +231,11 @@ pub mod agent_permissions; /// on [`agent_state_subject`] only. pub mod agent_login; +/// The bucket swarm-controller keeps the evaluated agent archetypes in — one +/// key per source flake. The names and the stored type are unconditional; +/// opening needs `kv`. +pub mod archetypes; + #[cfg(feature = "kv")] mod kv; From 0382c65386c8336789a5013bc727bd558e5281f9 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 17 Oct 2026 00:38:13 +0200 Subject: [PATCH 2/3] archetypes: reviewer holds execution so it can run hive-forge and git The reviewer now has messaging, inbox and execution. execution is what gives an agent the bash server, without which it cannot run hive-forge or git to read a PR and post a review. meta is dropped: get_agent_meta is not part of reviewing a PR. Refs #5386 --- docs/swarm/archetypes.md | 10 +++++----- nix/archetypes/default.nix | 4 ++-- nix/module-eval/builder.nix | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/swarm/archetypes.md b/docs/swarm/archetypes.md index 3b58317f..c9708a6c 100644 --- a/docs/swarm/archetypes.md +++ b/docs/swarm/archetypes.md @@ -13,9 +13,9 @@ fields as the form shows them. Nothing records which archetype you picked. The hyperhive flake ships these: -| archetype | what it sets | -| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `reviewer` | permissions `messaging`, `meta`, `inbox`: the default set without `execution`. It has no shell, so it can't run `hive-forge`, `git` or any command. | +| archetype | what it sets | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `reviewer` | permissions `messaging`, `inbox`, `execution`: the default set without `meta`. With `execution` it can run `hive-forge`, `git` and other commands. | ## Where an archetype comes from @@ -26,7 +26,7 @@ hyperhiveArchetypes.reviewer = { module = { }; meta = { description = "Reviews what it is sent."; - permissions = [ "messaging" "meta" "inbox" ]; + permissions = [ "messaging" "inbox" "execution" ]; limits = { memory_max = "4G"; }; inference = { effort = "high"; }; }; @@ -79,7 +79,7 @@ The stored value: "archetypes": { "reviewer": { "description": "…", - "permissions": ["messaging", "meta", "inbox"] + "permissions": ["messaging", "inbox", "execution"] } }, "refused": { "broken": "limits: …" } diff --git a/nix/archetypes/default.nix b/nix/archetypes/default.nix index 718b42b6..fbde8638 100644 --- a/nix/archetypes/default.nix +++ b/nix/archetypes/default.nix @@ -9,11 +9,11 @@ reviewer = { module = { }; meta = { - description = "Reviews what it is sent. No shell: it holds the default permissions minus execution, so it cannot run hive-forge, git or any other command."; + description = "Reviews pull requests and replies. It can run hive-forge and git (execution), and holds messaging and inbox; it has no get_agent_meta (meta)."; permissions = [ "messaging" - "meta" "inbox" + "execution" ]; }; }; diff --git a/nix/module-eval/builder.nix b/nix/module-eval/builder.nix index 4249d087..42b2aa87 100644 --- a/nix/module-eval/builder.nix +++ b/nix/module-eval/builder.nix @@ -126,7 +126,7 @@ let # output and prints it as JSON. A derivation or a path in `meta` would # make that evaluation build or copy something; a function would not # print at all. - name = "the built-in archetypes carry a module and a meta of plain data, and reviewer has no shell"; + name = "the built-in archetypes carry a module and a meta of plain data, and reviewer can run commands"; ok = let plain = @@ -146,7 +146,9 @@ let lib.all (a: a ? module && a.meta.description != "") (lib.attrValues archetypes) && plain metas && lib.elem "messaging" reviewer - && !(lib.elem "execution" reviewer); + && lib.elem "inbox" reviewer + && lib.elem "execution" reviewer + && !(lib.elem "meta" reviewer); } { # The responder denies a client id it does not know, and a NATS denial From a2b4320cc4197ffdbdb7c7799eb2265124fb281d Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 17 Oct 2026 00:41:01 +0200 Subject: [PATCH 3/3] create form: reset caps and inference with the archetype after a create A successful create left cpu, memory and inference at the picked archetype's values under a select reading none. The reset now goes through pickArchetype(""), the path picking none takes. Refs #5386 --- frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx b/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx index 8eaf846d..e9734be3 100644 --- a/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx +++ b/frontend/packages/swarm-ui/src/pages/CreateAgentForm.tsx @@ -239,7 +239,7 @@ export function CreateAgentForm({ const data = (await r.json()) as CreateAgentResponse; setResult({ status: "done", nodeId: data.node_id }); setName(""); - setArchetype(""); + pickArchetype(""); if (catalogue) setSelected( initialSelection(catalogue.all, catalogue.default, extraPermissions),