archetypes: create an agent from an archetype with no config repo

POST /api/agents takes an archetype the stored list holds, records it in
the new agent-archetype bucket, and queues no CreateRepo, AddRepoMember or
InitAgentConfigRepo for it. A destroy clears the key. Hives may read the
bucket.

Refs #5386

swarm-controller/src/main.rs

@@ -42,6 +42,7 @@ use swarm_authelia_bridge_sock::BridgeResponse;
 use utoipa::{OpenApi, ToSchema};
 use utoipa_axum::{router::OpenApiRouter, routes};
 
+mod agent_archetype;
 mod agent_control;
 mod agent_icon;
 mod agent_identity;
@@ -1455,12 +1456,13 @@ async fn set_agent_state(
 
     // After the declaration and never before it: the write above is the
     // destroy order, and until it lands the agent is not being torn down, so
-    // its credential and permissions have to keep working. Sequenced rather
-    // than spawned so both attempts are finished — and their outcomes
+    // its credential, permissions and archetype have to keep working. Sequenced
+    // rather than spawned so every attempt is finished — and their outcomes
     // logged — by the time the operator's call returns.
     if revokes_queue_credential(req.state) {
         revoke_or_complain(&hive, &agent).await;
         agent_permissions::clear_on_destroy(&state, &hive, &agent).await;
+        agent_archetype::clear_on_destroy(&state, &hive, &agent).await;
     }
     Ok(Json(render(&declaration)))
 }
@@ -1793,6 +1795,12 @@ struct CreateAgentRequest {
     #[serde(default)]
     #[schema(value_type = Option<Vec<String>>)]
     permissions: Option<Vec<hive_sh4re::permissions::Permission>>,
+    /// The archetype to build the agent from, one `GET /api/archetypes`
+    /// lists. The agent then gets no `agent-configs/<name>` repo: its hive
+    /// builds it from the archetype's module. Absent is a custom agent.
+    #[serde(default)]
+    #[schema(value_type = Option<Object>, example = json!({"source": "hyperhive", "name": "reviewer"}))]
+    archetype: Option<swarm_queue_client::agent_archetype::ArchetypeRef>,
 }
 
 /// Where the queued job landed — a caller polls `/api/jobq/graph` (or
@@ -1827,9 +1835,11 @@ struct CreateAgentResponse {
 /// validation error, not an idempotent no-op. `InitAgentConfigRepo` has no
 /// such second dependency: seeding files uses this daemon's own forge
 /// token, never the agent's, so it only ever needed the repo (mara, design
-/// review: it's a jobq graph, not a linear chain). Returns as soon as the
-/// graph is inserted — **not** once any of it has run; `run_swarm_node`
-/// does that asynchronously off the scheduler loop already running
+/// review: it's a jobq graph, not a linear chain). An agent created from an
+/// `archetype` has no config repo, so its graph has none of the three repo
+/// nodes. Returns as soon as the graph is inserted — **not** once any of it
+/// has run; `run_swarm_node` does that asynchronously off the scheduler loop
+/// already running
 /// (`spawn_jobq_worker`). The response reports `CreateIdentity`'s id, one
 /// of the graph's two roots — a caller watches the whole thing settle via
 /// `/api/jobq/graph`, which serves every root, not just this one.
@@ -1846,10 +1856,10 @@ struct CreateAgentResponse {
     request_body = CreateAgentRequest,
     responses(
         (status = 200, description = "job chain queued", body = CreateAgentResponse),
-        (status = 400, description = "`name` or `hive` is not a valid identifier, `name` is new and reserved or refused by the forge, `hive` is not in this swarm, `limits` or `inference` holds a value the matching `PUT` refuses, or a permission is listed twice (problem+json)", body = String),
+        (status = 400, description = "`name` or `hive` is not a valid identifier, `name` is new and reserved or refused by the forge, `hive` is not in this swarm, `limits` or `inference` holds a value the matching `PUT` refuses, a permission is listed twice, or `archetype` is not one the stored archetype list holds (problem+json)", body = String),
         (status = 409, description = "the swarm has already placed `name` on a different hive (problem+json)", body = String),
         (status = 422, description = "an unknown field in the body, in `limits`, or in `inference`, an `effort` that is not one of the five levels, or a `permissions` entry that is not a permission name"),
-        (status = 503, description = "the swarm queue is not connected, so whether `name` is placed on another hive is unknown, or `limits`, `inference` or `permissions` is set and cannot be stored (problem+json)", body = String),
+        (status = 503, description = "the swarm queue is not connected, so whether `name` is placed on another hive is unknown, or `limits`, `inference`, `permissions` or `archetype` is set and cannot be stored (problem+json)", body = String),
         (status = 500, description = "the job chain could not be queued, or another hive's wanted state could not be read (problem+json)", body = String),
     ),
     tag = "agents"
@@ -1864,7 +1874,7 @@ async fn create_agent(
 
     // Settings are checked before anything is created, so a bad value stops
     // the request rather than leaving an agent without them.
-    let settings = CreateSettings::parse(req.limits, req.inference, req.permissions)
+    let settings = CreateSettings::parse(req.limits, req.inference, req.permissions, req.archetype)
         .map_err(|why| error_problem(axum::http::StatusCode::BAD_REQUEST, &why))?;
 
     // Two checks, and the second is the one that makes the field worth
@@ -1928,11 +1938,7 @@ async fn create_agent(
     }
     // Checked after the roster, so a mistyped hive stays a 400 while the queue
     // is down.
-    let settings_queue = if settings.any() {
-        Some(schedules::queue(&state).map_err(schedules::refused)?)
-    } else {
-        None
-    };
+    let settings_queue = settings.queue(&state).await?;
 
     // Agent names are one swarm-wide namespace: identity, forge user, store
     // path and matrix id all carry the bare name. The same name on the same
@@ -1965,6 +1971,7 @@ async fn create_agent(
                 settings.limits.as_ref(),
                 settings.inference.as_ref(),
                 settings.permissions.as_ref(),
+                settings.archetype.as_ref(),
             )
             .await?
         }
@@ -1975,7 +1982,9 @@ async fn create_agent(
             .jobq
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner);
-        sched.insert_job(None, |b| declare_agent_job(b, &agent, &hive))
+        sched.insert_job(None, |b| {
+            declare_agent_job(b, &agent, &hive, settings.archetype.is_some())
+        })
     };
     let ids = match queued {
         Ok(ids) => ids,
@@ -2003,27 +2012,55 @@ struct CreateSettings {
     limits: Option<swarm_queue_client::agent_limits::AgentCaps>,
     inference: Option<swarm_queue_client::agent_inference::AgentInference>,
     permissions: Option<swarm_queue_client::agent_permissions::AgentPermissions>,
+    archetype: Option<swarm_queue_client::agent_archetype::ArchetypeRef>,
 }
 
 impl CreateSettings {
     /// Each setting given, or why its `PUT` would refuse the first bad one.
+    /// An archetype is checked here for its shape only; whether the swarm
+    /// lists it is [`agent_archetype::check`].
     fn parse(
         limits: Option<agent_limits::Caps>,
         inference: Option<inference::InferenceSettings>,
         permissions: Option<Vec<hive_sh4re::permissions::Permission>>,
+        archetype: Option<swarm_queue_client::agent_archetype::ArchetypeRef>,
     ) -> Result<Self, String> {
+        if let Some(a) = &archetype {
+            for part in [&a.source, &a.name] {
+                hive_types::Ident::parse(part)
+                    .map_err(|why| format!("archetype {part:?}: {why}"))?;
+            }
+        }
         Ok(Self {
             limits: limits.map(agent_limits::Caps::into_stored).transpose()?,
             inference: inference
                 .map(inference::InferenceSettings::into_stored)
                 .transpose()?,
             permissions: permissions.map(agent_permissions::at_create).transpose()?,
+            archetype,
         })
     }
 
     /// Whether any setting is given.
     fn any(&self) -> bool {
         self.limits.is_some() || self.inference.is_some() || self.permissions.is_some()
+            || self.archetype.is_some()
+    }
+
+    /// The queue to store the settings through, `None` when none is given.
+    /// An archetype is refused here unless the stored list holds it.
+    async fn queue(
+        &self,
+        state: &AppState,
+    ) -> Result<Option<async_nats::Client>, problem_details::ProblemDetails> {
+        if !self.any() {
+            return Ok(None);
+        }
+        let client = schedules::queue(state).map_err(schedules::refused)?;
+        if let Some(archetype) = &self.archetype {
+            agent_archetype::check(&client, archetype).await?;
+        }
+        Ok(Some(client))
     }
 }
 
@@ -2034,17 +2071,19 @@ struct StoredSettings {
     limits: agent_limits::Replaced,
     inference: inference::Replaced,
     permissions: agent_permissions::Replaced,
+    archetype: agent_archetype::Replaced,
 }
 
 impl StoredSettings {
-    /// Store whichever of `limits`, `permissions` and `inference` are given, in
-    /// that order. A failure on one puts back the ones stored before it.
+    /// Store whichever of `limits`, `permissions`, `inference` and `archetype` are
+    /// given, in that order. A failure on one puts back the ones stored before it.
     async fn store(
         client: &async_nats::Client,
         agent: &str,
         limits: Option<&swarm_queue_client::agent_limits::AgentCaps>,
         inference: Option<&swarm_queue_client::agent_inference::AgentInference>,
         permissions: Option<&swarm_queue_client::agent_permissions::AgentPermissions>,
+        archetype: Option<&swarm_queue_client::agent_archetype::ArchetypeRef>,
     ) -> Result<Self, problem_details::ProblemDetails> {
         let mut stored = Self::default();
         if let Some(limits) = limits {
@@ -2068,6 +2107,15 @@ impl StoredSettings {
                 }
             }
         }
+        if let Some(archetype) = archetype {
+            match agent_archetype::store(client, agent, archetype).await {
+                Ok(replaced) => stored.archetype = replaced,
+                Err(e) => {
+                    stored.roll_back(client, agent).await;
+                    return Err(e);
+                }
+            }
+        }
         Ok(stored)
     }
 
@@ -2075,6 +2123,7 @@ impl StoredSettings {
         agent_limits::restore(client, agent, self.limits).await;
         inference::restore(client, agent, self.inference).await;
         agent_permissions::restore(client, agent, self.permissions).await;
+        agent_archetype::restore(client, agent, self.archetype).await;
     }
 }
 
@@ -2430,6 +2479,7 @@ fn declare_agent_job(
     b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
     agent: &str,
     hive: &str,
+    archetype: bool,
 ) -> Vec<hive_jobq::builder::NodeGuid> {
     let create_identity = b.node(SwarmNodeKind::CreateIdentity {
         agent: agent.to_owned(),
@@ -2440,22 +2490,31 @@ fn declare_agent_job(
     let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
         agent: agent.to_owned(),
     });
-    let create_repo = b
-        .node(SwarmNodeKind::CreateRepo {
-            agent: agent.to_owned(),
-        })
-        .after_ok(create_identity);
-    // `AddRepoMember` needs both parents: the repo to add a collaborator to,
-    // and the forge user to add as one — adding a nonexistent user is a
-    // Forgejo validation error, not an idempotent no-op.
-    // `InitAgentConfigRepo` needs only the repo — see the doc comment above
-    // for why.
-    let _add_repo_member = b
-        .node(SwarmNodeKind::AddRepoMember {
+    // An archetype agent has no config repo: its hive builds it from the
+    // archetype's module, so there is no repo to create, share or seed.
+    let init_config = (!archetype).then(|| {
+        let create_repo = b
+            .node(SwarmNodeKind::CreateRepo {
+                agent: agent.to_owned(),
+            })
+            .after_ok(create_identity);
+        // `AddRepoMember` needs both parents: the repo to add a collaborator
+        // to, and the forge user to add as one — adding a nonexistent user is
+        // a Forgejo validation error, not an idempotent no-op.
+        // `InitAgentConfigRepo` needs only the repo — see the doc comment
+        // above for why.
+        let _add_repo_member = b
+            .node(SwarmNodeKind::AddRepoMember {
+                agent: agent.to_owned(),
+            })
+            .after_ok(create_repo)
+            .after_ok(create_forge_user);
+        b.node(SwarmNodeKind::InitAgentConfigRepo {
             agent: agent.to_owned(),
         })
         .after_ok(create_repo)
-        .after_ok(create_forge_user);
+        .guid()
+    });
     // Read on `internal/docs`, a repo that exists whatever this agent is, so
     // it needs only the user. Not a parent of the deploy: the agent reads docs
     // once running, and a failed grant is retried by `forge::docs_grant::spawn`.
@@ -2464,11 +2523,6 @@ fn declare_agent_job(
             agent: agent.to_owned(),
         })
         .after_ok(create_forge_user);
-    let init_config = b
-        .node(SwarmNodeKind::InitAgentConfigRepo {
-            agent: agent.to_owned(),
-        })
-        .after_ok(create_repo);
     // The agent's own identity at the swarm secret store, minted and
     // published at swarm level so no hive ever needs the capability to mint
     // one. After `create_identity` because the certificate names a principal
@@ -2507,7 +2561,8 @@ fn declare_agent_job(
     // deploys by reading that repo, so a deploy asked for any earlier would
     // find nothing to build. This is the edge that makes creating an agent at
     // swarm level actually put it on a hive, rather than leaving a
-    // provisioned name nobody runs.
+    // provisioned name nobody runs. An archetype agent has no repo, so its
+    // deploy waits on the mints and the declaration alone.
     //
     // `after_any` on the mint, not `after_ok`: a hive cannot pass down a
     // certificate the swarm has not published, so the deploy must not
@@ -2526,13 +2581,12 @@ fn declare_agent_job(
     // queue at all fails this node — but `TriggerDeploy` has nothing to
     // publish to on such a host either, so the strict edge cancels a node
     // that could not have succeeded anyway.
-    let _trigger_deploy = b
+    let trigger_deploy = b
         .node(SwarmNodeKind::TriggerDeploy {
             hive: hive.to_owned(),
             agent: agent.to_owned(),
             rev: None,
         })
-        .after_ok(init_config)
         .after_any(mint_identity)
         .after_any(mint_forge_token)
         // `after_any` for the same reason: the container reads this token
@@ -2541,6 +2595,9 @@ fn declare_agent_job(
         // and only this node fails, by name.
         .after_any(mint_matrix)
         .after_ok(set_wanted);
+    if let Some(init_config) = init_config {
+        let _ = trigger_deploy.after_ok(init_config);
+    }
     vec![create_identity.guid()]
 }
 
@@ -3904,6 +3961,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await
@@ -3945,6 +4003,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await
@@ -4108,6 +4167,7 @@ mod tests {
                 Some(&caps(r#"{"cpu_quota":"400%"}"#)),
                 Some(&inference),
                 None,
+                None,
             )
             .await;
             assert!(stored.is_err(), "{agent}: the inference write must fail");
@@ -4246,6 +4306,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await
@@ -5374,6 +5435,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await
@@ -5408,6 +5470,168 @@ mod tests {
         );
     }
 
+    /// Node labels of the graph [`super::declare_agent_job`] builds, and the
+    /// labels the deploy node waits on.
+    fn creation_graph(archetype: bool) -> (Vec<String>, Vec<String>) {
+        use hive_jobq_wire::WireNode as _;
+
+        let mut sched = hive_jobq::scheduler::Scheduler::new(
+            hive_jobq::Graph::new(),
+            hive_jobq::resources::ResourceTable::new(),
+        );
+        sched
+            .insert_job(None, |b| {
+                super::declare_agent_job(b, "atlas", "pr1ma", archetype)
+            })
+            .expect("the graph inserts");
+        let graph = sched.graph();
+        let labels = graph.nodes().map(|n| n.payload.label()).collect();
+        let deploy = graph
+            .nodes()
+            .find(|n| n.payload.label() == "trigger_deploy")
+            .expect("every creation deploys");
+        let waits = deploy
+            .deps
+            .iter()
+            .filter_map(|d| match d {
+                hive_jobq::Dep::Node { id, .. } => graph.node(*id).map(|n| n.payload.label()),
+                hive_jobq::Dep::Resource { .. } => None,
+            })
+            .collect();
+        (labels, waits)
+    }
+
+    /// An archetype agent's graph has no repo nodes and its deploy waits on
+    /// no repo; a custom agent's has all three, and its deploy waits on the
+    /// seeded repo.
+    #[test]
+    fn an_archetype_agent_is_created_without_a_config_repo() {
+        const REPO_NODES: [&str; 3] = ["create_repo", "add_repo_member", "init_agent_config_repo"];
+
+        let (custom, custom_waits) = creation_graph(false);
+        for repo in REPO_NODES {
+            assert!(custom.iter().any(|l| l == repo), "custom lacks {repo}");
+        }
+        assert!(custom_waits.iter().any(|l| l == "init_agent_config_repo"));
+
+        let (archetype, archetype_waits) = creation_graph(true);
+        for repo in REPO_NODES {
+            assert!(!archetype.iter().any(|l| l == repo), "archetype has {repo}");
+        }
+        for kept in [
+            "create_identity",
+            "create_forge_user",
+            "grant_docs_read",
+            "mint_agent_identity",
+            "mint_agent_forge_token",
+            "mint_agent_matrix_account",
+            "set_agent_wanted",
+        ] {
+            assert!(archetype.iter().any(|l| l == kept), "archetype lacks {kept}");
+            assert!(custom.iter().any(|l| l == kept), "custom lacks {kept}");
+        }
+        assert!(archetype_waits.iter().any(|l| l == "set_agent_wanted"));
+        assert_eq!(archetype.len() + REPO_NODES.len(), custom.len());
+    }
+
+    /// Seed the stored archetype list with `reviewer` under `hyperhive`.
+    async fn seed_archetypes(client: &async_nats::Client) {
+        let record = swarm_queue_client::archetypes::SourceRecord {
+            source: "/nix/store/h".to_owned(),
+            archetypes: std::collections::BTreeMap::from([(
+                "reviewer".to_owned(),
+                swarm_queue_client::archetypes::ArchetypeMeta {
+                    description: "reviews".to_owned(),
+                    ..Default::default()
+                },
+            )]),
+            refused: std::collections::BTreeMap::new(),
+        };
+        swarm_queue_client::archetypes::open_or_create(client)
+            .await
+            .expect("bucket")
+            .put(
+                swarm_queue_client::archetypes::BUILTIN_SOURCE,
+                serde_json::to_vec(&record).expect("encodes").into(),
+            )
+            .await
+            .expect("seeded");
+    }
+
+    /// Through the real handler and a real `nats-server`: a listed archetype
+    /// lands under the agent's key as the hive reads it, and the queued graph
+    /// has no config repo.
+    #[tokio::test]
+    async fn a_creation_from_an_archetype_records_it_and_queues_no_repo() {
+        use hive_jobq_wire::WireNode as _;
+
+        let nats = crate::test_nats::TestNats::start();
+        let client = nats.client().await;
+        seed_archetypes(&client).await;
+        let state = crate::agent_limits::tests::state(&client);
+        let req: super::CreateAgentRequest = serde_json::from_str(
+            r#"{"name":"atlas","hive":"pr1ma",
+                "archetype":{"source":"hyperhive","name":"reviewer"}}"#,
+        )
+        .expect("body parses");
+        let _queued = super::create_agent(axum::extract::State(state.clone()), axum::Json(req))
+            .await
+            .expect("a listed archetype goes through");
+
+        let stored = swarm_queue_client::agent_archetype::open_or_create(&client)
+            .await
+            .expect("bucket")
+            .get("atlas")
+            .await
+            .expect("read")
+            .expect("a value");
+        assert_eq!(&stored[..], br#"{"source":"hyperhive","name":"reviewer"}"#);
+        let labels: Vec<String> = state
+            .jobq
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .graph()
+            .nodes()
+            .map(|n| n.payload.label())
+            .collect();
+        assert!(labels.iter().any(|l| l == "trigger_deploy"), "{labels:?}");
+        assert!(!labels.iter().any(|l| l == "create_repo"), "{labels:?}");
+    }
+
+    /// An archetype the stored list does not hold is a 400 that stores and
+    /// queues nothing, and so is a name that is not an identifier.
+    #[tokio::test]
+    async fn an_unlisted_archetype_is_refused_before_anything_is_stored() {
+        let nats = crate::test_nats::TestNats::start();
+        let client = nats.client().await;
+        seed_archetypes(&client).await;
+        let state = crate::agent_limits::tests::state(&client);
+        for archetype in [
+            r#"{"source":"hyperhive","name":"nope"}"#,
+            r#"{"source":"elsewhere","name":"reviewer"}"#,
+            r#"{"source":"hyperhive","name":"a.b"}"#,
+        ] {
+            let req: super::CreateAgentRequest = serde_json::from_str(&format!(
+                r#"{{"name":"atlas","hive":"pr1ma","permissions":["messaging"],
+                    "archetype":{archetype}}}"#
+            ))
+            .expect("body parses");
+            let err = super::create_agent(axum::extract::State(state.clone()), axum::Json(req))
+                .await
+                .expect_err("an unlisted archetype must be refused");
+            assert_eq!(
+                err.status,
+                Some(axum::http::StatusCode::BAD_REQUEST),
+                "{archetype}: {err:?}"
+            );
+        }
+        assert_eq!(node_count(&state.jobq), 0, "nothing may be queued");
+        let permissions = swarm_queue_client::agent_permissions::open_or_create(&client)
+            .await
+            .expect("bucket");
+        assert_eq!(permissions.get("atlas").await.expect("read"), None);
+    }
+
     /// The ordering the operator's ruling requires: a hive cannot pass down
     /// a certificate the swarm has not published, so the deploy message must
     /// not leave before the mint is terminal.
@@ -5428,6 +5652,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await
@@ -5478,6 +5703,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await
@@ -5536,6 +5762,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await
@@ -5741,6 +5968,7 @@ mod tests {
                 limits: None,
                 inference: None,
                 permissions: None,
+                archetype: None,
             }),
         )
         .await