controller: turn an archetype agent into a custom one

swarm-controller/src/jobq.rs

@@ -57,6 +57,15 @@ pub(crate) enum SwarmNodeKind {
     /// swarm routes a deploy message to — it belongs on the node that
     /// sends that message, not on this one.
     InitAgentConfigRepo { agent: String },
+    /// Seed the agent's config repo with the archetype's module, for turn
+    /// into custom. See `agent_custom::seed`.
+    SeedArchetypeConfig {
+        agent: String,
+        archetype: swarm_wire::agent_archetype::ArchetypeRef,
+    },
+    /// Delete the agent's archetype key, which makes it custom. See
+    /// `agent_custom::drop_archetype`.
+    DropArchetype { agent: String },
     /// Mint `agent`'s own client certificate for the swarm secret store,
     /// publish it there, grant it, and log in with it. See
     /// `agent_identity::mint_and_verify` — including why this node does not
@@ -168,6 +177,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
             SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(),
             SwarmNodeKind::GrantDocsRead { .. } => "grant_docs_read".to_owned(),
             SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(),
+            SwarmNodeKind::SeedArchetypeConfig { .. } => "seed_archetype_config".to_owned(),
+            SwarmNodeKind::DropArchetype { .. } => "drop_archetype".to_owned(),
             SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(),
             SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(),
             SwarmNodeKind::MintAgentMatrixAccount { .. } => "mint_agent_matrix_account".to_owned(),
@@ -202,6 +213,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
             | SwarmNodeKind::AddRepoMember { agent }
             | SwarmNodeKind::GrantDocsRead { agent }
             | SwarmNodeKind::InitAgentConfigRepo { agent }
+            | SwarmNodeKind::DropArchetype { agent }
             | SwarmNodeKind::MintAgentIdentity { agent }
             | SwarmNodeKind::MintAgentForgeToken { agent }
             | SwarmNodeKind::MintAgentMatrixAccount { agent }
@@ -211,6 +223,9 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
             SwarmNodeKind::BuildMeta { agent, pr } => {
                 serde_json::json!({ "agent": agent, "pr": pr })
             }
+            SwarmNodeKind::SeedArchetypeConfig { agent, archetype } => {
+                serde_json::json!({ "agent": agent, "archetype": archetype.name })
+            }
             SwarmNodeKind::MintHiveSenderToken { hive }
             | SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
             SwarmNodeKind::MintClassifierForgeToken
@@ -404,6 +419,9 @@ async fn run_node_work(
         },
         SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await,
         SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await,
+        SwarmNodeKind::SeedArchetypeConfig { .. } | SwarmNodeKind::DropArchetype { .. } => {
+            convert_node(&kind, forge, deps.queue.as_ref()).await
+        }
         SwarmNodeKind::MintAgentIdentity { agent } => {
             mint_identity(deps.wanted.as_deref(), &agent).await
         }
@@ -592,6 +610,44 @@ async fn init_agent_config_repo(
     }
 }
 
+/// The `SeedArchetypeConfig` and `DropArchetype` arms, lifted out for the
+/// same reason. Any other kind is not this function's to run.
+async fn convert_node(
+    kind: &SwarmNodeKind,
+    forge: Option<Arc<forge::Client>>,
+    queue: Option<&async_nats::Client>,
+) -> hive_jobq::scheduler::Outcome {
+    use hive_jobq::scheduler::Outcome;
+
+    let done = |result: Result<()>| match result {
+        Ok(()) => Outcome::Done,
+        Err(e) => Outcome::Failed(format!("{e:#}")),
+    };
+    match kind {
+        SwarmNodeKind::SeedArchetypeConfig { agent, archetype } => {
+            let Some(client) = forge else {
+                return Outcome::Failed(
+                    "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
+                     not in the swarm secret store)"
+                        .to_owned(),
+                );
+            };
+            done(
+                forge_node(
+                    "seed_archetype_config",
+                    crate::agent_custom::seed(client.as_ref(), agent, archetype),
+                )
+                .await,
+            )
+        }
+        SwarmNodeKind::DropArchetype { agent } => match queue {
+            Some(client) => done(crate::agent_custom::drop_archetype(client, agent).await),
+            None => Outcome::Failed("no swarm queue is configured on this host".to_owned()),
+        },
+        other => unreachable!("{other:?} is not a conversion node"),
+    }
+}
+
 /// The `RenewAgentQueueCredential` arm, lifted out for the same reason.
 async fn renew_queue_credential(agent: &str) -> hive_jobq::scheduler::Outcome {
     use hive_jobq::scheduler::Outcome;
@@ -1421,6 +1477,65 @@ pub(crate) fn declare_agent_job(
     vec![create_identity.guid()]
 }
 
+/// The sub-DAG turning the archetype agent `agent` into a custom one: its
+/// config repo is created and seeded, and only then is its archetype key
+/// deleted, so a failed step leaves the agent archetype-built. The meta
+/// update and the deploy follow the delete (`after_ok`), as in
+/// [`merged_deploy_job`]. Returns the repo node, the handle the response
+/// reports.
+///
+/// `CreateRepo`, `CreateForgeUser` and `AddRepoMember` are no-ops for what
+/// already exists, so the same graph can be queued again after a failure.
+///
+/// With `archetype` `None` the agent is already custom: its key was deleted and
+/// a later step failed, so the graph is the meta update and the deploy alone,
+/// and the handle is the update. A build that fails after the key delete leaves
+/// the agent custom, which is how that failure is retried.
+pub(crate) fn declare_custom_job(
+    b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
+    agent: &str,
+    hive: &str,
+    archetype: Option<&swarm_wire::agent_archetype::ArchetypeRef>,
+) -> Vec<hive_jobq::builder::NodeGuid> {
+    let Some(archetype) = archetype else {
+        let update_meta = b
+            .node(SwarmNodeKind::UpdateMeta)
+            .needs(SwarmResourceKind::Meta);
+        let _deploy = gated_deploy(b, hive, agent, None, None, Some(update_meta.guid()));
+        return vec![update_meta.guid()];
+    };
+    let create_repo = b.node(SwarmNodeKind::CreateRepo {
+        agent: agent.to_owned(),
+    });
+    let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
+        agent: agent.to_owned(),
+    });
+    let add_repo_member = b
+        .node(SwarmNodeKind::AddRepoMember {
+            agent: agent.to_owned(),
+        })
+        .after_ok(create_repo)
+        .after_ok(create_forge_user);
+    let seed = b
+        .node(SwarmNodeKind::SeedArchetypeConfig {
+            agent: agent.to_owned(),
+            archetype: archetype.clone(),
+        })
+        .after_ok(create_repo);
+    let drop_archetype = b
+        .node(SwarmNodeKind::DropArchetype {
+            agent: agent.to_owned(),
+        })
+        .after_ok(seed)
+        .after_ok(add_repo_member);
+    let update_meta = b
+        .node(SwarmNodeKind::UpdateMeta)
+        .needs(SwarmResourceKind::Meta)
+        .after_ok(drop_archetype);
+    let _deploy = gated_deploy(b, hive, agent, None, None, Some(update_meta.guid()));
+    vec![create_repo.guid()]
+}
+
 /// Insert one job per agent: `CreateForgeUser`, then `MintAgentForgeToken`
 /// `after_ok` it — the same edge [`declare_agent_job`] uses. Returns the mint
 /// nodes' ids.
@@ -2389,6 +2504,87 @@ mod tests {
         let _ = super::merged_deploy_job(b, "pr1ma", "atlas", "abc", 7);
     }
 
+    fn custom_dag(b: &TestBuilder) {
+        let archetype = swarm_wire::agent_archetype::ArchetypeRef {
+            source: "hyperhive".to_owned(),
+            name: "reviewer".to_owned(),
+        };
+        let _ = super::declare_custom_job(b, "atlas", "pr1ma", Some(&archetype));
+    }
+
+    /// Turn into custom: with every node succeeding the agent is seeded, loses
+    /// its key, and is deployed.
+    #[tokio::test]
+    async fn a_conversion_seeds_then_drops_the_key_then_deploys() {
+        use hive_jobq::State;
+
+        let done = run_dag_failing(custom_dag, None).await;
+        for label in [
+            "create_repo",
+            "seed_archetype_config",
+            "drop_archetype",
+            "update_meta",
+            "trigger_deploy",
+        ] {
+            assert_eq!(state_of(&done, label), State::Done, "{label}");
+        }
+        // The key is deleted only after the seed and the member grant, and
+        // each edge refuses a failure. The deploy edge is the control that
+        // the lookup finds an `after_ok` edge in this graph.
+        for (from, on) in [
+            ("drop_archetype", "seed_archetype_config"),
+            ("drop_archetype", "add_repo_member"),
+            ("update_meta", "drop_archetype"),
+            ("trigger_deploy", "update_meta"),
+        ] {
+            assert!(
+                !edge_accepts_failure(custom_dag, from, on),
+                "{from} -> {on}"
+            );
+        }
+    }
+
+    /// The retry of a conversion whose build failed after the key was deleted:
+    /// with no archetype the graph is the meta update and the deploy, with no
+    /// repo, seed or key delete. The conversion is the control that those
+    /// nodes exist when there is one.
+    #[tokio::test]
+    async fn a_retry_of_a_custom_agent_is_only_the_update_and_the_deploy() {
+        use hive_jobq::State;
+
+        fn retry_dag(b: &TestBuilder) {
+            let _ = super::declare_custom_job(b, "atlas", "pr1ma", None);
+        }
+        let done = run_dag_failing(retry_dag, None).await;
+        let labels: Vec<&str> = done.iter().map(|(l, _)| l.as_str()).collect();
+        for absent in ["create_repo", "seed_archetype_config", "drop_archetype"] {
+            assert!(!labels.contains(&absent), "{absent} in {labels:?}");
+        }
+        assert_eq!(state_of(&done, "update_meta"), State::Done);
+        assert_eq!(state_of(&done, "trigger_deploy"), State::Done);
+        let converting = run_dag_failing(custom_dag, None).await;
+        assert!(converting.iter().any(|(l, _)| l == "drop_archetype"));
+        // A failed build leaves the deploy skipped, so a retry has work to do.
+        let red = run_dag_failing(retry_dag, Some("build_meta")).await;
+        assert_eq!(state_of(&red, "trigger_deploy"), State::Skipped);
+    }
+
+    /// A seed that fails leaves the key in place and nothing deployed; the
+    /// same graph, queued again with the seed succeeding, completes.
+    #[tokio::test]
+    async fn a_failed_seed_keeps_the_key_and_a_retry_completes() {
+        use hive_jobq::State;
+
+        let failed = run_dag_failing(custom_dag, Some("seed_archetype_config")).await;
+        assert_eq!(state_of(&failed, "seed_archetype_config"), State::Failed);
+        for label in ["drop_archetype", "update_meta", "trigger_deploy"] {
+            assert_eq!(state_of(&failed, label), State::Skipped, "{label}");
+        }
+        let retried = run_dag_failing(custom_dag, None).await;
+        assert_eq!(state_of(&retried, "drop_archetype"), State::Done);
+        assert_eq!(state_of(&retried, "trigger_deploy"), State::Done);
+    }
+
     /// A deploy needs the meta update to succeed: its edge refuses a failed
     /// update, in both DAGs that queue one.
     #[test]