swarm-builder: build each agent config PR head and post a commit status

A new `swarm-builder` service, in its own container on the host that sets
`deploy.builder.enable` (defaulted on by `allSwarmServices` where the host
has the builder's store leaf, like the review classifier).

When an `agent-configs/<agent>` PR is opened, reopened or synchronized,
swarm-controller queues a `build_agent` jobq node. The node sets a
`swarm-builder` commit status to pending on the PR head, sends
`{agent, rev, pr}` to the builder over NATS request-reply
(`$SWARM.builder.build`), and writes success (store path), failure (last
line nix printed) or error (no builder / no reply) on the head. The node
fails when the build does, so the Jobs page matches the PR.

The builder builds nix/builder/check-flake's `check` with every input
overridden: hyperhive and nixpkgs to the host's own
`c0re.hyperhiveFlake`/`c0re.nixpkgsFlake` (what a hive there renders into
its meta flake), and the agent to the PR head on the forge. Evaluation is
pure and runs in the container; builds go to the host nix daemon as an
untrusted client. The private config repo is fetched with the token of a
new read-only `builder` forge account (in the `hives` team), which the
controller mints into `swarm/services/builder/forge/token`; git gets it
from a credential helper reading the child's environment, so it is in no
argument or URL.

Credentials: a `builder` store leaf minted by glue-bao-tls on the store
host, a `swarm-builder` reader policy (forge token + its own OIDC client
secret), an authelia machine client, and a `--builder-client` grant in the
queue's auth-callout responder. The controller gains a grant on the token
path and the secret publisher a deny on it.

This is the agent-config build check atlas scoped on #5081 (slice 3):
nothing is pushed to a cache, nothing is written to KV, and a red status
gates no deploy. The check composes agent-base + the config without the
values a hive adds (CA, docs, claude-code path, ports, input follows), so
it can disagree with a hive's own build.

swarm-builder/src/main.rs

@@ -0,0 +1,190 @@
+//! The swarm builder's build service.
+//!
+//! Connects to the swarm queue as its own OIDC client, takes build requests
+//! from swarm-controller on [`swarm_queue_client::builder::SUBJECT`], builds
+//! each agent config at its commit, answers on the request's reply subject,
+//! and logs one line per build. It writes nothing to the forge: the
+//! controller turns the reply into a commit status.
+//!
+//! The store identity comes from the `BAO_*` environment. The queue client
+//! secret is read at start and again when authelia refuses it, the forge token
+//! once per build, both from the store under that identity.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+
+use anyhow::{Context, Result, bail};
+use futures_util::StreamExt as _;
+use swarm_queue_client::builder::{BUILD_DEADLINE, BuildAgent, QUEUE_GROUP, Reply, SUBJECT};
+use swarm_queue_client::stored_secret::StoredSecret;
+use swarm_queue_client::{ClientSecret, QueueConfig};
+use swarm_secret_client::forge::Credential;
+
+mod nix_build;
+
+/// The cert-auth role this service logs in to the store as. `swarm-bao.nix`
+/// writes it, matching the leaf's subject.
+const CERT_ROLE: &str = "swarm-builder";
+
+/// The environment prefix `swarm-builder.nix` sets.
+const PREFIX: &str = "SWARM_BUILDER";
+
+/// Seconds to wait after each failed read of the queue client secret. A
+/// longer outage fails the start, and the unit's `Restart=` tries again.
+const RETRY_DELAYS_S: [u64; 6] = [1, 2, 4, 8, 16, 30];
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    hive_log::init();
+    swarm_queue_client::install_crypto_provider();
+    let var = |name: &str| {
+        let full = format!("{PREFIX}_{name}");
+        std::env::var(&full).with_context(|| format!("{full} is unset"))
+    };
+    let client_id = var("OIDC_CLIENT_ID")?;
+    let builder = Arc::new(nix_build::Builder {
+        nix: PathBuf::from(var("NIX")?),
+        check_flake: var("CHECK_FLAKE")?,
+        hyperhive: var("HYPERHIVE_FLAKE")?,
+        nixpkgs: var("NIXPKGS_FLAKE")?,
+        forge_url: var("FORGE_URL")?.trim_end_matches('/').to_owned(),
+        config_org: var("CONFIG_ORG")?,
+        deadline: BUILD_DEADLINE,
+    });
+    let secret = client_secret_with_retry(&client_id).await?;
+    let reread_id = client_id.clone();
+    let secret = StoredSecret::new(client_id.clone(), secret, move || {
+        let client_id = reread_id.clone();
+        async move { client_secret(&client_id).await.map_err(Into::into) }
+    });
+    let queue = swarm_queue_client::connect(QueueConfig {
+        url: var("NATS_URL")?,
+        token_endpoint: var("OIDC_TOKEN_ENDPOINT")?,
+        client_id,
+        client_secret: ClientSecret::Stored(secret),
+        ca_file: std::env::var(format!("{PREFIX}_OIDC_CA_FILE"))
+            .ok()
+            .map(PathBuf::from),
+    })
+    .await?;
+    let mut requests = queue
+        .queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
+        .await
+        .with_context(|| format!("subscribing to {SUBJECT}"))?;
+    tracing::info!(subject = SUBJECT, "builder: awaiting builds");
+
+    while let Some(msg) = requests.next().await {
+        let Some(reply_to) = msg.reply.clone() else {
+            tracing::warn!("a build arrived with no reply subject; dropping it");
+            continue;
+        };
+        let queue = queue.clone();
+        let builder = Arc::clone(&builder);
+        tokio::spawn(async move {
+            let reply = handle(&msg.payload, &builder).await;
+            match serde_json::to_vec(&reply) {
+                Ok(body) => {
+                    if let Err(e) = queue.publish(reply_to, body.into()).await {
+                        tracing::warn!(error = %e, "publishing a build's reply failed");
+                    }
+                }
+                Err(e) => tracing::warn!(error = %e, "encoding a build's reply failed"),
+            }
+        });
+    }
+    bail!("the subscription to {SUBJECT} ended")
+}
+
+/// Decode one request, build it and log the build.
+async fn handle(payload: &[u8], builder: &nix_build::Builder) -> Reply {
+    let job: BuildAgent = match serde_json::from_slice(payload) {
+        Ok(job) => job,
+        Err(e) => {
+            tracing::warn!(error = %e, "an undecodable build; answering with the error");
+            return Reply::Failed {
+                error_tail: format!("the builder could not decode the request: {e}"),
+            };
+        }
+    };
+    let reply = match forge_token().await {
+        Ok(token) => builder.build(&job, &token).await,
+        Err(e) => Reply::Failed {
+            error_tail: format!("{e:#}"),
+        },
+    };
+    log_build(&job, &reply);
+    reply
+}
+
+/// The one journal line a build leaves.
+fn log_build(job: &BuildAgent, reply: &Reply) {
+    match reply {
+        Reply::Built { out_path } => tracing::info!(
+            agent = %job.agent,
+            rev = %job.rev,
+            pr = job.pr,
+            %out_path,
+            "agent config built"
+        ),
+        Reply::Failed { error_tail } => tracing::warn!(
+            agent = %job.agent,
+            rev = %job.rev,
+            pr = job.pr,
+            %error_tail,
+            "agent config build failed"
+        ),
+    }
+}
+
+/// The forge token the builder fetches config repos with, read on every
+/// build so a rotation takes effect on the next one.
+async fn forge_token() -> Result<String> {
+    let path = swarm_secret_client::builder::forge_token_path()?;
+    let store = swarm_secret_client::SecretStore::from_env(CERT_ROLE)
+        .await
+        .context("logging in to the secret store")?;
+    let stored: Option<Credential> = store
+        .read_optional(&path)
+        .await
+        .with_context(|| format!("reading {path}"))?;
+    match stored {
+        Some(c) if !c.value.trim().is_empty() => Ok(c.value.trim().to_owned()),
+        _ => bail!("nothing at {path}: swarm-controller has not minted the builder's forge token"),
+    }
+}
+
+/// The queue client secret, read from the store, retried through a store that
+/// is restarting alongside this service.
+async fn client_secret_with_retry(client_id: &str) -> Result<String> {
+    for delay in RETRY_DELAYS_S.map(Duration::from_secs) {
+        match client_secret(client_id).await {
+            Ok(secret) => return Ok(secret),
+            Err(e) => {
+                tracing::warn!(error = %format!("{e:#}"), retry_in = ?delay, "reading the queue client secret failed");
+                tokio::time::sleep(delay).await;
+            }
+        }
+    }
+    client_secret(client_id).await
+}
+
+async fn client_secret(client_id: &str) -> Result<String> {
+    use swarm_secret_client::builder::{ClientSecret, client_secret_path};
+
+    let path = client_secret_path(client_id)?;
+    let store = swarm_secret_client::SecretStore::from_env(CERT_ROLE)
+        .await
+        .context("logging in to the secret store")?;
+    let stored: Option<ClientSecret> = store
+        .read_optional(&path)
+        .await
+        .with_context(|| format!("reading {path}"))?;
+    match stored {
+        Some(s) if !s.value.trim().is_empty() => Ok(s.value.trim().to_owned()),
+        _ => bail!(
+            "nothing at {path}: swarm-secret-publish, on the host that runs authelia, has not \
+             published this service's client secret"
+        ),
+    }
+}