hyperhive/swarm-builder/src/main.rs

//! The swarm builder's build service.
//!
//! Connects to the swarm queue as its own OIDC client, takes build requests
//! from swarm-controller on [`swarm_queue_client::builder::SUBJECT`], builds
//! each agent config at its commit, answers on the request's reply subject,
//! and logs one line per build. It writes nothing to the forge: the
//! controller turns the reply into a commit status.
//!
//! The store identity comes from the `BAO_*` environment. The queue client
//! secret is read at start and again when authelia refuses it, the forge token
//! once per build, both from the store under that identity.

use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;

use anyhow::{Context, Result, bail};
use futures_util::StreamExt as _;
use swarm_queue_client::builder::{BUILD_DEADLINE, BuildAgent, QUEUE_GROUP, Reply, SUBJECT};
use swarm_queue_client::stored_secret::StoredSecret;
use swarm_queue_client::{ClientSecret, QueueConfig};
use swarm_secret_client::forge::Credential;

mod nix_build;

/// The cert-auth role this service logs in to the store as. `swarm-bao.nix`
/// writes it, matching the leaf's subject.
const CERT_ROLE: &str = "swarm-builder";

/// The environment prefix `swarm-builder.nix` sets.
const PREFIX: &str = "SWARM_BUILDER";

/// Seconds to wait after each failed read of the queue client secret. A
/// longer outage fails the start, and the unit's `Restart=` tries again.
const RETRY_DELAYS_S: [u64; 6] = [1, 2, 4, 8, 16, 30];

#[tokio::main]
async fn main() -> Result<()> {
    hive_log::init();
    swarm_queue_client::install_crypto_provider();
    let var = |name: &str| {
        let full = format!("{PREFIX}_{name}");
        std::env::var(&full).with_context(|| format!("{full} is unset"))
    };
    let client_id = var("OIDC_CLIENT_ID")?;
    let builder = Arc::new(nix_build::Builder {
        nix: PathBuf::from(var("NIX")?),
        check_flake: var("CHECK_FLAKE")?,
        hyperhive: var("HYPERHIVE_FLAKE")?,
        nixpkgs: var("NIXPKGS_FLAKE")?,
        forge_url: forge_url()?,
        config_org: var("CONFIG_ORG")?,
        deadline: BUILD_DEADLINE,
    });
    let secret = client_secret_with_retry(&client_id).await?;
    let reread_id = client_id.clone();
    let secret = StoredSecret::new(client_id.clone(), secret, move || {
        let client_id = reread_id.clone();
        async move { client_secret(&client_id).await.map_err(Into::into) }
    });
    let queue = swarm_queue_client::connect(QueueConfig {
        url: var("NATS_URL")?,
        token_endpoint: var("OIDC_TOKEN_ENDPOINT")?,
        client_id,
        client_secret: ClientSecret::Stored(secret),
        ca_file: std::env::var(format!("{PREFIX}_OIDC_CA_FILE"))
            .ok()
            .map(PathBuf::from),
    })
    .await?;
    let mut requests = queue
        .queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
        .await
        .with_context(|| format!("subscribing to {SUBJECT}"))?;
    tracing::info!(subject = SUBJECT, "builder: awaiting builds");

    // One build at a time. swarm-controller already sends one at a time; this
    // holds the bound for any request it did not serialise.
    let slot = Arc::new(tokio::sync::Semaphore::new(1));

    while let Some(msg) = requests.next().await {
        let Some(reply_to) = msg.reply.clone() else {
            tracing::warn!("a build arrived with no reply subject; dropping it");
            continue;
        };
        let queue = queue.clone();
        let slot = Arc::clone(&slot);
        let builder = Arc::clone(&builder);
        tokio::spawn(async move {
            let Ok(_permit) = slot.acquire().await else {
                return;
            };
            let reply = handle(&msg.payload, &builder).await;
            match serde_json::to_vec(&reply) {
                Ok(body) => {
                    if let Err(e) = queue.publish(reply_to, body.into()).await {
                        tracing::warn!(error = %e, "publishing a build's reply failed");
                    }
                }
                Err(e) => tracing::warn!(error = %e, "encoding a build's reply failed"),
            }
        });
    }
    bail!("the subscription to {SUBJECT} ended")
}

/// Decode one request, build it and log the build.
async fn handle(payload: &[u8], builder: &nix_build::Builder) -> Reply {
    let job: BuildAgent = match serde_json::from_slice(payload) {
        Ok(job) => job,
        Err(e) => {
            tracing::warn!(error = %e, "an undecodable build; answering with the error");
            return Reply::Failed {
                error_tail: format!("the builder could not decode the request: {e}"),
            };
        }
    };
    let reply = match forge_token().await {
        Ok(token) => builder.build(&job, &token).await,
        Err(e) => Reply::Failed {
            error_tail: format!("{e:#}"),
        },
    };
    log_build(&job, &reply);
    reply
}

/// The one journal line a build leaves.
fn log_build(job: &BuildAgent, reply: &Reply) {
    match reply {
        Reply::Built { out_path } => tracing::info!(
            agent = %job.agent,
            rev = %job.rev,
            pr = job.pr,
            %out_path,
            "agent config built"
        ),
        Reply::Failed { error_tail } => tracing::warn!(
            agent = %job.agent,
            rev = %job.rev,
            pr = job.pr,
            %error_tail,
            "agent config build failed"
        ),
    }
}

/// The forge token the builder fetches config repos with, read on every
/// build so a rotation takes effect on the next one.
async fn forge_token() -> Result<String> {
    let path = swarm_secret_client::builder::forge_token_path()?;
    let store = swarm_secret_client::SecretStore::from_env(CERT_ROLE)
        .await
        .context("logging in to the secret store")?;
    let stored: Option<Credential> = store
        .read_optional(&path)
        .await
        .with_context(|| format!("reading {path}"))?;
    match stored {
        Some(c) if !c.value.trim().is_empty() => Ok(c.value.trim().to_owned()),
        _ => bail!("nothing at {path}: swarm-controller has not minted the builder's forge token"),
    }
}

/// The queue client secret, read from the store, retried through a store that
/// is restarting alongside this service.
async fn client_secret_with_retry(client_id: &str) -> Result<String> {
    for delay in RETRY_DELAYS_S.map(Duration::from_secs) {
        match client_secret(client_id).await {
            Ok(secret) => return Ok(secret),
            Err(e) => {
                tracing::warn!(error = %format!("{e:#}"), retry_in = ?delay, "reading the queue client secret failed");
                tokio::time::sleep(delay).await;
            }
        }
    }
    client_secret(client_id).await
}

async fn client_secret(client_id: &str) -> Result<String> {
    use swarm_secret_client::builder::{ClientSecret, client_secret_path};

    let path = client_secret_path(client_id)?;
    let store = swarm_secret_client::SecretStore::from_env(CERT_ROLE)
        .await
        .context("logging in to the secret store")?;
    let stored: Option<ClientSecret> = store
        .read_optional(&path)
        .await
        .with_context(|| format!("reading {path}"))?;
    match stored {
        Some(s) if !s.value.trim().is_empty() => Ok(s.value.trim().to_owned()),
        _ => bail!(
            "nothing at {path}: swarm-secret-publish, on the host that runs authelia, has not \
             published this service's client secret"
        ),
    }
}

/// The forge's base URL, which must be `https`: the credential helper offers
/// the token to `https` requests for its host alone.
fn forge_url() -> Result<String> {
    let name = format!("{PREFIX}_FORGE_URL");
    let url = std::env::var(&name).with_context(|| format!("{name} is unset"))?;
    let url = url.trim_end_matches('/');
    if !url.starts_with("https://") || url["https://".len()..].contains('/') {
        bail!("{name} is {url:?}, not https://<host>[:port]");
    }
    Ok(url.to_owned())
}