A hung forge fetch inside `nix flake update hyperhive` (or any other meta git/nix child) now fails after its deadline with an error naming the command, the child is killed, and META_LOCK, BuildSlot and MetaWindow are released. The boot sweep already treats that failure as a warning and fans out the rebuilds. A wait on META_LOCK past 60s logs a warning naming the waiter and the holder.
427 lines
16 KiB
Rust
427 lines
16 KiB
Rust
//! Repo + org plumbing on the local Forgejo: the meta + knowledge repos,
|
|
//! meta mirroring (`push_meta`), collaborator grants, pull-mirrors, and
|
|
//! branch-protection rules. The typed API-client constructor + org-name
|
|
//! constants live in the module root (`super`).
|
|
|
|
use std::path::Path;
|
|
use std::sync::Mutex;
|
|
|
|
use anyhow::{Context, Result};
|
|
use forgejo_api::structs::{
|
|
AddCollaboratorOption, AddCollaboratorOptionPermission, CreateRepoOption, Repository,
|
|
};
|
|
use forgejo_api::{ApiErrorKind, ForgejoError};
|
|
use reqwest::StatusCode;
|
|
|
|
use crate::coordinator::Coordinator;
|
|
|
|
use super::{
|
|
CONFIG_ORG, api, bounded, core_auth_header, core_token, forge_git_url, forge_http_base,
|
|
is_present, reader_token,
|
|
};
|
|
|
|
/// Creation options for an empty repo defaulting to `main`.
|
|
fn repo_option(name: &str, private: bool) -> CreateRepoOption {
|
|
CreateRepoOption {
|
|
auto_init: Some(false),
|
|
default_branch: Some("main".to_owned()),
|
|
description: None,
|
|
gitignores: None,
|
|
issue_labels: None,
|
|
license: None,
|
|
name: name.to_owned(),
|
|
object_format_name: None,
|
|
private: Some(private),
|
|
readme: None,
|
|
template: None,
|
|
trust_model: None,
|
|
}
|
|
}
|
|
|
|
/// Whether a create-style call failed because the object already
|
|
/// exists. Forgejo signals this as HTTP 409 (conflict) or 422
|
|
/// (validation). The typed client surfaces those as
|
|
/// `ApiErrorKind::Other(409)` / `ValidationFailed` when the endpoint
|
|
/// spec lists the status, or as a bare `UnexpectedStatusCode`
|
|
/// otherwise — match all shapes defensively.
|
|
fn is_already_exists(e: &ForgejoError) -> bool {
|
|
match e {
|
|
ForgejoError::ApiError(api) => match api.error_kind() {
|
|
ApiErrorKind::ValidationFailed => true,
|
|
ApiErrorKind::Other(s) => *s == StatusCode::CONFLICT,
|
|
_ => false,
|
|
},
|
|
ForgejoError::UnexpectedStatusCode(s) => {
|
|
*s == StatusCode::CONFLICT || *s == StatusCode::UNPROCESSABLE_ENTITY
|
|
}
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Fold a repo-creation result's "already exists" (409 / 422) into
|
|
/// success. `label` is `<owner>/<name>` — purely for log + error
|
|
/// context.
|
|
fn created_or_exists(res: Result<Repository, ForgejoError>, label: &str) -> Result<()> {
|
|
match res {
|
|
Ok(_) => {
|
|
tracing::info!(%label, "forge: created repo");
|
|
Ok(())
|
|
}
|
|
Err(e) if is_already_exists(&e) => {
|
|
tracing::debug!(%label, "forge: repo already exists");
|
|
Ok(())
|
|
}
|
|
Err(e) => Err(e).with_context(|| format!("create repo {label}")),
|
|
}
|
|
}
|
|
|
|
/// Create a repo in the token-owner's own namespace. `token` belongs
|
|
/// to the user we want the repo owned by (we use `core`'s token for
|
|
/// `core/meta`). Idempotent.
|
|
pub async fn ensure_repo(name: &str, token: &str) -> Result<()> {
|
|
let label = format!("core/{name}");
|
|
let res = bounded(
|
|
&format!("create repo {label}"),
|
|
api(token)?.create_current_user_repo(repo_option(name, true)),
|
|
)
|
|
.await?;
|
|
created_or_exists(res, &label)
|
|
}
|
|
|
|
/// Dashboard-warning guard for the meta non-fast-forward condition.
|
|
/// Held while `core/meta` remote is ahead of our local mirror; cleared
|
|
/// automatically when the next push succeeds.
|
|
static META_NON_FF_GUARD: Mutex<Option<crate::warnings::WarningGuard>> = Mutex::new(None);
|
|
|
|
/// Push `dir` (the meta repo) to `core/meta` on the local forge.
|
|
/// Best-effort: returns Err which callers log + ignore. No-op when
|
|
/// the core token isn't present yet (forge container not provisioned).
|
|
pub async fn push_meta(dir: &Path) -> Result<()> {
|
|
let Some(token) = core_token() else {
|
|
return Ok(());
|
|
};
|
|
// Token-in-URL push. Forgejo accepts `oauth2:<token>` or just
|
|
// any-username:<token>; using `core` matches the owner so the
|
|
// remote name is self-describing.
|
|
//
|
|
// No --force: `meta.rs` uses regular `git commit` (append-only),
|
|
// so the push is always fast-forward in normal operation. Per
|
|
// operator directive, hive-c0re must not force-push anywhere. If
|
|
// the remote is somehow ahead (e.g. split-brain or manual push) we
|
|
// raise a dashboard warning and leave the remote intact rather than
|
|
// silently erasing its history.
|
|
let url = forge_git_url("core/meta");
|
|
let out = crate::lifecycle::run_with_deadline(
|
|
crate::lifecycle::git_command_authed(&core_auth_header(&token))
|
|
.current_dir(dir)
|
|
.args(["push", &url, "HEAD:main"]),
|
|
crate::lifecycle::FETCH_DEADLINE,
|
|
"git push core/meta",
|
|
)
|
|
.await?;
|
|
if !out.status.success() {
|
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
|
if stderr.contains("non-fast-forward") || stderr.contains("fetch first") {
|
|
let msg = "forge/core/meta push rejected (non-fast-forward): \
|
|
remote is ahead of local; leaving remote history intact";
|
|
tracing::warn!("{msg}");
|
|
// Raise a persistent dashboard banner; cleared on next successful push.
|
|
if let Ok(mut g) = META_NON_FF_GUARD.lock() {
|
|
*g = Some(crate::warnings::set_warning(
|
|
"forge-meta-non-ff",
|
|
"warn",
|
|
msg,
|
|
));
|
|
}
|
|
return Ok(());
|
|
}
|
|
anyhow::bail!(
|
|
"git push core/meta failed ({}): {}",
|
|
out.status,
|
|
stderr.trim()
|
|
);
|
|
}
|
|
// Successful push: clear any outstanding non-ff warning.
|
|
if let Ok(mut g) = META_NON_FF_GUARD.lock() {
|
|
*g = None;
|
|
}
|
|
tracing::info!("forge: pushed meta to core/meta");
|
|
Ok(())
|
|
}
|
|
|
|
/// Grant agent `name` read-only collaborator access to `core/meta` on
|
|
/// the forge so the agent can clone/fetch the meta flake. Idempotent:
|
|
/// re-adding an existing collaborator succeeds (Forgejo answers 204
|
|
/// either way).
|
|
pub async fn meta_read_access(name: &str, core_token: &str) -> Result<()> {
|
|
add_collaborator(
|
|
"core",
|
|
"meta",
|
|
name,
|
|
AddCollaboratorOptionPermission::Read,
|
|
core_token,
|
|
)
|
|
.await?;
|
|
tracing::info!(%name, "forge: granted meta read access");
|
|
Ok(())
|
|
}
|
|
|
|
/// Add the forge `core/meta.git` URL as the `meta` remote in the
|
|
/// agent's proposed config repo so the agent (and the manager) can
|
|
/// fetch the meta flake from the forge. Idempotent: no-op when the
|
|
/// remote already points at the right URL, or when the proposed repo
|
|
/// does not exist yet. No-op when the forge is not running.
|
|
pub async fn ensure_meta_remote(name: &str) -> Result<()> {
|
|
if !is_present().await {
|
|
return Ok(());
|
|
}
|
|
// A malformed name has no proposed config repo (repos are only created
|
|
// under a validated Ident), so there's nothing to wire — no-op.
|
|
let Ok(agent) = hive_types::Ident::parse(name) else {
|
|
return Ok(());
|
|
};
|
|
let proposed_dir = Coordinator::agent_proposed_dir(&agent);
|
|
if !proposed_dir.join(".git").exists() {
|
|
return Ok(());
|
|
}
|
|
let want = format!("{}/core/meta.git", forge_http_base());
|
|
let existing = crate::lifecycle::git_command()
|
|
.current_dir(&proposed_dir)
|
|
.args(["remote", "get-url", "meta"])
|
|
.output()
|
|
.await
|
|
.context("git remote get-url meta")?;
|
|
if existing.status.success() {
|
|
let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned();
|
|
if current == want {
|
|
return Ok(());
|
|
}
|
|
crate::lifecycle::git(&proposed_dir, &["remote", "set-url", "meta", &want]).await
|
|
} else {
|
|
crate::lifecycle::git(&proposed_dir, &["remote", "add", "meta", &want]).await
|
|
}
|
|
}
|
|
|
|
/// Reseed a revived agent's `applied` repo — missing its `.git` entirely,
|
|
/// e.g. after `destroy --purge` — by fetching `agent-configs/<name>`'s
|
|
/// `main` from the forge. Unlike `proposed` (seeded once at first spawn and
|
|
/// never touched again — see `lifecycle::setup_proposed`'s own doc
|
|
/// comment), forge `main` advances with every operator-merged config PR, so
|
|
/// this reflects the last merged config rather than a stale creation-time
|
|
/// snapshot. A config deployed through hive-c0re without a forge merge never
|
|
/// reaches it.
|
|
///
|
|
/// Best-effort: returns `false` (not an error) rather than bailing when there
|
|
/// is no [`reader_token`], `applied` already has a `.git` (nothing to do), or
|
|
/// any git step fails — callers fall back to a less-current source or the
|
|
/// original hard error. Only `true` once `applied` is
|
|
/// actually seeded and tagged `deployed/0`, mirroring first-spawn's own
|
|
/// `setup_applied` seed shape exactly.
|
|
pub async fn fetch_config_main_into_applied(name: &str) -> bool {
|
|
let Some(reader) = reader_token().await else {
|
|
return false;
|
|
};
|
|
let applied = crate::paths::applied_dir(name);
|
|
if applied.join(".git").exists() {
|
|
return false;
|
|
}
|
|
if let Err(e) = std::fs::create_dir_all(&applied) {
|
|
tracing::warn!(%name, error = ?e, "forge: applied reseed mkdir failed");
|
|
return false;
|
|
}
|
|
if let Err(e) = crate::lifecycle::git(&applied, &["init", "--initial-branch=main"]).await {
|
|
tracing::warn!(%name, error = ?e, "forge: applied reseed init failed");
|
|
return false;
|
|
}
|
|
let url = forge_git_url(&format!("{CONFIG_ORG}/{name}"));
|
|
let auth = reader.auth_header();
|
|
let out = crate::lifecycle::git_command_authed(&auth)
|
|
.current_dir(&applied)
|
|
.args([
|
|
"fetch",
|
|
"--no-tags",
|
|
"--update-head-ok",
|
|
&url,
|
|
"refs/heads/main:refs/heads/main",
|
|
])
|
|
.output()
|
|
.await;
|
|
match out {
|
|
Ok(o) if o.status.success() => {}
|
|
Ok(o) => {
|
|
tracing::warn!(
|
|
%name,
|
|
stderr = %String::from_utf8_lossy(&o.stderr).trim(),
|
|
"forge: applied reseed fetch failed"
|
|
);
|
|
return false;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(%name, error = ?e, "forge: applied reseed fetch failed");
|
|
return false;
|
|
}
|
|
}
|
|
if let Err(e) = crate::lifecycle::git_read_tree_reset(&applied, "refs/heads/main").await {
|
|
tracing::warn!(%name, error = ?e, "forge: applied reseed read-tree failed");
|
|
return false;
|
|
}
|
|
if let Err(e) = crate::lifecycle::git_tag(&applied, "deployed/0", "refs/heads/main").await {
|
|
tracing::warn!(%name, error = ?e, "forge: applied reseed tag failed");
|
|
return false;
|
|
}
|
|
tracing::info!(%name, "forge: reseeded applied repo from agent-configs main");
|
|
true
|
|
}
|
|
|
|
/// Fast-forward `applied/<name>/main` to the config repo's current live
|
|
/// main. Unlike [`fetch_config_main_into_applied`] (reseed-only — bails
|
|
/// if `applied` already has a `.git`), this is for an `applied` repo that
|
|
/// already exists and just needs to catch up: force-sets the local ref to
|
|
/// match live main rather than requiring a fast-forward or any review
|
|
/// gate — there is no PR to review on this path, so nothing to check
|
|
/// ancestry against. Contrast the merged-PR deploy flow's CAS'd,
|
|
/// ancestry-gated advance for a reviewed PR merge, which this is not.
|
|
///
|
|
/// Best-effort, same shape as [`fetch_config_main_into_applied`]: returns
|
|
/// `false` (not an error) when there is no [`reader_token`], `applied` has
|
|
/// no `.git` yet (nothing to fast-forward — reseed instead), or any git step
|
|
/// fails.
|
|
pub async fn fast_forward_applied_main(name: &str) -> bool {
|
|
let Some(reader) = reader_token().await else {
|
|
return false;
|
|
};
|
|
let applied = crate::paths::applied_dir(name);
|
|
if !applied.join(".git").exists() {
|
|
return false;
|
|
}
|
|
let url = forge_git_url(&format!("{CONFIG_ORG}/{name}"));
|
|
let auth = reader.auth_header();
|
|
let out = crate::lifecycle::git_command_authed(&auth)
|
|
.current_dir(&applied)
|
|
.args(["fetch", "--no-tags", &url, "refs/heads/main"])
|
|
.output()
|
|
.await;
|
|
match out {
|
|
Ok(o) if o.status.success() => {}
|
|
Ok(o) => {
|
|
tracing::warn!(
|
|
%name,
|
|
stderr = %String::from_utf8_lossy(&o.stderr).trim(),
|
|
"forge: applied fast-forward fetch failed"
|
|
);
|
|
return false;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(%name, error = ?e, "forge: applied fast-forward fetch failed");
|
|
return false;
|
|
}
|
|
}
|
|
if let Err(e) =
|
|
crate::lifecycle::git_update_ref(&applied, "refs/heads/main", "FETCH_HEAD").await
|
|
{
|
|
tracing::warn!(%name, error = ?e, "forge: applied fast-forward update-ref failed");
|
|
return false;
|
|
}
|
|
if let Err(e) = crate::lifecycle::git_read_tree_reset(&applied, "refs/heads/main").await {
|
|
tracing::warn!(%name, error = ?e, "forge: applied fast-forward read-tree failed");
|
|
return false;
|
|
}
|
|
tracing::info!(%name, "forge: fast-forwarded applied/main to agent-configs main");
|
|
true
|
|
}
|
|
|
|
/// Seed an agent's `proposed` repo from `agent-configs/<name>` on the forge.
|
|
///
|
|
/// When the swarm creates an agent it writes that agent's config to the forge
|
|
/// before any hive is told to deploy it, so the hive's job here is to **take**
|
|
/// that config rather than author a second one beside it. The local template
|
|
/// is the fallback, not the default.
|
|
///
|
|
/// Best-effort like [`fetch_config_main_into_applied`], and `false` is an
|
|
/// ordinary answer rather than an error: there is no [`reader_token`], the
|
|
/// forge is unreachable, or `agent-configs/<name>` has no `main` — the last of
|
|
/// which is the *normal* case right after `swarm-controller` creates the
|
|
/// agent, before it has pushed anything into the repo it just made.
|
|
///
|
|
/// `clone` rather than `init` + `fetch`: git removes a directory it created
|
|
/// when the clone fails, and cloning into an existing empty dir leaves no
|
|
/// `.git` behind either — so the caller's "does this repo already exist"
|
|
/// check reads exactly the same before and after a failed attempt.
|
|
pub async fn clone_config_into_proposed(dir: &Path, name: &str) -> bool {
|
|
let Some(reader) = reader_token().await else {
|
|
return false;
|
|
};
|
|
let url = forge_git_url(&format!("{CONFIG_ORG}/{name}"));
|
|
// `--branch main` fails outright against a repo with no commits, so an
|
|
// empty repo reads as "nothing to take" instead of cloning to an unborn
|
|
// HEAD that would then look like a seeded checkout.
|
|
let out = crate::lifecycle::git_command_authed(&reader.auth_header())
|
|
.args([
|
|
"clone",
|
|
"--branch",
|
|
"main",
|
|
&url,
|
|
&dir.display().to_string(),
|
|
])
|
|
.output()
|
|
.await;
|
|
match out {
|
|
Ok(o) if o.status.success() => {
|
|
tracing::info!(%name, "forge: seeded proposed repo from agent-configs main");
|
|
true
|
|
}
|
|
Ok(o) => {
|
|
// `info`, not `warn`: the normal case right after
|
|
// `swarm-controller` creates the agent, before it has pushed
|
|
// anything into the repo it just made, and nothing is wrong.
|
|
// Logged at all because "where did this agent's config come
|
|
// from" is worth answering from the journal, and proposed is
|
|
// seeded exactly once.
|
|
tracing::info!(
|
|
%name,
|
|
stderr = %String::from_utf8_lossy(&o.stderr).trim(),
|
|
"forge: no config to clone for this agent; seeding the template"
|
|
);
|
|
false
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(%name, error = ?e, "forge: proposed clone failed to run");
|
|
false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Add `user` as a collaborator on `owner/repo` at `permission`.
|
|
/// Idempotent: adding an existing collaborator just updates its
|
|
/// permission (Forgejo answers 204 either way; a 201 from older
|
|
/// versions is tolerated defensively).
|
|
async fn add_collaborator(
|
|
owner: &str,
|
|
repo: &str,
|
|
user: &str,
|
|
permission: AddCollaboratorOptionPermission,
|
|
token: &str,
|
|
) -> Result<()> {
|
|
let op = format!("add collaborator {user} to {owner}/{repo}");
|
|
let res = bounded(
|
|
&op,
|
|
api(token)?.repo_add_collaborator(
|
|
owner,
|
|
repo,
|
|
user,
|
|
AddCollaboratorOption {
|
|
permission: Some(permission),
|
|
},
|
|
),
|
|
)
|
|
.await?;
|
|
match res {
|
|
Ok(()) => {}
|
|
Err(ForgejoError::UnexpectedStatusCode(s)) if s == StatusCode::CREATED => {}
|
|
Err(e) => {
|
|
return Err(e).context(op);
|
|
}
|
|
}
|
|
tracing::debug!(%owner, %repo, %user, ?permission, "forge: collaborator set");
|
|
Ok(())
|
|
}
|