Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/hive-c0re/src/forge/repos.rs
atlas 7cde4e7b15 meta nix/git calls under META_LOCK time out instead of wedging every deploy
A hung forge fetch inside `nix flake update hyperhive` (or any other meta
git/nix child) now fails after its deadline with an error naming the command,
the child is killed, and META_LOCK, BuildSlot and MetaWindow are released. The
boot sweep already treats that failure as a warning and fans out the rebuilds.
A wait on META_LOCK past 60s logs a warning naming the waiter and the holder.
2026-10-20 12:01:14 +02:00

427 lines
16 KiB
Rust

//! Repo + org plumbing on the local Forgejo: the meta + knowledge repos,
//! meta mirroring (`push_meta`), collaborator grants, pull-mirrors, and
//! branch-protection rules. The typed API-client constructor + org-name
//! constants live in the module root (`super`).
use std::path::Path;
use std::sync::Mutex;
use anyhow::{Context, Result};
use forgejo_api::structs::{
AddCollaboratorOption, AddCollaboratorOptionPermission, CreateRepoOption, Repository,
};
use forgejo_api::{ApiErrorKind, ForgejoError};
use reqwest::StatusCode;
use crate::coordinator::Coordinator;
use super::{
CONFIG_ORG, api, bounded, core_auth_header, core_token, forge_git_url, forge_http_base,
is_present, reader_token,
};
/// Creation options for an empty repo defaulting to `main`.
fn repo_option(name: &str, private: bool) -> CreateRepoOption {
CreateRepoOption {
auto_init: Some(false),
default_branch: Some("main".to_owned()),
description: None,
gitignores: None,
issue_labels: None,
license: None,
name: name.to_owned(),
object_format_name: None,
private: Some(private),
readme: None,
template: None,
trust_model: None,
}
}
/// Whether a create-style call failed because the object already
/// exists. Forgejo signals this as HTTP 409 (conflict) or 422
/// (validation). The typed client surfaces those as
/// `ApiErrorKind::Other(409)` / `ValidationFailed` when the endpoint
/// spec lists the status, or as a bare `UnexpectedStatusCode`
/// otherwise — match all shapes defensively.
fn is_already_exists(e: &ForgejoError) -> bool {
match e {
ForgejoError::ApiError(api) => match api.error_kind() {
ApiErrorKind::ValidationFailed => true,
ApiErrorKind::Other(s) => *s == StatusCode::CONFLICT,
_ => false,
},
ForgejoError::UnexpectedStatusCode(s) => {
*s == StatusCode::CONFLICT || *s == StatusCode::UNPROCESSABLE_ENTITY
}
_ => false,
}
}
/// Fold a repo-creation result's "already exists" (409 / 422) into
/// success. `label` is `<owner>/<name>` — purely for log + error
/// context.
fn created_or_exists(res: Result<Repository, ForgejoError>, label: &str) -> Result<()> {
match res {
Ok(_) => {
tracing::info!(%label, "forge: created repo");
Ok(())
}
Err(e) if is_already_exists(&e) => {
tracing::debug!(%label, "forge: repo already exists");
Ok(())
}
Err(e) => Err(e).with_context(|| format!("create repo {label}")),
}
}
/// Create a repo in the token-owner's own namespace. `token` belongs
/// to the user we want the repo owned by (we use `core`'s token for
/// `core/meta`). Idempotent.
pub async fn ensure_repo(name: &str, token: &str) -> Result<()> {
let label = format!("core/{name}");
let res = bounded(
&format!("create repo {label}"),
api(token)?.create_current_user_repo(repo_option(name, true)),
)
.await?;
created_or_exists(res, &label)
}
/// Dashboard-warning guard for the meta non-fast-forward condition.
/// Held while `core/meta` remote is ahead of our local mirror; cleared
/// automatically when the next push succeeds.
static META_NON_FF_GUARD: Mutex<Option<crate::warnings::WarningGuard>> = Mutex::new(None);
/// Push `dir` (the meta repo) to `core/meta` on the local forge.
/// Best-effort: returns Err which callers log + ignore. No-op when
/// the core token isn't present yet (forge container not provisioned).
pub async fn push_meta(dir: &Path) -> Result<()> {
let Some(token) = core_token() else {
return Ok(());
};
// Token-in-URL push. Forgejo accepts `oauth2:<token>` or just
// any-username:<token>; using `core` matches the owner so the
// remote name is self-describing.
//
// No --force: `meta.rs` uses regular `git commit` (append-only),
// so the push is always fast-forward in normal operation. Per
// operator directive, hive-c0re must not force-push anywhere. If
// the remote is somehow ahead (e.g. split-brain or manual push) we
// raise a dashboard warning and leave the remote intact rather than
// silently erasing its history.
let url = forge_git_url("core/meta");
let out = crate::lifecycle::run_with_deadline(
crate::lifecycle::git_command_authed(&core_auth_header(&token))
.current_dir(dir)
.args(["push", &url, "HEAD:main"]),
crate::lifecycle::FETCH_DEADLINE,
"git push core/meta",
)
.await?;
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr);
if stderr.contains("non-fast-forward") || stderr.contains("fetch first") {
let msg = "forge/core/meta push rejected (non-fast-forward): \
remote is ahead of local; leaving remote history intact";
tracing::warn!("{msg}");
// Raise a persistent dashboard banner; cleared on next successful push.
if let Ok(mut g) = META_NON_FF_GUARD.lock() {
*g = Some(crate::warnings::set_warning(
"forge-meta-non-ff",
"warn",
msg,
));
}
return Ok(());
}
anyhow::bail!(
"git push core/meta failed ({}): {}",
out.status,
stderr.trim()
);
}
// Successful push: clear any outstanding non-ff warning.
if let Ok(mut g) = META_NON_FF_GUARD.lock() {
*g = None;
}
tracing::info!("forge: pushed meta to core/meta");
Ok(())
}
/// Grant agent `name` read-only collaborator access to `core/meta` on
/// the forge so the agent can clone/fetch the meta flake. Idempotent:
/// re-adding an existing collaborator succeeds (Forgejo answers 204
/// either way).
pub async fn meta_read_access(name: &str, core_token: &str) -> Result<()> {
add_collaborator(
"core",
"meta",
name,
AddCollaboratorOptionPermission::Read,
core_token,
)
.await?;
tracing::info!(%name, "forge: granted meta read access");
Ok(())
}
/// Add the forge `core/meta.git` URL as the `meta` remote in the
/// agent's proposed config repo so the agent (and the manager) can
/// fetch the meta flake from the forge. Idempotent: no-op when the
/// remote already points at the right URL, or when the proposed repo
/// does not exist yet. No-op when the forge is not running.
pub async fn ensure_meta_remote(name: &str) -> Result<()> {
if !is_present().await {
return Ok(());
}
// A malformed name has no proposed config repo (repos are only created
// under a validated Ident), so there's nothing to wire — no-op.
let Ok(agent) = hive_types::Ident::parse(name) else {
return Ok(());
};
let proposed_dir = Coordinator::agent_proposed_dir(&agent);
if !proposed_dir.join(".git").exists() {
return Ok(());
}
let want = format!("{}/core/meta.git", forge_http_base());
let existing = crate::lifecycle::git_command()
.current_dir(&proposed_dir)
.args(["remote", "get-url", "meta"])
.output()
.await
.context("git remote get-url meta")?;
if existing.status.success() {
let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned();
if current == want {
return Ok(());
}
crate::lifecycle::git(&proposed_dir, &["remote", "set-url", "meta", &want]).await
} else {
crate::lifecycle::git(&proposed_dir, &["remote", "add", "meta", &want]).await
}
}
/// Reseed a revived agent's `applied` repo — missing its `.git` entirely,
/// e.g. after `destroy --purge` — by fetching `agent-configs/<name>`'s
/// `main` from the forge. Unlike `proposed` (seeded once at first spawn and
/// never touched again — see `lifecycle::setup_proposed`'s own doc
/// comment), forge `main` advances with every operator-merged config PR, so
/// this reflects the last merged config rather than a stale creation-time
/// snapshot. A config deployed through hive-c0re without a forge merge never
/// reaches it.
///
/// Best-effort: returns `false` (not an error) rather than bailing when there
/// is no [`reader_token`], `applied` already has a `.git` (nothing to do), or
/// any git step fails — callers fall back to a less-current source or the
/// original hard error. Only `true` once `applied` is
/// actually seeded and tagged `deployed/0`, mirroring first-spawn's own
/// `setup_applied` seed shape exactly.
pub async fn fetch_config_main_into_applied(name: &str) -> bool {
let Some(reader) = reader_token().await else {
return false;
};
let applied = crate::paths::applied_dir(name);
if applied.join(".git").exists() {
return false;
}
if let Err(e) = std::fs::create_dir_all(&applied) {
tracing::warn!(%name, error = ?e, "forge: applied reseed mkdir failed");
return false;
}
if let Err(e) = crate::lifecycle::git(&applied, &["init", "--initial-branch=main"]).await {
tracing::warn!(%name, error = ?e, "forge: applied reseed init failed");
return false;
}
let url = forge_git_url(&format!("{CONFIG_ORG}/{name}"));
let auth = reader.auth_header();
let out = crate::lifecycle::git_command_authed(&auth)
.current_dir(&applied)
.args([
"fetch",
"--no-tags",
"--update-head-ok",
&url,
"refs/heads/main:refs/heads/main",
])
.output()
.await;
match out {
Ok(o) if o.status.success() => {}
Ok(o) => {
tracing::warn!(
%name,
stderr = %String::from_utf8_lossy(&o.stderr).trim(),
"forge: applied reseed fetch failed"
);
return false;
}
Err(e) => {
tracing::warn!(%name, error = ?e, "forge: applied reseed fetch failed");
return false;
}
}
if let Err(e) = crate::lifecycle::git_read_tree_reset(&applied, "refs/heads/main").await {
tracing::warn!(%name, error = ?e, "forge: applied reseed read-tree failed");
return false;
}
if let Err(e) = crate::lifecycle::git_tag(&applied, "deployed/0", "refs/heads/main").await {
tracing::warn!(%name, error = ?e, "forge: applied reseed tag failed");
return false;
}
tracing::info!(%name, "forge: reseeded applied repo from agent-configs main");
true
}
/// Fast-forward `applied/<name>/main` to the config repo's current live
/// main. Unlike [`fetch_config_main_into_applied`] (reseed-only — bails
/// if `applied` already has a `.git`), this is for an `applied` repo that
/// already exists and just needs to catch up: force-sets the local ref to
/// match live main rather than requiring a fast-forward or any review
/// gate — there is no PR to review on this path, so nothing to check
/// ancestry against. Contrast the merged-PR deploy flow's CAS'd,
/// ancestry-gated advance for a reviewed PR merge, which this is not.
///
/// Best-effort, same shape as [`fetch_config_main_into_applied`]: returns
/// `false` (not an error) when there is no [`reader_token`], `applied` has
/// no `.git` yet (nothing to fast-forward — reseed instead), or any git step
/// fails.
pub async fn fast_forward_applied_main(name: &str) -> bool {
let Some(reader) = reader_token().await else {
return false;
};
let applied = crate::paths::applied_dir(name);
if !applied.join(".git").exists() {
return false;
}
let url = forge_git_url(&format!("{CONFIG_ORG}/{name}"));
let auth = reader.auth_header();
let out = crate::lifecycle::git_command_authed(&auth)
.current_dir(&applied)
.args(["fetch", "--no-tags", &url, "refs/heads/main"])
.output()
.await;
match out {
Ok(o) if o.status.success() => {}
Ok(o) => {
tracing::warn!(
%name,
stderr = %String::from_utf8_lossy(&o.stderr).trim(),
"forge: applied fast-forward fetch failed"
);
return false;
}
Err(e) => {
tracing::warn!(%name, error = ?e, "forge: applied fast-forward fetch failed");
return false;
}
}
if let Err(e) =
crate::lifecycle::git_update_ref(&applied, "refs/heads/main", "FETCH_HEAD").await
{
tracing::warn!(%name, error = ?e, "forge: applied fast-forward update-ref failed");
return false;
}
if let Err(e) = crate::lifecycle::git_read_tree_reset(&applied, "refs/heads/main").await {
tracing::warn!(%name, error = ?e, "forge: applied fast-forward read-tree failed");
return false;
}
tracing::info!(%name, "forge: fast-forwarded applied/main to agent-configs main");
true
}
/// Seed an agent's `proposed` repo from `agent-configs/<name>` on the forge.
///
/// When the swarm creates an agent it writes that agent's config to the forge
/// before any hive is told to deploy it, so the hive's job here is to **take**
/// that config rather than author a second one beside it. The local template
/// is the fallback, not the default.
///
/// Best-effort like [`fetch_config_main_into_applied`], and `false` is an
/// ordinary answer rather than an error: there is no [`reader_token`], the
/// forge is unreachable, or `agent-configs/<name>` has no `main` — the last of
/// which is the *normal* case right after `swarm-controller` creates the
/// agent, before it has pushed anything into the repo it just made.
///
/// `clone` rather than `init` + `fetch`: git removes a directory it created
/// when the clone fails, and cloning into an existing empty dir leaves no
/// `.git` behind either — so the caller's "does this repo already exist"
/// check reads exactly the same before and after a failed attempt.
pub async fn clone_config_into_proposed(dir: &Path, name: &str) -> bool {
let Some(reader) = reader_token().await else {
return false;
};
let url = forge_git_url(&format!("{CONFIG_ORG}/{name}"));
// `--branch main` fails outright against a repo with no commits, so an
// empty repo reads as "nothing to take" instead of cloning to an unborn
// HEAD that would then look like a seeded checkout.
let out = crate::lifecycle::git_command_authed(&reader.auth_header())
.args([
"clone",
"--branch",
"main",
&url,
&dir.display().to_string(),
])
.output()
.await;
match out {
Ok(o) if o.status.success() => {
tracing::info!(%name, "forge: seeded proposed repo from agent-configs main");
true
}
Ok(o) => {
// `info`, not `warn`: the normal case right after
// `swarm-controller` creates the agent, before it has pushed
// anything into the repo it just made, and nothing is wrong.
// Logged at all because "where did this agent's config come
// from" is worth answering from the journal, and proposed is
// seeded exactly once.
tracing::info!(
%name,
stderr = %String::from_utf8_lossy(&o.stderr).trim(),
"forge: no config to clone for this agent; seeding the template"
);
false
}
Err(e) => {
tracing::warn!(%name, error = ?e, "forge: proposed clone failed to run");
false
}
}
}
/// Add `user` as a collaborator on `owner/repo` at `permission`.
/// Idempotent: adding an existing collaborator just updates its
/// permission (Forgejo answers 204 either way; a 201 from older
/// versions is tolerated defensively).
async fn add_collaborator(
owner: &str,
repo: &str,
user: &str,
permission: AddCollaboratorOptionPermission,
token: &str,
) -> Result<()> {
let op = format!("add collaborator {user} to {owner}/{repo}");
let res = bounded(
&op,
api(token)?.repo_add_collaborator(
owner,
repo,
user,
AddCollaboratorOption {
permission: Some(permission),
},
),
)
.await?;
match res {
Ok(()) => {}
Err(ForgejoError::UnexpectedStatusCode(s)) if s == StatusCode::CREATED => {}
Err(e) => {
return Err(e).context(op);
}
}
tracing::debug!(%owner, %repo, %user, ?permission, "forge: collaborator set");
Ok(())
}