A hung forge fetch inside `nix flake update hyperhive` (or any other meta git/nix child) now fails after its deadline with an error naming the command, the child is killed, and META_LOCK, BuildSlot and MetaWindow are released. The boot sweep already treats that failure as a warning and fans out the rebuilds. A wait on META_LOCK past 60s logs a warning naming the waiter and the holder.
264 lines
10 KiB
Rust
264 lines
10 KiB
Rust
//! Git shellout helpers for the per-agent proposed/applied repos: run
|
|
//! `git` with the hive-c0re identity, resolve/plant refs and tags, and
|
|
//! fetch proposal commits into the applied repo.
|
|
|
|
use std::path::Path;
|
|
use std::process::Output;
|
|
use std::time::Duration;
|
|
|
|
use anyhow::{Context, Result, bail};
|
|
use tokio::process::Command;
|
|
|
|
/// Deadline for a child that talks to a remote (forge fetch/push, `nix flake
|
|
/// update`/`lock`): a healthy one takes seconds, so ten minutes only ever
|
|
/// fires on a hang.
|
|
pub const FETCH_DEADLINE: Duration = Duration::from_mins(10);
|
|
|
|
/// Deadline for a git child that touches only the local repo.
|
|
pub const GIT_LOCAL_DEADLINE: Duration = Duration::from_mins(2);
|
|
|
|
/// Run `cmd` to completion and capture its output, failing once `deadline`
|
|
/// passes. `what` names the command in the error. The child is killed when the
|
|
/// deadline drops its future, so a hung remote cannot leave an orphan behind.
|
|
pub async fn run_with_deadline(
|
|
cmd: &mut Command,
|
|
deadline: Duration,
|
|
what: &str,
|
|
) -> Result<Output> {
|
|
// `kill_on_drop` is what turns the timeout's drop into a kill.
|
|
cmd.kill_on_drop(true);
|
|
tokio::time::timeout(deadline, cmd.output())
|
|
.await
|
|
.map_err(|_| anyhow::anyhow!("{what} timed out after {}s", deadline.as_secs_f64()))?
|
|
.with_context(|| what.to_owned())
|
|
}
|
|
|
|
const GIT_NAME: &str = "c0re";
|
|
const GIT_EMAIL: &str = "c0re@hyperhive.local";
|
|
|
|
/// Return the SHA of the root (oldest, no-parent) commit in a repo.
|
|
/// Used to seed the applied repo at the template baseline rather than at
|
|
/// `main`, so `deployed/0` records the template, not the manager's first commit.
|
|
pub(super) async fn git_root_commit(dir: &Path) -> Result<String> {
|
|
let out = git_command()
|
|
.current_dir(dir)
|
|
.args(["rev-list", "--max-parents=0", "HEAD"])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git rev-list --max-parents=0 HEAD in {}", dir.display()))?;
|
|
if !out.status.success() {
|
|
anyhow::bail!(
|
|
"git rev-list --max-parents=0 failed: {}",
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
|
}
|
|
|
|
pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> {
|
|
git(
|
|
dir,
|
|
&[
|
|
"-c",
|
|
&format!("user.name={GIT_NAME}"),
|
|
"-c",
|
|
&format!("user.email={GIT_EMAIL}"),
|
|
"commit",
|
|
"-m",
|
|
message,
|
|
],
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in
|
|
/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise.
|
|
///
|
|
/// `kill_on_drop(true)`: if the caller's future is dropped before the child
|
|
/// exits — e.g. a `tokio::time::timeout` around startup migration fires — the
|
|
/// git child is killed instead of orphaned (left retrying an unreachable
|
|
/// forge). No-op on normal completion, where the child has already exited.
|
|
#[must_use]
|
|
pub fn git_command() -> Command {
|
|
let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into());
|
|
let mut cmd = Command::new(exe);
|
|
cmd.kill_on_drop(true);
|
|
cmd
|
|
}
|
|
|
|
/// [`git_command`] carrying HTTP credentials **in its environment**.
|
|
///
|
|
/// `auth_header` is an `Authorization:` line (see
|
|
/// [`crate::forge::core_auth_header`]) handed to git as `http.extraHeader`
|
|
/// through `GIT_CONFIG_*`, which git reads exactly like a config file. The
|
|
/// alternative — userinfo in the remote URL — puts the secret in `argv`, and
|
|
/// `/proc/<pid>/cmdline` is world-readable while `/proc/<pid>/environ` is
|
|
/// owner-only. Same credential, materially smaller audience.
|
|
///
|
|
/// `GIT_CONFIG_COUNT` needs git >= 2.31; `HYPERHIVE_GIT` points at a pinned
|
|
/// nixpkgs git well past that.
|
|
#[must_use]
|
|
pub fn git_command_authed(auth_header: &str) -> Command {
|
|
let mut cmd = git_command();
|
|
cmd.env("GIT_CONFIG_COUNT", "1")
|
|
.env("GIT_CONFIG_KEY_0", "http.extraHeader")
|
|
.env("GIT_CONFIG_VALUE_0", auth_header);
|
|
cmd
|
|
}
|
|
|
|
/// Run `cmd` as `git <args>` in `dir` within `deadline`, erroring on a
|
|
/// non-zero exit.
|
|
async fn run(mut cmd: Command, dir: &Path, args: &[&str], deadline: Duration) -> Result<()> {
|
|
let what = format!("git {} in {}", args.join(" "), dir.display());
|
|
let out = run_with_deadline(cmd.current_dir(dir).args(args), deadline, &what).await?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"git {} failed ({}): {}",
|
|
args.join(" "),
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn git(dir: &Path, args: &[&str]) -> Result<()> {
|
|
run(git_command(), dir, args, GIT_LOCAL_DEADLINE).await
|
|
}
|
|
|
|
/// [`git`] against an authenticated remote — for the paths that talk to the
|
|
/// forge. The credential rides the environment, so `args` (and therefore the
|
|
/// error above) stay free of it.
|
|
pub async fn git_authed(dir: &Path, args: &[&str], auth_header: &str) -> Result<()> {
|
|
run(git_command_authed(auth_header), dir, args, FETCH_DEADLINE).await
|
|
}
|
|
|
|
/// Resolve `refname` (a tag, branch, or sha) in `dir` to its full sha.
|
|
pub async fn git_rev_parse(dir: &Path, refname: &str) -> Result<String> {
|
|
let out = git_command()
|
|
.current_dir(dir)
|
|
.args(["rev-parse", refname])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git rev-parse {refname} in {}", dir.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"git rev-parse {refname} failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
|
}
|
|
|
|
/// The contents of `path` at commit `rev` in `dir`, `None` when `rev` has no
|
|
/// such file. Reads the object database, not the working tree.
|
|
pub async fn git_show_file(dir: &Path, rev: &str, path: &str) -> Result<Option<String>> {
|
|
let spec = format!("{rev}:{path}");
|
|
let listed = git_command()
|
|
.current_dir(dir)
|
|
.args(["ls-tree", "--name-only", rev, "--", path])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git ls-tree {spec} in {}", dir.display()))?;
|
|
if !listed.status.success() {
|
|
bail!(
|
|
"git ls-tree {spec} failed ({}): {}",
|
|
listed.status,
|
|
String::from_utf8_lossy(&listed.stderr).trim()
|
|
);
|
|
}
|
|
if listed.stdout.iter().all(u8::is_ascii_whitespace) {
|
|
return Ok(None);
|
|
}
|
|
let shown = git_command()
|
|
.current_dir(dir)
|
|
.args(["show", &spec])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git show {spec} in {}", dir.display()))?;
|
|
if !shown.status.success() {
|
|
bail!(
|
|
"git show {spec} failed ({}): {}",
|
|
shown.status,
|
|
String::from_utf8_lossy(&shown.stderr).trim()
|
|
);
|
|
}
|
|
Ok(Some(String::from_utf8_lossy(&shown.stdout).into_owned()))
|
|
}
|
|
|
|
/// Plant a lightweight tag at `target`. Errors if the tag already
|
|
/// exists — we want loud failures on id reuse, not silent
|
|
/// overwrites.
|
|
pub async fn git_tag(dir: &Path, name: &str, target: &str) -> Result<()> {
|
|
git(dir, &["tag", name, target]).await
|
|
}
|
|
|
|
/// Replace working tree + index with the tree at `target` without
|
|
/// moving HEAD. `applied/main` stays pointing at the last known-good
|
|
/// `deployed/*` while we let `nixos-container update` evaluate the
|
|
/// candidate. On build failure callers reset back to HEAD; on
|
|
/// success they fast-forward main to `target`.
|
|
pub async fn git_read_tree_reset(dir: &Path, target: &str) -> Result<()> {
|
|
git(dir, &["read-tree", "--reset", "-u", target]).await
|
|
}
|
|
|
|
/// Hard-set a ref to `target`. Used to fast-forward `refs/heads/main`
|
|
/// to the just-deployed proposal commit. Uses `update-ref`, not
|
|
/// `branch -f`, so it works regardless of where HEAD currently sits.
|
|
pub async fn git_update_ref(dir: &Path, refname: &str, target: &str) -> Result<()> {
|
|
git(dir, &["update-ref", refname, target]).await
|
|
}
|
|
|
|
/// Compare-and-swap a ref: move `refname` to `new` only if it currently points
|
|
/// at `old`. `git update-ref <ref> <new> <old>` refuses — leaving the ref
|
|
/// untouched — when the current value differs. That refusal is the whole
|
|
/// difference between "advance this branch" and "overwrite whatever is there".
|
|
///
|
|
/// Prefer this over [`git_update_ref`] whenever the caller already knows the
|
|
/// value it believes the ref holds: a plain `update-ref` that raced another
|
|
/// writer discards the other writer's commits without a word.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns an error if the ref does not currently point at `old` (the CAS lost)
|
|
/// or if the `git` invocation itself fails.
|
|
pub async fn git_update_ref_cas(dir: &Path, refname: &str, new: &str, old: &str) -> Result<()> {
|
|
git(dir, &["update-ref", refname, new, old]).await
|
|
}
|
|
|
|
/// True when `ancestor` is reachable from `descendant` — i.e. moving a branch
|
|
/// from `ancestor` to `descendant` is a genuine fast-forward that discards
|
|
/// nothing.
|
|
///
|
|
/// Not the same question as "did this branch land upstream": a squash-merge
|
|
/// rewrites the commit, so `--is-ancestor` correctly answers `false` for a
|
|
/// branch whose *contents* were merged. This helper compares two commits with
|
|
/// real shared ancestry inside one repo, which is precisely what it decides.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns an error if `git` fails to run or either revision can't be resolved.
|
|
/// A clean "no, not an ancestor" is `Ok(false)`, not an error.
|
|
pub async fn git_is_ancestor(dir: &Path, ancestor: &str, descendant: &str) -> Result<bool> {
|
|
let out = git_command()
|
|
.current_dir(dir)
|
|
.args(["merge-base", "--is-ancestor", ancestor, descendant])
|
|
.output()
|
|
.await
|
|
.with_context(|| {
|
|
format!(
|
|
"git merge-base --is-ancestor {ancestor} {descendant} in {}",
|
|
dir.display()
|
|
)
|
|
})?;
|
|
match out.status.code() {
|
|
Some(0) => Ok(true),
|
|
Some(1) => Ok(false),
|
|
_ => bail!(
|
|
"git merge-base --is-ancestor {ancestor} {descendant} failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
),
|
|
}
|
|
}
|