diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index bc94862f..0aa2919d 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -20,6 +20,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use tokio::process::Command; +use tokio::sync::Mutex; use crate::lifecycle; @@ -28,6 +29,17 @@ const APPLIED_ROOT: &str = "/var/lib/hyperhive/applied"; const GIT_NAME: &str = "hive-c0re"; const GIT_EMAIL: &str = "hive-c0re@hyperhive"; +/// Single-writer lock around every meta-repo operation. Git isn't +/// safe to drive from concurrent processes against the same `.git/` +/// — two simultaneous `git add` / `commit` invocations race on +/// `.git/index.lock`; if either dies before releasing, the lock +/// sticks and the next operation hits "another git process seems to +/// be running" until somebody `rm`s it manually. Holding this mutex +/// across each public function's git+nix calls makes parallel +/// rebuilds (`auto_update` + dashboard-triggered + apply-commit) +/// take turns instead of colliding. +static META_LOCK: Mutex<()> = Mutex::const_new(()); + /// Where the manager sees this directory inside its container (RO bind). #[allow(dead_code)] // wired up by set_nspawn_flags in a follow-up commit pub const CONTAINER_MANAGER_META_MOUNT: &str = "/meta"; @@ -56,6 +68,7 @@ pub async fn sync_agents( operator_pronouns: &str, agents: &[AgentSpec], ) -> Result<()> { + let _guard = META_LOCK.lock().await; let dir = meta_dir(); std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?; @@ -104,6 +117,7 @@ pub async fn sync_agents( /// meta history only carries successful deploys. #[allow(dead_code)] // wired up by actions::run_apply_commit in a later commit pub async fn prepare_deploy(name: &str) -> Result<()> { + let _guard = META_LOCK.lock().await; let dir = meta_dir(); let input = format!("agent-{name}"); nix(&dir, &["flake", "update", &input]).await?; @@ -118,6 +132,7 @@ pub async fn prepare_deploy(name: &str) -> Result<()> { /// place (nothing staged → nothing to commit). #[allow(dead_code)] pub async fn finalize_deploy(name: &str, sha: &str, tag: &str) -> Result<()> { + let _guard = META_LOCK.lock().await; let dir = meta_dir(); if !has_staged_changes(&dir).await? { return Ok(()); @@ -131,6 +146,7 @@ pub async fn finalize_deploy(name: &str, sha: &str, tag: &str) -> Result<()> { /// captured in `applied/`'s annotated `failed/` tag. #[allow(dead_code)] pub async fn abort_deploy() -> Result<()> { + let _guard = META_LOCK.lock().await; let dir = meta_dir(); git(&dir, &["restore", "--staged", "flake.lock"]).await?; git(&dir, &["restore", "flake.lock"]).await @@ -157,6 +173,7 @@ async fn has_staged_changes(dir: &Path) -> Result { /// semantics — it always wants the latest main. #[allow(dead_code)] // wired up by lifecycle::rebuild in this commit pub async fn lock_update_for_rebuild(name: &str) -> Result<()> { + let _guard = META_LOCK.lock().await; let dir = meta_dir(); let input = format!("agent-{name}"); nix(&dir, &["flake", "update", &input]).await?; @@ -172,6 +189,7 @@ pub async fn lock_update_for_rebuild(name: &str) -> Result<()> { /// because the per-agent inputs aren't touched. #[allow(dead_code)] pub async fn lock_update_hyperhive() -> Result<()> { + let _guard = META_LOCK.lock().await; let dir = meta_dir(); nix(&dir, &["flake", "update", "hyperhive"]).await?; if git_is_clean(&dir).await? { diff --git a/hive-c0re/src/migrate.rs b/hive-c0re/src/migrate.rs index 33a7787f..06b69069 100644 --- a/hive-c0re/src/migrate.rs +++ b/hive-c0re/src/migrate.rs @@ -49,6 +49,17 @@ pub async fn run(coord: &Arc) -> Result<()> { tracing::info!("migration: {KILL_SWITCH} set — skipping"); return Ok(()); } + // Stale meta index lock: a previous hive-c0re crash mid-`git add` + // can leave `.git/index.lock` behind, which blocks every + // subsequent meta op until somebody `rm`s it manually. We just + // booted so nothing of ours is holding it; safe to clear. + let meta_lock = std::path::PathBuf::from("/var/lib/hyperhive/meta/.git/index.lock"); + if meta_lock.exists() { + match std::fs::remove_file(&meta_lock) { + Ok(()) => tracing::warn!("cleared stale meta/.git/index.lock"), + Err(e) => tracing::warn!(error = ?e, "clear stale meta lock failed"), + } + } let names = enumerate_agents().await; tracing::info!(count = names.len(), "migration: scanning");