Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2d277038a7 | ||
|
|
78f21ccc5d |
2 changed files with 29 additions and 0 deletions
|
|
@ -20,6 +20,7 @@ use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
use crate::lifecycle;
|
use crate::lifecycle;
|
||||||
|
|
||||||
|
|
@ -28,6 +29,17 @@ const APPLIED_ROOT: &str = "/var/lib/hyperhive/applied";
|
||||||
const GIT_NAME: &str = "hive-c0re";
|
const GIT_NAME: &str = "hive-c0re";
|
||||||
const GIT_EMAIL: &str = "hive-c0re@hyperhive";
|
const GIT_EMAIL: &str = "hive-c0re@hyperhive";
|
||||||
|
|
||||||
|
/// Single-writer lock around every meta-repo operation. Git isn't
|
||||||
|
/// safe to drive from concurrent processes against the same `.git/`
|
||||||
|
/// — two simultaneous `git add` / `commit` invocations race on
|
||||||
|
/// `.git/index.lock`; if either dies before releasing, the lock
|
||||||
|
/// sticks and the next operation hits "another git process seems to
|
||||||
|
/// be running" until somebody `rm`s it manually. Holding this mutex
|
||||||
|
/// across each public function's git+nix calls makes parallel
|
||||||
|
/// rebuilds (`auto_update` + dashboard-triggered + apply-commit)
|
||||||
|
/// take turns instead of colliding.
|
||||||
|
static META_LOCK: Mutex<()> = Mutex::const_new(());
|
||||||
|
|
||||||
/// Where the manager sees this directory inside its container (RO bind).
|
/// Where the manager sees this directory inside its container (RO bind).
|
||||||
#[allow(dead_code)] // wired up by set_nspawn_flags in a follow-up commit
|
#[allow(dead_code)] // wired up by set_nspawn_flags in a follow-up commit
|
||||||
pub const CONTAINER_MANAGER_META_MOUNT: &str = "/meta";
|
pub const CONTAINER_MANAGER_META_MOUNT: &str = "/meta";
|
||||||
|
|
@ -56,6 +68,7 @@ pub async fn sync_agents(
|
||||||
operator_pronouns: &str,
|
operator_pronouns: &str,
|
||||||
agents: &[AgentSpec],
|
agents: &[AgentSpec],
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
let _guard = META_LOCK.lock().await;
|
||||||
let dir = meta_dir();
|
let dir = meta_dir();
|
||||||
std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
|
std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
|
||||||
|
|
||||||
|
|
@ -104,6 +117,7 @@ pub async fn sync_agents(
|
||||||
/// meta history only carries successful deploys.
|
/// meta history only carries successful deploys.
|
||||||
#[allow(dead_code)] // wired up by actions::run_apply_commit in a later commit
|
#[allow(dead_code)] // wired up by actions::run_apply_commit in a later commit
|
||||||
pub async fn prepare_deploy(name: &str) -> Result<()> {
|
pub async fn prepare_deploy(name: &str) -> Result<()> {
|
||||||
|
let _guard = META_LOCK.lock().await;
|
||||||
let dir = meta_dir();
|
let dir = meta_dir();
|
||||||
let input = format!("agent-{name}");
|
let input = format!("agent-{name}");
|
||||||
nix(&dir, &["flake", "update", &input]).await?;
|
nix(&dir, &["flake", "update", &input]).await?;
|
||||||
|
|
@ -118,6 +132,7 @@ pub async fn prepare_deploy(name: &str) -> Result<()> {
|
||||||
/// place (nothing staged → nothing to commit).
|
/// place (nothing staged → nothing to commit).
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn finalize_deploy(name: &str, sha: &str, tag: &str) -> Result<()> {
|
pub async fn finalize_deploy(name: &str, sha: &str, tag: &str) -> Result<()> {
|
||||||
|
let _guard = META_LOCK.lock().await;
|
||||||
let dir = meta_dir();
|
let dir = meta_dir();
|
||||||
if !has_staged_changes(&dir).await? {
|
if !has_staged_changes(&dir).await? {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
|
|
@ -131,6 +146,7 @@ pub async fn finalize_deploy(name: &str, sha: &str, tag: &str) -> Result<()> {
|
||||||
/// captured in `applied/<n>`'s annotated `failed/<id>` tag.
|
/// captured in `applied/<n>`'s annotated `failed/<id>` tag.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn abort_deploy() -> Result<()> {
|
pub async fn abort_deploy() -> Result<()> {
|
||||||
|
let _guard = META_LOCK.lock().await;
|
||||||
let dir = meta_dir();
|
let dir = meta_dir();
|
||||||
git(&dir, &["restore", "--staged", "flake.lock"]).await?;
|
git(&dir, &["restore", "--staged", "flake.lock"]).await?;
|
||||||
git(&dir, &["restore", "flake.lock"]).await
|
git(&dir, &["restore", "flake.lock"]).await
|
||||||
|
|
@ -157,6 +173,7 @@ async fn has_staged_changes(dir: &Path) -> Result<bool> {
|
||||||
/// semantics — it always wants the latest main.
|
/// semantics — it always wants the latest main.
|
||||||
#[allow(dead_code)] // wired up by lifecycle::rebuild in this commit
|
#[allow(dead_code)] // wired up by lifecycle::rebuild in this commit
|
||||||
pub async fn lock_update_for_rebuild(name: &str) -> Result<()> {
|
pub async fn lock_update_for_rebuild(name: &str) -> Result<()> {
|
||||||
|
let _guard = META_LOCK.lock().await;
|
||||||
let dir = meta_dir();
|
let dir = meta_dir();
|
||||||
let input = format!("agent-{name}");
|
let input = format!("agent-{name}");
|
||||||
nix(&dir, &["flake", "update", &input]).await?;
|
nix(&dir, &["flake", "update", &input]).await?;
|
||||||
|
|
@ -172,6 +189,7 @@ pub async fn lock_update_for_rebuild(name: &str) -> Result<()> {
|
||||||
/// because the per-agent inputs aren't touched.
|
/// because the per-agent inputs aren't touched.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn lock_update_hyperhive() -> Result<()> {
|
pub async fn lock_update_hyperhive() -> Result<()> {
|
||||||
|
let _guard = META_LOCK.lock().await;
|
||||||
let dir = meta_dir();
|
let dir = meta_dir();
|
||||||
nix(&dir, &["flake", "update", "hyperhive"]).await?;
|
nix(&dir, &["flake", "update", "hyperhive"]).await?;
|
||||||
if git_is_clean(&dir).await? {
|
if git_is_clean(&dir).await? {
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,17 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
|
||||||
tracing::info!("migration: {KILL_SWITCH} set — skipping");
|
tracing::info!("migration: {KILL_SWITCH} set — skipping");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
// Stale meta index lock: a previous hive-c0re crash mid-`git add`
|
||||||
|
// can leave `.git/index.lock` behind, which blocks every
|
||||||
|
// subsequent meta op until somebody `rm`s it manually. We just
|
||||||
|
// booted so nothing of ours is holding it; safe to clear.
|
||||||
|
let meta_lock = std::path::PathBuf::from("/var/lib/hyperhive/meta/.git/index.lock");
|
||||||
|
if meta_lock.exists() {
|
||||||
|
match std::fs::remove_file(&meta_lock) {
|
||||||
|
Ok(()) => tracing::warn!("cleared stale meta/.git/index.lock"),
|
||||||
|
Err(e) => tracing::warn!(error = ?e, "clear stale meta lock failed"),
|
||||||
|
}
|
||||||
|
}
|
||||||
let names = enumerate_agents().await;
|
let names = enumerate_agents().await;
|
||||||
tracing::info!(count = names.len(), "migration: scanning");
|
tracing::info!(count = names.len(), "migration: scanning");
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue