//! This agent's own clone of the hive-wide `internal/knowledge` repo, kept //! current without the hive. //! //! The swarm controller publishes one empty event on //! [`swarm_queue_client::KNOWLEDGE_SUBJECT`] when the forge webhook says the //! repo moved. This task subscribes to it with the agent's own queue //! connection and, on each event, fast-forwards a shallow clone under the //! harness dir. It also syncs once at start and then hourly, so a missed event //! (the queue was down, or this agent was) costs at most an hour of stale docs. //! //! When the clone moved, the agent gets a keyed `knowledge` todo with the //! `git diff --stat`, so a second change while the first is still unread //! updates one row instead of stacking. The first clone is silent: a fresh //! agent has nothing to be told changed. //! //! Best-effort throughout. The repo is public, so a failed pull costs stale //! docs and a log line, never a turn. use std::path::Path; use std::sync::Arc; use std::time::Duration; use anyhow::{Context as _, Result, bail}; use futures_util::{FutureExt as _, StreamExt as _}; use tokio::process::Command; use tokio::sync::Notify; use crate::todos::Todos; /// Producing subsystem marker for the todo row. const SUBSYSTEM: &str = "harness"; /// Dedup key of the todo row: one row, whatever the number of changes. const KEY: &str = "knowledge"; /// How often to sync when no event arrives. const FALLBACK: Duration = Duration::from_hours(1); /// Longest a single git command may run. const GIT_TIMEOUT: Duration = Duration::from_mins(5); /// Org and repo of the knowledge repo on the forge. Same values as the swarm /// controller's webhook target. const REPO_PATH: &str = "internal/knowledge.git"; /// Start the sync task. Without a forge URL there is nothing to clone, so the /// task is not started (logged). pub fn spawn(todos: Arc, wake: Arc) { let Ok(base) = std::env::var("HIVE_FORGE_URL") else { tracing::warn!("HIVE_FORGE_URL is unset; not keeping a /knowledge clone"); return; }; tokio::spawn(run( repo_url(&base), crate::paths::knowledge_dir(), todos, wake, )); } /// The clone URL for `internal/knowledge` on the forge at `base`. fn repo_url(base: &str) -> String { format!("{}/{REPO_PATH}", base.trim_end_matches('/')) } /// What a sync did to the clone. #[derive(Debug, PartialEq, Eq)] enum Outcome { /// Already at the remote's head. Unchanged, /// There was no clone; there is one now. Cloned, /// The clone moved; `stat` is `git diff --stat` between the two heads /// (empty if git could not produce one). Updated { stat: String }, } async fn run(url: String, dir: std::path::PathBuf, todos: Arc, wake: Arc) { // Subscribed before the first sync so an event published during it is not // lost. A queue that will not take the subscription leaves the hourly // sync, which is the whole fallback. let mut events = match crate::swarm_queue::client() .await .subscribe(swarm_queue_client::KNOWLEDGE_SUBJECT) .await { Ok(sub) => Some(sub), Err(e) => { tracing::warn!(error = %e, "knowledge sync: subscribe failed; syncing hourly only"); None } }; let mut tick = tokio::time::interval(FALLBACK); loop { // The first tick fires at once: that is the boot sync. tokio::select! { _ = tick.tick() => {} msg = async { match events.as_mut() { Some(sub) => sub.next().await, None => std::future::pending().await, } } => { if msg.is_none() { tracing::warn!("knowledge sync: event stream ended; syncing hourly only"); events = None; continue; } // A burst of pushes queues one event each; one sync covers // them all. if let Some(sub) = events.as_mut() { while let Some(Some(_)) = sub.next().now_or_never() {} } } } match tokio::time::timeout(GIT_TIMEOUT, sync(&url, &dir)).await { Ok(Ok(Outcome::Updated { stat })) => announce(&todos, &wake, &stat), Ok(Ok(outcome)) => tracing::info!(?outcome, "knowledge sync"), Ok(Err(e)) => tracing::warn!(error = ?e, "knowledge sync failed"), Err(_) => tracing::warn!("knowledge sync timed out"), } } } /// Raise the `knowledge` todo for a change and wake the turn loop. fn announce(todos: &Todos, wake: &Notify, stat: &str) { match todos.upsert(SUBSYSTEM, Some(KEY), &summary(stat), None, true) { Ok((_, true)) => wake.notify_one(), Ok((_, false)) => {} Err(e) => tracing::warn!(error = ?e, "knowledge sync: todo upsert failed"), } } /// The todo text for a change with the given `diff --stat`. fn summary(stat: &str) -> String { if stat.is_empty() { "/knowledge was updated; see the repo for what changed.".to_owned() } else { format!("/knowledge was updated:\n{stat}") } } /// Bring the clone in `dir` to the remote's head, cloning it first when there /// is no usable clone. /// /// "No usable clone" includes a `dir` whose HEAD cannot be read: a clone that /// was killed half-way (the timeout, a container stop) leaves `.git` behind /// with no commit, and treating that as a clone would fail every later sync /// for good. The clone is made beside `dir` and renamed into place, so a /// reader never sees a half-made tree either. async fn sync(url: &str, dir: &Path) -> Result { let before = if dir.join(".git").exists() { git(Some(dir), &["rev-parse", "HEAD"]).await.ok() } else { None }; let Some(before) = before else { let parent = dir.parent().context("the clone dir has no parent")?; std::fs::create_dir_all(parent).context("create the clone's parent dir")?; let building = dir.with_extension("new"); let _ = std::fs::remove_dir_all(&building); git( None, &["clone", "--depth", "1", url, &building.to_string_lossy()], ) .await?; let _ = std::fs::remove_dir_all(dir); std::fs::rename(&building, dir).context("move the new clone into place")?; return Ok(Outcome::Cloned); }; git(Some(dir), &["fetch", "--depth", "1", "origin"]).await?; // Reset rather than merge: this clone is read-only, so a rewritten remote // history must not wedge it. git(Some(dir), &["reset", "--hard", "FETCH_HEAD"]).await?; let after = git(Some(dir), &["rev-parse", "HEAD"]).await?; if before == after { return Ok(Outcome::Unchanged); } let stat = git(Some(dir), &["diff", "--stat", &before, &after]) .await .unwrap_or_default(); Ok(Outcome::Updated { stat }) } /// Run `git ` (in `dir` when given) and return its trimmed stdout. async fn git(dir: Option<&Path>, args: &[&str]) -> Result { let mut cmd = Command::new("git"); cmd.args(args) // A repo that wants credentials must fail, not wait on a terminal. .env("GIT_TERMINAL_PROMPT", "0") .kill_on_drop(true); if let Some(dir) = dir { cmd.current_dir(dir); } let out = cmd.output().await.context("run git")?; if !out.status.success() { bail!( "git {} failed: {}", args.first().copied().unwrap_or_default(), String::from_utf8_lossy(&out.stderr).trim() ); } Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) } #[cfg(test)] mod tests { use super::*; fn run_git(dir: &Path, args: &[&str]) { let status = std::process::Command::new("git") .args([ "-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false", ]) .args(args) .current_dir(dir) .status() .expect("git runs"); assert!(status.success(), "git {args:?}"); } fn commit(dir: &Path, file: &str, text: &str) { std::fs::write(dir.join(file), text).unwrap(); run_git(dir, &["add", "."]); run_git(dir, &["commit", "-q", "-m", file]); } #[test] fn the_clone_url_has_one_slash_whatever_the_base() { assert_eq!( repo_url("http://forge.t"), "http://forge.t/internal/knowledge.git" ); assert_eq!( repo_url("http://forge.t/"), "http://forge.t/internal/knowledge.git" ); } #[test] fn the_todo_says_what_changed_and_still_reads_without_a_stat() { assert!(summary(" a.md | 2 +-").contains("a.md")); assert!(summary("").contains("updated")); } #[tokio::test] async fn a_sync_clones_then_reports_a_change_then_reports_none() { let tmp = tempfile::tempdir().unwrap(); let remote = tmp.path().join("remote"); std::fs::create_dir(&remote).unwrap(); run_git(&remote, &["init", "-q", "-b", "main"]); commit(&remote, "README.md", "one\n"); let url = format!("file://{}", remote.display()); let clone = tmp.path().join("clone"); assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Cloned); assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged); commit(&remote, "NEW.md", "two\n"); match sync(&url, &clone).await.unwrap() { Outcome::Updated { stat } => assert!(stat.contains("NEW.md"), "{stat}"), other => panic!("expected an update, got {other:?}"), } assert!(clone.join("NEW.md").exists()); assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged); } #[tokio::test] async fn a_clone_killed_half_way_is_replaced_not_stuck() { let tmp = tempfile::tempdir().unwrap(); let remote = tmp.path().join("remote"); std::fs::create_dir(&remote).unwrap(); run_git(&remote, &["init", "-q", "-b", "main"]); commit(&remote, "README.md", "one\n"); let url = format!("file://{}", remote.display()); let clone = tmp.path().join("clone"); // What a killed `git clone` leaves: a `.git` with no commit. std::fs::create_dir(&clone).unwrap(); run_git(&clone, &["init", "-q"]); assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Cloned); assert!(clone.join("README.md").exists()); assert!(!tmp.path().join("clone.new").exists()); assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged); } #[test] fn two_changes_before_the_agent_reads_leave_one_todo_row() { let tmp = tempfile::tempdir().unwrap(); let todos = Todos::open(&tmp.path().join("state.sqlite")).unwrap(); let wake = Notify::new(); announce(&todos, &wake, " a.md | 1 +"); announce(&todos, &wake, " b.md | 1 +"); let rows = todos.list(Some(SUBSYSTEM)).unwrap(); assert_eq!(rows.len(), 1); assert!(rows[0].summary.contains("b.md")); } }