hive-agent: keep an own /knowledge clone current from the swarm knowledge event
This commit is contained in:
parent
ff4e42f1ae
commit
989d1d578c
3 changed files with 347 additions and 1 deletions
306
hive-agent/src/knowledge_sync.rs
Normal file
306
hive-agent/src/knowledge_sync.rs
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
//! This agent's own clone of the hive-wide `internal/knowledge` repo, kept
|
||||
//! current without the hive.
|
||||
//!
|
||||
//! The swarm controller publishes one empty event on
|
||||
//! [`swarm_queue_client::KNOWLEDGE_SUBJECT`] when the forge webhook says the
|
||||
//! repo moved. This task subscribes to it with the agent's own queue
|
||||
//! connection and, on each event, fast-forwards a shallow clone under the
|
||||
//! harness dir. It also syncs once at start and then hourly, so a missed event
|
||||
//! (the queue was down, or this agent was) costs at most an hour of stale docs.
|
||||
//!
|
||||
//! When the clone moved, the agent gets a keyed `knowledge` todo with the
|
||||
//! `git diff --stat`, so a second change while the first is still unread
|
||||
//! updates one row instead of stacking. The first clone is silent: a fresh
|
||||
//! agent has nothing to be told changed.
|
||||
//!
|
||||
//! Best-effort throughout. The repo is public, so a failed pull costs stale
|
||||
//! docs and a log line, never a turn.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use futures_util::{FutureExt as _, StreamExt as _};
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use crate::todos::Todos;
|
||||
|
||||
/// Producing subsystem marker for the todo row.
|
||||
const SUBSYSTEM: &str = "harness";
|
||||
|
||||
/// Dedup key of the todo row: one row, whatever the number of changes.
|
||||
const KEY: &str = "knowledge";
|
||||
|
||||
/// How often to sync when no event arrives.
|
||||
const FALLBACK: Duration = Duration::from_hours(1);
|
||||
|
||||
/// Longest a single git command may run.
|
||||
const GIT_TIMEOUT: Duration = Duration::from_mins(5);
|
||||
|
||||
/// Org and repo of the knowledge repo on the forge. Same values as the swarm
|
||||
/// controller's webhook target.
|
||||
const REPO_PATH: &str = "internal/knowledge.git";
|
||||
|
||||
/// Start the sync task. Without a forge URL there is nothing to clone, so the
|
||||
/// task is not started (logged).
|
||||
pub fn spawn(todos: Arc<Todos>, wake: Arc<Notify>) {
|
||||
let Ok(base) = std::env::var("HIVE_FORGE_URL") else {
|
||||
tracing::warn!("HIVE_FORGE_URL is unset; not keeping a /knowledge clone");
|
||||
return;
|
||||
};
|
||||
tokio::spawn(run(
|
||||
repo_url(&base),
|
||||
crate::paths::knowledge_dir(),
|
||||
todos,
|
||||
wake,
|
||||
));
|
||||
}
|
||||
|
||||
/// The clone URL for `internal/knowledge` on the forge at `base`.
|
||||
fn repo_url(base: &str) -> String {
|
||||
format!("{}/{REPO_PATH}", base.trim_end_matches('/'))
|
||||
}
|
||||
|
||||
/// What a sync did to the clone.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum Outcome {
|
||||
/// Already at the remote's head.
|
||||
Unchanged,
|
||||
/// There was no clone; there is one now.
|
||||
Cloned,
|
||||
/// The clone moved; `stat` is `git diff --stat` between the two heads
|
||||
/// (empty if git could not produce one).
|
||||
Updated { stat: String },
|
||||
}
|
||||
|
||||
async fn run(url: String, dir: std::path::PathBuf, todos: Arc<Todos>, wake: Arc<Notify>) {
|
||||
// Subscribed before the first sync so an event published during it is not
|
||||
// lost. A queue that will not take the subscription leaves the hourly
|
||||
// sync, which is the whole fallback.
|
||||
let mut events = match crate::swarm_queue::client()
|
||||
.await
|
||||
.subscribe(swarm_queue_client::KNOWLEDGE_SUBJECT)
|
||||
.await
|
||||
{
|
||||
Ok(sub) => Some(sub),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "knowledge sync: subscribe failed; syncing hourly only");
|
||||
None
|
||||
}
|
||||
};
|
||||
let mut tick = tokio::time::interval(FALLBACK);
|
||||
loop {
|
||||
// The first tick fires at once: that is the boot sync.
|
||||
tokio::select! {
|
||||
_ = tick.tick() => {}
|
||||
msg = async {
|
||||
match events.as_mut() {
|
||||
Some(sub) => sub.next().await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
} => {
|
||||
if msg.is_none() {
|
||||
tracing::warn!("knowledge sync: event stream ended; syncing hourly only");
|
||||
events = None;
|
||||
continue;
|
||||
}
|
||||
// A burst of pushes queues one event each; one sync covers
|
||||
// them all.
|
||||
if let Some(sub) = events.as_mut() {
|
||||
while let Some(Some(_)) = sub.next().now_or_never() {}
|
||||
}
|
||||
}
|
||||
}
|
||||
match tokio::time::timeout(GIT_TIMEOUT, sync(&url, &dir)).await {
|
||||
Ok(Ok(Outcome::Updated { stat })) => announce(&todos, &wake, &stat),
|
||||
Ok(Ok(outcome)) => tracing::info!(?outcome, "knowledge sync"),
|
||||
Ok(Err(e)) => tracing::warn!(error = ?e, "knowledge sync failed"),
|
||||
Err(_) => tracing::warn!("knowledge sync timed out"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Raise the `knowledge` todo for a change and wake the turn loop.
|
||||
fn announce(todos: &Todos, wake: &Notify, stat: &str) {
|
||||
match todos.upsert(SUBSYSTEM, Some(KEY), &summary(stat), None, true) {
|
||||
Ok((_, true)) => wake.notify_one(),
|
||||
Ok((_, false)) => {}
|
||||
Err(e) => tracing::warn!(error = ?e, "knowledge sync: todo upsert failed"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The todo text for a change with the given `diff --stat`.
|
||||
fn summary(stat: &str) -> String {
|
||||
if stat.is_empty() {
|
||||
"/knowledge was updated; see the repo for what changed.".to_owned()
|
||||
} else {
|
||||
format!("/knowledge was updated:\n{stat}")
|
||||
}
|
||||
}
|
||||
|
||||
/// Bring the clone in `dir` to the remote's head, cloning it first when there
|
||||
/// is no usable clone.
|
||||
///
|
||||
/// "No usable clone" includes a `dir` whose HEAD cannot be read: a clone that
|
||||
/// was killed half-way (the timeout, a container stop) leaves `.git` behind
|
||||
/// with no commit, and treating that as a clone would fail every later sync
|
||||
/// for good. The clone is made beside `dir` and renamed into place, so a
|
||||
/// reader never sees a half-made tree either.
|
||||
async fn sync(url: &str, dir: &Path) -> Result<Outcome> {
|
||||
let before = if dir.join(".git").exists() {
|
||||
git(Some(dir), &["rev-parse", "HEAD"]).await.ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let Some(before) = before else {
|
||||
let parent = dir.parent().context("the clone dir has no parent")?;
|
||||
std::fs::create_dir_all(parent).context("create the clone's parent dir")?;
|
||||
let building = dir.with_extension("new");
|
||||
let _ = std::fs::remove_dir_all(&building);
|
||||
git(
|
||||
None,
|
||||
&["clone", "--depth", "1", url, &building.to_string_lossy()],
|
||||
)
|
||||
.await?;
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
std::fs::rename(&building, dir).context("move the new clone into place")?;
|
||||
return Ok(Outcome::Cloned);
|
||||
};
|
||||
git(Some(dir), &["fetch", "--depth", "1", "origin"]).await?;
|
||||
// Reset rather than merge: this clone is read-only, so a rewritten remote
|
||||
// history must not wedge it.
|
||||
git(Some(dir), &["reset", "--hard", "FETCH_HEAD"]).await?;
|
||||
let after = git(Some(dir), &["rev-parse", "HEAD"]).await?;
|
||||
if before == after {
|
||||
return Ok(Outcome::Unchanged);
|
||||
}
|
||||
let stat = git(Some(dir), &["diff", "--stat", &before, &after])
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
Ok(Outcome::Updated { stat })
|
||||
}
|
||||
|
||||
/// Run `git <args>` (in `dir` when given) and return its trimmed stdout.
|
||||
async fn git(dir: Option<&Path>, args: &[&str]) -> Result<String> {
|
||||
let mut cmd = Command::new("git");
|
||||
cmd.args(args)
|
||||
// A repo that wants credentials must fail, not wait on a terminal.
|
||||
.env("GIT_TERMINAL_PROMPT", "0")
|
||||
.kill_on_drop(true);
|
||||
if let Some(dir) = dir {
|
||||
cmd.current_dir(dir);
|
||||
}
|
||||
let out = cmd.output().await.context("run git")?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"git {} failed: {}",
|
||||
args.first().copied().unwrap_or_default(),
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn run_git(dir: &Path, args: &[&str]) {
|
||||
let status = std::process::Command::new("git")
|
||||
.args([
|
||||
"-c",
|
||||
"user.name=t",
|
||||
"-c",
|
||||
"user.email=t@t",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
])
|
||||
.args(args)
|
||||
.current_dir(dir)
|
||||
.status()
|
||||
.expect("git runs");
|
||||
assert!(status.success(), "git {args:?}");
|
||||
}
|
||||
|
||||
fn commit(dir: &Path, file: &str, text: &str) {
|
||||
std::fs::write(dir.join(file), text).unwrap();
|
||||
run_git(dir, &["add", "."]);
|
||||
run_git(dir, &["commit", "-q", "-m", file]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_clone_url_has_one_slash_whatever_the_base() {
|
||||
assert_eq!(
|
||||
repo_url("http://forge.t"),
|
||||
"http://forge.t/internal/knowledge.git"
|
||||
);
|
||||
assert_eq!(
|
||||
repo_url("http://forge.t/"),
|
||||
"http://forge.t/internal/knowledge.git"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_todo_says_what_changed_and_still_reads_without_a_stat() {
|
||||
assert!(summary(" a.md | 2 +-").contains("a.md"));
|
||||
assert!(summary("").contains("updated"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_sync_clones_then_reports_a_change_then_reports_none() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote");
|
||||
std::fs::create_dir(&remote).unwrap();
|
||||
run_git(&remote, &["init", "-q", "-b", "main"]);
|
||||
commit(&remote, "README.md", "one\n");
|
||||
let url = format!("file://{}", remote.display());
|
||||
let clone = tmp.path().join("clone");
|
||||
|
||||
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Cloned);
|
||||
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged);
|
||||
|
||||
commit(&remote, "NEW.md", "two\n");
|
||||
match sync(&url, &clone).await.unwrap() {
|
||||
Outcome::Updated { stat } => assert!(stat.contains("NEW.md"), "{stat}"),
|
||||
other => panic!("expected an update, got {other:?}"),
|
||||
}
|
||||
assert!(clone.join("NEW.md").exists());
|
||||
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_clone_killed_half_way_is_replaced_not_stuck() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote");
|
||||
std::fs::create_dir(&remote).unwrap();
|
||||
run_git(&remote, &["init", "-q", "-b", "main"]);
|
||||
commit(&remote, "README.md", "one\n");
|
||||
let url = format!("file://{}", remote.display());
|
||||
let clone = tmp.path().join("clone");
|
||||
|
||||
// What a killed `git clone` leaves: a `.git` with no commit.
|
||||
std::fs::create_dir(&clone).unwrap();
|
||||
run_git(&clone, &["init", "-q"]);
|
||||
|
||||
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Cloned);
|
||||
assert!(clone.join("README.md").exists());
|
||||
assert!(!tmp.path().join("clone.new").exists());
|
||||
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_changes_before_the_agent_reads_leave_one_todo_row() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let todos = Todos::open(&tmp.path().join("state.sqlite")).unwrap();
|
||||
let wake = Notify::new();
|
||||
|
||||
announce(&todos, &wake, " a.md | 1 +");
|
||||
announce(&todos, &wake, " b.md | 1 +");
|
||||
|
||||
let rows = todos.list(Some(SUBSYSTEM)).unwrap();
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert!(rows[0].summary.contains("b.md"));
|
||||
}
|
||||
}
|
||||
|
|
@ -18,6 +18,7 @@ mod disk_watch;
|
|||
mod events;
|
||||
mod harness_state;
|
||||
mod identity;
|
||||
mod knowledge_sync;
|
||||
mod login;
|
||||
mod login_watch;
|
||||
mod mcp_config;
|
||||
|
|
@ -575,6 +576,7 @@ async fn serve_main<S: Surface>(socket: &Path, poll_ms: u64) -> Result<()> {
|
|||
// for why this is strictly better than a host-side kick.
|
||||
if let Some(store) = &todos_store {
|
||||
boot_announce::announce(&label, store, &todo_wake);
|
||||
knowledge_sync::spawn(store.clone(), todo_wake.clone());
|
||||
}
|
||||
if matches!(initial, LoginState::NeedsLogin) {
|
||||
login::wait_for_login(
|
||||
|
|
|
|||
|
|
@ -39,6 +39,13 @@ pub fn harness_dir() -> PathBuf {
|
|||
hive_agent_sock::paths::harness_dir()
|
||||
}
|
||||
|
||||
/// This agent's own clone of the hive-wide `internal/knowledge` repo, kept
|
||||
/// current by [`crate::knowledge_sync`].
|
||||
#[must_use]
|
||||
pub fn knowledge_dir() -> PathBuf {
|
||||
harness_dir().join("knowledge")
|
||||
}
|
||||
|
||||
/// Consolidated harness-local state db — todos + reminders, one table
|
||||
/// each — mutable per-agent state the harness owns, kept apart
|
||||
/// from the usage db. Each store's `open()` only
|
||||
|
|
|
|||
Loading…
Reference in a new issue