Watch
0
0
Fork
You've already forked hyperhive
0

hive-agent: keep an own /knowledge clone current from the swarm knowledge event

This commit is contained in:
damocles 2026-11-22 17:09:48 +01:00 • committed by mara
commit 989d1d578c
5 changed files with 347 additions and 1 deletions

View file

@ -66,6 +66,24 @@ read token when it has one (see
[forge-accounts.md](forge-accounts.md#how-a-hive-reads-the-forge)) and clones the public [forge-accounts.md](forge-accounts.md#how-a-hive-reads-the-forge)) and clones the public
repo anonymously otherwise. repo anonymously otherwise.
### The agent's own clone
Each agent harness also keeps its own shallow clone of `internal/knowledge`
(`hive-agent/src/knowledge_sync.rs`), under the harness dir as `knowledge/`,
without going through the hive. The agent's queue token may subscribe to
`$SWARM.knowledge` (and not publish it), so the harness hears the same event
as the hive: on it, once at start and hourly as a fallback, it fetches and
resets the clone to the remote head (`GIT_TERMINAL_PROMPT=0`, a five-minute
limit per git command, anonymous because the repo is public). When the head
moved it raises one keyed `knowledge` todo carrying the `git diff --stat`, so
a second change before the agent reads the first updates the same row; the
first clone raises nothing. A failed or timed-out sync logs a warning, and the
next trigger retries it.
This runs next to the host mount for now: `/knowledge` is still the host's
bind mount and the `/knowledge updated` broadcast above still goes out, until
a later change removes the host side.
## State ## State
<!-- vale write-good.Passive = NO --> <!-- vale write-good.Passive = NO -->

View file

@ -0,0 +1,306 @@
//! This agent's own clone of the hive-wide `internal/knowledge` repo, kept
//! current without the hive.
//!
//! The swarm controller publishes one empty event on
//! [`swarm_queue_client::KNOWLEDGE_SUBJECT`] when the forge webhook says the
//! repo moved. This task subscribes to it with the agent's own queue
//! connection and, on each event, fast-forwards a shallow clone under the
//! harness dir. It also syncs once at start and then hourly, so a missed event
//! (the queue was down, or this agent was) costs at most an hour of stale docs.
//!
//! When the clone moved, the agent gets a keyed `knowledge` todo with the
//! `git diff --stat`, so a second change while the first is still unread
//! updates one row instead of stacking. The first clone is silent: a fresh
//! agent has nothing to be told changed.
//!
//! Best-effort throughout. The repo is public, so a failed pull costs stale
//! docs and a log line, never a turn.
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context as _, Result, bail};
use futures_util::{FutureExt as _, StreamExt as _};
use tokio::process::Command;
use tokio::sync::Notify;
use crate::todos::Todos;
/// Producing subsystem marker for the todo row.
const SUBSYSTEM: &str = "harness";
/// Dedup key of the todo row: one row, whatever the number of changes.
const KEY: &str = "knowledge";
/// How often to sync when no event arrives.
const FALLBACK: Duration = Duration::from_hours(1);
/// Longest a single git command may run.
const GIT_TIMEOUT: Duration = Duration::from_mins(5);
/// Org and repo of the knowledge repo on the forge. Same values as the swarm
/// controller's webhook target.
const REPO_PATH: &str = "internal/knowledge.git";
/// Start the sync task. Without a forge URL there is nothing to clone, so the
/// task is not started (logged).
pub fn spawn(todos: Arc<Todos>, wake: Arc<Notify>) {
let Ok(base) = std::env::var("HIVE_FORGE_URL") else {
tracing::warn!("HIVE_FORGE_URL is unset; not keeping a /knowledge clone");
return;
};
tokio::spawn(run(
repo_url(&base),
crate::paths::knowledge_dir(),
todos,
wake,
));
}
/// The clone URL for `internal/knowledge` on the forge at `base`.
fn repo_url(base: &str) -> String {
format!("{}/{REPO_PATH}", base.trim_end_matches('/'))
}
/// What a sync did to the clone.
#[derive(Debug, PartialEq, Eq)]
enum Outcome {
/// Already at the remote's head.
Unchanged,
/// There was no clone; there is one now.
Cloned,
/// The clone moved; `stat` is `git diff --stat` between the two heads
/// (empty if git could not produce one).
Updated { stat: String },
}
async fn run(url: String, dir: std::path::PathBuf, todos: Arc<Todos>, wake: Arc<Notify>) {
// Subscribed before the first sync so an event published during it is not
// lost. A queue that will not take the subscription leaves the hourly
// sync, which is the whole fallback.
let mut events = match crate::swarm_queue::client()
.await
.subscribe(swarm_queue_client::KNOWLEDGE_SUBJECT)
.await
{
Ok(sub) => Some(sub),
Err(e) => {
tracing::warn!(error = %e, "knowledge sync: subscribe failed; syncing hourly only");
None
}
};
let mut tick = tokio::time::interval(FALLBACK);
loop {
// The first tick fires at once: that is the boot sync.
tokio::select! {
_ = tick.tick() => {}
msg = async {
match events.as_mut() {
Some(sub) => sub.next().await,
None => std::future::pending().await,
}
} => {
if msg.is_none() {
tracing::warn!("knowledge sync: event stream ended; syncing hourly only");
events = None;
continue;
}
// A burst of pushes queues one event each; one sync covers
// them all.
if let Some(sub) = events.as_mut() {
while let Some(Some(_)) = sub.next().now_or_never() {}
}
}
}
match tokio::time::timeout(GIT_TIMEOUT, sync(&url, &dir)).await {
Ok(Ok(Outcome::Updated { stat })) => announce(&todos, &wake, &stat),
Ok(Ok(outcome)) => tracing::info!(?outcome, "knowledge sync"),
Ok(Err(e)) => tracing::warn!(error = ?e, "knowledge sync failed"),
Err(_) => tracing::warn!("knowledge sync timed out"),
}
}
}
/// Raise the `knowledge` todo for a change and wake the turn loop.
fn announce(todos: &Todos, wake: &Notify, stat: &str) {
match todos.upsert(SUBSYSTEM, Some(KEY), &summary(stat), None, true) {
Ok((_, true)) => wake.notify_one(),
Ok((_, false)) => {}
Err(e) => tracing::warn!(error = ?e, "knowledge sync: todo upsert failed"),
}
}
/// The todo text for a change with the given `diff --stat`.
fn summary(stat: &str) -> String {
if stat.is_empty() {
"/knowledge was updated; see the repo for what changed.".to_owned()
} else {
format!("/knowledge was updated:\n{stat}")
}
}
/// Bring the clone in `dir` to the remote's head, cloning it first when there
/// is no usable clone.
///
/// "No usable clone" includes a `dir` whose HEAD cannot be read: a clone that
/// was killed half-way (the timeout, a container stop) leaves `.git` behind
/// with no commit, and treating that as a clone would fail every later sync
/// for good. The clone is made beside `dir` and renamed into place, so a
/// reader never sees a half-made tree either.
async fn sync(url: &str, dir: &Path) -> Result<Outcome> {
let before = if dir.join(".git").exists() {
git(Some(dir), &["rev-parse", "HEAD"]).await.ok()
} else {
None
};
let Some(before) = before else {
let parent = dir.parent().context("the clone dir has no parent")?;
std::fs::create_dir_all(parent).context("create the clone's parent dir")?;
let building = dir.with_extension("new");
let _ = std::fs::remove_dir_all(&building);
git(
None,
&["clone", "--depth", "1", url, &building.to_string_lossy()],
)
.await?;
let _ = std::fs::remove_dir_all(dir);
std::fs::rename(&building, dir).context("move the new clone into place")?;
return Ok(Outcome::Cloned);
};
git(Some(dir), &["fetch", "--depth", "1", "origin"]).await?;
// Reset rather than merge: this clone is read-only, so a rewritten remote
// history must not wedge it.
git(Some(dir), &["reset", "--hard", "FETCH_HEAD"]).await?;
let after = git(Some(dir), &["rev-parse", "HEAD"]).await?;
if before == after {
return Ok(Outcome::Unchanged);
}
let stat = git(Some(dir), &["diff", "--stat", &before, &after])
.await
.unwrap_or_default();
Ok(Outcome::Updated { stat })
}
/// Run `git <args>` (in `dir` when given) and return its trimmed stdout.
async fn git(dir: Option<&Path>, args: &[&str]) -> Result<String> {
let mut cmd = Command::new("git");
cmd.args(args)
// A repo that wants credentials must fail, not wait on a terminal.
.env("GIT_TERMINAL_PROMPT", "0")
.kill_on_drop(true);
if let Some(dir) = dir {
cmd.current_dir(dir);
}
let out = cmd.output().await.context("run git")?;
if !out.status.success() {
bail!(
"git {} failed: {}",
args.first().copied().unwrap_or_default(),
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
}
#[cfg(test)]
mod tests {
use super::*;
fn run_git(dir: &Path, args: &[&str]) {
let status = std::process::Command::new("git")
.args([
"-c",
"user.name=t",
"-c",
"user.email=t@t",
"-c",
"commit.gpgsign=false",
])
.args(args)
.current_dir(dir)
.status()
.expect("git runs");
assert!(status.success(), "git {args:?}");
}
fn commit(dir: &Path, file: &str, text: &str) {
std::fs::write(dir.join(file), text).unwrap();
run_git(dir, &["add", "."]);
run_git(dir, &["commit", "-q", "-m", file]);
}
#[test]
fn the_clone_url_has_one_slash_whatever_the_base() {
assert_eq!(
repo_url("http://forge.t"),
"http://forge.t/internal/knowledge.git"
);
assert_eq!(
repo_url("http://forge.t/"),
"http://forge.t/internal/knowledge.git"
);
}
#[test]
fn the_todo_says_what_changed_and_still_reads_without_a_stat() {
assert!(summary(" a.md | 2 +-").contains("a.md"));
assert!(summary("").contains("updated"));
}
#[tokio::test]
async fn a_sync_clones_then_reports_a_change_then_reports_none() {
let tmp = tempfile::tempdir().unwrap();
let remote = tmp.path().join("remote");
std::fs::create_dir(&remote).unwrap();
run_git(&remote, &["init", "-q", "-b", "main"]);
commit(&remote, "README.md", "one\n");
let url = format!("file://{}", remote.display());
let clone = tmp.path().join("clone");
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Cloned);
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged);
commit(&remote, "NEW.md", "two\n");
match sync(&url, &clone).await.unwrap() {
Outcome::Updated { stat } => assert!(stat.contains("NEW.md"), "{stat}"),
other => panic!("expected an update, got {other:?}"),
}
assert!(clone.join("NEW.md").exists());
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged);
}
#[tokio::test]
async fn a_clone_killed_half_way_is_replaced_not_stuck() {
let tmp = tempfile::tempdir().unwrap();
let remote = tmp.path().join("remote");
std::fs::create_dir(&remote).unwrap();
run_git(&remote, &["init", "-q", "-b", "main"]);
commit(&remote, "README.md", "one\n");
let url = format!("file://{}", remote.display());
let clone = tmp.path().join("clone");
// What a killed `git clone` leaves: a `.git` with no commit.
std::fs::create_dir(&clone).unwrap();
run_git(&clone, &["init", "-q"]);
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Cloned);
assert!(clone.join("README.md").exists());
assert!(!tmp.path().join("clone.new").exists());
assert_eq!(sync(&url, &clone).await.unwrap(), Outcome::Unchanged);
}
#[test]
fn two_changes_before_the_agent_reads_leave_one_todo_row() {
let tmp = tempfile::tempdir().unwrap();
let todos = Todos::open(&tmp.path().join("state.sqlite")).unwrap();
let wake = Notify::new();
announce(&todos, &wake, " a.md | 1 +");
announce(&todos, &wake, " b.md | 1 +");
let rows = todos.list(Some(SUBSYSTEM)).unwrap();
assert_eq!(rows.len(), 1);
assert!(rows[0].summary.contains("b.md"));
}
}

View file

@ -18,6 +18,7 @@ mod disk_watch;
mod events; mod events;
mod harness_state; mod harness_state;
mod identity; mod identity;
mod knowledge_sync;
mod login; mod login;
mod login_watch; mod login_watch;
mod mcp_config; mod mcp_config;
@ -575,6 +576,7 @@ async fn serve_main<S: Surface>(socket: &Path, poll_ms: u64) -> Result<()> {
// for why this is strictly better than a host-side kick. // for why this is strictly better than a host-side kick.
if let Some(store) = &todos_store { if let Some(store) = &todos_store {
boot_announce::announce(&label, store, &todo_wake); boot_announce::announce(&label, store, &todo_wake);
knowledge_sync::spawn(store.clone(), todo_wake.clone());
} }
if matches!(initial, LoginState::NeedsLogin) { if matches!(initial, LoginState::NeedsLogin) {
login::wait_for_login( login::wait_for_login(

View file

@ -39,6 +39,13 @@ pub fn harness_dir() -> PathBuf {
hive_agent_sock::paths::harness_dir() hive_agent_sock::paths::harness_dir()
} }
/// This agent's own clone of the hive-wide `internal/knowledge` repo, kept
/// current by [`crate::knowledge_sync`].
#[must_use]
pub fn knowledge_dir() -> PathBuf {
harness_dir().join("knowledge")
}
/// Consolidated harness-local state db — todos + reminders, one table /// Consolidated harness-local state db — todos + reminders, one table
/// each — mutable per-agent state the harness owns, kept apart /// each — mutable per-agent state the harness owns, kept apart
/// from the usage db. Each store's `open()` only /// from the usage db. Each store's `open()` only

View file

@ -186,6 +186,11 @@ impl Policy {
format!("{}.>", swarm_queue_client::agent_inbox_prefix(agent)), format!("{}.>", swarm_queue_client::agent_inbox_prefix(agent)),
swarm_queue_client::agent_input_subject(agent), swarm_queue_client::agent_input_subject(agent),
swarm_queue_client::agent_control_subject(agent), swarm_queue_client::agent_control_subject(agent),
// The knowledge event: the agent pulls its own clone of the
// shared repo when it hears the repo moved. One empty event
// for every subscriber, so no per-agent subject; the agent
// cannot publish it (nothing here grants that).
swarm_queue_client::KNOWLEDGE_SUBJECT.to_owned(),
]), ]),
// The controller sends a control command as a request, and the // The controller sends a control command as a request, and the
// harness answers it. `allow_responses` lets it reply to that one // harness answers it. `allow_responses` lets it reply to that one
@ -1577,7 +1582,7 @@ mod tests {
} }
#[test] #[test]
fn an_agent_token_may_subscribe_to_its_own_inbox_input_and_control_only() { fn an_agent_token_may_subscribe_to_its_own_inbox_input_control_and_the_knowledge_event_only() {
use swarm_queue_client::{agent_control_subject, agent_inbox_prefix, agent_input_subject}; use swarm_queue_client::{agent_control_subject, agent_inbox_prefix, agent_input_subject};
let p = policy() let p = policy()
@ -1587,6 +1592,14 @@ mod tests {
assert!(may_subscribe(&atlas, &reply(&agent_inbox_prefix("atlas")))); assert!(may_subscribe(&atlas, &reply(&agent_inbox_prefix("atlas"))));
assert!(may_subscribe(&atlas, &agent_input_subject("atlas"))); assert!(may_subscribe(&atlas, &agent_input_subject("atlas")));
assert!(may_subscribe(&atlas, &agent_control_subject("atlas"))); assert!(may_subscribe(&atlas, &agent_control_subject("atlas")));
assert!(may_subscribe(&atlas, swarm_queue_client::KNOWLEDGE_SUBJECT));
// Hearing the event is all an agent gets; publishing it is the
// controller's alone.
assert!(
!atlas
.publish
.contains(&swarm_queue_client::KNOWLEDGE_SUBJECT.to_owned())
);
for refused in [ for refused in [
">".to_owned(), ">".to_owned(),
"$SWARM.term.>".to_owned(), "$SWARM.term.>".to_owned(),