1b4e764e5c
The swarm settings gain `runtime` (`api-key`, the default, or `acp`) and an `acp` provider record: an https base URL, the provider's model, and its prices per million tokens. A PUT now changes only the fields its body names, so the settings form that sends model, effort and limit leaves the runtime alone; `null` clears `effort` or `acp`. The body stays strict, and the stored document keeps #5192's lenient read. The controller sends the runtime in each run request. On `acp` the classifier runs one prompt on a new `opencode acp` session per classification through hive-runtime's AcpRuntime, in a scratch directory, with a config rendered from the settings. opencode runs `read` and other tools without asking, so OPENCODE_PERMISSION {"*":"deny"} takes every tool away from the model, and a deny-all policy refuses any request on top. The title call, which resends the prompt, is turned off. The deadline cancels the turn, and a turn that does not end within 15s of that is dropped, which also covers an agent stuck before its prompt. The answer is the final assistant text, as ACP has no result event. The provider key is read per run from swarm/services/review-classifier/acp-provider, which the bao UI's admins login may create and update, and reaches opencode through AcpRuntime::with_provider_key and the trampoline's allowlist. Each ACP classification's cost is computed from the configured prices and the reported tokens, and added to hyperhive.review_classifier.cost.usage labelled repo, pr, model and currency. Part of #5096
467 lines
18 KiB
Rust
467 lines
18 KiB
Rust
//! One classification on the ACP runtime: `opencode acp`, driven through
|
|
//! [`hive_runtime::AcpRuntime`] for one prompt on a new session, in a
|
|
//! directory made for it and removed after it.
|
|
//!
|
|
//! The agent has no tool at all. opencode runs some tools, `read` among them,
|
|
//! without asking, so a deny-all permission policy alone would never see them:
|
|
//! [`DENY_ALL`] in `OPENCODE_PERMISSION` makes opencode offer the model no
|
|
//! tool, and the policy refuses whatever is still asked. A tool would let a
|
|
//! PR's diff talk the model into reading this unit's credentials into its
|
|
//! `reason`, which the controller posts on the forge.
|
|
//!
|
|
//! The rendered config names one provider and model, from the run, and turns
|
|
//! off opencode's title call, which would send the prompt a second time.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use hive_runtime::{
|
|
AcpCommand, AcpRuntime, Config, PermissionAsk, PermissionPolicy, Runtime as _, Sink,
|
|
};
|
|
use serde_json::{Value, json};
|
|
use swarm_queue_client::review_classifier::{Acp as Provider, Runtime, TryOut};
|
|
|
|
use crate::cost::{Cost, CostSink};
|
|
|
|
/// The variable the provider key reaches `opencode` in.
|
|
pub const PROVIDER_KEY_VAR: &str = "ACP_PROVIDER_API_KEY";
|
|
|
|
/// `OPENCODE_PERMISSION`: every tool denied, which opencode merges over its
|
|
/// config.
|
|
const DENY_ALL: &str = r#"{"*":"deny"}"#;
|
|
|
|
/// The provider's id in the rendered config.
|
|
const PROVIDER_ID: &str = "review-classifier";
|
|
|
|
/// How long the agent may send nothing before its turn is cancelled.
|
|
const IDLE_TIMEOUT: Duration = Duration::from_mins(2);
|
|
|
|
/// How long a turn cancelled at the deadline has to end before it is dropped,
|
|
/// which interrupts the agent: past `hive-runtime`'s own grace before it sends
|
|
/// SIGINT. A cancel reaches only a turn whose prompt has been sent, so an
|
|
/// agent stuck before that needs the drop.
|
|
pub const CANCEL_WAIT: Duration = Duration::from_secs(15);
|
|
|
|
/// The `opencode` binary the classifications run on, started through the
|
|
/// trampoline that clears its environment.
|
|
pub struct Acp<S> {
|
|
pub program: PathBuf,
|
|
/// This binary, which execs [`Self::program`] when started as the
|
|
/// trampoline.
|
|
pub trampoline: PathBuf,
|
|
/// How long one classification may take before it is cancelled.
|
|
pub deadline: Duration,
|
|
/// [`CANCEL_WAIT`] outside the tests.
|
|
pub cancel_wait: Duration,
|
|
pub cost: S,
|
|
}
|
|
|
|
impl<S: CostSink> crate::run::Model for Acp<S> {
|
|
async fn ask(&self, prompt: &str, key: &str, run: &TryOut) -> Result<String, String> {
|
|
let Runtime::Acp(provider) = &run.runtime else {
|
|
return Err("not an ACP run".to_owned());
|
|
};
|
|
let dir = crate::run::scratch_dir();
|
|
let outcome = self.ask_in(&dir, prompt, key, run, provider).await;
|
|
if let Err(e) = tokio::fs::remove_dir_all(&dir).await {
|
|
tracing::warn!(dir = %dir.display(), error = %e, "removing a run's directory failed");
|
|
}
|
|
outcome
|
|
}
|
|
}
|
|
|
|
impl<S: CostSink> Acp<S> {
|
|
async fn ask_in(
|
|
&self,
|
|
dir: &Path,
|
|
prompt: &str,
|
|
key: &str,
|
|
run: &TryOut,
|
|
provider: &Provider,
|
|
) -> Result<String, String> {
|
|
let system = dir.join("system.md");
|
|
let config_file = dir.join("opencode.json");
|
|
tokio::fs::create_dir_all(dir)
|
|
.await
|
|
.and(tokio::fs::write(&system, crate::prompt::SYSTEM).await)
|
|
.and(tokio::fs::write(&config_file, opencode_config(provider).to_string()).await)
|
|
.map_err(|e| format!("preparing the run's directory: {e}"))?;
|
|
let runtime = AcpRuntime::new(
|
|
self.command(dir, &config_file),
|
|
dir.join("session"),
|
|
deny_all(),
|
|
hive_claude::NeverCompact,
|
|
)
|
|
.with_provider_key(key.to_owned());
|
|
let canceller = runtime
|
|
.canceller()
|
|
.ok_or("the ACP runtime offers no cancel handle")?;
|
|
let config = Config {
|
|
effort: run.effort.map(|e| e.as_str().to_owned()),
|
|
cwd: Some(dir.to_owned()),
|
|
system_prompt_file: Some(system),
|
|
idle_timeout: Some(IDLE_TIMEOUT),
|
|
..Config::default()
|
|
};
|
|
let sink = FinalText::default();
|
|
let turn = runtime.run(&config, prompt, &sink);
|
|
tokio::pin!(turn);
|
|
match tokio::time::timeout(self.deadline, &mut turn).await {
|
|
Ok(Ok(progress)) => {
|
|
self.record_cost(run, provider, &progress.telemetry.cost);
|
|
sink.take()
|
|
}
|
|
Ok(Err(e)) => Err(format!("the ACP agent failed: {e}")),
|
|
Err(_) => {
|
|
let _ = canceller.cancel();
|
|
let _ = tokio::time::timeout(self.cancel_wait, turn).await;
|
|
Err(format!(
|
|
"no answer within {} seconds",
|
|
self.deadline.as_secs()
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One classification's invocation: the trampoline as the program, with
|
|
/// the real `opencode` and the run's own variables in its environment.
|
|
fn command(&self, dir: &Path, config_file: &Path) -> AcpCommand {
|
|
let env: BTreeMap<String, String> = [
|
|
(
|
|
crate::child_env::EXEC_OPENCODE,
|
|
self.program.display().to_string(),
|
|
),
|
|
("HOME", dir.display().to_string()),
|
|
("OPENCODE_CONFIG", config_file.display().to_string()),
|
|
("OPENCODE_PERMISSION", DENY_ALL.to_owned()),
|
|
("OPENCODE_DISABLE_PROJECT_CONFIG", "1".to_owned()),
|
|
]
|
|
.into_iter()
|
|
.map(|(k, v)| (k.to_owned(), v))
|
|
.collect();
|
|
AcpCommand {
|
|
command: self.trampoline.display().to_string(),
|
|
args: vec!["acp".to_owned()],
|
|
env,
|
|
api_key_env: Some(PROVIDER_KEY_VAR.to_owned()),
|
|
}
|
|
}
|
|
|
|
/// Record what `usage` cost at the run's prices. An agent that reports no
|
|
/// tokens leaves the classification's cost unknown, not zero.
|
|
fn record_cost(&self, run: &TryOut, provider: &Provider, usage: &hive_runtime::TokenUsage) {
|
|
if *usage == hive_runtime::TokenUsage::default() {
|
|
tracing::warn!(
|
|
repo = %run.repo, pr = run.pr,
|
|
"the ACP agent reported no token counts, so this classification's cost is unknown"
|
|
);
|
|
return;
|
|
}
|
|
self.cost.record(&Cost {
|
|
repo: run.repo.clone(),
|
|
pr: run.pr,
|
|
model: provider.model.clone(),
|
|
amount: crate::cost::amount(&provider.prices, usage),
|
|
});
|
|
}
|
|
}
|
|
|
|
/// Refuses every permission request.
|
|
fn deny_all() -> PermissionPolicy {
|
|
Arc::new(|_: &PermissionAsk<'_>| false)
|
|
}
|
|
|
|
/// The `opencode.json` of a run on `provider`.
|
|
fn opencode_config(provider: &Provider) -> Value {
|
|
let prices = &provider.prices;
|
|
json!({
|
|
"$schema": "https://opencode.ai/config.json",
|
|
"autoupdate": false,
|
|
"share": "disabled",
|
|
"model": format!("{PROVIDER_ID}/{}", provider.model),
|
|
"enabled_providers": [PROVIDER_ID],
|
|
"agent": { "title": { "disable": true } },
|
|
"compaction": { "auto": false },
|
|
"provider": {
|
|
PROVIDER_ID: {
|
|
"npm": "@ai-sdk/openai-compatible",
|
|
"name": "review classifier provider",
|
|
"options": {
|
|
"baseURL": provider.base_url,
|
|
// opencode substitutes it from its environment.
|
|
"apiKey": format!("{{env:{PROVIDER_KEY_VAR}}}"),
|
|
},
|
|
"models": {
|
|
provider.model.as_str(): {
|
|
"name": provider.model,
|
|
"cost": {
|
|
"input": prices.input,
|
|
"output": prices.output,
|
|
"cache_read": prices.cache_read.unwrap_or(prices.input),
|
|
"cache_write": prices.cache_write.unwrap_or(prices.input),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
/// Keeps the turn's last assistant text: the model's final message. An ACP
|
|
/// turn has no `result` event to take it from.
|
|
#[derive(Default)]
|
|
struct FinalText(Mutex<Option<String>>);
|
|
|
|
impl FinalText {
|
|
fn take(&self) -> Result<String, String> {
|
|
self.0
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.take()
|
|
.ok_or_else(|| "the ACP agent ended without an answer".to_owned())
|
|
}
|
|
}
|
|
|
|
impl Sink for FinalText {
|
|
fn on_event(&self, event: &Value) {
|
|
if event["type"] != "assistant" {
|
|
return;
|
|
}
|
|
let mut texts = event["message"]["content"]
|
|
.as_array()
|
|
.into_iter()
|
|
.flatten()
|
|
.filter(|block| block["type"] == "text")
|
|
.filter_map(|block| block["text"].as_str());
|
|
if let Some(text) = texts.next_back() {
|
|
*self
|
|
.0
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(text.to_owned());
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::os::unix::fs::PermissionsExt as _;
|
|
|
|
use swarm_queue_client::review_classifier::Prices;
|
|
|
|
use super::*;
|
|
use crate::run::Model as _;
|
|
|
|
/// An ACP agent in plain `sh`, logging to `<log>.*`. It writes its
|
|
/// environment and a copy of its config at start, appends each method
|
|
/// it is sent to `<log>.methods`, and answers a prompt with a text chunk
|
|
/// holding a verdict, then `end_turn` with 1000 input and 200 output
|
|
/// tokens. A prompt holding `ASK-READ` first asks permission for a
|
|
/// `read` and appends the reply to `<log>.permission`; one holding
|
|
/// `STALL` is answered only once it is cancelled; one holding `MUTE`
|
|
/// gets the answer with no token counts.
|
|
const AGENT: &str = r#"#!/bin/sh
|
|
log="$1"
|
|
env > "$log.env"
|
|
cp "$OPENCODE_CONFIG" "$log.config"
|
|
stalled=
|
|
while IFS= read -r line; do
|
|
id=${line#*\"id\":}; id=${id%%[,\}]*}
|
|
method=$(printf '%s' "$line" | sed -n 's/.*"method":"\([^"]*\)".*/\1/p')
|
|
printf '%s\n' "$method" >> "$log.methods"
|
|
case "$method" in
|
|
initialize)
|
|
printf '{"jsonrpc":"2.0","id":%s,"result":{"protocolVersion":1,"agentCapabilities":{"mcpCapabilities":{"http":true}}}}\n' "$id" ;;
|
|
session/new)
|
|
printf '{"jsonrpc":"2.0","id":%s,"result":{"sessionId":"s1"}}\n' "$id" ;;
|
|
session/cancel)
|
|
[ -n "$stalled" ] && printf '{"jsonrpc":"2.0","id":%s,"result":{"stopReason":"cancelled"}}\n' "$stalled" ;;
|
|
session/prompt)
|
|
case "$line" in *STALL*) stalled=$id; continue ;; esac
|
|
case "$line" in *ASK-READ*)
|
|
printf '{"jsonrpc":"2.0","id":900,"method":"session/request_permission","params":{"sessionId":"s1","toolCall":{"toolCallId":"t1","kind":"read","title":"read"},"options":[{"optionId":"yes","kind":"allow_once","name":"Allow"},{"optionId":"no","kind":"reject_once","name":"Reject"}]}}\n'
|
|
read -r reply
|
|
printf '%s\n' "$reply" >> "$log.permission" ;;
|
|
esac
|
|
printf '{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s1","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"{\\"verdict\\":\\"match\\",\\"certainty\\":90,\\"reason\\":\\"ok\\"}"}}}}\n'
|
|
case "$line" in
|
|
*MUTE*) printf '{"jsonrpc":"2.0","id":%s,"result":{"stopReason":"end_turn"}}\n' "$id" ;;
|
|
*) printf '{"jsonrpc":"2.0","id":%s,"result":{"stopReason":"end_turn","usage":{"inputTokens":1000,"outputTokens":200}}}\n' "$id" ;;
|
|
esac ;;
|
|
esac
|
|
done
|
|
"#;
|
|
|
|
#[derive(Default)]
|
|
struct Costs(Mutex<Vec<Cost>>);
|
|
|
|
impl CostSink for Arc<Costs> {
|
|
fn record(&self, cost: &Cost) {
|
|
self.0.lock().expect("unpoisoned").push(cost.clone());
|
|
}
|
|
}
|
|
|
|
/// An [`Acp`] whose trampoline is [`AGENT`], logging under `dir`.
|
|
fn acp(dir: &Path, deadline: Duration) -> (Acp<Arc<Costs>>, Arc<Costs>) {
|
|
let script = dir.join("agent.sh");
|
|
let log = dir.join("log");
|
|
std::fs::write(
|
|
&script,
|
|
AGENT.replacen("log=\"$1\"", &format!("log='{}'", log.display()), 1),
|
|
)
|
|
.expect("written");
|
|
std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).expect("chmod");
|
|
let costs = Arc::new(Costs::default());
|
|
let acp = Acp {
|
|
program: PathBuf::from("/nix/store/x-opencode/bin/opencode"),
|
|
trampoline: script,
|
|
deadline,
|
|
cancel_wait: Duration::from_millis(500),
|
|
cost: Arc::clone(&costs),
|
|
};
|
|
(acp, costs)
|
|
}
|
|
|
|
fn read(dir: &Path, suffix: &str) -> String {
|
|
std::fs::read_to_string(dir.join(format!("log.{suffix}"))).unwrap_or_default()
|
|
}
|
|
|
|
fn run() -> TryOut {
|
|
TryOut {
|
|
operator: "alice".to_owned(),
|
|
repo: "o/r".to_owned(),
|
|
pr: 7,
|
|
head_sha: "abc".to_owned(),
|
|
commits: Vec::new(),
|
|
diff: String::new(),
|
|
classifications: Vec::new(),
|
|
model: "haiku".to_owned(),
|
|
effort: None,
|
|
runtime: Runtime::Acp(Provider {
|
|
base_url: "https://llm.example/v1".to_owned(),
|
|
model: "qwen".to_owned(),
|
|
prices: Prices {
|
|
input: 3.0,
|
|
output: 15.0,
|
|
cache_read: None,
|
|
cache_write: None,
|
|
},
|
|
}),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn the_answer_is_the_final_text_and_its_cost_is_labelled_with_the_pr() {
|
|
let dir = tempfile::tempdir().expect("a dir");
|
|
let (acp, costs) = acp(dir.path(), Duration::from_mins(1));
|
|
|
|
let answer = acp.ask("classify", "p", &run()).await;
|
|
assert_eq!(
|
|
answer.as_deref(),
|
|
Ok(r#"{"verdict":"match","certainty":90,"reason":"ok"}"#)
|
|
);
|
|
let costs = costs.0.lock().expect("unpoisoned");
|
|
assert_eq!(costs.len(), 1);
|
|
let cost = &costs[0];
|
|
assert_eq!(
|
|
(cost.repo.as_str(), cost.pr, cost.model.as_str()),
|
|
("o/r", 7, "qwen")
|
|
);
|
|
assert!((cost.amount - 0.006).abs() < 1e-12, "{}", cost.amount);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn opencode_gets_the_key_no_tools_and_the_runs_provider() {
|
|
let dir = tempfile::tempdir().expect("a dir");
|
|
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
|
|
|
|
acp.ask("classify", "p", &run()).await.expect("an answer");
|
|
let env = read(dir.path(), "env");
|
|
for line in [
|
|
"ACP_PROVIDER_API_KEY=p",
|
|
r#"OPENCODE_PERMISSION={"*":"deny"}"#,
|
|
"OPENCODE_DISABLE_PROJECT_CONFIG=1",
|
|
"SWARM_REVIEW_CLASSIFIER_EXEC_OPENCODE=/nix/store/x-opencode/bin/opencode",
|
|
] {
|
|
assert!(env.lines().any(|l| l == line), "{line} not in:\n{env}");
|
|
}
|
|
let config: Value = serde_json::from_str(&read(dir.path(), "config")).expect("JSON");
|
|
let provider = &config["provider"][PROVIDER_ID];
|
|
assert_eq!(config["model"], "review-classifier/qwen");
|
|
assert_eq!(provider["options"]["baseURL"], "https://llm.example/v1");
|
|
assert_eq!(provider["options"]["apiKey"], "{env:ACP_PROVIDER_API_KEY}");
|
|
assert_eq!(provider["models"]["qwen"]["cost"]["output"], 15.0);
|
|
assert_eq!(config["agent"]["title"]["disable"], true);
|
|
// The run's directory is gone with the run.
|
|
let home = env
|
|
.lines()
|
|
.find_map(|l| l.strip_prefix("HOME="))
|
|
.expect("a HOME");
|
|
assert!(!Path::new(home).exists(), "{home}");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_permission_request_is_refused() {
|
|
let dir = tempfile::tempdir().expect("a dir");
|
|
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
|
|
|
|
acp.ask("ASK-READ", "p", &run()).await.expect("an answer");
|
|
let reply: Value =
|
|
serde_json::from_str(read(dir.path(), "permission").trim()).expect("a JSON reply");
|
|
assert_eq!(reply["result"]["outcome"]["optionId"], "no");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn the_deadline_cancels_the_turn() {
|
|
let dir = tempfile::tempdir().expect("a dir");
|
|
let (acp, costs) = acp(dir.path(), Duration::from_millis(500));
|
|
|
|
let answer = acp.ask("STALL", "p", &run()).await;
|
|
assert_eq!(answer, Err("no answer within 0 seconds".to_owned()));
|
|
assert!(
|
|
read(dir.path(), "methods")
|
|
.lines()
|
|
.any(|m| m == "session/cancel")
|
|
);
|
|
assert!(costs.0.lock().expect("unpoisoned").is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn an_agent_stuck_before_its_prompt_is_dropped_at_the_deadline() {
|
|
let dir = tempfile::tempdir().expect("a dir");
|
|
let (acp, _) = acp(dir.path(), Duration::from_millis(500));
|
|
let script = dir.path().join("agent.sh");
|
|
let unanswered = std::fs::read_to_string(&script).expect("written").replacen(
|
|
" initialize)",
|
|
" never)",
|
|
1,
|
|
);
|
|
std::fs::write(&script, unanswered).expect("written");
|
|
|
|
let answer = tokio::time::timeout(Duration::from_secs(10), acp.ask("one", "p", &run()))
|
|
.await
|
|
.expect("the deadline ends the ask");
|
|
assert_eq!(answer, Err("no answer within 0 seconds".to_owned()));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn no_token_counts_record_no_cost() {
|
|
let dir = tempfile::tempdir().expect("a dir");
|
|
let (acp, costs) = acp(dir.path(), Duration::from_mins(1));
|
|
|
|
acp.ask("MUTE", "p", &run()).await.expect("an answer");
|
|
assert!(costs.0.lock().expect("unpoisoned").is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_claude_run_is_refused() {
|
|
let dir = tempfile::tempdir().expect("a dir");
|
|
let (acp, _) = acp(dir.path(), Duration::from_mins(1));
|
|
let mut claude = run();
|
|
claude.runtime = Runtime::ApiKey;
|
|
|
|
assert!(acp.ask("classify", "p", &claude).await.is_err());
|
|
assert_eq!(read(dir.path(), "methods"), "", "no agent started");
|
|
}
|
|
}
|