hive-agent: no hard-coded context window sizes, the window is what the API reports
This commit is contained in:
parent
4a109fa038
commit
1bfee54414
7 changed files with 47 additions and 124 deletions
|
|
@ -20,8 +20,8 @@ use swarm_wire::agent_inference::Source;
|
|||
use tokio::sync::broadcast;
|
||||
|
||||
use crate::harness_state::{
|
||||
configured_effort, configured_model, context_window_tokens, local_effort, local_model,
|
||||
read_harness_state, write_harness_state,
|
||||
configured_effort, configured_model, local_effort, local_model, read_harness_state,
|
||||
write_harness_state,
|
||||
};
|
||||
|
||||
const CHANNEL_CAPACITY: usize = 256;
|
||||
|
|
@ -361,11 +361,11 @@ pub struct Bus {
|
|||
last_turn_ended_unix: Arc<AtomicI64>,
|
||||
/// Per-inference context-window size as reported by the Anthropic API
|
||||
/// in the stream-json `result` event (`modelUsage.*.contextWindow`).
|
||||
/// Set by the stdout pump on every completed turn. Takes precedence
|
||||
/// over the per-model defaults in `harness_state::context_window_tokens`
|
||||
/// for compaction watermark calculations — it reflects the actual
|
||||
/// limit the model enforces, which may differ from the default
|
||||
/// (e.g. 200 k active window on a 1 M cache-enabled model).
|
||||
/// Set by the stdout pump on every completed turn. The only source of the
|
||||
/// window: there is no per-model table, so it is `None` until the first
|
||||
/// turn of this process completes, and everything that divides by it
|
||||
/// (the compaction watermarks, the context fill the swarm shows) waits
|
||||
/// for it.
|
||||
api_context_window: Arc<Mutex<Option<u64>>>,
|
||||
/// The swarm's soft context limit, as a percent of the context window
|
||||
/// (`soft_context_percent` of the agent's `agent-inference` key): the
|
||||
|
|
@ -717,16 +717,6 @@ impl Bus {
|
|||
*self.soft_context_percent.lock().unwrap()
|
||||
}
|
||||
|
||||
/// The effective context window for `model`: the API-reported window if a
|
||||
/// turn has completed (`api_context_window`), else the per-model default
|
||||
/// ([`context_window_tokens`]). Single accessor so the state + dashboard
|
||||
/// endpoints agree by construction.
|
||||
#[must_use]
|
||||
pub fn effective_context_window(&self, model: &str) -> u64 {
|
||||
self.api_context_window()
|
||||
.unwrap_or_else(|| context_window_tokens(model))
|
||||
}
|
||||
|
||||
/// Walk a stream-json value for `tool_use` blocks and bump the
|
||||
/// per-turn counter for each one we find. Called by the stdout
|
||||
/// pump on every parsed line. Cheap when the line isn't an
|
||||
|
|
|
|||
|
|
@ -224,47 +224,11 @@ pub fn configured_effort() -> Option<&'static str> {
|
|||
.map(|s| &*Box::leak(s.into_boxed_str()))
|
||||
}
|
||||
|
||||
/// Context-window size in tokens per model family, keyed by a substring
|
||||
/// of the model name (`"sonnet"` matches `"claude-sonnet-4-5"`).
|
||||
const CONTEXT_WINDOW_DEFAULTS: [(&str, u64); 3] = [
|
||||
("haiku", 200_000),
|
||||
("sonnet", 1_000_000),
|
||||
("opus", 1_000_000),
|
||||
];
|
||||
|
||||
/// Context-window size in tokens for a given model name, used until a
|
||||
/// completed turn reports the real window (`Bus::effective_context_window`
|
||||
/// prefers that one).
|
||||
///
|
||||
/// Resolution order (first match wins):
|
||||
/// 1. [`CONTEXT_WINDOW_DEFAULTS`] — the family key, case-insensitively,
|
||||
/// is a substring of the model name.
|
||||
/// 2. `HIVE_CONTEXT_WINDOW_TOKENS` — single global override (any model).
|
||||
/// 3. Hard fallback: `200_000`.
|
||||
#[must_use]
|
||||
pub fn context_window_tokens(model: &str) -> u64 {
|
||||
let m = model.to_ascii_lowercase();
|
||||
if let Some((_, tokens)) = CONTEXT_WINDOW_DEFAULTS
|
||||
.iter()
|
||||
.find(|(family, _)| m.contains(family))
|
||||
{
|
||||
return *tokens;
|
||||
}
|
||||
// Global override (single value, any model).
|
||||
if let Ok(s) = std::env::var("HIVE_CONTEXT_WINDOW_TOKENS")
|
||||
&& let Ok(v) = s.trim().parse::<u64>()
|
||||
&& v > 0
|
||||
{
|
||||
return v;
|
||||
}
|
||||
200_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DEFAULT_EFFORT, DEFAULT_MODEL, Source, context_window_tokens, effort_from, model_from,
|
||||
read_harness_state, write_harness_state,
|
||||
DEFAULT_EFFORT, DEFAULT_MODEL, Source, effort_from, model_from, read_harness_state,
|
||||
write_harness_state,
|
||||
};
|
||||
|
||||
/// A state file an older harness wrote, with the `active_model` nothing
|
||||
|
|
@ -310,17 +274,6 @@ mod tests {
|
|||
assert_eq!(stored["needs_login"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn context_window_defaults_per_model_family() {
|
||||
assert_eq!(context_window_tokens("claude-haiku-4-5"), 200_000);
|
||||
assert_eq!(context_window_tokens("haiku"), 200_000);
|
||||
assert_eq!(context_window_tokens("claude-sonnet-4-5"), 1_000_000);
|
||||
assert_eq!(context_window_tokens("Claude-Sonnet-4-6"), 1_000_000);
|
||||
assert_eq!(context_window_tokens("sonnet"), 1_000_000);
|
||||
assert_eq!(context_window_tokens("claude-opus-4-1"), 1_000_000);
|
||||
assert_eq!(context_window_tokens("opus"), 1_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compiled_default_effort_matches_claude_settings() {
|
||||
// The value baked into prompts/claude-settings.json.
|
||||
|
|
|
|||
|
|
@ -1064,7 +1064,7 @@ async fn handle_turn<S: Surface>(
|
|||
let ended_at = chrono::Utc::now().timestamp();
|
||||
let duration_ms = i64::try_from(started_instant.elapsed().as_millis()).unwrap_or(i64::MAX);
|
||||
let (open_threads, open_reminders) = S::post_turn_counts(socket).await;
|
||||
let context_window = bus.effective_context_window(&model_at_start);
|
||||
let context_window = bus.api_context_window();
|
||||
let row = serve_common::build_row(serve_common::TurnRowArgs {
|
||||
started_at,
|
||||
ended_at,
|
||||
|
|
|
|||
|
|
@ -273,15 +273,18 @@ pub fn record_swarm_queue_publish_skipped(site: &'static str) {
|
|||
.add(1, &[KeyValue::new("site", site)]);
|
||||
}
|
||||
|
||||
/// Record the context window's fill at turn end: `used` tokens of `size`,
|
||||
/// the two numbers the web UI's ctx% divides. No-op when OTEL isn't
|
||||
/// configured, same as [`record`].
|
||||
pub fn record_context(used: u64, size: u64) {
|
||||
/// Record the context window's fill at turn end: `used` tokens, and the
|
||||
/// window's `size` once the API has reported one (until then there is nothing
|
||||
/// to divide by, so no size is recorded rather than a guessed one). No-op when
|
||||
/// OTEL isn't configured, same as [`record`].
|
||||
pub fn record_context(used: u64, size: Option<u64>) {
|
||||
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
|
||||
return;
|
||||
};
|
||||
inst.context_used.record(used, &[]);
|
||||
inst.context_size.record(size, &[]);
|
||||
if let Some(size) = size {
|
||||
inst.context_size.record(size, &[]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Add what an ACP agent reported spending since the last call. No-op when
|
||||
|
|
|
|||
|
|
@ -91,8 +91,9 @@ struct AgentStateMsg {
|
|||
/// on, resolved from that turn's assistant events. `None` before any
|
||||
/// turn has completed.
|
||||
resolved_model: Option<String>,
|
||||
/// Effective context-window token budget for the current model.
|
||||
context_window_tokens: u64,
|
||||
/// The context window the API last reported, in tokens; `null` until the
|
||||
/// first turn of this process completes.
|
||||
context_window_tokens: Option<u64>,
|
||||
/// Last-inference token usage from the most recent completed turn — the
|
||||
/// current context-window occupancy. `None` until the first turn ends.
|
||||
ctx_usage: Option<hive_claude::TokenUsage>,
|
||||
|
|
@ -178,7 +179,7 @@ fn snapshot(bus: &Bus, login_state: &Mutex<LoginState>) -> AgentStateMsg {
|
|||
turn_state,
|
||||
turn_state_since: iso8601_utc(since_unix),
|
||||
agent_state: observed_state(),
|
||||
context_window_tokens: bus.effective_context_window(&model),
|
||||
context_window_tokens: bus.api_context_window(),
|
||||
inference: Effective {
|
||||
model,
|
||||
effort: bus.effort(),
|
||||
|
|
@ -407,7 +408,7 @@ mod tests {
|
|||
effort_source: Source::AgentNix,
|
||||
},
|
||||
resolved_model: Some("claude-sonnet-4-5-20260805".to_owned()),
|
||||
context_window_tokens: 200_000,
|
||||
context_window_tokens: Some(200_000),
|
||||
ctx_usage: Some(usage(&serde_json::json!({
|
||||
"input_tokens": 11,
|
||||
"output_tokens": 22,
|
||||
|
|
|
|||
|
|
@ -444,13 +444,12 @@ fn compact(bus: &Bus, wake_prompt: Option<String>) -> Response {
|
|||
message: "compact refused: no completed turn yet — nothing to compact".to_owned(),
|
||||
};
|
||||
};
|
||||
let model = bus.model();
|
||||
let window = bus.effective_context_window(&model);
|
||||
if window == 0 {
|
||||
let Some(window) = bus.api_context_window() else {
|
||||
return Response::Err {
|
||||
message: "compact refused: effective context window is unknown (0)".to_owned(),
|
||||
message: "compact refused: the context window is unknown until a turn reports it"
|
||||
.to_owned(),
|
||||
};
|
||||
}
|
||||
};
|
||||
// Token counts stay well under 2^53 in practice, so the f64 conversion is exact.
|
||||
// The ratio is only compared against a fixed threshold, so a last-place rounding error cannot change the outcome.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
|
|
|
|||
|
|
@ -262,29 +262,14 @@ pub fn stall_sleep_secs() -> u64 {
|
|||
env_u64_positive("HIVE_STALL_SLEEP_SECS", DEFAULT_STALL_SLEEP_SECS)
|
||||
}
|
||||
|
||||
/// Resolve the effective context-window size for watermark calculations.
|
||||
/// Priority order (first wins):
|
||||
/// 1. API-reported window from the last `result` event's `modelUsage.*.contextWindow`.
|
||||
/// 2. `harness_state::context_window_tokens` (per-model defaults, then the
|
||||
/// `HIVE_CONTEXT_WINDOW_TOKENS` override, then 200 000).
|
||||
///
|
||||
/// The API-reported window is the authoritative per-inference active
|
||||
/// context limit. It reflects what the model actually enforces — which
|
||||
/// for models with large prompt caches (e.g. 1 M total cache) may be
|
||||
/// significantly smaller than the cache capacity (e.g. 200 k active window
|
||||
/// for `claude-sonnet-4-6`).
|
||||
fn effective_context_window(bus: &Bus) -> u64 {
|
||||
bus.api_context_window()
|
||||
.unwrap_or_else(|| crate::harness_state::context_window_tokens(&bus.model()))
|
||||
}
|
||||
|
||||
/// Resolve the auto-reset watermark. Priority order:
|
||||
/// 1. `HIVE_AUTO_RESET_WATERMARK_TOKENS` env var (explicit override).
|
||||
/// 2. 50% of `effective_context_window(bus)`.
|
||||
/// 2. 50% of the window the API last reported, or `0` before one has been.
|
||||
///
|
||||
/// `0` disables auto-reset entirely.
|
||||
fn auto_reset_watermark_tokens(bus: &Bus) -> u64 {
|
||||
env_u64("HIVE_AUTO_RESET_WATERMARK_TOKENS").unwrap_or_else(|| effective_context_window(bus) / 2)
|
||||
env_u64("HIVE_AUTO_RESET_WATERMARK_TOKENS")
|
||||
.unwrap_or_else(|| bus.api_context_window().map_or(0, |window| window / 2))
|
||||
}
|
||||
|
||||
/// Resolve the assumed cache TTL: `HIVE_CACHE_TTL_SECS` if set, else
|
||||
|
|
@ -353,10 +338,9 @@ impl CompactionPolicy for SoftLimitPolicy {
|
|||
/// Construct the agent's durable session, with a percent-of-window compaction
|
||||
/// policy that checkpoints (`CHECKPOINT_PROMPT`) before compacting; on claude
|
||||
/// also a constant title and the on-disk store. Called once at serve-loop start.
|
||||
/// `percent` comes from a boot-time env var and `default_window` is only a
|
||||
/// fallback for turns where the model didn't report a window, so a single
|
||||
/// build at startup is fine; the swarm's soft limit is read from the bus per
|
||||
/// decision.
|
||||
/// `percent` comes from a boot-time env var, so a single build at startup is
|
||||
/// fine; the window is the one each turn reports (no turn that reports none
|
||||
/// compacts), and the swarm's soft limit is read from the bus per decision.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
|
|
@ -365,7 +349,7 @@ pub fn make_session(bus: &Bus) -> Result<AgentSession> {
|
|||
let policy = SoftLimitPolicy {
|
||||
percent: PercentPolicy {
|
||||
percent: compact_percent(),
|
||||
default_window: Some(effective_context_window(bus)),
|
||||
default_window: None,
|
||||
checkpoint_prompt: Some(CHECKPOINT_PROMPT.to_string()),
|
||||
},
|
||||
bus: bus.clone(),
|
||||
|
|
|
|||
Loading…
Reference in a new issue