From 216eb8ae9b79f6151f8f4b1b1082f12f02bf34e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Sun, 5 Jul 2026 23:04:48 +0200 Subject: [PATCH] docs(turn-loop): split into index + claude-invocation/config/mcp sub-pages, condense the loop --- CLAUDE.md | 5 +- docs/turn-loop.md | 720 ++-------------------------- docs/turn-loop/claude-invocation.md | 241 ++++++++++ docs/turn-loop/config.md | 199 ++++++++ docs/turn-loop/mcp.md | 201 ++++++++ hive-ag3nt/src/mcp.rs | 2 +- hive-ag3nt/src/prompt.rs | 4 +- 7 files changed, 696 insertions(+), 676 deletions(-) create mode 100644 docs/turn-loop/claude-invocation.md create mode 100644 docs/turn-loop/config.md create mode 100644 docs/turn-loop/mcp.md diff --git a/CLAUDE.md b/CLAUDE.md index ddb1d70c..9b52b3f4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,7 +86,10 @@ read them à la carte. - **"How does the per-agent terminal classify + colour events?"** → [`docs/terminal-rendering.md`](docs/terminal-rendering.md). - **"How does claude get its prompt and what tools does it have?"** → - [`docs/turn-loop.md`](docs/turn-loop.md). + [`docs/turn-loop.md`](docs/turn-loop.md) (index: the loop, binary shape, + turn outcomes; sub-pages: + [`claude-invocation`](docs/turn-loop/claude-invocation.md), + [`config`](docs/turn-loop/config.md), [`mcp`](docs/turn-loop/mcp.md)). - **"How do config changes flow from manager to operator to container?"** → [`docs/approvals.md`](docs/approvals.md). - **"What state survives destroy / purge / restart?"** → diff --git a/docs/turn-loop.md b/docs/turn-loop.md index 70aee411..0b70e486 100644 --- a/docs/turn-loop.md +++ b/docs/turn-loop.md @@ -18,46 +18,37 @@ Each agent harness (`hive serve` — one binary for all agents) runs: over stdin. 5. Stream stdout (JSON lines) into the bus as `LiveEvent::Stream(value)`. Pump stderr as `Note`. -6. Wait for claude to exit. Compaction is two-pronged — *reactive* - on `Prompt is too long` and *proactive* on a context watermark - (see [Compaction](#compaction) below). **Rate-limit detection**: - on stderr the harness does a raw-line match for `429` / - `rate_limit` markers; on stdout it only fires on parsed - `{"type":"error"}` JSON events (avoiding false positives when - agents discuss `rate_limit_error` in conversation text). On - detection the harness sets the `rate_limited` sentinel - (`Bus::emit_status("rate_limited")`), sleeps - `HIVE_RATE_LIMIT_SLEEP_SECS` (default 300), then retries. - The dashboard and per-agent page show a `⊘ rate limited` badge - while the harness is parked. **Auth-failed detection**: both - stdout and stderr pumps also match - `AUTH_FAIL_MARKERS` (`"authentication_failed"`, `401`, etc.). - On the first 401, `drive_turn` retries the same prompt once - immediately (transient token-refresh races and brief API hiccups - can cause a 401 that clears on retry). Only if the retry also - returns `AuthFailed` does `drive_turn` bubble it up to the serve - loop, which then writes `{state_dir}/hyperhive-needs-login`, - emits `needs_login_idle` status, requeues the inflight message - (so it replays after re-auth), and parks in `wait_for_login` — - the same path used at boot. The operator re-authenticates via - the per-agent web UI login flow; on success the sentinel is - cleared and the queued message drives the next turn normally. - **Mtime-snapshot resumption**: `wait_for_login` - snapshots the `~/.claude/` dir (newest mtime + count over the - credential files in `login::CRED_FILE_NAMES`) at entry and only - resumes when that snapshot advances — not just when credentials - exist on disk. This prevents a silent infinite-401 loop: stale - credentials already on disk at the time of the 401 no longer cause - an immediate false-resume. The `DirSnapshot` struct tracks both - axes; either a mtime advance OR a file-count change triggers resume - (the count axis handles filesystems where `modified()` errors on - every file). Boot detection (`login::has_session`, deciding Online - vs NeedsLogin) keys off the same `CRED_FILE_NAMES` set — the files - `/logout` deletes — so leftover session-history files don't read as - a live session after a logout + container recreate. +6. Wait for claude to exit and classify the turn's outcome from the + stream + exit — success, compaction, rate-limit, auth-failure, or + hard failure. The outcome drives the post-turn action (see + [Turn outcomes](#turn-outcomes)); compaction is handled inside the + session (see + [Compaction](turn-loop/claude-invocation.md#compaction)). Rate-limit + and auth-failure detection is described [below](#failure-detection-and-login). 7. Emit `LiveEvent::TurnEnd { ok, note }`. Sleep `poll_ms` to avoid tight loops on transient failures. +### Failure detection and login + +- **Rate limit** — a `429` / `rate_limit` marker on stderr, or a parsed + `{"type":"error"}` rate-limit event on stdout (conversation-text + mentions don't count), sets the `rate_limited` sentinel, parks for + `HIVE_RATE_LIMIT_SLEEP_SECS` (default 300), then retries. The UI shows + a `⊘ rate limited` badge while parked. +- **Auth failure (401)** — `drive_turn` retries once (transient + token-refresh races clear on retry); a second `AuthFailed` writes + `{state_dir}/hyperhive-needs-login`, requeues the message, and parks in + `wait_for_login` — the same path as a cold boot with no session. The + operator re-auths via the per-agent web UI; the queued message then + drives the next turn. +- **Login detection** — both boot (`login::has_session`, Online vs + NeedsLogin) and `wait_for_login`'s resume check key off the credential + files in `login::CRED_FILE_NAMES` (the set `/logout` deletes). + `wait_for_login` resumes only when that set changes (a new file or a + newer mtime), so stale credentials on disk at the 401 don't trigger an + instant false-resume, and leftover session-history files don't read as a + live session after a logout + container recreate. + ## Harness binary shape One `hive` binary for all agents. The earlier split into @@ -99,7 +90,8 @@ loop (`serve_loop` / `handle_turn` / `wake`) has no per-role branches. `HIVE_LABEL` (default `"hive"` for standalone runs; the meta flake sets it unconditionally for any container-deployed agent; see `docs/conventions.md::Hive identity` for the env stack), -opens turn-stats sqlite, prepares the on-boot files (see below), +opens turn-stats sqlite, prepares the on-boot files (see +[claude-invocation](turn-loop/claude-invocation.md#on-boot-files)), installs claude plugins, spawns `forge_notify::run` + `web_ui::serve`, and either drops into `serve_loop` directly (`Online`) or parks on the login flow first (`NeedsLogin`). @@ -139,637 +131,21 @@ isn't needed (this is the `request_next_turn` contract — "no effect if a new inbox message arrives before this turn ends"). The `should_self_continue` predicate encodes exactly that decision. -## The claude invocation - -``` -claude --print --verbose --output-format stream-json --model \ - --effort --resume # or --name <title> on first use \ - --system-prompt-file /run/hive/claude-system-prompt.md \ - --mcp-config /run/hive/claude-mcp-config.json --strict-mcp-config \ - --tools <builtins> --allowedTools <builtins+mcp> -# wake prompt piped over stdin -``` - -**Crate split.** The generic subprocess mechanics — spawning -`claude --print`, streaming + classifying stream-json, session -lookup/archive, and the durable-session compaction loop — live in the -reusable **`hive-claude`** crate (`hive_claude::{Claude, InfiniteSession, -Attach, CompactionPolicy, PercentPolicy, Telemetry, Sink, SessionStore}`; -see `hive-claude/README.md`). `hive_ag3nt::turn` is the hyperhive **policy -layer** on top: it builds the per-turn config from the bus, bridges the -output stream onto the event bus (`BusSink`), and owns the compaction / -auto-reset / retry decisions in `drive_turn`. The lib returns everything it -parsed from a turn (usage, cost, context window, resolved model) as -`Telemetry`, which the policy layer applies to the bus. - -Hive-enforced settings ship at `/etc/claude-code/managed-settings.json` -(claude-code's canonical managed-settings path — precedence #1, -read-only, un-overridable), wired in `nix/templates/harness-base.nix` -from the `prompts/claude-settings.json` asset. `effortLevel` is -deliberately not in that file — effort is controlled live via the -`--effort` flag (`HIVE_DEFAULT_EFFORT` / the per-agent UI slider), which -managed scope would otherwise lock. - -`<name>` is read from `Bus::model()` on each turn. The initial -default is set by `hyperhive.model` in the agent's `agent.nix` -(NixOS option; propagates via `HIVE_DEFAULT_MODEL` env var; falls -back to `"haiku"` if unset). The operator can flip it at runtime -with `/model <name>` in the web terminal — the next turn picks it -up. The choice is persisted to `/harness/hyperhive-model` so it -survives restart; override path: `HYPERHIVE_MODEL_FILE` env var -for tests. - -Context-window size is looked up per-model via -`events::context_window_tokens(model)`. Resolution order (first -match wins): - -1. `HIVE_CONTEXT_WINDOW_TOKENS_<KEY>` env var, where `KEY` - (lowercased) is a substring of the active model name. Injected - by the meta flake from `services.hyperhive.c0re.contextWindowTokens` - (host-level NixOS option, defaults: haiku=200k, sonnet=1M, - opus=1M). Override these for all agents at once without a - per-agent config change. -2. `HIVE_CONTEXT_WINDOW_TOKENS` — single global override for any - model (useful in dev / test). -3. Hard fallback: `200_000` (conservative; only reached outside - NixOS where the env vars aren't set). - -The effective window drives watermarks and is exposed at runtime -via `/api/state.context_window_tokens` so the UI can show a -percentage-of-window ctx badge. - -**Session identity — a constant title.** Every turn keys on one fixed, -harness-owned session title (`turn::session_title()`, default -`hive-session`, override `HIVE_SESSION_TITLE`). The durable -`hive_claude::InfiniteSession` (built once by the serve loop via -`turn::make_session`, then reused) `--resume <title>`s it; the *first* use -(bootstrap, post-archive, post-purge) misses and the session re-runs the -same prompt once with `--name <title>` to mint it. That single self-heal -rule is the whole identity system — there is **no** scraped session-id -file. Because the -title is constant, `/compact` and its post-compact retry provably target -the same session (killing the old "compact ran on a different/empty -session" bug), and a `choom` invocation in the same cwd can't hijack the -context (it won't carry our title). claude stores sessions in -`~/.claude/projects/<cwd-slug>/<uuid>.jsonl` (bind-mounted persistently); -`--name` writes the title into the file as a `custom-title` event, which -is what `--resume <title>` resolves against. We never pass bare -`--continue` (it resumes the *latest* session in the cwd — the hijack -vector). Auto-compact and auto-memory are disabled via the managed -settings at `/etc/claude-code/managed-settings.json` because hyperhive -owns compaction — see [Compaction](#compaction) below. - -**Session reset** is available via `POST /api/new-session` (or -`/new-session` slash command). It does *not* touch the session inline — -that would race a mid-write claude process. Instead `Bus::request_session_reset()` -sets a one-shot flag consumed at the next turn boundary by `drive_turn`, -which **archives** the current session: the backing `<uuid>.jsonl` is -renamed to `<uuid>.jsonl.archived` (dropped out of claude's `*.jsonl` -resolution glob, history preserved on disk, only the file carrying *our* -title — any `choom` session sharing the cwd is left alone). The next -turn's `--resume <title>` then misses and self-heals into a fresh session. - -### Compaction - -claude's own in-session auto-compact is off (via the managed settings -at `/etc/claude-code/managed-settings.json`); hyperhive owns it. The -`hive_claude::InfiniteSession` keeps the session alive across the context -window with two triggers baked into its `run`: - -- **Reactive** — claude-code prints `Prompt is too long`. The session is - *already* past the window, so no turn can run on it — the session - `/compact`s straight away and retries the same wake-up prompt once. No - notes-checkpoint turn is possible here: the detail is gone. If the retry - *still* overflows, `run` surfaces `Error::PromptTooLong`; `drive_turn` - then archives the session (session lifecycle stays hive-side) and the - serve loop requeues the message so it redelivers into a fresh session - (see [Turn outcomes](#turn-outcomes) — the wake prompt itself is tiny, so - the overflow was the accumulated context the archive clears). -- **Proactive** — a turn finishes cleanly but the last inference's context - size crossed the policy watermark. While the session is still healthy it - runs one synthetic *notes-checkpoint* turn (`CHECKPOINT_PROMPT` — - "context is filling up, flush durable state into `/state` now") and - *then* `/compact`s, so the agent can persist in-flight state before the - detail collapses into a summary. - -The **when** is a `hive_claude::CompactionPolicy` injected by the harness: -`turn::make_session` builds a `PercentPolicy` that compacts once the -model-reported context fill reaches `HIVE_COMPACT_WATERMARK_PERCENT` -(default **75%**), falling back to `events::context_window_tokens(model)` -for the window on turns the model didn't report one. `0` disables proactive -compaction (the reactive path always applies). The proactive path is -best-effort — a failed checkpoint or `/compact` never fails the turn that -already succeeded. - -The operator can force a compaction any time via `POST /api/compact`. It's -**deferred**: the handler sets `Bus::request_compact()` and returns -immediately; the harness runs the `/compact` at the next turn boundary -(end of the in-flight turn in `drive_turn`, or — when the agent is idle — -in `turn::run_pending_compact` on the serve loop's next empty poll). This -lets `/api/compact` work mid-turn instead of only when idle, without racing -a live claude process. - -To disable proactive compaction for a specific agent, use the nix option: - -```nix -hyperhive.autoCompact = false; # default true -``` - -Setting `autoCompact = false` sets `HIVE_COMPACT_WATERMARK_TOKENS=0`, which -the percent resolver still honours as a disable. Useful for large-context -models (sonnet/opus) where the 75% heuristic fires before the session is -actually full — the reactive path (compact-on-overflow at the hard limit) -still applies. - -- **Auto session-reset** — a third path (`turn::maybe_auto_reset`, - pre-turn) that fires when both conditions hold: context is ≥ a watermark - (`HIVE_AUTO_RESET_WATERMARK_TOKENS`, default **50% of - `context_window_tokens(model)`**) AND the time since the last turn - exceeds the assumed prompt-cache TTL (`HIVE_CACHE_TTL_SECS`, default - `3600`). Claude's prompt cache goes cold after a while; once it's cold, - `--resume`-ing a large session pays the full re-upload cost with no - benefit over starting fresh. So `drive_turn` **archives** the current - session (same mechanism as the operator reset — rename `<uuid>.jsonl` → - `.archived`) so the next turn's `--resume <title>` misses and starts - fresh. Unlike proactive compaction the session is dropped entirely, not - compacted — and *no* preceding checkpoint turn runs, because any turn - before the reset would just re-warm the cache and defeat the purpose. - Set `HIVE_AUTO_RESET_WATERMARK_TOKENS=0` to disable. Auto-reset and the - operator reset are mutually exclusive per turn (both archive → fresh - turn), so an explicit operator reset short-circuits the heuristic. - -The child runs with `cwd = /state` (when the bind exists; falls -back to the parent's cwd in dev), so any relative path in a tool -call (`Read foo.md`, `Bash ls`, `Write notes.md`) lands in the -agent's durable bind-mounted dir. CLAUDE.md auto-load walks -upward from `/state` — drop a per-agent CLAUDE.md there if you -want long-term hints that survive destroy/recreate. - -The wake prompt is intentionally minimal: the popped message's -`from`/`body`, prefixed with a `[msg #<id>]` broker-row-id marker -(so the agent knows what id to pass to `ack_until` for bulk triage; -transient pings show no marker because their sentinel id 0 has nothing -to ack), plus an inline `({unread} more pending — drain via …)` hint -when `unread > 0`. Claude drives any further `recv`/`send` itself via -the embedded MCP server. - -Whenever hive-c0re starts / restarts / rebuilds a container, it -also drops a `system` message into the agent's inbox via -`Coordinator::kick_agent` — a one-line "you were just (re)started, -check /state/ for your notes, your session is intact". The -next turn picks it up like any other inbox message. - -### On-boot files - -`hive_ag3nt::turn::write_*` writes two files next to the per-agent -socket at `/run/hive/` once at startup: - -- `claude-mcp-config.json` — by default re-invokes the running binary - as `mcp` stdio child (so the same binary serves as harness + MCP - server per turn). When `hyperhive.mcp.httpPort` is set in the - agent's NixOS config, the config instead points claude at the - persistent `hive-mcp-http` daemon (`http://127.0.0.1:{port}/mcp`) - — no stdio child per turn; trades the per-turn re-registration race - for a hard dependency on the daemon's uptime (`Restart=always`). -- `claude-system-prompt.md` — rendered from - `hive-ag3nt/prompts/system.md` by `hive_ag3nt::prompt::render`: - HTML-comment markers (`<!-- role:agent -->...<!-- /role:agent -->`, - same for `role:manager`) gate the role-specific blocks; everything - else is shared. Five placeholders are then - substituted: `{label}` (short agent name), `{qualified_label}` - (hive-qualified `name@domain` form), `{operator_pronouns}`, - `{hive_identity}` (e.g. `` on hive `pr1ma` ``; empty when - `hyperhive.hiveName` is unset), and `{swarm_identity}` (same - shape for the swarm). Pronouns come from `HIVE_OPERATOR_PRONOUNS` - env (set by the meta flake from - `services.hyperhive.c0re.operatorPronouns`, default `she/her`). - When `hyperhive.docs.enable` is set, `HIVE_DOCS_DIR` is present - in the environment and `render()` appends a one-sentence pointer - telling the agent the docs are mounted at that path (in lieu of - the old CLAUDE.md-in-docs-dir approach, which was dropped in - favour of this direct injection). - Passed via `--system-prompt-file`. - - **Marker grammar.** `<!-- role:X -->` opens a block; any - `<!-- /role:X -->` closes the current block. The renderer always uses - role `agent`, so blocks with other role tags are elided. Nesting is NOT - supported — a stray opener with no closer runs until end of file. - Whitespace inside markers is tolerated (`<!--role:foo-->` parses the - same as `<!-- role:foo -->`). Content outside any marker is always - included. Today's `system.md` carries no markers (single agent role) — - the grammar stays wired for a future manager / multi-role prompt. - - **`hive_identity` / `swarm_identity` shape.** Each carries a - leading space + backticked name (` on hive \`pr1ma\``, - ` in swarm \`constellat1on\``) when the corresponding env var - is set, otherwise empty string. The independence lets the - template drop one or both into the opener prose without - breaking single-hive deployments that never set the option; - the renderer also treats `Some("")` from a caller as `None` so - empty-string env vars and missing env vars round-trip the - same way. - -The per-turn plumbing lives in `hive_ag3nt::turn`: `write_mcp_config` / -`write_system_prompt` (on-boot files), `make_session` (builds the durable -`InfiniteSession`, once), `drive_turn` (the policy state machine — -reset/auto-reset, the turn, 401-retry, deferred-compact-at-turn-end), -`run_pending_compact` (idle operator compact), `BusSink` (stream → bus + -`Telemetry` applied via `apply_telemetry`), `emit_turn_end`, `session_title` -/ `session_store` / `archive_session` (identity + turn-boundary reset). The -actual claude spawn, stream classification, and the reactive/proactive -compaction loop are in the `hive-claude` crate. Login-wait -(`wait_for_login`) lives in `hive_ag3nt::login`. - -### Reference docs (`hyperhive.docs.enable`) - -```nix -hyperhive.docs.enable = true; # default: false (true for the manager agent) -``` - -Makes the hyperhive `docs/` tree available inside the container at a nix -store path read from `$HIVE_DOCS_DIR`, and injects a single pointer -sentence into the agent's system prompt so it knows the docs exist and -where to find them. The tree is served by `claude --add-dir` so the full -markdown is readable during every turn. - -Enabled by default only for the root/manager agent (`manager.nix`). Any -agent can opt in by adding the line above to its `agent.nix`. - -The `docs/` source is a narrow flake input (`hyperhive-docs`) tracked -separately from the main `hyperhive` flake so editing docs re-locks only -that input — not every agent's container gets rebuilt on a doc-only change. - -### Agent icon - -```nix -hyperhive.icon = ./icon.svg; # default: null (falls back to shared hyperhive logo) -``` - -Path to an SVG file used as this agent's visual identity — shown in -the per-agent page header, as the page favicon, and uploaded to the -agent's Forgejo profile avatar (via the `forge-avatar-sync` boot -unit) and Matrix profile avatar (set by `hive-matrix-daemon` over its -live Client). Commit the SVG next to `agent.nix` in the config repo -and reference it as a relative path. - -When `null` (the default), the agent falls back to the shared -hyperhive branding mark. The harness serves whichever icon is active -at `GET /icon` on the per-agent web port. - -### `user.passwordlessSudo` - -```nix -hyperhive.user.passwordlessSudo = true; # default -``` - -Grants the per-agent unix user passwordless `sudo` (`NOPASSWD: ALL`). -Enabled by default so claude's shell tools work for operations that -need root inside the container (`systemctl`, package managers in dev -shells, etc.) — the same privilege surface the previous root-user shape -had, now elevated explicitly rather than implicitly. - -Set to `false` for agents that should be strictly unprivileged. -Any tool invocation that needs root then fails loudly with the standard -sudo rejection rather than silently succeeding — easier to audit. - -`hyperhive.user.uid`, `hyperhive.user.gid`, and -`hyperhive.user.name` are the companion options; see -`docs/agent-hierarchy.md` — "Harness systemd unit shape" for the full -`user.*` surface. - -### Dashboard links - -```nix -hyperhive.dashboardLinks = [ - { label = "Stats"; icon = "📊"; url = "http://localhost:9001/stats"; } - { label = "Scratchpad"; url = "http://localhost:8080"; } -]; -``` - -Declares extra navigation links that appear on the agent's dashboard -card and in the per-agent page header alongside the built-in forge / -config / container links. Each entry has: - -| Field | Required | Description | -|-------|----------|-------------| -| `label` | yes | Display text shown in the icon strip tooltip and meta-nav. | -| `url` | yes | Absolute URL — may include a different port (the dashboard renders it as a plain anchor). | -| `icon` | no | Emoji or short glyph prefix. Defaults to empty string. | - -The list is written to `<state>/hyperhive-dashboard-links.json` by a -one-shot systemd unit at container boot. `hive-c0re` reads the file on -each container-view snapshot and attaches the links to the agent card -(`kind = External`) without any code change. Omitting the option -(default empty) produces no extra links. - -### Custom static files - -```nix -hyperhive.frontend.extraFiles = { - "games/bitburner" = { - source = ./bitburner-dist; # path relative to agent.nix - # target defaults to attribute name: "games/bitburner" - }; - "my-page" = { - source = ./my-page.html; - target = "my-page.html"; # explicit override - }; -}; -``` - -Layers additional files over the default per-agent web UI dist. Each -attribute defines one overlay entry: - -- **`source`** — a Nix path (file or directory) copied into the merged - static tree. Evaluated at nix build time; the resulting derivation is - pointed at by `HIVE_STATIC_DIR`. -- **`target`** — destination path within the merged tree, used as both - the served URL prefix (`/<target>/…`) and the on-disk layout. - Defaults to the attribute name. Forward slashes create nested layouts - (`"games/bitburner"` serves at `/games/bitburner/…`). - -Constraints: `target` must start with an alphanumeric or `_` and -contain only alphanumerics, `_`, `.`, `/`, `-`. `..` segments are -rejected by a config assertion. The merge step refuses to overwrite -files already present in the default dist — pick a target name that -does not collide with existing paths (`static/`, `index.html`, etc.). - -The default dist ships at `hyperhive.frontend.dist` (the -`hyperhive-frontend` package output, read-only). To replace the -entire UI rather than layer on top, override `frontend.dist` directly. - -### Connectivity overrides - -Two `hyperhive.forge.*` / `hyperhive.matrix.*` options override where -the per-agent daemons connect. Both rarely need changing on a standard -single-host deploy, but are useful for multi-hive or custom-network -setups. - -```nix -hyperhive.forge.url = "http://localhost:3000"; # default -hyperhive.matrix.url = "http://localhost:8008"; # default -``` - -**`hyperhive.forge.url`** — base URL of the Forgejo instance. Used by -a one-shot boot unit (`tea-login`) that writes `~/.config/tea/config.yml` -directly from the agent's `forge-token`, so `tea` and `hive-forge` -work without an interactive auth step. The unit is a no-op when -`forge-token` is absent. Override when the agent should connect to a -Forgejo on a different host or port (e.g. a swarm peer's forge). -Validated: must be an `http://` or `https://` URL or the empty string. - -**`hyperhive.matrix.url`** — homeserver URL used by -`hive-matrix-daemon` when connecting via the matrix-sdk. Default -(`localhost:8008`) is overridden by hive-c0re at deploy time to the -gateway-routed `matrix.<domain>` URL so isolated agents can reach the -homeserver. Override per-agent when an agent should talk to a -different homeserver — for example a remote hive's tuwunel reached -over a VPN, or an external Matrix server for a federation-only agent. - -### Claude Code plugins - -The harness installs Claude Code plugins before the serve loop opens. -Two per-agent `agent.nix` options control this: - -```nix -hyperhive.claudeMarketplaces = [ "anthropics/claude-plugins-official" ]; # default -hyperhive.claudePlugins = [ "formatter@my-marketplace" ]; # default: [] -hyperhive.claudePluginsAutoUpdate = false; # default -``` - -- **`claudeMarketplaces`** — list of marketplace sources passed to - `claude plugin marketplace add <source>`. The official Anthropic - marketplace is pre-configured by default; override or extend to add - custom marketplaces. Idempotent — re-adding an existing source is - a no-op. -- **`claudePlugins`** — list of plugin specs passed to - `claude plugin install <spec>`. Empty by default. Each spec is - installed on every boot (`install` is expected to be idempotent); - failures log a warning but do not abort boot. -- **`claudePluginsAutoUpdate`** — when `true`, runs - `claude plugin marketplace update` before installing plugins to pull - the latest index. Disabled by default to keep boot times short and - plugin versions pinned. - -### `cargo.shortMessages` - -```nix -hyperhive.cargo.shortMessages = true; # default -``` - -When enabled (the default), the harness injects a `cargo` shell -function into `/etc/hyperhive/bash-env.sh` that transparently appends -`--message-format short` to compile subcommands (`build`, `check`, -`clippy`, `test`, `run`, `doc`, `bench`, `install`, `rustc`, `fix`). -This suppresses the per-crate progress lines that flood the response -window, leaving only warnings and errors. - -The function handles `+toolchain` selectors (`cargo +nightly build`) -and passes through cleanly when `--message-format` is already present. -Non-compile subcommands (`new`, `add`, third-party `cargo-*`) are -left untouched. - -Set to `false` for agents that parse cargo's JSON output -programmatically and do not pass `--message-format json` themselves. - -## MCP surface - -The harness ships an embedded MCP server (rmcp 1.7). Claude launches -it as a stdio child via `--mcp-config`. The hyperhive socket name is -`hyperhive`, so the tools land in claude as `mcp__hyperhive__<tool>`. - -Tool access is gated by tool groups (`HIVE_TOOL_GROUPS`). The default -preset (`AGENT_DEFAULT`) includes `messaging`, `meta`, `inbox`, and -`execution`. Privileged groups (`lifecycle`, `approvals`, `scheduling`, -`diagnostics`) are opt-in via the P3RM1SS10NS tab. - -### Core tools (always available) - -**Messaging** (`messaging` group): `send(to, body, in_reply_to?)`, -`recv(wait_seconds?, max?)`, `ask(question, options?, multi?, -ttl_seconds?, to?)`, `answer(id, answer)`, `ack_until(up_to)`. - -- `send` — message a peer (logical name) or the operator - (`to: "operator"`). Use `to: "<parent>"` to address the topology - parent without hardcoding the label; the broker resolves the - sentinel at delivery time. Optional `in_reply_to: i64` links the - message to a prior id for thread rendering. Per-agent - `hyperhive.allowedRecipients` (default: empty = unrestricted) limits - which names `send` accepts — useful for sandboxing: set - `[ "operator" ]` to restrict a sub-agent to operator messages only - (the topology parent is always reachable regardless of this list — - that carve-out is structural, keyed on parent relationship, not name). -- `recv` — drain inbox. Without `wait_seconds` (or `0`) returns - immediately. Positive value parks the turn up to that many seconds - (cap 180) — incoming messages wake instantly. `max` (default 1, cap - 5) drains up to N rows; `wait_seconds` applies to the first, then - drains up to `max` total. Each returned row is prefixed with - `[msg #<id>]` (broker row id; note the highest id seen, then pass - it to `ack_until` to bulk-triage the batch). **Graceful shutdown**: when the harness - receives a stop signal, the inbox becomes fenced and `recv` returns an - explicit `from: "graceful-stop"` message instead of an empty inbox. - This unmissably directs the agent to flush durable state (`/state` - files) and end the turn — the container exits when the turn completes. - The graceful-stop turn takes the same post-turn compaction path as any - other turn: if the context crossed the watermark the harness runs a - notes-checkpoint turn and then `/compact`. Compacting before shutdown - keeps a later cold start cheap instead of re-uploading a huge transcript. -- `ask` — surface a structured question to the operator (default) or - a peer agent (`to: "<agent>"`). Non-blocking — returns a question - id; the answer arrives as a `question_answered` system event in the - asker's inbox. `options` is advisory; `multi=true` renders as - checkboxes; `ttl_seconds` auto-cancels with answer `[expired]`. -- `answer` — respond to a `question_asked` event routed to this - agent. Strict authorisation: only the declared target can answer. -- `ack_until(up_to)` — bulk-mark inbox rows handled: every row with - broker id `<= up_to` is stamped as acked in a single UPDATE. - Recipient-scoped (agents can only ack their own rows). Use when a - restart redelivers a large backlog of already-handled messages: read - the highest `[msg #N]` from the set you've actually processed, then - `ack_until(N)` to prevent re-pop. Acked rows never redeliver. - Transient pings (sentinel id 0) have nothing to ack and show no marker. - -**System messages** (from sender `system`): lifecycle and Q&A events -delivered as regular inbox messages (same `recv` path; body is a JSON -object with an `event` discriminant field). The **submitting agent** -(the root agent for top-level containers; an agent with the `approvals` -tool group for its own subtree) receives lifecycle events (`spawned`, -`rebuilt`, `killed`, `destroyed`, `container_crash`, `needs_login`, -`logged_in`, `config_ready`, `needs_update`, `approval_resolved`). Any -agent receives Q&A events when it is the declared target -(`question_asked`) or the asker (`question_answered`). Full payload -shapes and routing logic in -[`docs/approvals.md` § Helper events](../approvals.md#helper-events-to-the-submitting-agent). - -**Inbox** (`inbox` group): `get_loose_ends(agent?)`, -`cancel_loose_end(kind, id)`, `remind(message, delay_seconds? | -at_unix_timestamp?)`, `request_next_turn()`. - -- `get_loose_ends(agent?)` — list pending questions (asked/owed), - scheduled reminders, and active local tasks published by external MCP - daemons (e.g. running bash tasks from `hive-bash-mcp`). Each row - carries an id + kind for `cancel_loose_end`. Omit `agent` to list - your own threads. Pass `agent: "<name>"` to inspect a direct child - agent (always accessible per topology enforcement); non-children - require the `query_agent_state` capability. The `"*"` hive-wide - query is not available on the agent socket. -- `cancel_loose_end` — withdraw a `question` (posts `[cancelled by - <self>]`), hard-delete a `reminder`, or cancel a pending `approval` - row. Agents may only cancel rows they own; the `approval` kind is - further restricted to the root agent (`ruth`) server-side. -- `remind` — schedule a reminder in this agent's own inbox. Large - payloads spill to `/agents/<self>/state/reminders/`. Pending count - capped at 50 per agent (`HIVE_REMIND_MAX_PENDING_PER_AGENT`). -- `request_next_turn` — ask the harness to start another turn - immediately after this one ends, even if the inbox is empty. - Next turn fires with `from: "self"` and `body: "continue"`. - -**Meta** (`meta` group): `set_status(text)`, `get_agent_meta(name?)`. - -- `set_status` — set a free-text status string visible on the - dashboard. Single line, ≤ 200 chars. Persisted to - `{state_dir}/hyperhive-status`. Pass `""` to clear. -- `get_agent_meta` — fetch identity + status metadata for an agent: - `{ name, hyperhive_rev, running, status_text, status_set_at, - hive_name?, swarm_name?, matrix_accounts? }`. `matrix_accounts` is a - list of matrix identities the agent can act as (`name`, `user_id?`, - `homeserver`); omitted for agents with no matrix provisioning. Omit - `name` to query self. - -### Privileged tools (by tool group) - -- **Bash execution** (`execution`) — background shell tasks. See - [`docs/tools/bash.md`](tools/bash.md). -- **Lifecycle + config** (`lifecycle`, `approvals`) — manage child - agents, spawn new ones, apply config commits. See - [`docs/tools/lifecycle.md`](tools/lifecycle.md). -- **Scheduling + diagnostics** (`scheduling`, `diagnostics`) — - scheduled prompts, `get_logs`. See - [`docs/tools/scheduling.md`](tools/scheduling.md). -- **Forge repos** (`forge`) — `create_repo` — the only agent path to - create a repo under the `agents/` org (direct forge token creation is - disabled for agents). The repo is created in the c0re-owned `agents` - org; the calling agent gets write collaborator access; the default - branch is branch-protected (operator-team must approve merges, so the - agent cannot self-merge). Opt-in; not in any default preset. - See [`docs/tools/forge.md — Repo management`](tools/forge.md). -- **Web egress** (`web_tools`) — enables Claude's built-in `WebFetch` - and `WebSearch` tools (not MCP tools; added directly to the - `--allowedTools` list). Off by default; add the group in the - P3RM1SS10NS tab and rebuild to enable. -- **Capability-gated** — `get_host_journal` (requires - `read_host_journal` capability set via the P3RM1SS10NS tab; - orthogonal to tool groups). Full list of capabilities and their - effects in [`docs/conventions.md#capabilities`](../docs/conventions.md). - Also documented in [`docs/tools/scheduling.md`](tools/scheduling.md). -- **Matrix MCP + extra servers** — `mcp__matrix__*` tools and - per-agent extra MCP config. See - [`docs/tools/matrix.md`](tools/matrix.md). - -### Waking the agent from inside the container - -External MCP servers (and any other in-container process) can -inject a wake-up event into the agent's inbox via the per-agent -socket at `/run/hive/mcp.sock`. Two equivalent paths: - -- **Shell out to `hive wake --from <label> --body <text>`** - (use `--body -` to read body from stdin). Already on the - container's `PATH` since the harness binary is in - `systemPackages`. Convenient for shell-script integrations and - co-process daemons (matrix bridge, webhook listeners, scrapers). - -- **Speak the wire protocol directly** — JSON-line over the - unix socket: `{"cmd":"wake","from":"matrix","body":"new dm - from @alice"}\n`. Same shape as any other `AgentRequest`; - see `hive-sh4re::AgentRequest::Wake`. - -The wake event lands in the broker as `{from:<label>, -to:<agent>, body}`, waking whatever `recv` call the harness -is currently blocked on. The next turn fires with the wake -prompt formed from that message. - -Identity = socket: anything that can connect to -`/run/hive/mcp.sock` is implicitly trusted to inject these — -the bind-mount is the agent's own container only. - -### Authoritative state - -`hive_ag3nt::events::Bus` carries the current turn-loop state in -addition to the broadcast channel and the events history. Variants: - -- `Idle` — sitting on `Recv` waiting for mail. -- `Thinking` — `claude --print` is running for a turn. -- `Compacting` — operator-triggered `/compact` is in flight. - -The harness flips state at the relevant transitions -(`set_state(Thinking)` before `drive_turn`, `set_state(Idle)` -after; `set_state(Compacting)` around an idle operator compact in -`turn::run_pending_compact`). Exposed via `/api/state.turn_state` + -`turn_state_since` (unix seconds); the agent page renders this rather -than deriving from SSE events. - -### Tool envelope - -`mcp::run_tool_envelope`: every MCP tool handler logs the request, -runs the body, logs the result. Pre-/post-log only — the inbox -status hint moved to the wake prompt + UI header. - -### Tool whitelist (`mcp_config::ALLOWED_BUILTIN_TOOLS`) - -- Allowed built-ins: `Edit`, `Glob`, `Grep`, `Read`, `Write`. -- Tool-group-gated built-ins: `WebFetch`, `WebSearch` (added when the - `web_tools` tool group is enabled — see P3RM1SS10NS tab). -- Denied by omission or the managed-settings deny list - (`/etc/claude-code/managed-settings.json`): `Bash`, `Task`, - `NotebookEdit`, `TodoWrite`. -- Allowed MCP tools: as listed above (by tool group). - -`Bash` is disallowed — shell execution goes through -`mcp__bash__run` (background tasks with structured output + -task-id tracking) instead of an interactive shell. The bash MCP server -(`run` / `status` / `kill`) uses `allowedTools = ["*"]` so all -`mcp__bash__*` tools are always available regardless of tool groups. - -`WebFetch` / `WebSearch` are off by default; enable the `web_tools` -tool group in the P3RM1SS10NS tab and rebuild the agent to enable them. - +## Sub-pages + +The rest lives in three topic pages under [`turn-loop/`](turn-loop/): + +- **[claude-invocation.md](turn-loop/claude-invocation.md)** — how the harness + spawns `claude --print` each turn, the two-pronged compaction (reactive + + proactive), and the on-boot files it materialises (`--mcp-config`, + `--system-prompt-file`). +- **[config.md](turn-loop/config.md)** — the optional per-agent knobs the meta + flake wires in (reference docs, icon, passwordless sudo, dashboard links, + custom static files, connectivity overrides, claude plugins, cargo message + filtering). +- **[mcp.md](turn-loop/mcp.md)** — the MCP tool surface claude sees: core tools, + privileged tool groups, self-wake, authoritative state, the tool envelope, + and the built-in tool whitelist. + +Per-subsystem impl detail lives in each module's `//!` doc-comment; these pages +describe present-state behaviour + wiring, not line-level mechanics. diff --git a/docs/turn-loop/claude-invocation.md b/docs/turn-loop/claude-invocation.md new file mode 100644 index 00000000..e44d050e --- /dev/null +++ b/docs/turn-loop/claude-invocation.md @@ -0,0 +1,241 @@ +# The claude invocation + +``` +claude --print --verbose --output-format stream-json --model <name> \ + --effort <level> --resume <title> # or --name <title> on first use \ + --system-prompt-file /run/hive/claude-system-prompt.md \ + --mcp-config /run/hive/claude-mcp-config.json --strict-mcp-config \ + --tools <builtins> --allowedTools <builtins+mcp> +# wake prompt piped over stdin +``` + +**Crate split.** The generic subprocess mechanics — spawning +`claude --print`, streaming + classifying stream-json, session +lookup/archive, and the durable-session compaction loop — live in the +reusable **`hive-claude`** crate (`hive_claude::{Claude, InfiniteSession, +Attach, CompactionPolicy, PercentPolicy, Telemetry, Sink, SessionStore}`; +see `hive-claude/README.md`). `hive_ag3nt::turn` is the hyperhive **policy +layer** on top: it builds the per-turn config from the bus, bridges the +output stream onto the event bus (`BusSink`), and owns the compaction / +auto-reset / retry decisions in `drive_turn`. The lib returns everything it +parsed from a turn (usage, cost, context window, resolved model) as +`Telemetry`, which the policy layer applies to the bus. + +Hive-enforced settings ship at `/etc/claude-code/managed-settings.json` +(claude-code's canonical managed-settings path — precedence #1, +read-only, un-overridable), wired in `nix/templates/harness-base.nix` +from the `prompts/claude-settings.json` asset. `effortLevel` is +deliberately not in that file — effort is controlled live via the +`--effort` flag (`HIVE_DEFAULT_EFFORT` / the per-agent UI slider), which +managed scope would otherwise lock. + +`<name>` is read from `Bus::model()` on each turn. The initial +default is set by `hyperhive.model` in the agent's `agent.nix` +(NixOS option; propagates via `HIVE_DEFAULT_MODEL` env var; falls +back to `"haiku"` if unset). The operator can flip it at runtime +with `/model <name>` in the web terminal — the next turn picks it +up. The choice is persisted to `/harness/hyperhive-model` so it +survives restart; override path: `HYPERHIVE_MODEL_FILE` env var +for tests. + +Context-window size is looked up per-model via +`events::context_window_tokens(model)`. Resolution order (first +match wins): + +1. `HIVE_CONTEXT_WINDOW_TOKENS_<KEY>` env var, where `KEY` + (lowercased) is a substring of the active model name. Injected + by the meta flake from `services.hyperhive.c0re.contextWindowTokens` + (host-level NixOS option, defaults: haiku=200k, sonnet=1M, + opus=1M). Override these for all agents at once without a + per-agent config change. +2. `HIVE_CONTEXT_WINDOW_TOKENS` — single global override for any + model (useful in dev / test). +3. Hard fallback: `200_000` (conservative; only reached outside + NixOS where the env vars aren't set). + +The effective window drives watermarks and is exposed at runtime +via `/api/state.context_window_tokens` so the UI can show a +percentage-of-window ctx badge. + +**Session identity — a constant title.** Every turn keys on one fixed, +harness-owned session title (`turn::session_title()`, default +`hive-session`, override `HIVE_SESSION_TITLE`). The durable +`hive_claude::InfiniteSession` (built once by the serve loop via +`turn::make_session`, then reused) `--resume <title>`s it; the *first* use +(bootstrap, post-archive, post-purge) misses and the session re-runs the +same prompt once with `--name <title>` to mint it. That single self-heal +rule is the whole identity system — there is **no** scraped session-id +file. Because the +title is constant, `/compact` and its post-compact retry provably target +the same session (killing the old "compact ran on a different/empty +session" bug), and a `choom` invocation in the same cwd can't hijack the +context (it won't carry our title). claude stores sessions in +`~/.claude/projects/<cwd-slug>/<uuid>.jsonl` (bind-mounted persistently); +`--name` writes the title into the file as a `custom-title` event, which +is what `--resume <title>` resolves against. We never pass bare +`--continue` (it resumes the *latest* session in the cwd — the hijack +vector). Auto-compact and auto-memory are disabled via the managed +settings at `/etc/claude-code/managed-settings.json` because hyperhive +owns compaction — see [Compaction](#compaction) below. + +**Session reset** is available via `POST /api/new-session` (or +`/new-session` slash command). It does *not* touch the session inline — +that would race a mid-write claude process. Instead `Bus::request_session_reset()` +sets a one-shot flag consumed at the next turn boundary by `drive_turn`, +which **archives** the current session: the backing `<uuid>.jsonl` is +renamed to `<uuid>.jsonl.archived` (dropped out of claude's `*.jsonl` +resolution glob, history preserved on disk, only the file carrying *our* +title — any `choom` session sharing the cwd is left alone). The next +turn's `--resume <title>` then misses and self-heals into a fresh session. + +## Compaction + +claude's own in-session auto-compact is off (via the managed settings +at `/etc/claude-code/managed-settings.json`); hyperhive owns it. The +`hive_claude::InfiniteSession` keeps the session alive across the context +window with two triggers baked into its `run`: + +- **Reactive** — claude-code prints `Prompt is too long`. The session is + *already* past the window, so no turn can run on it — the session + `/compact`s straight away and retries the same wake-up prompt once. No + notes-checkpoint turn is possible here: the detail is gone. If the retry + *still* overflows, `run` surfaces `Error::PromptTooLong`; `drive_turn` + then archives the session (session lifecycle stays hive-side) and the + serve loop requeues the message so it redelivers into a fresh session + (see [Turn outcomes](../turn-loop.md#turn-outcomes) — the wake prompt itself is tiny, so + the overflow was the accumulated context the archive clears). +- **Proactive** — a turn finishes cleanly but the last inference's context + size crossed the policy watermark. While the session is still healthy it + runs one synthetic *notes-checkpoint* turn (`CHECKPOINT_PROMPT` — + "context is filling up, flush durable state into `/state` now") and + *then* `/compact`s, so the agent can persist in-flight state before the + detail collapses into a summary. + +The **when** is a `hive_claude::CompactionPolicy` injected by the harness: +`turn::make_session` builds a `PercentPolicy` that compacts once the +model-reported context fill reaches `HIVE_COMPACT_WATERMARK_PERCENT` +(default **75%**), falling back to `events::context_window_tokens(model)` +for the window on turns the model didn't report one. `0` disables proactive +compaction (the reactive path always applies). The proactive path is +best-effort — a failed checkpoint or `/compact` never fails the turn that +already succeeded. + +The operator can force a compaction any time via `POST /api/compact`. It's +**deferred**: the handler sets `Bus::request_compact()` and returns +immediately; the harness runs the `/compact` at the next turn boundary +(end of the in-flight turn in `drive_turn`, or — when the agent is idle — +in `turn::run_pending_compact` on the serve loop's next empty poll). This +lets `/api/compact` work mid-turn instead of only when idle, without racing +a live claude process. + +To disable proactive compaction for a specific agent, use the nix option: + +```nix +hyperhive.autoCompact = false; # default true +``` + +Setting `autoCompact = false` sets `HIVE_COMPACT_WATERMARK_TOKENS=0`, which +the percent resolver still honours as a disable. Useful for large-context +models (sonnet/opus) where the 75% heuristic fires before the session is +actually full — the reactive path (compact-on-overflow at the hard limit) +still applies. + +- **Auto session-reset** — a third path (`turn::maybe_auto_reset`, + pre-turn) that fires when both conditions hold: context is ≥ a watermark + (`HIVE_AUTO_RESET_WATERMARK_TOKENS`, default **50% of + `context_window_tokens(model)`**) AND the time since the last turn + exceeds the assumed prompt-cache TTL (`HIVE_CACHE_TTL_SECS`, default + `3600`). Claude's prompt cache goes cold after a while; once it's cold, + `--resume`-ing a large session pays the full re-upload cost with no + benefit over starting fresh. So `drive_turn` **archives** the current + session (same mechanism as the operator reset — rename `<uuid>.jsonl` → + `.archived`) so the next turn's `--resume <title>` misses and starts + fresh. Unlike proactive compaction the session is dropped entirely, not + compacted — and *no* preceding checkpoint turn runs, because any turn + before the reset would just re-warm the cache and defeat the purpose. + Set `HIVE_AUTO_RESET_WATERMARK_TOKENS=0` to disable. Auto-reset and the + operator reset are mutually exclusive per turn (both archive → fresh + turn), so an explicit operator reset short-circuits the heuristic. + +The child runs with `cwd = /state` (when the bind exists; falls +back to the parent's cwd in dev), so any relative path in a tool +call (`Read foo.md`, `Bash ls`, `Write notes.md`) lands in the +agent's durable bind-mounted dir. CLAUDE.md auto-load walks +upward from `/state` — drop a per-agent CLAUDE.md there if you +want long-term hints that survive destroy/recreate. + +The wake prompt is intentionally minimal: the popped message's +`from`/`body`, prefixed with a `[msg #<id>]` broker-row-id marker +(so the agent knows what id to pass to `ack_until` for bulk triage; +transient pings show no marker because their sentinel id 0 has nothing +to ack), plus an inline `({unread} more pending — drain via …)` hint +when `unread > 0`. Claude drives any further `recv`/`send` itself via +the embedded MCP server. + +Whenever hive-c0re starts / restarts / rebuilds a container, it +also drops a `system` message into the agent's inbox via +`Coordinator::kick_agent` — a one-line "you were just (re)started, +check /state/ for your notes, your session is intact". The +next turn picks it up like any other inbox message. + +## On-boot files + +`hive_ag3nt::turn::write_*` writes two files next to the per-agent +socket at `/run/hive/` once at startup: + +- `claude-mcp-config.json` — by default re-invokes the running binary + as `mcp` stdio child (so the same binary serves as harness + MCP + server per turn). When `hyperhive.mcp.httpPort` is set in the + agent's NixOS config, the config instead points claude at the + persistent `hive-mcp-http` daemon (`http://127.0.0.1:{port}/mcp`) + — no stdio child per turn; trades the per-turn re-registration race + for a hard dependency on the daemon's uptime (`Restart=always`). +- `claude-system-prompt.md` — rendered from + `hive-ag3nt/prompts/system.md` by `hive_ag3nt::prompt::render`: + HTML-comment markers (`<!-- role:agent -->...<!-- /role:agent -->`, + same for `role:manager`) gate the role-specific blocks; everything + else is shared. Five placeholders are then + substituted: `{label}` (short agent name), `{qualified_label}` + (hive-qualified `name@domain` form), `{operator_pronouns}`, + `{hive_identity}` (e.g. `` on hive `pr1ma` ``; empty when + `hyperhive.hiveName` is unset), and `{swarm_identity}` (same + shape for the swarm). Pronouns come from `HIVE_OPERATOR_PRONOUNS` + env (set by the meta flake from + `services.hyperhive.c0re.operatorPronouns`, default `she/her`). + When `hyperhive.docs.enable` is set, `HIVE_DOCS_DIR` is present + in the environment and `render()` appends a one-sentence pointer + telling the agent the docs are mounted at that path (in lieu of + the old CLAUDE.md-in-docs-dir approach, which was dropped in + favour of this direct injection). + Passed via `--system-prompt-file`. + + **Marker grammar.** `<!-- role:X -->` opens a block; any + `<!-- /role:X -->` closes the current block. The renderer always uses + role `agent`, so blocks with other role tags are elided. Nesting is NOT + supported — a stray opener with no closer runs until end of file. + Whitespace inside markers is tolerated (`<!--role:foo-->` parses the + same as `<!-- role:foo -->`). Content outside any marker is always + included. Today's `system.md` carries no markers (single agent role) — + the grammar stays wired for a future manager / multi-role prompt. + + **`hive_identity` / `swarm_identity` shape.** Each carries a + leading space + backticked name (` on hive \`pr1ma\``, + ` in swarm \`constellat1on\``) when the corresponding env var + is set, otherwise empty string. The independence lets the + template drop one or both into the opener prose without + breaking single-hive deployments that never set the option; + the renderer also treats `Some("")` from a caller as `None` so + empty-string env vars and missing env vars round-trip the + same way. + +The per-turn plumbing lives in `hive_ag3nt::turn`: `write_mcp_config` / +`write_system_prompt` (on-boot files), `make_session` (builds the durable +`InfiniteSession`, once), `drive_turn` (the policy state machine — +reset/auto-reset, the turn, 401-retry, deferred-compact-at-turn-end), +`run_pending_compact` (idle operator compact), `BusSink` (stream → bus + +`Telemetry` applied via `apply_telemetry`), `emit_turn_end`, `session_title` +/ `session_store` / `archive_session` (identity + turn-boundary reset). The +actual claude spawn, stream classification, and the reactive/proactive +compaction loop are in the `hive-claude` crate. Login-wait +(`wait_for_login`) lives in `hive_ag3nt::login`. + diff --git a/docs/turn-loop/config.md b/docs/turn-loop/config.md new file mode 100644 index 00000000..ad74b988 --- /dev/null +++ b/docs/turn-loop/config.md @@ -0,0 +1,199 @@ +# Agent config knobs + +Optional per-agent knobs the meta flake wires into the container from +`services.hyperhive.agents.<name>`, read at boot or per turn by the harness. +Absent means the default. (The claude spawn + compaction themselves live in +[claude-invocation](claude-invocation.md).) + +## Reference docs (`hyperhive.docs.enable`) + +```nix +hyperhive.docs.enable = true; # default: false (true for the manager agent) +``` + +Makes the hyperhive `docs/` tree available inside the container at a nix +store path read from `$HIVE_DOCS_DIR`, and injects a single pointer +sentence into the agent's system prompt so it knows the docs exist and +where to find them. The tree is served by `claude --add-dir` so the full +markdown is readable during every turn. + +Enabled by default only for the root/manager agent (`manager.nix`). Any +agent can opt in by adding the line above to its `agent.nix`. + +The `docs/` source is a narrow flake input (`hyperhive-docs`) tracked +separately from the main `hyperhive` flake so editing docs re-locks only +that input — not every agent's container gets rebuilt on a doc-only change. + +## Agent icon + +```nix +hyperhive.icon = ./icon.svg; # default: null (falls back to shared hyperhive logo) +``` + +Path to an SVG file used as this agent's visual identity — shown in +the per-agent page header, as the page favicon, and uploaded to the +agent's Forgejo profile avatar (via the `forge-avatar-sync` boot +unit) and Matrix profile avatar (set by `hive-matrix-daemon` over its +live Client). Commit the SVG next to `agent.nix` in the config repo +and reference it as a relative path. + +When `null` (the default), the agent falls back to the shared +hyperhive branding mark. The harness serves whichever icon is active +at `GET /icon` on the per-agent web port. + +## `user.passwordlessSudo` + +```nix +hyperhive.user.passwordlessSudo = true; # default +``` + +Grants the per-agent unix user passwordless `sudo` (`NOPASSWD: ALL`). +Enabled by default so claude's shell tools work for operations that +need root inside the container (`systemctl`, package managers in dev +shells, etc.) — the same privilege surface the previous root-user shape +had, now elevated explicitly rather than implicitly. + +Set to `false` for agents that should be strictly unprivileged. +Any tool invocation that needs root then fails loudly with the standard +sudo rejection rather than silently succeeding — easier to audit. + +`hyperhive.user.uid`, `hyperhive.user.gid`, and +`hyperhive.user.name` are the companion options; see +`docs/agent-hierarchy.md` — "Harness systemd unit shape" for the full +`user.*` surface. + +## Dashboard links + +```nix +hyperhive.dashboardLinks = [ + { label = "Stats"; icon = "📊"; url = "http://localhost:9001/stats"; } + { label = "Scratchpad"; url = "http://localhost:8080"; } +]; +``` + +Declares extra navigation links that appear on the agent's dashboard +card and in the per-agent page header alongside the built-in forge / +config / container links. Each entry has: + +| Field | Required | Description | +|-------|----------|-------------| +| `label` | yes | Display text shown in the icon strip tooltip and meta-nav. | +| `url` | yes | Absolute URL — may include a different port (the dashboard renders it as a plain anchor). | +| `icon` | no | Emoji or short glyph prefix. Defaults to empty string. | + +The list is written to `<state>/hyperhive-dashboard-links.json` by a +one-shot systemd unit at container boot. `hive-c0re` reads the file on +each container-view snapshot and attaches the links to the agent card +(`kind = External`) without any code change. Omitting the option +(default empty) produces no extra links. + +## Custom static files + +```nix +hyperhive.frontend.extraFiles = { + "games/bitburner" = { + source = ./bitburner-dist; # path relative to agent.nix + # target defaults to attribute name: "games/bitburner" + }; + "my-page" = { + source = ./my-page.html; + target = "my-page.html"; # explicit override + }; +}; +``` + +Layers additional files over the default per-agent web UI dist. Each +attribute defines one overlay entry: + +- **`source`** — a Nix path (file or directory) copied into the merged + static tree. Evaluated at nix build time; the resulting derivation is + pointed at by `HIVE_STATIC_DIR`. +- **`target`** — destination path within the merged tree, used as both + the served URL prefix (`/<target>/…`) and the on-disk layout. + Defaults to the attribute name. Forward slashes create nested layouts + (`"games/bitburner"` serves at `/games/bitburner/…`). + +Constraints: `target` must start with an alphanumeric or `_` and +contain only alphanumerics, `_`, `.`, `/`, `-`. `..` segments are +rejected by a config assertion. The merge step refuses to overwrite +files already present in the default dist — pick a target name that +does not collide with existing paths (`static/`, `index.html`, etc.). + +The default dist ships at `hyperhive.frontend.dist` (the +`hyperhive-frontend` package output, read-only). To replace the +entire UI rather than layer on top, override `frontend.dist` directly. + +## Connectivity overrides + +Two `hyperhive.forge.*` / `hyperhive.matrix.*` options override where +the per-agent daemons connect. Both rarely need changing on a standard +single-host deploy, but are useful for multi-hive or custom-network +setups. + +```nix +hyperhive.forge.url = "http://localhost:3000"; # default +hyperhive.matrix.url = "http://localhost:8008"; # default +``` + +**`hyperhive.forge.url`** — base URL of the Forgejo instance. Used by +a one-shot boot unit (`tea-login`) that writes `~/.config/tea/config.yml` +directly from the agent's `forge-token`, so `tea` and `hive-forge` +work without an interactive auth step. The unit is a no-op when +`forge-token` is absent. Override when the agent should connect to a +Forgejo on a different host or port (e.g. a swarm peer's forge). +Validated: must be an `http://` or `https://` URL or the empty string. + +**`hyperhive.matrix.url`** — homeserver URL used by +`hive-matrix-daemon` when connecting via the matrix-sdk. Default +(`localhost:8008`) is overridden by hive-c0re at deploy time to the +gateway-routed `matrix.<domain>` URL so isolated agents can reach the +homeserver. Override per-agent when an agent should talk to a +different homeserver — for example a remote hive's tuwunel reached +over a VPN, or an external Matrix server for a federation-only agent. + +## Claude Code plugins + +The harness installs Claude Code plugins before the serve loop opens. +Two per-agent `agent.nix` options control this: + +```nix +hyperhive.claudeMarketplaces = [ "anthropics/claude-plugins-official" ]; # default +hyperhive.claudePlugins = [ "formatter@my-marketplace" ]; # default: [] +hyperhive.claudePluginsAutoUpdate = false; # default +``` + +- **`claudeMarketplaces`** — list of marketplace sources passed to + `claude plugin marketplace add <source>`. The official Anthropic + marketplace is pre-configured by default; override or extend to add + custom marketplaces. Idempotent — re-adding an existing source is + a no-op. +- **`claudePlugins`** — list of plugin specs passed to + `claude plugin install <spec>`. Empty by default. Each spec is + installed on every boot (`install` is expected to be idempotent); + failures log a warning but do not abort boot. +- **`claudePluginsAutoUpdate`** — when `true`, runs + `claude plugin marketplace update` before installing plugins to pull + the latest index. Disabled by default to keep boot times short and + plugin versions pinned. + +## `cargo.shortMessages` + +```nix +hyperhive.cargo.shortMessages = true; # default +``` + +When enabled (the default), the harness injects a `cargo` shell +function into `/etc/hyperhive/bash-env.sh` that transparently appends +`--message-format short` to compile subcommands (`build`, `check`, +`clippy`, `test`, `run`, `doc`, `bench`, `install`, `rustc`, `fix`). +This suppresses the per-crate progress lines that flood the response +window, leaving only warnings and errors. + +The function handles `+toolchain` selectors (`cargo +nightly build`) +and passes through cleanly when `--message-format` is already present. +Non-compile subcommands (`new`, `add`, third-party `cargo-*`) are +left untouched. + +Set to `false` for agents that parse cargo's JSON output +programmatically and do not pass `--message-format json` themselves. + diff --git a/docs/turn-loop/mcp.md b/docs/turn-loop/mcp.md new file mode 100644 index 00000000..02d707ed --- /dev/null +++ b/docs/turn-loop/mcp.md @@ -0,0 +1,201 @@ +# MCP surface + +The harness ships an embedded MCP server (rmcp 1.7). Claude launches +it as a stdio child via `--mcp-config`. The hyperhive socket name is +`hyperhive`, so the tools land in claude as `mcp__hyperhive__<tool>`. + +Tool access is gated by tool groups (`HIVE_TOOL_GROUPS`). The default +preset (`AGENT_DEFAULT`) includes `messaging`, `meta`, `inbox`, and +`execution`. Privileged groups (`lifecycle`, `approvals`, `scheduling`, +`diagnostics`) are opt-in via the P3RM1SS10NS tab. + +## Core tools (always available) + +**Messaging** (`messaging` group): `send(to, body, in_reply_to?)`, +`recv(wait_seconds?, max?)`, `ask(question, options?, multi?, +ttl_seconds?, to?)`, `answer(id, answer)`, `ack_until(up_to)`. + +- `send` — message a peer (logical name) or the operator + (`to: "operator"`). Use `to: "<parent>"` to address the topology + parent without hardcoding the label; the broker resolves the + sentinel at delivery time. Optional `in_reply_to: i64` links the + message to a prior id for thread rendering. Per-agent + `hyperhive.allowedRecipients` (default: empty = unrestricted) limits + which names `send` accepts — useful for sandboxing: set + `[ "operator" ]` to restrict a sub-agent to operator messages only + (the topology parent is always reachable regardless of this list — + that carve-out is structural, keyed on parent relationship, not name). +- `recv` — drain inbox. Without `wait_seconds` (or `0`) returns + immediately. Positive value parks the turn up to that many seconds + (cap 180) — incoming messages wake instantly. `max` (default 1, cap + 5) drains up to N rows; `wait_seconds` applies to the first, then + drains up to `max` total. Each returned row is prefixed with + `[msg #<id>]` (broker row id; note the highest id seen, then pass + it to `ack_until` to bulk-triage the batch). **Graceful shutdown**: when the harness + receives a stop signal, the inbox becomes fenced and `recv` returns an + explicit `from: "graceful-stop"` message instead of an empty inbox. + This unmissably directs the agent to flush durable state (`/state` + files) and end the turn — the container exits when the turn completes. + The graceful-stop turn takes the same post-turn compaction path as any + other turn: if the context crossed the watermark the harness runs a + notes-checkpoint turn and then `/compact`. Compacting before shutdown + keeps a later cold start cheap instead of re-uploading a huge transcript. +- `ask` — surface a structured question to the operator (default) or + a peer agent (`to: "<agent>"`). Non-blocking — returns a question + id; the answer arrives as a `question_answered` system event in the + asker's inbox. `options` is advisory; `multi=true` renders as + checkboxes; `ttl_seconds` auto-cancels with answer `[expired]`. +- `answer` — respond to a `question_asked` event routed to this + agent. Strict authorisation: only the declared target can answer. +- `ack_until(up_to)` — bulk-mark inbox rows handled: every row with + broker id `<= up_to` is stamped as acked in a single UPDATE. + Recipient-scoped (agents can only ack their own rows). Use when a + restart redelivers a large backlog of already-handled messages: read + the highest `[msg #N]` from the set you've actually processed, then + `ack_until(N)` to prevent re-pop. Acked rows never redeliver. + Transient pings (sentinel id 0) have nothing to ack and show no marker. + +**System messages** (from sender `system`): lifecycle and Q&A events +delivered as regular inbox messages (same `recv` path; body is a JSON +object with an `event` discriminant field). The **submitting agent** +(the root agent for top-level containers; an agent with the `approvals` +tool group for its own subtree) receives lifecycle events (`spawned`, +`rebuilt`, `killed`, `destroyed`, `container_crash`, `needs_login`, +`logged_in`, `config_ready`, `needs_update`, `approval_resolved`). Any +agent receives Q&A events when it is the declared target +(`question_asked`) or the asker (`question_answered`). Full payload +shapes and routing logic in +[`docs/approvals.md` § Helper events](../approvals.md#helper-events-to-the-submitting-agent). + +**Inbox** (`inbox` group): `get_loose_ends(agent?)`, +`cancel_loose_end(kind, id)`, `remind(message, delay_seconds? | +at_unix_timestamp?)`, `request_next_turn()`. + +- `get_loose_ends(agent?)` — list pending questions (asked/owed), + scheduled reminders, and active local tasks published by external MCP + daemons (e.g. running bash tasks from `hive-bash-mcp`). Each row + carries an id + kind for `cancel_loose_end`. Omit `agent` to list + your own threads. Pass `agent: "<name>"` to inspect a direct child + agent (always accessible per topology enforcement); non-children + require the `query_agent_state` capability. The `"*"` hive-wide + query is not available on the agent socket. +- `cancel_loose_end` — withdraw a `question` (posts `[cancelled by + <self>]`), hard-delete a `reminder`, or cancel a pending `approval` + row. Agents may only cancel rows they own; the `approval` kind is + further restricted to the root agent (`ruth`) server-side. +- `remind` — schedule a reminder in this agent's own inbox. Large + payloads spill to `/agents/<self>/state/reminders/`. Pending count + capped at 50 per agent (`HIVE_REMIND_MAX_PENDING_PER_AGENT`). +- `request_next_turn` — ask the harness to start another turn + immediately after this one ends, even if the inbox is empty. + Next turn fires with `from: "self"` and `body: "continue"`. + +**Meta** (`meta` group): `set_status(text)`, `get_agent_meta(name?)`. + +- `set_status` — set a free-text status string visible on the + dashboard. Single line, ≤ 200 chars. Persisted to + `{state_dir}/hyperhive-status`. Pass `""` to clear. +- `get_agent_meta` — fetch identity + status metadata for an agent: + `{ name, hyperhive_rev, running, status_text, status_set_at, + hive_name?, swarm_name?, matrix_accounts? }`. `matrix_accounts` is a + list of matrix identities the agent can act as (`name`, `user_id?`, + `homeserver`); omitted for agents with no matrix provisioning. Omit + `name` to query self. + +## Privileged tools (by tool group) + +- **Bash execution** (`execution`) — background shell tasks. See + [`docs/tools/bash.md`](../tools/bash.md). +- **Lifecycle + config** (`lifecycle`, `approvals`) — manage child + agents, spawn new ones, apply config commits. See + [`docs/tools/lifecycle.md`](../tools/lifecycle.md). +- **Scheduling + diagnostics** (`scheduling`, `diagnostics`) — + scheduled prompts, `get_logs`. See + [`docs/tools/scheduling.md`](../tools/scheduling.md). +- **Forge repos** (`forge`) — `create_repo` — the only agent path to + create a repo under the `agents/` org (direct forge token creation is + disabled for agents). The repo is created in the c0re-owned `agents` + org; the calling agent gets write collaborator access; the default + branch is branch-protected (operator-team must approve merges, so the + agent cannot self-merge). Opt-in; not in any default preset. + See [`docs/tools/forge.md — Repo management`](../tools/forge.md). +- **Web egress** (`web_tools`) — enables Claude's built-in `WebFetch` + and `WebSearch` tools (not MCP tools; added directly to the + `--allowedTools` list). Off by default; add the group in the + P3RM1SS10NS tab and rebuild to enable. +- **Capability-gated** — `get_host_journal` (requires + `read_host_journal` capability set via the P3RM1SS10NS tab; + orthogonal to tool groups). Full list of capabilities and their + effects in [`docs/conventions.md#capabilities`](../conventions.md). + Also documented in [`docs/tools/scheduling.md`](../tools/scheduling.md). +- **Matrix MCP + extra servers** — `mcp__matrix__*` tools and + per-agent extra MCP config. See + [`docs/tools/matrix.md`](../tools/matrix.md). + +## Waking the agent from inside the container + +External MCP servers (and any other in-container process) can +inject a wake-up event into the agent's inbox via the per-agent +socket at `/run/hive/mcp.sock`. Two equivalent paths: + +- **Shell out to `hive wake --from <label> --body <text>`** + (use `--body -` to read body from stdin). Already on the + container's `PATH` since the harness binary is in + `systemPackages`. Convenient for shell-script integrations and + co-process daemons (matrix bridge, webhook listeners, scrapers). + +- **Speak the wire protocol directly** — JSON-line over the + unix socket: `{"cmd":"wake","from":"matrix","body":"new dm + from @alice"}\n`. Same shape as any other `AgentRequest`; + see `hive-sh4re::AgentRequest::Wake`. + +The wake event lands in the broker as `{from:<label>, +to:<agent>, body}`, waking whatever `recv` call the harness +is currently blocked on. The next turn fires with the wake +prompt formed from that message. + +Identity = socket: anything that can connect to +`/run/hive/mcp.sock` is implicitly trusted to inject these — +the bind-mount is the agent's own container only. + +## Authoritative state + +`hive_ag3nt::events::Bus` carries the current turn-loop state in +addition to the broadcast channel and the events history. Variants: + +- `Idle` — sitting on `Recv` waiting for mail. +- `Thinking` — `claude --print` is running for a turn. +- `Compacting` — operator-triggered `/compact` is in flight. + +The harness flips state at the relevant transitions +(`set_state(Thinking)` before `drive_turn`, `set_state(Idle)` +after; `set_state(Compacting)` around an idle operator compact in +`turn::run_pending_compact`). Exposed via `/api/state.turn_state` + +`turn_state_since` (unix seconds); the agent page renders this rather +than deriving from SSE events. + +## Tool envelope + +`mcp::run_tool_envelope`: every MCP tool handler logs the request, +runs the body, logs the result. Pre-/post-log only — the inbox +status hint moved to the wake prompt + UI header. + +## Tool whitelist (`mcp_config::ALLOWED_BUILTIN_TOOLS`) + +- Allowed built-ins: `Edit`, `Glob`, `Grep`, `Read`, `Write`. +- Tool-group-gated built-ins: `WebFetch`, `WebSearch` (added when the + `web_tools` tool group is enabled — see P3RM1SS10NS tab). +- Denied by omission or the managed-settings deny list + (`/etc/claude-code/managed-settings.json`): `Bash`, `Task`, + `NotebookEdit`, `TodoWrite`. +- Allowed MCP tools: as listed above (by tool group). + +`Bash` is disallowed — shell execution goes through +`mcp__bash__run` (background tasks with structured output + +task-id tracking) instead of an interactive shell. The bash MCP server +(`run` / `status` / `kill`) uses `allowedTools = ["*"]` so all +`mcp__bash__*` tools are always available regardless of tool groups. + +`WebFetch` / `WebSearch` are off by default; enable the `web_tools` +tool group in the P3RM1SS10NS tab and rebuild the agent to enable them. + diff --git a/hive-ag3nt/src/mcp.rs b/hive-ag3nt/src/mcp.rs index 35b2252e..2f7fd95b 100644 --- a/hive-ag3nt/src/mcp.rs +++ b/hive-ag3nt/src/mcp.rs @@ -380,7 +380,7 @@ fn loose_end_kind_label(kind: hive_sh4re::CancelLooseEndKind) -> &'static str { /// (it would be a stale snapshot from before the stop) so the status /// line is implicitly `<none>` in that case — but the explicit /// `running: no` line tells the caller WHY. See -/// `docs/turn-loop.md::Sub-agent tools` (`get_agent_meta`). +/// `docs/turn-loop/mcp.md::Core tools` (`get_agent_meta`). #[must_use] pub fn format_agent_meta(resp: Result<hive_sh4re::Response, anyhow::Error>) -> String { match resp { diff --git a/hive-ag3nt/src/prompt.rs b/hive-ag3nt/src/prompt.rs index 5c2e8e01..4e37ce05 100644 --- a/hive-ag3nt/src/prompt.rs +++ b/hive-ag3nt/src/prompt.rs @@ -2,7 +2,7 @@ //! HTML-comment markers gating role-specific blocks; this module //! assembles the final prompt (always "agent" role — there is only one //! role). Marker grammar + placeholder substitution rules in -//! `docs/turn-loop.md::On-boot files` (`claude-system-prompt.md`). +//! `docs/turn-loop/claude-invocation.md::On-boot files` (`claude-system-prompt.md`). use std::path::{Path, PathBuf}; @@ -16,7 +16,7 @@ use anyhow::{Context, Result}; /// [`hive_sh4re::assets::prompt_template`] /// (`$HIVE_ASSETS_DIR/prompts/system.md`). Substitution placeholders + /// marker grammar documented in -/// `docs/turn-loop.md::On-boot files` (`claude-system-prompt.md`). +/// `docs/turn-loop/claude-invocation.md::On-boot files` (`claude-system-prompt.md`). #[must_use] pub fn render( template: &str,