Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/agent-modules/agent-service.nix
atlas fa56931a99 hive-runtime: respawn the ACP agent when its provider key changed after a failed turn
The provider key was resolved once, at spawn. A provider rejecting it
shows only as an empty end_turn or an idle timeout, neither of which
drops the agent, so a key rotated in the store never reached a running
opencode until the process died for another reason.

After a turn fails with EmptyEndTurn or IdleTimeout, the runtime now
resolves the key once more. When that yields a key that differs from
the one the agent was spawned with, the agent is dropped (SIGINT to its
process group, as every other respawn) and the next turn spawns it with
the new key and reloads the recorded session. No key read (inherited
env value, no store, a failed or timed-out read, nothing stored) keeps
the running agent. The failed turn's error is returned unchanged.

Part of #5017 (slice A).
2026-10-13 08:07:38 +02:00

564 lines
24 KiB
Nix

# The hive-agent harness service itself, plus the per-agent knobs it
# reads from its environment: model selection, effort level,
# compaction watermark, and the extra reverse-proxies of the per-agent
# web UI.
{
pkgs,
lib,
config,
...
}:
let
userName = config.services.hyperhive.agent.user.name;
homeDir = "/home/${userName}";
acp = config.services.hyperhive.agent.acp;
isAcp = config.services.hyperhive.agent.runtime == "acp";
preset = if acp.preset == null then null else acp.presets.${acp.preset} or null;
oc = acp.opencode;
# Tools opencode must not offer, mirroring claude's built-in allow-list
# (hive_sh4re::permissions): no built-in shell (shell is `mcp__bash__run`),
# no nested agents, no in-session todo list, no web search. `webfetch`
# asks, and the harness answers per the `web_tools` tool group. Passed as
# OPENCODE_PERMISSION because opencode merges that over every config file,
# including ones the agent can write.
opencodePermission = {
bash = "deny";
task = "deny";
todowrite = "deny";
websearch = "deny";
webfetch = "ask";
};
opencodeConfig = pkgs.writeText "opencode.json" (
builtins.toJSON {
"$schema" = "https://opencode.ai/config.json";
autoupdate = false;
share = "disabled";
model = "${oc.provider.id}/${oc.model}";
provider.${oc.provider.id} = {
npm = "@ai-sdk/openai-compatible";
inherit (oc.provider) name;
options = {
baseURL = oc.provider.baseUrl;
# Substituted by opencode from its environment at startup, so the
# key stays in backendEnvironmentFile and out of the store.
apiKey = "{env:${oc.provider.apiKeyEnv}}";
};
# `limit.context` is what opencode reports as the window in its
# `usage_update`, i.e. the harness's ctx %.
models.${oc.model} = {
name = oc.model;
limit = {
context = oc.contextWindow;
output = oc.outputLimit;
};
};
};
}
);
in
{
options.services.hyperhive.agent.model = lib.mkOption {
type = lib.types.str;
default = "haiku";
example = "sonnet";
description = ''
Claude model for this agent. Sets the `HIVE_DEFAULT_MODEL`
environment variable; the harness applies it at boot and it takes
priority over a `hyperhive-model` file in the harness dir. A model the
swarm sets for the agent (the swarm UI's agents page, or
`swarmctl agent inference set`) wins over this option.
Valid values are the short model names that `claude --model` accepts:
`"haiku"`, `"sonnet"`, `"opus"` (or any future identifier). Context
window sizes are looked up at runtime from the
`HIVE_CONTEXT_WINDOW_TOKENS_<KEY_UPPER>` env vars injected by the
meta flake; override sizes via `services.hyperhive.c0re.contextWindowTokens`
on the host.
'';
};
options.services.hyperhive.agent.availableModels = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"haiku"
"sonnet"
"opus"
];
example = [
"sonnet"
"opus"
];
description = ''
Models this agent may run. `services.hyperhive.agent.model` must be one
of them, and the subagent daemon refuses a `start` or `continue` on any
other model (rendered into its `HIVE_AVAILABLE_MODELS`) unless the swarm
sets `subagent_models` for the agent. An empty list restricts nothing.
Values are the short model names that `claude --model` accepts:
`"haiku"`, `"sonnet"`, `"opus"` (or any future identifier).
'';
};
options.services.hyperhive.agent.effortLevel = lib.mkOption {
type = lib.types.enum [
"low"
"medium"
"high"
"xhigh"
"max"
];
default = "medium";
example = "high";
description = ''
Baseline claude effort level for this agent. Rendered into the
`HIVE_DEFAULT_EFFORT` environment variable; the harness resolves
effort as an existing `hyperhive-effort` file → this env → built-in
`"medium"`, and passes the result to `claude --effort` at turn launch.
An effort the swarm sets for the agent (the swarm UI's agents page, or
`swarmctl agent inference set`) wins over all of these.
Ascending scale: `"low"` (minimal thinking budget), `"medium"`
(default — balanced), `"high"` (platform default), `"xhigh"`
(recommended for autonomous coding on capable models), `"max"`
(maximum thinking budget, highest cost).
'';
};
options.services.hyperhive.agent.autoCompact = lib.mkOption {
type = lib.types.bool;
default = true;
description = ''
Enable proactive watermark-based compaction. When `true` (the
default) the harness automatically runs a notes-checkpoint turn
followed by `/compact` once the context window crosses 75% of
the model's limit, keeping later turns from hitting the hard
overflow path. Set to `false` to disable proactive compaction
entirely (`HIVE_COMPACT_WATERMARK_TOKENS=0`); the reactive path
(compact-on-overflow when the session is already past the limit)
still applies.
Disable for agents that run large-context models (sonnet/opus)
where the heuristic fires too early and discards useful history
before the session is actually close to the limit.
'';
};
options.services.hyperhive.agent.useApiKey = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Authenticate this agent's `claude` invocations with an API key
(`ANTHROPIC_API_KEY`/`ANTHROPIC_BASE_URL`, e.g. OpenRouter) rather
than a Claude OAuth session. Sets `HIVE_USE_API_KEY=1`, which the
harness reads (`hive_agent::login::using_api_key`) to report
`LoginState::Online` at boot without checking `~/.claude/` — an
api-key agent has no OAuth session to wait for, so it must never
park the turn loop expecting one. Also stamped into the consolidated
harness state file so hive-c0re's dashboard stops reading an empty
`~/.claude/` as "needs login" for this agent (see
`hive_c0re::container_view`'s `needs_login` computation).
Set this AND `services.hyperhive.agent.backendEnvironmentFile` together — this
option changes what the harness believes about its own login state,
the other actually supplies the credentials `claude` reads. Neither
is useful alone.
'';
};
options.services.hyperhive.agent.backendEnvironmentFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/agents/myagent/harness/openrouter.env";
description = ''
Path (outside the nix store) to a systemd `EnvironmentFile` loaded
by the harness service — the mechanism for supplying
`ANTHROPIC_API_KEY`/`ANTHROPIC_BASE_URL` (or any other backend
credential `claude`/the harness reads from the environment) without
baking a secret into the nix store.
The file must use systemd `EnvironmentFile` syntax: one `KEY=value`
pair per line, no `export`, no shell quoting needed for simple
values. Example contents:
```
ANTHROPIC_API_KEY=sk-or-v1-...
ANTHROPIC_BASE_URL=https://openrouter.ai/api/v1
```
Place the file inside the agent's bind-mounted **harness** dir (e.g.
`/agents/<name>/harness/openrouter.env`, `$HYPERHIVE_HARNESS_DIR`),
not `state/` — `harness/` survives container rebuilds exactly like
`state/` does, but is never bind-mounted into another agent's
container (unlike `state/`, which a `ManageRootAgent` holder gets
read-write for recovery — see `docs/agent-lifecycle/persistence.md`'s "Cross-agent access to
state"), so this credential is reachable by nothing but this agent
and the host. Permissions should be `0600`, owned by the agent's
unix user. Loaded with a leading `-` (optional `EnvironmentFile`),
so a path that doesn't exist yet — an operator setting this option
before creating the file, or a fresh host rebuild before state is
restored — makes systemd skip it rather than refuse to start the
harness. See `services.hyperhive.agent.useApiKey`'s doc for the option this one is
paired with. The subagent daemon loads the same file on an ACP agent
and on an agent with `useApiKey` set, so its subagents authenticate
the same way the harness does.
On an ACP agent using the `opencode` preset, a provider key this file
leaves unset is read from the swarm secret store at
`swarm/agents/<agent>/acp-provider`, field `api_key`, if one is stored.
After a turn that ends empty or idle, the store is read again, and a
key that changed there gets the agent respawned with it.
'';
};
options.services.hyperhive.agent.runtime = lib.mkOption {
type = lib.types.enum [
"claude"
"acp"
];
default = "claude";
example = "acp";
description = ''
What drives this agent's turns. `"claude"` runs `claude --print`.
`"acp"` runs the Agent Client Protocol agent described by
`services.hyperhive.agent.acp`, as one long-lived child of the
harness, and turns `services.hyperhive.agent.useApiKey` on by default:
the agent authenticates to its own provider, so there is no Claude
login to wait for.
On `"acp"`, `model` and `effortLevel` are set on the session before
each prompt, but only when the session offers that value (otherwise
the agent keeps the model it is configured with). `autoCompact` applies
as it does on `"claude"`. The web UI's cancel button and `/compact`
both work.
Subagents (`hive-subagent-daemon`) run on the same runtime. On
`"acp"`, each subagent run spawns its own copy of the ACP agent, which
also gets `services.hyperhive.agent.backendEnvironmentFile`.
'';
};
options.services.hyperhive.agent.acp = {
preset = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "opencode";
description = ''
Name of an entry in `services.hyperhive.agent.acp.presets` whose
`command`, `args` and `env` become the defaults of the options of
the same name here.
'';
};
command = lib.mkOption {
type = lib.types.str;
default = "";
example = lib.literalExpression ''"''${pkgs.opencode}/bin/opencode"'';
description = "Program the harness spawns as its ACP agent (`HIVE_ACP_COMMAND`).";
};
args = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "acp" ];
description = "Arguments for `command` (`HIVE_ACP_ARGS`, as JSON).";
};
env = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
description = ''
Environment for the ACP agent only, on top of the harness's own
(`HIVE_ACP_ENV`, as JSON). Rendered into the nix store: never put a
credential here. Put it in `services.hyperhive.agent.backendEnvironmentFile`,
which the agent inherits through the harness.
'';
};
presets = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options = {
command = lib.mkOption { type = lib.types.str; };
args = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
};
env = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
};
};
}
);
description = ''
Named ACP agent setups `services.hyperhive.agent.acp.preset` can
select. `opencode` runs `opencode acp` against the OpenAI-compatible
provider set in `services.hyperhive.agent.acp.opencode`.
'';
};
opencode = {
provider = {
id = lib.mkOption {
type = lib.types.str;
default = "provider";
example = "hetzner";
description = "Provider id in opencode's config; the model is addressed as `<id>/<model>`.";
};
name = lib.mkOption {
type = lib.types.str;
default = oc.provider.id;
defaultText = lib.literalExpression "config.services.hyperhive.agent.acp.opencode.provider.id";
description = "Display name of the provider.";
};
baseUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "https://inference.hetzner.com/api/v1";
description = "Base URL of the OpenAI-compatible API.";
};
apiKeyEnv = lib.mkOption {
type = lib.types.str;
default = "ACP_PROVIDER_API_KEY";
description = ''
Environment variable opencode reads the provider's API key from.
Set it in `services.hyperhive.agent.backendEnvironmentFile`, or store
the key in the swarm secret store at
`swarm/agents/<agent>/acp-provider`, field `api_key`.
'';
};
};
model = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "Qwen/Qwen3.6-35B-A3B-FP8";
description = "Model id, as the provider's API names it.";
};
contextWindow = lib.mkOption {
type = lib.types.ints.positive;
default = 131072;
example = 262144;
description = "The model's context window in tokens; the harness's ctx % is measured against it.";
};
outputLimit = lib.mkOption {
type = lib.types.ints.positive;
default = 32768;
description = "Maximum output tokens per model response.";
};
};
};
options.services.hyperhive.agent.extraWebProxies = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
example = lib.literalExpression ''{ "stats" = "http://127.0.0.1:3737"; }'';
description = ''
Transparent reverse-proxies mounted under `/extra/` in the per-agent web UI.
Each attribute name becomes the sub-path and the value is the upstream.
E.g. `{ "stats" = "http://127.0.0.1:3737"; }` mounts a proxy at
`/agent/<name>/extra/stats/` that forwards to port 3737 with the prefix
stripped. All user-declared proxies live under `/extra/` so they can
never conflict with native agent endpoints (`/api/*`, `/events/*`, etc.).
The upstream value is either an `http(s)://` URL or a Unix domain
socket, spelled `unix:<path>` (e.g. `unix:/run/myapp/http.sock`) — for
agents whose secondary web server only listens on a UDS.
Intended for agents that run secondary web servers in the same container.
Static assets served by the secondary app must use relative paths to
resolve correctly under the sub-path prefix.
Sets the `HIVE_EXTRA_WEB_PROXIES` environment variable (JSON object)
on the harness service unit.
'';
};
config = {
assertions = [
# services.hyperhive.agent.model must be a non-empty string — an empty value causes
# the harness to pass an invalid model flag to claude.
{
assertion = config.services.hyperhive.agent.model != "";
message = "services.hyperhive.agent.model must not be empty (set it to e.g. \"haiku\" or \"sonnet\")";
}
# The current model must be one this agent may run; the subagent daemon
# checks its own picks against the same list.
{
assertion =
config.services.hyperhive.agent.availableModels == [ ]
|| builtins.elem config.services.hyperhive.agent.model config.services.hyperhive.agent.availableModels;
message =
"services.hyperhive.agent.model (\"${config.services.hyperhive.agent.model}\") must be one of "
+ "services.hyperhive.agent.availableModels ([ ${lib.concatStringsSep " " config.services.hyperhive.agent.availableModels} ]) "
+ "— add it to the list or change the model.";
}
{
assertion = acp.preset == null || preset != null;
message =
"services.hyperhive.agent.acp.preset (\"${toString acp.preset}\") names no entry in "
+ "services.hyperhive.agent.acp.presets ([ ${lib.concatStringsSep " " (lib.attrNames acp.presets)} ]).";
}
{
assertion = !isAcp || acp.command != "";
message = "services.hyperhive.agent.runtime = \"acp\" needs services.hyperhive.agent.acp.command (or acp.preset).";
}
{
assertion = acp.preset != "opencode" || (oc.provider.baseUrl != null && oc.model != null);
message = "services.hyperhive.agent.acp.preset = \"opencode\" needs acp.opencode.provider.baseUrl and acp.opencode.model.";
}
];
services.hyperhive.agent.useApiKey = lib.mkIf isAcp (lib.mkDefault true);
services.hyperhive.agent.acp = {
presets.opencode = {
command = "${pkgs.opencode}/bin/opencode";
args = [ "acp" ];
env = {
OPENCODE_CONFIG = "${opencodeConfig}";
OPENCODE_PERMISSION = builtins.toJSON opencodePermission;
# Keeps a config file in the agent's working directory from
# overriding the one above.
OPENCODE_DISABLE_PROJECT_CONFIG = "1";
};
};
command = lib.mkIf (preset != null) (lib.mkDefault preset.command);
args = lib.mkIf (preset != null) (lib.mkDefault preset.args);
env = lib.mkIf (preset != null) (lib.mkDefault preset.env);
};
# HIVE_DEFAULT_MODEL is the model the agent runs while the swarm sets
# none; it wins over a `hyperhive-model` file.
environment.variables = {
HIVE_DEFAULT_MODEL = config.services.hyperhive.agent.model;
# Per-agent baseline effort (see services.hyperhive.agent.effortLevel). The
# harness resolves a `hyperhive-effort` file → this env → "medium"
# and passes it to claude --effort at turn launch.
HIVE_DEFAULT_EFFORT = config.services.hyperhive.agent.effortLevel;
}
// lib.optionalAttrs (!config.services.hyperhive.agent.autoCompact) {
# Zero watermark disables proactive compaction; the reactive path
# (compact-on-overflow) still fires when the session is truly full.
HIVE_COMPACT_WATERMARK_TOKENS = "0";
};
# Harness systemd unit. Unit shape (PATH wrapper-dir trick, env vars,
# RuntimeDirectory, User=, standalone-eval fallbacks):
# docs/turn-loop/README.md::Harness systemd unit shape. PATH /bin
# auto-append behaviour: docs/process/gotchas.md::systemd.services.*.path
# appends /bin to every entry.
systemd.services.hive-agent =
let
binary = "hive-agent";
in
{
description = "${binary} harness";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
# No start limit: a harness waiting out a queue outage must keep
# restarting however long it lasts, not stop at `start-limit-hit`.
# `[Unit]` setting, hence here and not in `serviceConfig`.
startLimitIntervalSec = 0;
# `/run/wrappers` before `/run/current-system/sw` so setuid
# `sudo` resolves first. Passing the bare prefixes (no trailing
# `/bin`) is intentional — see docs pointer above.
path = [
"/run/wrappers"
"/run/current-system/sw"
];
environment = {
SHELL = "${pkgs.bashInteractive}/bin/bash";
HOME = homeDir;
HIVE_STATIC_DIR = "${config.services.hyperhive.agent.frontend.mergedDist}";
HIVE_ASSETS_DIR = "${config.services.hyperhive.agent.packages.assets}/share/hyperhive";
# Unix-socket path for the harness web UI. All agents always bind
# here; there is no TCP fallback. Path matches
# `hive_c0re::agent_sockets::socket_path_for(name)` so lifecycle
# bind-mounts and gateway upstream config stay in sync.
HIVE_WEB_SOCKET = "/run/hive-agent/${userName}/web.sock";
# In-agent socket (loose-ends v2): the harness binds this and
# serves the `hive-agent-sock` todo protocol to the in-container
# producers (matrix daemon) + the MCP bridge (`get_loose_ends`).
# Same per-agent runtime dir as the web socket so all agent-user
# services in this container can reach it; purely in-container
# (never bind-mounted to the host — unlike hive-c0re's mcp.sock).
HIVE_AGENT_SOCKET = "/run/hive-agent/${userName}/agent.sock";
# Loopback URL of the persistent `hive-mcp-http` daemon that
# `render_claude_config` points claude at for the built-in
# surface (HTTP is the sole transport — no per-turn stdio child).
# Kept in sync with the `hive-mcp-http` unit's `--http` port
# (see ./mcp.nix) via the same option. Always set — network
# isolation is unconditional, so a fixed per-container port is
# collision-free.
HYPERHIVE_MCP_HTTP_PORT = toString config.services.hyperhive.agent.mcp.httpPort;
}
// lib.optionalAttrs config.services.hyperhive.agent.gui.enable {
# Tells the harness which fixed VNC port weston bound, and (by
# its presence) that gui is enabled — the harness `/screen/ws`
# relay reads this instead of a runtime marker file. The port is
# container-local + fixed (network isolation is unconditional),
# so the same value for every gui agent is fine. See
# ./weston-vnc.nix::services.hyperhive.agent.gui.vncPort.
HIVE_GUI_VNC_PORT = toString config.services.hyperhive.agent.gui.vncPort;
}
// lib.optionalAttrs (config.services.hyperhive.agent.extraWebProxies != { }) {
# JSON object {"<path>": "<upstream>"} for the transparent
# reverse-proxies. See `services.hyperhive.agent.extraWebProxies` option
# and `web_ui/proxy.rs::extra_proxy_service`.
HIVE_EXTRA_WEB_PROXIES = builtins.toJSON config.services.hyperhive.agent.extraWebProxies;
}
// lib.optionalAttrs config.services.hyperhive.agent.useApiKey {
# Tells the harness not to wait for a Claude OAuth session — see
# `services.hyperhive.agent.useApiKey`'s own description for the full mechanism.
HIVE_USE_API_KEY = "1";
}
// lib.optionalAttrs isAcp {
# Read by `hive_runtime::RuntimeSpec`; unset means claude.
HIVE_RUNTIME = "acp";
HIVE_ACP_COMMAND = acp.command;
HIVE_ACP_ARGS = builtins.toJSON acp.args;
HIVE_ACP_ENV = builtins.toJSON acp.env;
}
// lib.optionalAttrs (isAcp && acp.preset == "opencode") {
# Read by `hive_runtime`, which fills it from the store when the
# environment leaves it unset.
HIVE_ACP_API_KEY_ENV = oc.provider.apiKeyEnv;
};
serviceConfig = {
ExecStart = "${config.services.hyperhive.agent.packages.hive-agent}/bin/${binary}";
# Pin the journal identity to the binary name (otherwise systemd
# derives SyslogIdentifier from the ExecStart basename).
SyslogIdentifier = binary;
# The harness exits when it cannot reach the swarm queue
# (`hive-agent`'s `swarm_queue`), and a swarm-wide outage does that to
# every agent at once: 30s keeps those restarts from hammering the
# store and the queue.
Restart = "on-failure";
RestartSec = 30;
# A LOWER OOMScoreAdjust means LESS likely to be killed, so this
# negative value puts the harness — and the agent's own `claude`,
# which it spawns as a child and which therefore inherits the
# value — last in line under memory pressure, behind
# `hive-subagent-daemon`'s `+500` (see ./mcp.nix). Losing a
# subagent costs one restartable task; losing this costs the
# session that was supervising it. Not `-1000`, which would
# exempt the harness entirely and leave the kernel with nothing
# to kill in a container whose only large process is this one.
OOMScoreAdjust = -500;
# Per-service runtime dir owned by `User=` below; the harness
# writes its regenerated claude-{mcp-config,settings,system-prompt}
# files here (`paths::config_dir`). Separate from /run/hive,
# which holds hive-c0re's mcp.sock.
RuntimeDirectory = "hive-config";
User = userName;
Group = userName;
}
// lib.optionalAttrs (config.services.hyperhive.agent.backendEnvironmentFile != null) {
# See `services.hyperhive.agent.backendEnvironmentFile`'s own description for
# the file shape and the leading-`-` rationale.
EnvironmentFile = "-${config.services.hyperhive.agent.backendEnvironmentFile}";
};
};
};
}