systemd's default OOMPolicy=stop tears the whole unit down the moment the kernel kills any process in its cgroup, so a single over-large subagent takes the daemon and every sibling session with it — measured on a live agent, both units show OOMPolicy=stop today, which is exactly the "every live subagent session was cut mid-turn" symptom. That blast radius is also what would make the preceding commit a bad trade: deliberately putting this unit first in the OOM queue is only an improvement if losing one subagent isn't losing all of them. OOMPolicy= continue scopes the loss to the process the kernel actually chose, and leaves the daemon alive to report the kill instead of vanishing and being restarted with an empty session map. Refs #4316
430 lines
20 KiB
Nix
430 lines
20 KiB
Nix
# The MCP tool surface: the built-in hyperhive server (persistent
|
|
# streamable-http daemon), the bash-task backend daemon (also a
|
|
# persistent streamable-http MCP server, auto-injected into
|
|
# `extraMcpServers`), the `extraMcpServers` option itself (stdio or http,
|
|
# per entry), and the send-recipient allowlist.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
userName = config.hyperhive.user.name;
|
|
# This container's own effective `MemoryMax=` in bytes, baked in per
|
|
# agent by meta.rs's flake render — see
|
|
# `hyperhive.claudeMemoryMaxBytes` in ./claude-settings.nix. `null`
|
|
# when the cap is `infinity` or a RAM percentage, i.e. when the module
|
|
# has no byte count to size anything against.
|
|
containerMemoryMaxBytes = config.hyperhive.claudeMemoryMaxBytes;
|
|
# Two thirds of the container's cap, as the soft ceiling on everything
|
|
# the subagent daemon runs. The daemon spawns nested `claude` sessions
|
|
# as plain children, so its cgroup already *is* the "all subagents"
|
|
# cgroup and a unit-level ceiling bounds the set without a slice.
|
|
#
|
|
# `MemoryHigh=` and not `MemoryMax=`: this throttles rather than walls.
|
|
# Past it the kernel reclaims aggressively and the cgroup stalls, so a
|
|
# subagent that overshoots gets visibly slow and the remaining third
|
|
# stays available for the agent's own turn — but a subagent that
|
|
# genuinely needs more than two thirds still gets it when the container
|
|
# has the memory to spare, which is what keeps overprovisioning
|
|
# (several agents that rarely compile at the same time) working.
|
|
subagentMemoryHigh = containerMemoryMaxBytes * 2 / 3;
|
|
in
|
|
{
|
|
options.hyperhive.allowedRecipients = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
example = [
|
|
"alice"
|
|
"manager"
|
|
];
|
|
description = ''
|
|
Names this agent is allowed to `send` to via
|
|
`mcp__hyperhive__send`. Empty list (the default) means
|
|
unrestricted — the agent can message any peer, the
|
|
operator, or the manager. Non-empty list constrains the
|
|
surface: only the listed names + the manager (always
|
|
allowed) get through; anything else returns an error
|
|
string to claude without touching the broker. The
|
|
operator (`operator`) needs to be in the list if the
|
|
agent should be able to surface output on the
|
|
dashboard.
|
|
|
|
Useful for sandboxing untrusted sub-agents — set
|
|
`[ "manager" ]` to scope them to manager-only chatter.
|
|
The manager itself is always exempt; this option only
|
|
affects sub-agent `send`.
|
|
'';
|
|
};
|
|
|
|
options.hyperhive.extraMcpServers = lib.mkOption {
|
|
type = lib.types.attrsOf (
|
|
lib.types.submodule {
|
|
options = {
|
|
type = lib.mkOption {
|
|
type = lib.types.enum [
|
|
"stdio"
|
|
"http"
|
|
];
|
|
default = "stdio";
|
|
description = ''
|
|
Transport for this MCP server. `"stdio"` (the default) spawns
|
|
`command` as a fresh child process every turn, talking
|
|
JSON-RPC over its stdin/stdout — existing entries need zero
|
|
changes to keep this behaviour. `"http"` points claude at a
|
|
long-lived streamable-http `url` instead: no per-turn spawn,
|
|
no re-registration race, same shape as the built-in
|
|
hyperhive surface (`hive-mcp-http`) — use this for a server
|
|
backed by an always-on daemon. `command`/`args`/`env` only
|
|
apply to `"stdio"`; `url` only to `"http"`.
|
|
'';
|
|
};
|
|
command = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = ''
|
|
Absolute path to the MCP server binary. Use `''${pkgs.foo}/bin/foo`
|
|
or `/run/current-system/sw/bin/foo`. Required when
|
|
`type = "stdio"`; ignored (leave `null`) for `"http"`.
|
|
'';
|
|
};
|
|
args = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
description = "Args passed to the MCP server binary. `\"stdio\"` only.";
|
|
};
|
|
env = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.str;
|
|
default = { };
|
|
description = "Environment variables for the MCP server child process. `\"stdio\"` only.";
|
|
};
|
|
url = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = ''
|
|
Streamable-http URL (e.g. `http://127.0.0.1:8791/mcp`) of the
|
|
always-on daemon serving this MCP surface. Required when
|
|
`type = "http"`; ignored (leave `null`) for `"stdio"`.
|
|
'';
|
|
};
|
|
allowedTools = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "*" ];
|
|
example = [
|
|
"send_message"
|
|
"join_room"
|
|
];
|
|
description = ''
|
|
Tool names this MCP server is auto-approved to call via
|
|
`--allowedTools`. Single entry `"*"` (the default) means
|
|
"every tool from this server" — convenient but trusting.
|
|
Tighten to a specific list when you only want a subset.
|
|
Names are bare (e.g. `send_message`); the harness prepends
|
|
`mcp__<server-key>__` at build time.
|
|
'';
|
|
};
|
|
};
|
|
}
|
|
);
|
|
default = { };
|
|
example = lib.literalExpression ''
|
|
{
|
|
matrix = {
|
|
command = "/run/current-system/sw/bin/mcp-matrix";
|
|
args = [ "--config" "/state/matrix.toml" ];
|
|
env.MATRIX_HOMESERVER = "https://matrix.example.org";
|
|
allowedTools = [ "send_message" "join_room" ];
|
|
};
|
|
bash = {
|
|
type = "http";
|
|
url = "http://127.0.0.1:8791/mcp";
|
|
};
|
|
}
|
|
'';
|
|
description = ''
|
|
Extra MCP servers claude sees alongside the hyperhive tool surface.
|
|
Keys are the server names (claude addresses tools as
|
|
`mcp__<key>__<tool>`). Rendered to `/etc/hyperhive/extra-mcp.json`
|
|
at activation time; the harness reads that file at boot and merges
|
|
it into `--mcp-config` + `--allowedTools`. Take effect on the
|
|
agent's next harness restart (no operator approval needed beyond
|
|
whatever brought the new agent.nix into deployed/*).
|
|
|
|
The `bash` entry above is illustrative only — it's auto-injected
|
|
below (`hyperhive.extraMcpServers.bash` via `lib.mkDefault`) already
|
|
tracking `hyperhive.mcp.bashHttpPort`, so overriding it directly
|
|
with a hardcoded port (as shown) is unusual and will drift if
|
|
`bashHttpPort` is changed separately; bump `bashHttpPort` instead.
|
|
'';
|
|
};
|
|
|
|
options.hyperhive.mcp.httpPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 8790;
|
|
example = 8791;
|
|
description = ''
|
|
Loopback port the built-in hyperhive MCP surface is served on. HTTP
|
|
is the *sole* transport for the built-in surface: a
|
|
long-lived `hive-mcp-http` systemd unit runs
|
|
`hive-agent-mcp --http 127.0.0.1:<port>` and `render_claude_config`
|
|
points claude at the stable `http://127.0.0.1:<port>/mcp` URL. That
|
|
URL survives the per-turn claude re-spawn (and a host-side hive-c0re
|
|
restart — each tool call dials the control socket fresh), so there
|
|
is no per-turn MCP re-registration race (a resumed stdio child could
|
|
emit its first tool call before that turn's async
|
|
`initialize`/`tools-list` completed, stranding the agent with `No
|
|
such tool` — the http endpoint eliminates that). `matrix` stays a
|
|
stdio bridge; `bash` runs its own persistent http listener (see
|
|
`hyperhive.mcp.bashHttpPort`).
|
|
|
|
Bound loopback-only; the rmcp streamable-http transport's default
|
|
`allowed_hosts` (`localhost` / `127.0.0.1` / `::1`) rejects Host
|
|
headers from anywhere else, so no auth token is required for a
|
|
container-local endpoint.
|
|
|
|
Failure-mode note: with no stdio fallback, if `hive-mcp-http` is
|
|
down claude hits a dead URL until the unit restarts (guarded by
|
|
`Restart=always`, `RestartSec=3`). Intended shape: no per-turn race
|
|
while up, a bounded self-healing gap while restarting.
|
|
|
|
Safe as a single fixed default across all agents: each container
|
|
runs in its own private network namespace (isolation is always-on —
|
|
see docs/networking/network.md), so `127.0.0.1:<port>` is per-container-private
|
|
and cannot collide across agents. Override only if a container-local
|
|
service already occupies this port.
|
|
|
|
Must match `mcp_config::DEFAULT_MCP_HTTP_PORT` (the harness always
|
|
exports `HYPERHIVE_MCP_HTTP_PORT`, so the const is only a fallback).
|
|
'';
|
|
};
|
|
|
|
options.hyperhive.mcp.bashHttpPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 8791;
|
|
example = 8792;
|
|
description = ''
|
|
Loopback port `hive-bash-daemon` serves its MCP tools
|
|
(`run`/`status`/`kill`) on. Same shape as `hyperhive.mcp.httpPort`
|
|
for the built-in surface: HTTP is the *sole* transport (no stdio
|
|
bridge — the daemon that owns the subprocess runner serves the MCP
|
|
tools directly in-process), `Restart = "always"` keeps the listener
|
|
self-healing, and loopback-only binding means no auth token is
|
|
needed (same `allowed_hosts` reasoning as `hyperhive.mcp.httpPort`).
|
|
Safe as a single fixed default across all agents (private
|
|
per-container network namespace — see docs/networking/network.md).
|
|
'';
|
|
};
|
|
|
|
options.hyperhive.mcp.subagentHttpPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 8793;
|
|
example = 8794;
|
|
description = ''
|
|
Loopback port `hive-subagent-daemon` serves its MCP tools
|
|
(`start`/`continue`/`status`/`interrupt`) on. Independent daemon (own crate,
|
|
`hive-subagent-mcp`) — a subagent spawns a full nested `claude`
|
|
process, a much heavier capability than a bash command, worth its own
|
|
deployable/restartable unit. Same shape/reasoning as
|
|
`hyperhive.mcp.bashHttpPort` otherwise: sole transport, self-healing
|
|
restart, loopback-only so no auth token is needed. Shipped default-on
|
|
for every agent today, same as `bash` — expected to become a real
|
|
opt-in capability gate later, not yet.
|
|
'';
|
|
};
|
|
|
|
config = {
|
|
# Assert the transport-specific required field is actually set —
|
|
# `command`/`url` are both `nullOr` so the submodule schema stays
|
|
# backward-compatible for existing stdio entries, but a `null` in the
|
|
# field the chosen `type` actually needs is a config mistake, not a
|
|
# valid "unset".
|
|
assertions =
|
|
lib.mapAttrsToList (name: spec: {
|
|
assertion = spec.type != "stdio" || spec.command != null;
|
|
message = "hyperhive.extraMcpServers.${name}: type = \"stdio\" requires `command` to be set";
|
|
}) config.hyperhive.extraMcpServers
|
|
++ lib.mapAttrsToList (name: spec: {
|
|
assertion = spec.type != "http" || spec.url != null;
|
|
message = "hyperhive.extraMcpServers.${name}: type = \"http\" requires `url` to be set";
|
|
}) config.hyperhive.extraMcpServers;
|
|
|
|
# Auto-inject the built-in bash MCP server — always present, every
|
|
# agent needs bash tools. `lib.mkDefault` so the operator's own
|
|
# agent.nix can override the entry. (The matrix sibling lives in
|
|
# ./matrix.nix, gated on hyperhive.matrix.enable.) `hive-bash-daemon`
|
|
# serves its MCP tools directly over streamable-http (no stdio bridge,
|
|
# no round-trip socket) — see the `hive-bash-daemon` service below.
|
|
hyperhive.extraMcpServers.bash = lib.mkDefault {
|
|
type = "http";
|
|
url = "http://127.0.0.1:${toString config.hyperhive.mcp.bashHttpPort}/mcp";
|
|
allowedTools = [ "*" ];
|
|
};
|
|
|
|
# Auto-inject the subagent MCP server — default-on for every agent for
|
|
# now (operator's call: "default on for now, should be a capability
|
|
# later" — not gated behind an enable option yet, unlike `matrix.nix`'s
|
|
# pattern). `lib.mkDefault` so an agent.nix can still override/disable
|
|
# the entry in the meantime.
|
|
hyperhive.extraMcpServers.subagent = lib.mkDefault {
|
|
type = "http";
|
|
url = "http://127.0.0.1:${toString config.hyperhive.mcp.subagentHttpPort}/mcp";
|
|
allowedTools = [ "*" ];
|
|
};
|
|
|
|
environment.etc."hyperhive/extra-mcp.json".text = builtins.toJSON config.hyperhive.extraMcpServers;
|
|
|
|
environment.etc."hyperhive/send-allow.json".text =
|
|
builtins.toJSON config.hyperhive.allowedRecipients;
|
|
|
|
# Bash task runner daemon — long-running process that owns subprocess
|
|
# monitoring + completion wake signals, and serves the MCP tools
|
|
# (`run`/`status`/`kill`) directly over streamable-http on
|
|
# `hyperhive.mcp.bashHttpPort` — no stdio bridge, no per-turn spawn.
|
|
systemd.services.hive-bash-daemon = {
|
|
description = "bash task runner + MCP daemon for hive-bash";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "hive-agent.service" ];
|
|
# The daemon runs every bash task via `Command::new("bash")` and the
|
|
# commands themselves (hive-forge, git, jq, …) resolve from PATH.
|
|
# A standalone daemon has no inherited agent PATH, so without this
|
|
# `bash` itself isn't found (spawn fails with ENOENT, the task is
|
|
# marked done in 0s with no output / no .out/.err). Mirror the
|
|
# harness unit's PATH: NixOS appends `/bin` to each entry →
|
|
# /run/wrappers/bin (setuid sudo) + /run/current-system/sw/bin
|
|
# (bash, coreutils, hive-forge, …).
|
|
path = [
|
|
"/run/wrappers"
|
|
"/run/current-system/sw"
|
|
];
|
|
environment = {
|
|
# In-agent todo socket the harness serves (loose-ends v2): the
|
|
# runner pushes bash-task todos here (upsert while active, keyless
|
|
# 'done' on completion) instead of firing a c0re wake. Must match
|
|
# the harness's HIVE_AGENT_SOCKET (agent-service.nix).
|
|
HIVE_AGENT_SOCKET = "/run/hive-agent/${userName}/agent.sock";
|
|
RUST_LOG = "info";
|
|
# HYPERHIVE_HARNESS_DIR and HYPERHIVE_STATE_DIR are already
|
|
# injected via systemd.globalEnvironment by the meta flake
|
|
# (set to /agents/<name>/harness and /agents/<name>/state
|
|
# respectively). The daemon uses these to derive its task +
|
|
# loose-ends dir paths; `hive_agent_sock::paths::harness_dir`
|
|
# panics loudly if HYPERHIVE_HARNESS_DIR is unset rather than
|
|
# deriving a fallback, since every service here always gets it.
|
|
};
|
|
serviceConfig = {
|
|
ExecStart = "${config.hyperhive.packages.hive-bash-daemon}/bin/hive-bash-daemon --http 127.0.0.1:${toString config.hyperhive.mcp.bashHttpPort}";
|
|
SyslogIdentifier = "hive-bash-daemon";
|
|
# `always` (not `on-failure`): since the MCP tools are served
|
|
# in-process now, a down window is total loss of bash tools with
|
|
# no stdio fallback — same reasoning as `hive-mcp-http` below.
|
|
Restart = "always";
|
|
RestartSec = 3;
|
|
User = userName;
|
|
Group = userName;
|
|
};
|
|
};
|
|
|
|
# Subagent task runner daemon — independent of `hive-bash-daemon` (own
|
|
# crate, own process): spawns nested claude sessions on request, serves
|
|
# the `start`/`continue`/`status`/`interrupt` MCP tools directly over
|
|
# streamable-http on `hyperhive.mcp.subagentHttpPort`. No task files —
|
|
# this daemon's only state is an in-memory map of currently-running
|
|
# processes, live only as long as the process is (see
|
|
# `hive-subagent-mcp/src/session.rs`'s module doc); a restart stops
|
|
# whatever's running, the actual claude session survives independently.
|
|
systemd.services.hive-subagent-daemon = {
|
|
description = "subagent task runner + MCP daemon for hive-subagent";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "hive-agent.service" ];
|
|
# A subagent task runs its own nested `claude` invocation (via
|
|
# `hive-claude`), which needs to resolve `claude` itself off PATH —
|
|
# same reasoning as `hive-bash-daemon`'s `path` above, even though
|
|
# this daemon never shells out to `bash -c` directly.
|
|
path = [
|
|
"/run/wrappers"
|
|
"/run/current-system/sw"
|
|
];
|
|
environment = {
|
|
# Same in-agent todo socket as `hive-bash-daemon` — both push task
|
|
# todos to the one harness socket. Must match the harness's
|
|
# HIVE_AGENT_SOCKET (agent-service.nix).
|
|
HIVE_AGENT_SOCKET = "/run/hive-agent/${userName}/agent.sock";
|
|
RUST_LOG = "info";
|
|
# HYPERHIVE_HARNESS_DIR / HYPERHIVE_STATE_DIR: see
|
|
# `hive-bash-daemon`'s own comment above — same global injection,
|
|
# same reasoning.
|
|
};
|
|
serviceConfig = {
|
|
ExecStart = "${config.hyperhive.packages.hive-subagent-daemon}/bin/hive-subagent-daemon --http 127.0.0.1:${toString config.hyperhive.mcp.subagentHttpPort}";
|
|
SyslogIdentifier = "hive-subagent-daemon";
|
|
# `always`, same reasoning as `hive-bash-daemon`: the MCP tools are
|
|
# served in-process, so a down window is total loss of
|
|
# `start`/`continue`/`status`/`interrupt` with no stdio fallback.
|
|
Restart = "always";
|
|
RestartSec = 3;
|
|
# A HIGHER OOMScoreAdjust means MORE likely to be killed: the
|
|
# kernel adds it to the badness score it derives from the
|
|
# process's memory footprint, then kills the highest scorer. So
|
|
# the positive value here and the negative one on `hive-agent`
|
|
# put this unit first in line — a subagent turn can be retried,
|
|
# the agent's own turn is the thing everything else hangs off.
|
|
# `+500` against `hive-agent`'s `-500` is a full half of the
|
|
# score range apart, enough that a fat subagent outranks a fatter
|
|
# harness rather than merely tying with it. Every nested `claude`
|
|
# inherits the value from the daemon, so the ordering covers the
|
|
# whole subtree and not just this process.
|
|
OOMScoreAdjust = 500;
|
|
# systemd's default is `stop`, which tears the whole unit down as
|
|
# soon as the kernel kills *any* process in its cgroup — so one
|
|
# over-large subagent takes the daemon and every sibling session
|
|
# with it, which is the blast radius that makes an OOM here
|
|
# expensive. `continue` scopes the loss to the process the kernel
|
|
# actually chose and leaves the daemon serving, which is also what
|
|
# lets it report the kill rather than vanish and be restarted.
|
|
OOMPolicy = "continue";
|
|
User = userName;
|
|
Group = userName;
|
|
}
|
|
// lib.optionalAttrs (containerMemoryMaxBytes != null) {
|
|
MemoryHigh = toString subagentMemoryHigh;
|
|
};
|
|
};
|
|
|
|
# Persistent streamable-http MCP daemon for the built-in hyperhive
|
|
# surface — the *sole* transport for that surface; always
|
|
# wired. Long-lived so claude reconnects to the stable URL each turn
|
|
# instead of respawning + re-registering a stdio subprocess (the
|
|
# per-turn MCP registration race). It dials the control socket
|
|
# (`/run/hive/mcp.sock`, the harness binaries' default) fresh on every
|
|
# tool call, so a host-side hive-c0re restart is transparent.
|
|
# `before = hive-agent` so the URL is already listening by the time
|
|
# the harness renders the first turn's config; the harness/claude also
|
|
# reconnect on their own, so ordering is a latency nicety not a hard
|
|
# correctness dep.
|
|
systemd.services.hive-mcp-http = {
|
|
description = "persistent streamable-http MCP daemon for the hyperhive surface";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "hive-agent.service" ];
|
|
environment.RUST_LOG = "info";
|
|
# In-agent todo socket the harness serves (loose-ends v2): the
|
|
# `get_loose_ends` handler dials it to merge this agent's local todos
|
|
# with the static loose-ends from hive-c0re.
|
|
environment.HIVE_AGENT_SOCKET = "/run/hive-agent/${userName}/agent.sock";
|
|
serviceConfig = {
|
|
ExecStart = "${config.hyperhive.packages.hive-agent-mcp}/bin/hive-agent-mcp --http 127.0.0.1:${toString config.hyperhive.mcp.httpPort}";
|
|
SyslogIdentifier = "hive-mcp-http";
|
|
# `always` (not `on-failure`): this endpoint is load-bearing — the
|
|
# sole hyperhive-MCP transport, so a down window is total
|
|
# hyperhive-MCP loss with no stdio fallback and no per-turn
|
|
# self-heal (the URL just stays dead). `always` also covers any
|
|
# unforeseen clean-return path and restarts after a stray SIGTERM
|
|
# stops it out from under the harness.
|
|
Restart = "always";
|
|
RestartSec = 3;
|
|
User = userName;
|
|
Group = userName;
|
|
};
|
|
};
|
|
};
|
|
}
|