hyperhive/nix/agent-modules/claude-settings.nix
atlas fbeff69fd7 fix(otel): stop delivering the hive's upstream token to agents
The host-side collector is the only path telemetry leaves a hive, so
HYPERHIVE_OTEL_HEADERS_CREDENTIAL is never emitted and everything
downstream of it is unreachable. What made it worth removing rather than
leaving inert is what it looked like to a reader: a complete,
well-commented mechanism for writing the hive's upstream credential into
a file the agent can read, described in the present tense. Anyone auditing
"can an agent obtain the OTEL token?" had to reconstruct the whole env-var
chain to find out the answer is no.

Gone: the per-agent `hyperhive.otel.headersCredential` option, the
`hive-otel-header` oneshot that merged OTEL_EXPORTER_OTLP_HEADERS into the
agent's own settings.json, and meta.rs's field, env read and render.

⚠️ Scoped by NAMESPACE, not by name. `hyperhive.otel.headersCredential`
(per-agent) and `services.hyperhive.otel.headersCredential` (host) are
different options sharing a leaf name — the host one is read by
`stats/otel_metrics.rs` for c0re's own container-resource exporter and
stays. Sweeping the string would have taken out working code.

The comment above `otelSettingsEnv` now states the property rather than
the absence: there is no auth header and no mechanism to add one, because
an agent exports to the hive's own collector and nothing an agent can read
is a secret to the swarm. The old behaviour is named in the past tense so
it reads as removed rather than overlooked.

meta.rs's assertions that pinned the injection are deleted rather than
adjusted; the surrounding test keeps covering extraResourceAttributes and
the endpoint/protocol injection, which are live.
2026-08-18 22:23:35 +02:00

417 lines
18 KiB
Nix

# Everything that shapes claude-code's own configuration inside the
# container: the managed settings json (base env + OTEL), the
# onboarding/trust seed, the runtime OTEL auth-header injection, and
# the plugin/marketplace install lists the harness reads at boot.
{
pkgs,
lib,
config,
...
}:
let
userName = config.hyperhive.user.name;
homeDir = "/home/${userName}";
# Hive-wide OpenTelemetry config (host-driven; baked in per-agent by
# meta.rs `otel_config`).
otelCfg = config.hyperhive.otel;
# Hive/swarm display names, read from the per-agent options meta.rs
# renders (NOT from `environment.variables` — those carry the same names
# at *runtime* only, so reading them here silently yielded "unknown" on
# every agent while the process env held the right answer). `null` means
# the hive did not name itself; "unknown" is then an honest label rather
# than a guess.
hiveDisplayName =
if config.hyperhive.hiveName == null then "unknown" else config.hyperhive.hiveName;
swarmDisplayName =
if config.hyperhive.swarmName == null then "unknown" else config.hyperhive.swarmName;
# Effective per-agent MemoryMax=, in bytes, injected by meta.rs's
# per-agent flake render (`hyperhive.claudeMemoryMaxBytes`). `null`
# when the effective cap is unbounded ("infinity") or a RAM
# percentage — see `resource_limits::effective_memory_bytes`.
memoryMaxBytes = config.hyperhive.claudeMemoryMaxBytes;
# Base claude-code environment applied to every agent regardless of OTEL.
# Shipped via the managed settings `env` block so claude and `hivectl
# choom` both inherit them without a launch wrapper.
baseClaudeEnv = {
# Suppress analytics, survey pings, and other non-essential outbound
# traffic — agents are headless and don't need any of that.
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC = "1";
DO_NOT_TRACK = "1";
# Disable claude's self-update machinery; package management is nix's job.
DISABLE_AUTOUPDATER = "1";
DISABLE_UPDATES = "1";
# Keep plugin updates synchronized on install; prefer HTTPS for fetches.
CLAUDE_CODE_PLUGIN_PREFER_HTTPS = "1";
CLAUDE_CODE_SYNC_PLUGIN_INSTALL = "1";
FORCE_AUTOUPDATE_PLUGINS = "1";
# Suppress the "install GitHub app" prompt — not applicable in-hive.
DISABLE_INSTALL_GITHUB_APP_COMMAND = "1";
# Disable Anthropic's hosted claude.ai MCP servers; the hive supplies its own.
ENABLE_CLAUDEAI_MCP_SERVERS = "0";
# Resume an interrupted turn on reconnect (recovers from transient MCP flaps).
CLAUDE_CODE_RESUME_INTERRUPTED_TURN = "1";
# Use the simpler system prompt variant suited to headless operation.
CLAUDE_CODE_SIMPLE_SYSTEM_PROMPT = "1";
# Tag remote-control sessions with "<hive>-<agent>" for identification.
CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX = "${hiveDisplayName}-${userName}";
}
# Bun/JavaScriptCore (the runtime under claude-code's `.claude-wrapped`
# binary) sizes its default heap ceiling off the container's visible
# `/proc/meminfo`, which under nspawn is the *host's* physical RAM, not
# the systemd `MemoryMax=` cgroup cap actually enforced on this
# container — the cap lives on the host's outer cgroup and is invisible
# from inside (confirmed: `/sys/fs/cgroup/memory.max` reads `max` at
# every level from in here). That mismatch lets JSC's heap grow well
# past the real wall before GC kicks in hard, causing choom sessions to
# hang. Pin JSC's ceiling to 75% of the *actual* effective cap instead,
# once it's known at build time.
// lib.optionalAttrs (memoryMaxBytes != null) {
BUN_JSC_forceRAMSize = toString (memoryMaxBytes * 75 / 100);
};
# OTEL environment Claude Code reads to export metrics/logs/traces.
# Shipped via the managed claude settings json (below), which claude
# auto-discovers for BOTH the harness turn-loop and `hivectl choom` —
# so telemetry parity is declarative, with no launch wrapper.
#
# There is no auth header here, and no mechanism to add one. An agent
# exports to the hive's own collector, which is the only thing holding
# a credential for anything upstream; nothing an agent can read is a
# secret to the swarm. An earlier revision forwarded the operator's
# upstream token into this container and merged it into the agent's own
# `~/.claude/settings.json` — which handed every agent the hive's
# credential, and was removed with the direct-export path it served.
otelSettingsEnv = {
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
# Attach feedback-survey data to the OTEL pipeline.
CLAUDE_CODE_ENABLE_FEEDBACK_SURVEY_FOR_OTEL = "1";
OTEL_METRICS_EXPORTER = "otlp";
OTEL_LOGS_EXPORTER = "otlp";
OTEL_TRACES_EXPORTER = "otlp";
OTEL_EXPORTER_OTLP_PROTOCOL = otelCfg.protocol;
OTEL_EXPORTER_OTLP_ENDPOINT = otelCfg.endpoint;
# Force CUMULATIVE temporality — Claude Code defaults to DELTA,
# which Prometheus/Mimir-family backends (incl. grafana-lgtm)
# silently drop without a deltatocumulative processor.
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
OTEL_RESOURCE_ATTRIBUTES =
"service.name=hyperhive-agent,agent=${userName},hive=${hiveDisplayName},swarm=${swarmDisplayName}"
+ lib.optionalString (otelCfg.extraResourceAttributes != "") ",${otelCfg.extraResourceAttributes}";
# Include the Claude Code version label in emitted metrics.
OTEL_METRICS_INCLUDE_VERSION = "1";
}
// lib.optionalAttrs (otelCfg.metricIntervalMs != null) {
OTEL_METRIC_EXPORT_INTERVAL = toString otelCfg.metricIntervalMs;
};
in
{
# OTEL stats export is configured ONCE at host level via
# `services.hyperhive.otel.*` (see nix/host-modules/hive-c0re.nix) and
# injected into every agent's build by the meta-flake renderer
# (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are
# the build-time implementation surface that injection writes into;
# they are not meant to be set directly in an agent.nix. Marked
# `internal` so the host option is the only documented operator knob.
options.hyperhive.otel = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
internal = true;
description = ''
Export this agent's Claude Code stats (token usage, cost, tool
calls) to an OTLP endpoint via Claude Code's built-in
OpenTelemetry. Each agent's harness exports directly to the
collector, so it keeps working even when hive-c0re is down.
Host-driven: set `services.hyperhive.otel.enable` instead.
'';
};
endpoint = lib.mkOption {
type = lib.types.str;
default = "";
internal = true;
description = ''
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`.
Host-driven via `services.hyperhive.otel.endpoint`.
'';
};
protocol = lib.mkOption {
type = lib.types.enum [
"http/protobuf"
"http/json"
"grpc"
];
default = "http/protobuf";
internal = true;
description = ''
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
Host-driven via `services.hyperhive.otel.protocol`.
'';
};
extraResourceAttributes = lib.mkOption {
type = lib.types.str;
default = "";
internal = true;
description = ''
Extra comma-separated entries appended to
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
`service.name` / `agent` / `hive` / `swarm` labels.
Host-driven via `services.hyperhive.otel.extraResourceAttributes`.
'';
};
metricIntervalMs = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
internal = true;
description = ''
Metric export interval in milliseconds, set as
`OTEL_METRIC_EXPORT_INTERVAL`. Null leaves Claude Code's 60s
default. Host-driven via `services.hyperhive.otel.metricIntervalMs`.
'';
};
debug = lib.mkOption {
type = lib.types.bool;
default = false;
internal = true;
description = ''
Emit OTEL SDK diagnostics to stderr (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`).
Host-driven via `services.hyperhive.otel.debug`.
'';
};
};
# Build-time implementation surface for the JSC-heap-ceiling fix:
# meta.rs's per-agent flake render injects this from the effective
# `MemoryMax=` (per-agent `resource-limits.json` override, else the
# hive-wide `services.hyperhive.agentMemoryMax`) — see
# `resource_limits::effective_memory_bytes_from`. Not meant to be set
# directly in an agent.nix, same convention as `hyperhive.otel.*`
# above; the host option (or `hivectl agent <name> set-limits`) is
# the real operator knob, and this only reflects the value baked in at
# the agent's *last rebuild* — `set-limits` still applies the
# cgroup cap live via a drop-in reload, but this derived heap ceiling
# needs a rebuild to pick up a new value.
options.hyperhive.hiveName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
internal = true;
description = ''
Human-readable hive name, rendered per-agent by
`meta.rs::render_flake` from the host's
`services.hyperhive.hiveName`. Baked into the OTEL resource
attributes at build time, which is why it is an option and not
just the `HYPERHIVE_HIVE_NAME` env var: the env var is read at
runtime, this is read during evaluation, and wiring only one of
the two is how every agent ended up reporting `hive=unknown`.
`null` means the hive did not name itself.
'';
};
options.hyperhive.swarmName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
internal = true;
description = ''
Human-readable swarm name, rendered per-agent by
`meta.rs::render_flake` from the host's
`services.hyperhive.swarm.name`. Same build-time/runtime split as
`hyperhive.hiveName`.
`null` means the hive is not part of a named swarm.
'';
};
options.hyperhive.claudeMemoryMaxBytes = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
internal = true;
description = ''
Effective per-agent memory cap in bytes, when it's a plain
byte-size value (null for an unbounded or percentage-based cap).
Used to derive `BUN_JSC_forceRAMSize` in `baseClaudeEnv`.
'';
};
options.hyperhive.claudeMarketplaces = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"anthropics/claude-plugins-official"
"${config.hyperhive.packages.claude-plugins}"
];
defaultText = lib.literalMD ''
`[ "anthropics/claude-plugins-official" "''${hyperhive.packages.claude-plugins}" ]`
(the flake's own local-path plugin marketplace)
'';
example = [
"anthropics/claude-plugins-official"
"anthropics/claude-plugins-community"
];
description = ''
Claude Code plugin marketplaces to add at harness boot. Each
entry is passed to `claude plugin marketplace add <source>`
(`owner/repo`, full git URL, or local path). Idempotent
re-adding an existing marketplace is treated as success.
Required before `hyperhive.claudePlugins` entries that
reference a marketplace (e.g. `foo@claude-plugins-official`).
Rendered to `/etc/hyperhive/claude-marketplaces.json`.
Defaults to Anthropic's official marketplace plus hyperhive's
own `claude-plugins` nix package (see
`nix/packages/claude-plugins.nix`) a local-path marketplace
built as a plain nix store path, registered under the name
`hyperhive` (from its `marketplace.json`, not the store path
itself, so plugin specs stay stable across rebuilds). No forge
repo or git remote needed to ship a hive-authored skill; agents
get both marketplaces out of the box without any per-agent.nix
wiring.
'';
};
options.hyperhive.claudePlugins = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"skill-creator@claude-plugins-official"
"base@hyperhive"
];
example = [
"formatter@my-marketplace"
"thinking-tools@anthropics"
];
description = ''
Claude Code plugins to install at harness boot. Each entry is
passed verbatim to `claude plugin install <spec>` once per
container start, before the turn loop opens. `claude plugin
install` is expected to be idempotent, so reinstalling on every
boot is cheap. Failures log a warning but do not abort boot a
missing plugin is preferable to a non-serving agent. Rendered to
`/etc/hyperhive/claude-plugins.json`; the harness reads it via
`plugins::install_configured`.
Defaults to Anthropic's `skill-creator` (teaches an agent to
write, refine, and evaluate its own skills) plus hyperhive's own
`base` plugin one plugin bundling every skill that applies to
*all* agents regardless of role (currently just `state-hygiene`,
read-before-write + dated-archive discipline for durable
notes/state files; more all-agent skills land as additional
skills inside this same plugin, not new plugins a skill that
only some agents need gets its own specialized plugin instead).
Agents get both out of the box, matching the default
marketplaces above.
Note that a per-agent definition REPLACES this default rather
than extending it (ordinary NixOS list-option semantics, same as
`claudeMarketplaces`). An agent that wants extra plugins AND the
defaults should list both `skill-creator@claude-plugins-official`
and `base@hyperhive` alongside them.
'';
};
options.hyperhive.claudePluginsAutoUpdate = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
When true, the harness runs `claude plugin marketplace update`
before installing plugins at boot, pulling the latest index from
all configured marketplaces. Disabled by default most agents
want pinned plugin versions and the network round-trip adds to
boot time. Enable for agents that should always install the latest
available version of their plugins.
'';
};
config = {
environment.etc."hyperhive/claude-plugins.json".text =
builtins.toJSON config.hyperhive.claudePlugins;
environment.etc."hyperhive/claude-marketplaces.json".text =
builtins.toJSON config.hyperhive.claudeMarketplaces;
environment.etc."hyperhive/claude-plugins-auto-update.json".text =
builtins.toJSON config.hyperhive.claudePluginsAutoUpdate;
# Hive-enforced claude settings. claude-code auto-discovers managed
# settings at this canonical Linux path (precedence #1, read-only,
# un-overridable by user/project/CLI) — so the harness doesn't pass
# `--settings` or copy the blob per turn. effortLevel is
# deliberately NOT shipped here: effort is controlled live via the
# `--effort` CLI flag (HIVE_DEFAULT_EFFORT / the per-agent UI slider),
# which managed scope would otherwise override and lock.
# Hive-enforced settings merged with a per-agent `env` block at BUILD
# time via `jq` (not eval-time `readFile`, which would be import-from-
# derivation). The `env` block is always present: `baseClaudeEnv` sets
# behaviour flags and the remote-control session prefix for every agent;
# `otelSettingsEnv` is merged on top when OTEL is enabled. claude-code
# auto-discovers this file in every context (harness turn-loop AND
# `hivectl choom`) so no launch wrapper is needed.
environment.etc."claude-code/managed-settings.json".source =
let
baseSettings = "${config.hyperhive.packages.assets}/share/hyperhive/prompts/claude-settings.json";
# Merge base env (always) with OTEL env (when enabled). jq is always
# run — `baseClaudeEnv` contains per-agent values (e.g.
# CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX) that can't live in the
# static store asset.
allEnv =
baseClaudeEnv
// lib.optionalAttrs otelCfg.enable otelSettingsEnv
// lib.optionalAttrs (otelCfg.enable && otelCfg.debug) {
# SDK diagnostics — noisy; only on when services.hyperhive.otel.debug = true.
CLAUDE_CODE_OTEL_DIAG_STDERR = "1";
};
in
pkgs.runCommand "managed-settings.json" { nativeBuildInputs = [ pkgs.jq ]; } ''
jq --argjson env ${lib.escapeShellArg (builtins.toJSON allEnv)} \
'. + { env: $env }' ${baseSettings} > "$out"
'';
# Seed claude's onboarding + per-project trust state once. claude only
# marks `hasCompletedOnboarding` (global) and the project trust dialog
# as accepted when run *interactively*; the harness only ever runs it
# headless (`--print`) and `claude auth login` doesn't set them either.
# So the first interactive launch (`hivectl choom`) would drop the
# operator into the onboarding/trust walkthrough despite valid OAuth
# creds. This oneshot is the single place hyperhive touches
# `~/.claude.json`: it runs before the harness (so nothing races it),
# is idempotent (skips when the flags are already set), and is
# best-effort (`before`, not a hard dep — a failed seed leaves the file
# untouched and the harness still starts). Credentials live in the
# separate `~/.claude/.credentials.json`, so this never touches secrets.
systemd.services.hive-claude-onboarding = {
description = "Seed claude onboarding + project-trust so choom skips the walkthrough";
wantedBy = [ "multi-user.target" ];
before = [ "hive-agent.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = userName;
Group = userName;
ExecStart = pkgs.writeShellScript "hive-claude-onboarding" ''
set -eu
umask 077
cfg=${homeDir}/.claude.json
dir=/agents/${userName}/state
base='{}'
[ -s "$cfg" ] && base="$(cat "$cfg")"
# Idempotent: nothing to do when already onboarded + trusted.
if printf '%s' "$base" | ${pkgs.jq}/bin/jq -e \
--arg d "$dir" \
'.hasCompletedOnboarding == true and (.projects[$d].hasTrustDialogAccepted == true)' \
>/dev/null 2>&1; then
exit 0
fi
printf '%s' "$base" | ${pkgs.jq}/bin/jq \
--arg d "$dir" \
'.hasCompletedOnboarding = true
| .projects[$d].hasTrustDialogAccepted = true
| .projects[$d].hasCompletedProjectOnboarding = true' \
> "$cfg.tmp"
mv "$cfg.tmp" "$cfg"
chmod 0600 "$cfg"
'';
};
};
};
}