Watch
0
0
Fork
You've already forked hyperhive
0

remove otel.metricIntervalMs and the host-level otel.debug

Both are ignored with a warning through the same shim, now
nix/host-modules/hive-options-removed.nix.

- otel.metricIntervalMs: claude in every agent and hive-c0re's
  container-resource exporter run at the 60s default. The agent-level
  services.hyperhive.agent.otel.metricIntervalMs, set only by the meta
  render, goes too (warning shim in nix/agent-modules/otel-removed.nix).
- otel.debug: services.hyperhive.agent.otel.debug stays, documented and
  set in an agent's own config; the meta render no longer emits it.
This commit is contained in:
atlas 2026-10-16 17:29:33 +02:00 • committed by mara
commit 0a5f35bda3
14 changed files with 136 additions and 152 deletions

View file

@ -325,7 +325,7 @@ labels (`hive`, `swarm`, …) ride on the resource via
<!-- vale write-good.Passive = NO --> <!-- vale write-good.Passive = NO -->
Cadence follows `metricIntervalMs` (default 60s). Transport is OTLP/HTTP The export cadence is a fixed 60s. Transport is OTLP/HTTP
(JSON) to the hive collector's bridge address, with no auth header — that (JSON) to the hive collector's bridge address, with no auth header — that
first hop is unauthenticated for every producer on this host, and the upstream first hop is unauthenticated for every producer on this host, and the upstream
credential stays on the swarm tier. credential stays on the swarm tier.
@ -409,8 +409,8 @@ window used by agent** panel (health tab) divides one by the other.
Resource attributes (`service.name`, `agent`, `swarm`) come from the Resource attributes (`service.name`, `agent`, `swarm`) come from the
same container-wide `OTEL_RESOURCE_ATTRIBUTES` as everything else in this same container-wide `OTEL_RESOURCE_ATTRIBUTES` as everything else in this
section, and the swarm's collector adds `hive` — nothing extra to configure. section, and the swarm's collector adds `hive` — nothing extra to configure.
Cadence follows `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` (default 60s, same variable + default as Cadence follows `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` (default 60s, the same
`hive-c0re`'s container-resource export above) — that only controls how often cadence as `hive-c0re`'s container-resource export above). That only controls how often
the harness flushes the batched points to the collector, not how often it the harness flushes the batched points to the collector, not how often it
records them (every turn, always). records them (every turn, always).

View file

@ -555,10 +555,6 @@ struct OtelConfig {
endpoint: String, endpoint: String,
protocol: String, protocol: String,
extra_resource_attributes: Option<String>, extra_resource_attributes: Option<String>,
metric_interval_ms: Option<u64>,
/// `HYPERHIVE_OTEL_DEBUG=1` → `services.hyperhive.agent.otel.debug = true` →
/// `CLAUDE_CODE_OTEL_DIAG_STDERR=1` in every agent's env.
debug: bool,
} }
/// Read the hive-wide OTEL config from env, or `None` when OTEL is off. /// Read the hive-wide OTEL config from env, or `None` when OTEL is off.
@ -578,19 +574,10 @@ fn otel_config() -> Option<OtelConfig> {
let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES") let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES")
.ok() .ok()
.filter(|v| !v.is_empty()); .filter(|v| !v.is_empty());
let metric_interval_ms = std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.filter(|v| *v > 0);
let debug = std::env::var("HYPERHIVE_OTEL_DEBUG")
.ok()
.is_some_and(|v| v == "1");
Some(OtelConfig { Some(OtelConfig {
endpoint, endpoint,
protocol, protocol,
extra_resource_attributes, extra_resource_attributes,
metric_interval_ms,
debug,
}) })
} }
@ -902,17 +889,6 @@ where
esc(attrs) esc(attrs)
); );
} }
if let Some(ms) = otel.metric_interval_ms {
// Int option — emit a bare numeric literal (no quotes). `ms` is a
// parsed u64, so it can't inject anything into the rendered nix.
let _ = writeln!(
out,
" services.hyperhive.agent.otel.metricIntervalMs = {ms};"
);
}
if otel.debug {
out.push_str(" services.hyperhive.agent.otel.debug = true;\n");
}
} }
// Agent-facing service URLs (`services.hyperhive.agent.forge.url`, `services.hyperhive.agent.matrix.url`): // Agent-facing service URLs (`services.hyperhive.agent.forge.url`, `services.hyperhive.agent.matrix.url`):
// emit the host's real values as build-time agent config, the same // emit the host's real values as build-time agent config, the same

View file

@ -40,8 +40,8 @@ use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
use super::container_stats::{self, ContainerResource}; use super::container_stats::{self, ContainerResource};
use super::warnings; use super::warnings;
/// Default export cadence when `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` is unset. /// Export cadence, and the snapshot refresher's.
const DEFAULT_INTERVAL: Duration = Duration::from_mins(1); const EXPORT_INTERVAL: Duration = Duration::from_mins(1);
/// Latest container-stats snapshot: written by the async refresher, read by /// Latest container-stats snapshot: written by the async refresher, read by
/// the sync observable-instrument callbacks. /// the sync observable-instrument callbacks.
@ -69,7 +69,7 @@ pub fn spawn_exporter(hyperhive_flake: &str) {
let snapshot = SNAPSHOT let snapshot = SNAPSHOT
.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) .get_or_init(|| Arc::new(Mutex::new(Vec::new())))
.clone(); .clone();
let interval = interval(); let interval = EXPORT_INTERVAL;
// Async refresher: keep the shared snapshot current for the sync callbacks. // Async refresher: keep the shared snapshot current for the sync callbacks.
let refresh = snapshot.clone(); let refresh = snapshot.clone();
@ -124,10 +124,9 @@ fn build_provider(
.with_protocol(Protocol::HttpJson) .with_protocol(Protocol::HttpJson)
.build() .build()
.context("build OTLP metric exporter")?; .context("build OTLP metric exporter")?;
// Drive the reader's export at `interval` so `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` // Drive the reader's export at `interval`, the same cadence as the
// is the real export cadence (not just the snapshot-refresh cadence). The // refresher. The refresher gathers first, so the snapshot is populated
// refresher runs at the same interval, gather-first, so the snapshot is // before the first export.
// populated before the first export.
let reader = PeriodicReader::builder(exporter) let reader = PeriodicReader::builder(exporter)
.with_interval(interval) .with_interval(interval)
.build(); .build();
@ -400,15 +399,6 @@ fn endpoint() -> Option<String> {
.filter(|s| !s.is_empty()) .filter(|s| !s.is_empty())
} }
/// Export cadence from `HYPERHIVE_OTEL_METRIC_INTERVAL_MS`, else the default.
fn interval() -> Duration {
std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS")
.ok()
.and_then(|s| s.trim().parse::<u64>().ok())
.filter(|ms| *ms > 0)
.map_or(DEFAULT_INTERVAL, Duration::from_millis)
}
/// Extra resource attributes from `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES` /// Extra resource attributes from `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`
/// (`key=value,key=value`), same env the agent-side export reads. /// (`key=value,key=value`), same env the agent-side export reads.
fn resource_attributes() -> Vec<(String, String)> { fn resource_attributes() -> Vec<(String, String)> {

View file

@ -219,7 +219,7 @@ in
baseClaudeEnv baseClaudeEnv
// lib.optionalAttrs otelCfg.enable otelSettingsEnv // lib.optionalAttrs otelCfg.enable otelSettingsEnv
// lib.optionalAttrs (otelCfg.enable && otelCfg.debug) { // lib.optionalAttrs (otelCfg.enable && otelCfg.debug) {
# SDK diagnostics — noisy; only on when services.hyperhive.otel.debug = true. # SDK diagnostics — noisy; only on when services.hyperhive.agent.otel.debug = true.
CLAUDE_CODE_OTEL_DIAG_STDERR = "1"; CLAUDE_CODE_OTEL_DIAG_STDERR = "1";
}; };
in in

View file

@ -51,6 +51,7 @@ in
./memory-cap.nix ./memory-cap.nix
./network.nix ./network.nix
./otel.nix ./otel.nix
./otel-removed.nix
./packages.nix ./packages.nix
./queue.nix ./queue.nix
./queue-identity.nix ./queue-identity.nix

View file

@ -0,0 +1,27 @@
# Removal shim for `services.hyperhive.agent.otel.metricIntervalMs`.
#
# A definition is ignored and produces a warning, so an agent config, or a
# meta flake rendered before the option went, keeps evaluating. claude runs
# at the OTEL SDK's default export cadence.
#
# Self-contained and deletable: one file to remove when the deprecation
# window closes.
{ lib, config, ... }:
let
value = config.services.hyperhive.agent.otel.metricIntervalMs;
in
{
options.services.hyperhive.agent.otel.metricIntervalMs = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
visible = false;
internal = true;
description = "Removed and ignored. `null` is \"not set\".";
};
config.warnings = lib.optional (value != null) ''
services.hyperhive.agent.otel.metricIntervalMs is removed and ignored:
claude exports metrics at the SDK default of 60 seconds. Remove this
setting.
'';
}

View file

@ -1,5 +1,6 @@
# OpenTelemetry wiring for an agent container: the per-agent options the # OpenTelemetry wiring for an agent container: the per-agent options the
# meta flake injects (host-driven from `services.hyperhive.otel.*`), the # meta flake injects (host-driven from `services.hyperhive.otel.*`) plus
# `debug`, which the agent's own config sets, the
# environment every OTLP producer inside the container reads, and the # environment every OTLP producer inside the container reads, and the
# collector that forwards this container's journal to the hive. # collector that forwards this container's journal to the hive.
# #
@ -62,9 +63,6 @@ let
# silently drop without a deltatocumulative processor. # silently drop without a deltatocumulative processor.
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
OTEL_RESOURCE_ATTRIBUTES = resourceAttributes; OTEL_RESOURCE_ATTRIBUTES = resourceAttributes;
}
// lib.optionalAttrs (cfg.metricIntervalMs != null) {
OTEL_METRIC_EXPORT_INTERVAL = toString cfg.metricIntervalMs;
}; };
# The exporter's NAME picks the wire protocol, so `protocol` has to be # The exporter's NAME picks the wire protocol, so `protocol` has to be
@ -131,25 +129,14 @@ in
''; '';
}; };
metricIntervalMs = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
internal = true;
description = ''
Metric export interval in milliseconds, set as
`OTEL_METRIC_EXPORT_INTERVAL`. Null leaves the SDK default (60s
for Claude Code). Host-driven via
`services.hyperhive.otel.metricIntervalMs`.
'';
};
debug = lib.mkOption { debug = lib.mkOption {
type = lib.types.bool; type = lib.types.bool;
default = false; default = false;
internal = true;
description = '' description = ''
Emit OTEL SDK diagnostics to stderr (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`). Emit claude's OTEL SDK diagnostics to stderr
Host-driven via `services.hyperhive.otel.debug`. (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`). Set it in this agent's own
config while troubleshooting its collector connection; it is noisy,
so leave it off otherwise. Only takes effect when `enable` is true.
''; '';
}; };
}; };

View file

@ -1,46 +0,0 @@
# Removal shim for `services.hyperhive.c0re.operatorPronouns` and
# `services.hyperhive.c0re.contextWindowTokens`.
#
# A definition of either is ignored and produces a warning, so a host that
# still sets one keeps evaluating. The agents' system prompt
# (hive-agent/prompts/system.md) names the operator's pronouns itself, and
# the harness (hive-agent/src/harness_state.rs) holds the per-model context
# windows it uses until a turn reports the model's own.
#
# Self-contained and deletable, like ./swarm-peers-removed.nix: one file to
# remove when the deprecation window closes.
{ lib, config, ... }:
let
cfg = config.services.hyperhive.c0re;
in
{
options.services.hyperhive.c0re = {
operatorPronouns = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
visible = false;
internal = true;
description = "Removed and ignored. `null` is \"not set\".";
};
contextWindowTokens = lib.mkOption {
type = lib.types.nullOr (lib.types.attrsOf lib.types.int);
default = null;
visible = false;
internal = true;
description = "Removed and ignored. `null` is \"not set\".";
};
};
config.warnings =
lib.optional (cfg.operatorPronouns != null) ''
services.hyperhive.c0re.operatorPronouns is removed and ignored: the
operator's pronouns will be a per-operator profile setting. Until then
every agent's system prompt says she/her. Remove this setting.
''
++ lib.optional (cfg.contextWindowTokens != null) ''
services.hyperhive.c0re.contextWindowTokens is removed and ignored: the
agent harness uses the context window the model reports after each
turn, and before the first one its own defaults (haiku 200000, sonnet
1000000, opus 1000000 tokens). Remove this setting.
'';
}

View file

@ -85,7 +85,7 @@
./swarm-victoriametrics.nix ./swarm-victoriametrics.nix
./swarm-wireguard.nix ./swarm-wireguard.nix
./swarm.nix ./swarm.nix
./c0re-options-removed.nix ./hive-options-removed.nix
./host-ca-removed.nix ./host-ca-removed.nix
./nats-callout-removed.nix ./nats-callout-removed.nix
./swarm-required-services.nix ./swarm-required-services.nix

View file

@ -125,12 +125,6 @@ in
# The rule: the collector holding the credential achieves nothing while # The rule: the collector holding the credential achieves nothing while
# anything else hands out a copy, so there is exactly one holder and it # anything else hands out a copy, so there is exactly one holder and it
# is on the host. # is on the host.
// lib.optionalAttrs (otel.metricIntervalMs != null) {
HYPERHIVE_OTEL_METRIC_INTERVAL_MS = toString otel.metricIntervalMs;
}
// lib.optionalAttrs otel.debug {
HYPERHIVE_OTEL_DEBUG = "1";
}
) )
// { // {
# In-cluster forge URL — the gateway vhost (`forge.<domain>`), which # In-cluster forge URL — the gateway vhost (`forge.<domain>`), which

View file

@ -0,0 +1,62 @@
# Removal shim for four host options: `c0re.operatorPronouns`,
# `c0re.contextWindowTokens`, `otel.metricIntervalMs` and `otel.debug`.
#
# A definition of any of them is ignored and produces a warning, so a host
# that still sets one keeps evaluating. The agents' system prompt
# (hive-agent/prompts/system.md) names the operator's pronouns itself, the
# harness (hive-agent/src/harness_state.rs) holds the per-model context
# windows it uses until a turn reports the model's own, every OTEL exporter
# on the hive runs at its SDK's default cadence, and an agent's own config
# sets `services.hyperhive.agent.otel.debug`.
#
# Self-contained and deletable, like ./swarm-peers-removed.nix: one file to
# remove when the deprecation window closes.
{ lib, config, ... }:
let
cfg = config.services.hyperhive;
# `null` is "not set".
ignored =
type:
lib.mkOption {
type = lib.types.nullOr type;
default = null;
visible = false;
internal = true;
description = "Removed and ignored.";
};
removed =
path: value: why:
lib.optional (value != null) ''
services.hyperhive.${path} is removed and ignored: ${why} Remove this
setting.
'';
in
{
options.services.hyperhive = {
c0re.operatorPronouns = ignored lib.types.str;
c0re.contextWindowTokens = ignored (lib.types.attrsOf lib.types.int);
otel.metricIntervalMs = ignored lib.types.ints.positive;
otel.debug = ignored lib.types.bool;
};
config.warnings =
removed "c0re.operatorPronouns" cfg.c0re.operatorPronouns ''
the operator's pronouns will be a per-operator profile setting. Until
then every agent's system prompt says she/her.
''
++ removed "c0re.contextWindowTokens" cfg.c0re.contextWindowTokens ''
the agent harness uses the context window the model reports after each
turn, and before the first one its own defaults (haiku 200000, sonnet
1000000, opus 1000000 tokens).
''
++ removed "otel.metricIntervalMs" cfg.otel.metricIntervalMs ''
claude in every agent and hive-c0re's container-resource exporter both
export metrics at the SDK default of 60 seconds.
''
++ removed "otel.debug" cfg.otel.debug ''
to see one agent's OTEL SDK diagnostics, set
services.hyperhive.agent.otel.debug = true in that agent's own config.
'';
}

View file

@ -158,18 +158,6 @@ in
''; '';
}; };
debug = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Emit OTEL SDK diagnostic messages to every agent's stderr by
setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when
troubleshooting collector connectivity or endpoint config;
leave off in normal operation to avoid noise in agent logs.
Only meaningful when `enable` is true.
'';
};
collector.upstreamHeaderName = lib.mkOption { collector.upstreamHeaderName = lib.mkOption {
type = lib.types.str; type = lib.types.str;
default = "Authorization"; default = "Authorization";
@ -306,23 +294,6 @@ in
copied into the store and never passed in argv. copied into the store and never passed in argv.
''; '';
}; };
metricIntervalMs = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
example = 10000;
description = ''
Metric export interval in milliseconds, set as
`OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's
default is 60000 (60s). Leave `null` to use that default.
Each agent runs claude as a short-lived per-turn process; claude
force-flushes metrics on shutdown, so this is not required for
metrics to be exported, but a lower value gives more frequent
intermediate flushes within long turns. Cosmetic, not a
correctness knob.
'';
};
}; };
config = lib.mkIf config.services.hyperhive.otel.enable ( config = lib.mkIf config.services.hyperhive.otel.enable (

View file

@ -38,11 +38,13 @@ let
# The same agent over the other wire protocol. An exporter's NAME is what # The same agent over the other wire protocol. An exporter's NAME is what
# selects it, so this is where a defined exporter and the pipeline's # selects it, so this is where a defined exporter and the pipeline's
# reference to it can drift apart. # reference to it can drift apart. Also an agent config still setting the
# removed metric interval.
agentOtelGrpc = agent { agentOtelGrpc = agent {
otel.enable = true; otel.enable = true;
otel.endpoint = agentBridge; otel.endpoint = agentBridge;
otel.protocol = "grpc"; otel.protocol = "grpc";
otel.metricIntervalMs = 10000;
}; };
agentNoOtel = shared.agentBare; agentNoOtel = shared.agentBare;
@ -116,6 +118,22 @@ let
in in
(s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ]; (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ];
} }
{
# Control: `agentOtel` sets no interval. The endpoint lookup shows the
# environment it reads is the one the OTEL SDKs get.
name = "an agent config setting the removed metric interval evaluates, warns, and exports none";
ok =
let
warned = lib.any (
w: lib.hasInfix "services.hyperhive.agent.otel.metricIntervalMs is removed and ignored" w
);
env = machine: machine.systemd.globalEnvironment;
in
warned agentOtelGrpc.warnings
&& !(warned agentOtel.warnings)
&& (env agentOtelGrpc) ? OTEL_EXPORTER_OTLP_ENDPOINT
&& !((env agentOtelGrpc) ? OTEL_METRIC_EXPORT_INTERVAL);
}
]; ];
in in
runGroup "agent-otel" cases runGroup "agent-otel" cases

View file

@ -49,11 +49,13 @@ let
# Every hive defaults the agents' queue address to the queue's name, so the # Every hive defaults the agents' queue address to the queue's name, so the
# only hive without one is a hive told to have none. Also the host still # only hive without one is a hive told to have none. Also the host still
# setting the two removed `c0re` options; no case on its queue reads them. # setting the four removed host options; no case on its queue reads them.
noAgentQueue = hiveWithController { noAgentQueue = hiveWithController {
deploy.hive-controller.queue.agentNatsUrl = null; deploy.hive-controller.queue.agentNatsUrl = null;
c0re.operatorPronouns = "they/them"; c0re.operatorPronouns = "they/them";
c0re.contextWindowTokens.sonnet = 900000; c0re.contextWindowTokens.sonnet = 900000;
otel.metricIntervalMs = 10000;
otel.debug = true;
}; };
# On the store's host, which mints the runner's store identity. # On the store's host, which mints the runner's store identity.
@ -243,17 +245,19 @@ let
!(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT); !(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT);
} }
{ {
# Control: `bare` sets neither option, and its serve.json carries the # Control: `bare` sets none of them, and its serve.json carries the
# dashboard port, so the key lookup reads a real document. # dashboard port, so the key lookup reads a real document.
name = "a definition of a removed c0re option evaluates, warns, and reaches no serve.json"; name = "a definition of a removed host option evaluates and warns, and the c0re ones reach no serve.json";
ok = ok =
let let
warned = warned =
m: option: m: option:
lib.any (w: lib.hasInfix "services.hyperhive.c0re.${option} is removed and ignored" w) m.warnings; lib.any (w: lib.hasInfix "services.hyperhive.${option} is removed and ignored" w) m.warnings;
options = [ options = [
"operatorPronouns" "c0re.operatorPronouns"
"contextWindowTokens" "c0re.contextWindowTokens"
"otel.metricIntervalMs"
"otel.debug"
]; ];
hasKey = m: key: lib.hasInfix "\"${key}\":" m.environment.etc."hyperhive/serve.json".text; hasKey = m: key: lib.hasInfix "\"${key}\":" m.environment.etc."hyperhive/serve.json".text;
noRemovedKeys = noRemovedKeys =