diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 299a16db..e586b7ff 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -325,7 +325,7 @@ labels (`hive`, `swarm`, …) ride on the resource via -Cadence follows `metricIntervalMs` (default 60s). Transport is OTLP/HTTP +The export cadence is a fixed 60s. Transport is OTLP/HTTP (JSON) to the hive collector's bridge address, with no auth header — that first hop is unauthenticated for every producer on this host, and the upstream credential stays on the swarm tier. @@ -409,8 +409,8 @@ window used by agent** panel (health tab) divides one by the other. Resource attributes (`service.name`, `agent`, `swarm`) come from the same container-wide `OTEL_RESOURCE_ATTRIBUTES` as everything else in this section, and the swarm's collector adds `hive` — nothing extra to configure. -Cadence follows `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` (default 60s, same variable + default as -`hive-c0re`'s container-resource export above) — that only controls how often +Cadence follows `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` (default 60s, the same +cadence as `hive-c0re`'s container-resource export above). That only controls how often the harness flushes the batched points to the collector, not how often it records them (every turn, always). diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 05e74ddc..46dab331 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -555,10 +555,6 @@ struct OtelConfig { endpoint: String, protocol: String, extra_resource_attributes: Option, - metric_interval_ms: Option, - /// `HYPERHIVE_OTEL_DEBUG=1` → `services.hyperhive.agent.otel.debug = true` → - /// `CLAUDE_CODE_OTEL_DIAG_STDERR=1` in every agent's env. - debug: bool, } /// Read the hive-wide OTEL config from env, or `None` when OTEL is off. @@ -578,19 +574,10 @@ fn otel_config() -> Option { let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES") .ok() .filter(|v| !v.is_empty()); - let metric_interval_ms = std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|v| *v > 0); - let debug = std::env::var("HYPERHIVE_OTEL_DEBUG") - .ok() - .is_some_and(|v| v == "1"); Some(OtelConfig { endpoint, protocol, extra_resource_attributes, - metric_interval_ms, - debug, }) } @@ -902,17 +889,6 @@ where esc(attrs) ); } - if let Some(ms) = otel.metric_interval_ms { - // Int option — emit a bare numeric literal (no quotes). `ms` is a - // parsed u64, so it can't inject anything into the rendered nix. - let _ = writeln!( - out, - " services.hyperhive.agent.otel.metricIntervalMs = {ms};" - ); - } - if otel.debug { - out.push_str(" services.hyperhive.agent.otel.debug = true;\n"); - } } // Agent-facing service URLs (`services.hyperhive.agent.forge.url`, `services.hyperhive.agent.matrix.url`): // emit the host's real values as build-time agent config, the same diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index 9fd8ec90..36ae37bb 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -40,8 +40,8 @@ use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use super::container_stats::{self, ContainerResource}; use super::warnings; -/// Default export cadence when `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` is unset. -const DEFAULT_INTERVAL: Duration = Duration::from_mins(1); +/// Export cadence, and the snapshot refresher's. +const EXPORT_INTERVAL: Duration = Duration::from_mins(1); /// Latest container-stats snapshot: written by the async refresher, read by /// the sync observable-instrument callbacks. @@ -69,7 +69,7 @@ pub fn spawn_exporter(hyperhive_flake: &str) { let snapshot = SNAPSHOT .get_or_init(|| Arc::new(Mutex::new(Vec::new()))) .clone(); - let interval = interval(); + let interval = EXPORT_INTERVAL; // Async refresher: keep the shared snapshot current for the sync callbacks. let refresh = snapshot.clone(); @@ -124,10 +124,9 @@ fn build_provider( .with_protocol(Protocol::HttpJson) .build() .context("build OTLP metric exporter")?; - // Drive the reader's export at `interval` so `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` - // is the real export cadence (not just the snapshot-refresh cadence). The - // refresher runs at the same interval, gather-first, so the snapshot is - // populated before the first export. + // Drive the reader's export at `interval`, the same cadence as the + // refresher. The refresher gathers first, so the snapshot is populated + // before the first export. let reader = PeriodicReader::builder(exporter) .with_interval(interval) .build(); @@ -400,15 +399,6 @@ fn endpoint() -> Option { .filter(|s| !s.is_empty()) } -/// Export cadence from `HYPERHIVE_OTEL_METRIC_INTERVAL_MS`, else the default. -fn interval() -> Duration { - std::env::var("HYPERHIVE_OTEL_METRIC_INTERVAL_MS") - .ok() - .and_then(|s| s.trim().parse::().ok()) - .filter(|ms| *ms > 0) - .map_or(DEFAULT_INTERVAL, Duration::from_millis) -} - /// Extra resource attributes from `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES` /// (`key=value,key=value`), same env the agent-side export reads. fn resource_attributes() -> Vec<(String, String)> { diff --git a/nix/agent-modules/claude-settings.nix b/nix/agent-modules/claude-settings.nix index 944ef489..c0dc3d6a 100644 --- a/nix/agent-modules/claude-settings.nix +++ b/nix/agent-modules/claude-settings.nix @@ -219,7 +219,7 @@ in baseClaudeEnv // lib.optionalAttrs otelCfg.enable otelSettingsEnv // lib.optionalAttrs (otelCfg.enable && otelCfg.debug) { - # SDK diagnostics — noisy; only on when services.hyperhive.otel.debug = true. + # SDK diagnostics — noisy; only on when services.hyperhive.agent.otel.debug = true. CLAUDE_CODE_OTEL_DIAG_STDERR = "1"; }; in diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index ffbfdb1c..03f889fb 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -51,6 +51,7 @@ in ./memory-cap.nix ./network.nix ./otel.nix + ./otel-removed.nix ./packages.nix ./queue.nix ./queue-identity.nix diff --git a/nix/agent-modules/otel-removed.nix b/nix/agent-modules/otel-removed.nix new file mode 100644 index 00000000..8c7802cc --- /dev/null +++ b/nix/agent-modules/otel-removed.nix @@ -0,0 +1,27 @@ +# Removal shim for `services.hyperhive.agent.otel.metricIntervalMs`. +# +# A definition is ignored and produces a warning, so an agent config, or a +# meta flake rendered before the option went, keeps evaluating. claude runs +# at the OTEL SDK's default export cadence. +# +# Self-contained and deletable: one file to remove when the deprecation +# window closes. +{ lib, config, ... }: +let + value = config.services.hyperhive.agent.otel.metricIntervalMs; +in +{ + options.services.hyperhive.agent.otel.metricIntervalMs = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + visible = false; + internal = true; + description = "Removed and ignored. `null` is \"not set\"."; + }; + + config.warnings = lib.optional (value != null) '' + services.hyperhive.agent.otel.metricIntervalMs is removed and ignored: + claude exports metrics at the SDK default of 60 seconds. Remove this + setting. + ''; +} diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index f42618ab..f080c8ea 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -1,5 +1,6 @@ # OpenTelemetry wiring for an agent container: the per-agent options the -# meta flake injects (host-driven from `services.hyperhive.otel.*`), the +# meta flake injects (host-driven from `services.hyperhive.otel.*`) plus +# `debug`, which the agent's own config sets, the # environment every OTLP producer inside the container reads, and the # collector that forwards this container's journal to the hive. # @@ -62,9 +63,6 @@ let # silently drop without a deltatocumulative processor. OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; OTEL_RESOURCE_ATTRIBUTES = resourceAttributes; - } - // lib.optionalAttrs (cfg.metricIntervalMs != null) { - OTEL_METRIC_EXPORT_INTERVAL = toString cfg.metricIntervalMs; }; # The exporter's NAME picks the wire protocol, so `protocol` has to be @@ -131,25 +129,14 @@ in ''; }; - metricIntervalMs = lib.mkOption { - type = lib.types.nullOr lib.types.ints.positive; - default = null; - internal = true; - description = '' - Metric export interval in milliseconds, set as - `OTEL_METRIC_EXPORT_INTERVAL`. Null leaves the SDK default (60s - for Claude Code). Host-driven via - `services.hyperhive.otel.metricIntervalMs`. - ''; - }; - debug = lib.mkOption { type = lib.types.bool; default = false; - internal = true; description = '' - Emit OTEL SDK diagnostics to stderr (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`). - Host-driven via `services.hyperhive.otel.debug`. + Emit claude's OTEL SDK diagnostics to stderr + (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`). Set it in this agent's own + config while troubleshooting its collector connection; it is noisy, + so leave it off otherwise. Only takes effect when `enable` is true. ''; }; }; diff --git a/nix/host-modules/c0re-options-removed.nix b/nix/host-modules/c0re-options-removed.nix deleted file mode 100644 index 00e7cc87..00000000 --- a/nix/host-modules/c0re-options-removed.nix +++ /dev/null @@ -1,46 +0,0 @@ -# Removal shim for `services.hyperhive.c0re.operatorPronouns` and -# `services.hyperhive.c0re.contextWindowTokens`. -# -# A definition of either is ignored and produces a warning, so a host that -# still sets one keeps evaluating. The agents' system prompt -# (hive-agent/prompts/system.md) names the operator's pronouns itself, and -# the harness (hive-agent/src/harness_state.rs) holds the per-model context -# windows it uses until a turn reports the model's own. -# -# Self-contained and deletable, like ./swarm-peers-removed.nix: one file to -# remove when the deprecation window closes. -{ lib, config, ... }: -let - cfg = config.services.hyperhive.c0re; -in -{ - options.services.hyperhive.c0re = { - operatorPronouns = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - visible = false; - internal = true; - description = "Removed and ignored. `null` is \"not set\"."; - }; - contextWindowTokens = lib.mkOption { - type = lib.types.nullOr (lib.types.attrsOf lib.types.int); - default = null; - visible = false; - internal = true; - description = "Removed and ignored. `null` is \"not set\"."; - }; - }; - - config.warnings = - lib.optional (cfg.operatorPronouns != null) '' - services.hyperhive.c0re.operatorPronouns is removed and ignored: the - operator's pronouns will be a per-operator profile setting. Until then - every agent's system prompt says she/her. Remove this setting. - '' - ++ lib.optional (cfg.contextWindowTokens != null) '' - services.hyperhive.c0re.contextWindowTokens is removed and ignored: the - agent harness uses the context window the model reports after each - turn, and before the first one its own defaults (haiku 200000, sonnet - 1000000, opus 1000000 tokens). Remove this setting. - ''; -} diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index be88c0b0..ef1a1760 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -85,7 +85,7 @@ ./swarm-victoriametrics.nix ./swarm-wireguard.nix ./swarm.nix - ./c0re-options-removed.nix + ./hive-options-removed.nix ./host-ca-removed.nix ./nats-callout-removed.nix ./swarm-required-services.nix diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index b114a898..291ea52c 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -125,12 +125,6 @@ in # The rule: the collector holding the credential achieves nothing while # anything else hands out a copy, so there is exactly one holder and it # is on the host. - // lib.optionalAttrs (otel.metricIntervalMs != null) { - HYPERHIVE_OTEL_METRIC_INTERVAL_MS = toString otel.metricIntervalMs; - } - // lib.optionalAttrs otel.debug { - HYPERHIVE_OTEL_DEBUG = "1"; - } ) // { # In-cluster forge URL — the gateway vhost (`forge.`), which diff --git a/nix/host-modules/hive-options-removed.nix b/nix/host-modules/hive-options-removed.nix new file mode 100644 index 00000000..a3902803 --- /dev/null +++ b/nix/host-modules/hive-options-removed.nix @@ -0,0 +1,62 @@ +# Removal shim for four host options: `c0re.operatorPronouns`, +# `c0re.contextWindowTokens`, `otel.metricIntervalMs` and `otel.debug`. +# +# A definition of any of them is ignored and produces a warning, so a host +# that still sets one keeps evaluating. The agents' system prompt +# (hive-agent/prompts/system.md) names the operator's pronouns itself, the +# harness (hive-agent/src/harness_state.rs) holds the per-model context +# windows it uses until a turn reports the model's own, every OTEL exporter +# on the hive runs at its SDK's default cadence, and an agent's own config +# sets `services.hyperhive.agent.otel.debug`. +# +# Self-contained and deletable, like ./swarm-peers-removed.nix: one file to +# remove when the deprecation window closes. +{ lib, config, ... }: +let + cfg = config.services.hyperhive; + + # `null` is "not set". + ignored = + type: + lib.mkOption { + type = lib.types.nullOr type; + default = null; + visible = false; + internal = true; + description = "Removed and ignored."; + }; + + removed = + path: value: why: + lib.optional (value != null) '' + services.hyperhive.${path} is removed and ignored: ${why} Remove this + setting. + ''; +in +{ + options.services.hyperhive = { + c0re.operatorPronouns = ignored lib.types.str; + c0re.contextWindowTokens = ignored (lib.types.attrsOf lib.types.int); + otel.metricIntervalMs = ignored lib.types.ints.positive; + otel.debug = ignored lib.types.bool; + }; + + config.warnings = + removed "c0re.operatorPronouns" cfg.c0re.operatorPronouns '' + the operator's pronouns will be a per-operator profile setting. Until + then every agent's system prompt says she/her. + '' + ++ removed "c0re.contextWindowTokens" cfg.c0re.contextWindowTokens '' + the agent harness uses the context window the model reports after each + turn, and before the first one its own defaults (haiku 200000, sonnet + 1000000, opus 1000000 tokens). + '' + ++ removed "otel.metricIntervalMs" cfg.otel.metricIntervalMs '' + claude in every agent and hive-c0re's container-resource exporter both + export metrics at the SDK default of 60 seconds. + '' + ++ removed "otel.debug" cfg.otel.debug '' + to see one agent's OTEL SDK diagnostics, set + services.hyperhive.agent.otel.debug = true in that agent's own config. + ''; +} diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index dda48021..5ab1e6de 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -158,18 +158,6 @@ in ''; }; - debug = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Emit OTEL SDK diagnostic messages to every agent's stderr by - setting `CLAUDE_CODE_OTEL_DIAG_STDERR=1`. Useful when - troubleshooting collector connectivity or endpoint config; - leave off in normal operation to avoid noise in agent logs. - Only meaningful when `enable` is true. - ''; - }; - collector.upstreamHeaderName = lib.mkOption { type = lib.types.str; default = "Authorization"; @@ -306,23 +294,6 @@ in copied into the store and never passed in argv. ''; }; - - metricIntervalMs = lib.mkOption { - type = lib.types.nullOr lib.types.ints.positive; - default = null; - example = 10000; - description = '' - Metric export interval in milliseconds, set as - `OTEL_METRIC_EXPORT_INTERVAL` for every agent. Claude Code's - default is 60000 (60s). Leave `null` to use that default. - - Each agent runs claude as a short-lived per-turn process; claude - force-flushes metrics on shutdown, so this is not required for - metrics to be exported, but a lower value gives more frequent - intermediate flushes within long turns. Cosmetic, not a - correctness knob. - ''; - }; }; config = lib.mkIf config.services.hyperhive.otel.enable ( diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index ad7df33b..5d75854b 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -38,11 +38,13 @@ let # The same agent over the other wire protocol. An exporter's NAME is what # selects it, so this is where a defined exporter and the pipeline's - # reference to it can drift apart. + # reference to it can drift apart. Also an agent config still setting the + # removed metric interval. agentOtelGrpc = agent { otel.enable = true; otel.endpoint = agentBridge; otel.protocol = "grpc"; + otel.metricIntervalMs = 10000; }; agentNoOtel = shared.agentBare; @@ -116,6 +118,22 @@ let in (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ]; } + { + # Control: `agentOtel` sets no interval. The endpoint lookup shows the + # environment it reads is the one the OTEL SDKs get. + name = "an agent config setting the removed metric interval evaluates, warns, and exports none"; + ok = + let + warned = lib.any ( + w: lib.hasInfix "services.hyperhive.agent.otel.metricIntervalMs is removed and ignored" w + ); + env = machine: machine.systemd.globalEnvironment; + in + warned agentOtelGrpc.warnings + && !(warned agentOtel.warnings) + && (env agentOtelGrpc) ? OTEL_EXPORTER_OTLP_ENDPOINT + && !((env agentOtelGrpc) ? OTEL_METRIC_EXPORT_INTERVAL); + } ]; in runGroup "agent-otel" cases diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index c32fc617..00b48a83 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -49,11 +49,13 @@ let # Every hive defaults the agents' queue address to the queue's name, so the # only hive without one is a hive told to have none. Also the host still - # setting the two removed `c0re` options; no case on its queue reads them. + # setting the four removed host options; no case on its queue reads them. noAgentQueue = hiveWithController { deploy.hive-controller.queue.agentNatsUrl = null; c0re.operatorPronouns = "they/them"; c0re.contextWindowTokens.sonnet = 900000; + otel.metricIntervalMs = 10000; + otel.debug = true; }; # On the store's host, which mints the runner's store identity. @@ -243,17 +245,19 @@ let !(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT); } { - # Control: `bare` sets neither option, and its serve.json carries the + # Control: `bare` sets none of them, and its serve.json carries the # dashboard port, so the key lookup reads a real document. - name = "a definition of a removed c0re option evaluates, warns, and reaches no serve.json"; + name = "a definition of a removed host option evaluates and warns, and the c0re ones reach no serve.json"; ok = let warned = m: option: - lib.any (w: lib.hasInfix "services.hyperhive.c0re.${option} is removed and ignored" w) m.warnings; + lib.any (w: lib.hasInfix "services.hyperhive.${option} is removed and ignored" w) m.warnings; options = [ - "operatorPronouns" - "contextWindowTokens" + "c0re.operatorPronouns" + "c0re.contextWindowTokens" + "otel.metricIntervalMs" + "otel.debug" ]; hasKey = m: key: lib.hasInfix "\"${key}\":" m.environment.etc."hyperhive/serve.json".text; noRemovedKeys =