feat(#1930): move otel stats export to host-level services.hyperhive.otel
This commit is contained in:
damocles
mara
parent
e18ddff0b0
commit
838cc9af9a
3 changed files with 303 additions and 38 deletions
|
|
@ -656,6 +656,48 @@ fn peer_ca_sources() -> Vec<String> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// Hive-wide OTEL config injected into every agent's build, read off
|
||||
/// hive-c0re's own unit env (set from `services.hyperhive.otel.*` in
|
||||
/// `nix/modules/hive-c0re.nix`). A present, non-empty
|
||||
/// `HYPERHIVE_OTEL_ENDPOINT` is the enable signal — the host module
|
||||
/// asserts the endpoint is set whenever `otel.enable` is true, so
|
||||
/// "endpoint present" == "OTEL on". The optional fields map to the
|
||||
/// matching host options and are only carried when set.
|
||||
struct OtelConfig {
|
||||
endpoint: String,
|
||||
protocol: String,
|
||||
extra_resource_attributes: Option<String>,
|
||||
headers_credential: Option<String>,
|
||||
}
|
||||
|
||||
/// Read the hive-wide OTEL config from env, or `None` when OTEL is off.
|
||||
/// Mirrors `hive_ca_source` — host state surfaced to the meta renderer
|
||||
/// so it can bake build-time `hyperhive.otel.*` config into each agent
|
||||
/// (the per-agent options harness-base.nix consumes). Returns `None`
|
||||
/// when the endpoint signal is absent so the renderer emits no
|
||||
/// `hyperhive.otel.*` lines and agents keep the disabled default.
|
||||
fn otel_config() -> Option<OtelConfig> {
|
||||
let endpoint = std::env::var("HYPERHIVE_OTEL_ENDPOINT")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())?;
|
||||
let protocol = std::env::var("HYPERHIVE_OTEL_PROTOCOL")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
.unwrap_or_else(|| "http/protobuf".to_owned());
|
||||
let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty());
|
||||
let headers_credential = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty());
|
||||
Some(OtelConfig {
|
||||
endpoint,
|
||||
protocol,
|
||||
extra_resource_attributes,
|
||||
headers_credential,
|
||||
})
|
||||
}
|
||||
|
||||
/// The ordered set of CA certs embedded next to the meta flake, as
|
||||
/// `(filename, host_source_path)`. The self-signed hive CA (when active)
|
||||
/// is `hive-ca.pem`; each peer CA is `peer-ca-<N>.pem` in declaration
|
||||
|
|
@ -907,6 +949,40 @@ where
|
|||
ca_refs.join(" ")
|
||||
);
|
||||
}
|
||||
// Hive-wide OTEL stats export (`services.hyperhive.otel.*`): inject the
|
||||
// build-time `hyperhive.otel.*` config harness-base.nix consumes (its
|
||||
// otelEnv + otelExecStart wrapper + LoadCredential). Host-driven, so
|
||||
// the same config lands on every agent; emitted only when enabled.
|
||||
// Mirrors the CA-cert injection above — host state -> build-time agent
|
||||
// module config.
|
||||
if let Some(otel) = otel_config() {
|
||||
let esc = |s: &str| s.replace('\\', "\\\\").replace('"', "\\\"");
|
||||
out.push_str(" hyperhive.otel.enable = true;\n");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" hyperhive.otel.endpoint = \"{}\";",
|
||||
esc(&otel.endpoint)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" hyperhive.otel.protocol = \"{}\";",
|
||||
esc(&otel.protocol)
|
||||
);
|
||||
if let Some(attrs) = &otel.extra_resource_attributes {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" hyperhive.otel.extraResourceAttributes = \"{}\";",
|
||||
esc(attrs)
|
||||
);
|
||||
}
|
||||
if let Some(cred) = &otel.headers_credential {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" hyperhive.otel.headersCredential = \"{}\";",
|
||||
esc(cred)
|
||||
);
|
||||
}
|
||||
}
|
||||
out.push_str(
|
||||
r#" # The harness service inside the container runs as a
|
||||
# non-root unix user named after the agent (`damocles`,
|
||||
|
|
@ -1445,4 +1521,89 @@ mod tests {
|
|||
"no certificateFiles reference without any CA signal:\n{without_ca}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_flake_injects_otel_when_signalled() {
|
||||
// services.hyperhive.otel.* -> HYPERHIVE_OTEL_* on hive-c0re's unit
|
||||
// -> injected as build-time hyperhive.otel.* into every agent. With
|
||||
// no endpoint signal, no hyperhive.otel lines are emitted (agents
|
||||
// keep the harness-base disabled default).
|
||||
//
|
||||
// SAFETY: single-threaded mutation of process env vars no other
|
||||
// test asserts on; restored before returning.
|
||||
let render = || {
|
||||
render_flake(
|
||||
"github:example/hyperhive",
|
||||
"path:/nix/store/aaaa-nixpkgs-source",
|
||||
"path:/nix/store/bbbb-nixpkgs-unstable-source",
|
||||
8000,
|
||||
"she/her",
|
||||
&std::collections::HashMap::new(),
|
||||
&[sample_spec("alice", false, 9001)],
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES");
|
||||
std::env::remove_var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL");
|
||||
std::env::set_var("HYPERHIVE_OTEL_ENDPOINT", "https://c.example/otel");
|
||||
std::env::set_var("HYPERHIVE_OTEL_PROTOCOL", "grpc");
|
||||
}
|
||||
let on_minimal = render();
|
||||
unsafe {
|
||||
std::env::set_var(
|
||||
"HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES",
|
||||
"deployment.environment=prod",
|
||||
);
|
||||
std::env::set_var(
|
||||
"HYPERHIVE_OTEL_HEADERS_CREDENTIAL",
|
||||
"/run/secrets/otel-headers",
|
||||
);
|
||||
}
|
||||
let on_full = render();
|
||||
unsafe {
|
||||
std::env::remove_var("HYPERHIVE_OTEL_ENDPOINT");
|
||||
std::env::remove_var("HYPERHIVE_OTEL_PROTOCOL");
|
||||
std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES");
|
||||
std::env::remove_var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL");
|
||||
}
|
||||
let off = render();
|
||||
|
||||
assert!(
|
||||
on_minimal.contains("hyperhive.otel.enable = true;"),
|
||||
"otel enable must be injected:\n{on_minimal}"
|
||||
);
|
||||
assert!(
|
||||
on_minimal.contains("hyperhive.otel.endpoint = \"https://c.example/otel\";"),
|
||||
"otel endpoint must be injected:\n{on_minimal}"
|
||||
);
|
||||
assert!(
|
||||
on_minimal.contains("hyperhive.otel.protocol = \"grpc\";"),
|
||||
"otel protocol must be injected:\n{on_minimal}"
|
||||
);
|
||||
// Optional fields absent when unset.
|
||||
assert!(
|
||||
!on_minimal.contains("hyperhive.otel.extraResourceAttributes"),
|
||||
"extraResourceAttributes must not appear when unset:\n{on_minimal}"
|
||||
);
|
||||
assert!(
|
||||
!on_minimal.contains("hyperhive.otel.headersCredential"),
|
||||
"headersCredential must not appear when unset:\n{on_minimal}"
|
||||
);
|
||||
|
||||
assert!(
|
||||
on_full.contains(
|
||||
"hyperhive.otel.extraResourceAttributes = \"deployment.environment=prod\";"
|
||||
),
|
||||
"extraResourceAttributes must be injected when set:\n{on_full}"
|
||||
);
|
||||
assert!(
|
||||
on_full.contains("hyperhive.otel.headersCredential = \"/run/secrets/otel-headers\";"),
|
||||
"headersCredential must be injected when set:\n{on_full}"
|
||||
);
|
||||
|
||||
assert!(
|
||||
!off.contains("hyperhive.otel"),
|
||||
"no otel lines when disabled:\n{off}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -187,6 +187,75 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# Hive-wide OTEL stats export. Set ONCE here at host level; the
|
||||
# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
|
||||
# HYPERHIVE_OTEL_* env exported below off hive-c0re's unit and injects
|
||||
# the matching `hyperhive.otel.*` build-time config into EVERY agent
|
||||
# (mirroring the CA-cert injection), so each agent's harness exports
|
||||
# its own Claude Code stats directly to the collector. There is no
|
||||
# per-agent opt-in — this is the single switch for the whole hive.
|
||||
options.services.hyperhive.otel = {
|
||||
enable = lib.mkEnableOption ''
|
||||
hive-wide export of every agent's Claude Code stats (token usage,
|
||||
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
|
||||
OpenTelemetry. One switch for all agents; each harness exports
|
||||
directly to the collector, so it keeps working even when hive-c0re
|
||||
is down
|
||||
'';
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "https://collector.example.com/otel";
|
||||
description = ''
|
||||
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`
|
||||
for every agent. Required when `enable` is true.
|
||||
'';
|
||||
};
|
||||
|
||||
protocol = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"http/protobuf"
|
||||
"http/json"
|
||||
"grpc"
|
||||
];
|
||||
default = "http/protobuf";
|
||||
description = ''
|
||||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||||
'';
|
||||
};
|
||||
|
||||
headersCredential = lib.mkOption {
|
||||
# `str`, not `path`: a `path`-typed relative literal is hash-copied
|
||||
# into the world-readable nix store at eval time, defeating the
|
||||
# point. Keep it a string + require an absolute runtime path so the
|
||||
# secret is only ever read from disk by systemd at start.
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/run/secrets/otel-headers";
|
||||
description = ''
|
||||
Absolute path to an operator-provided secret file whose contents
|
||||
become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
|
||||
`Authorization=Bearer <token>`). Loaded via systemd
|
||||
`LoadCredential` into each agent's unit-private credential store
|
||||
at runtime, so the token is never copied into the nix store or
|
||||
exposed in argv. Must be absolute. Leave null if the endpoint
|
||||
needs no auth header.
|
||||
'';
|
||||
};
|
||||
|
||||
extraResourceAttributes = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "deployment.environment=prod";
|
||||
description = ''
|
||||
Extra comma-separated entries appended to
|
||||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# Peer hives in the same swarm. Each entry declares a remote hive
|
||||
# reachable from this host. Serialised to JSON and injected as
|
||||
# `HYPERHIVE_PEERS` into the hive-c0re service and forwarded to agent
|
||||
|
|
@ -720,24 +789,31 @@ in
|
|||
config.services.hyperhive.swarm.wireguard.listenPort
|
||||
];
|
||||
|
||||
assertions = lib.mkIf config.services.hyperhive.swarm.wireguard.enable [
|
||||
{
|
||||
assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null;
|
||||
message = ''
|
||||
services.hyperhive.swarm.wireguard.enable requires
|
||||
services.hyperhive.swarm.wireguard.privateKeyFile to be set.
|
||||
Generate a key: wg genkey > /etc/wireguard/hive.key
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = config.services.hyperhive.swarm.wireguard.address != "";
|
||||
message = ''
|
||||
services.hyperhive.swarm.wireguard.enable requires
|
||||
services.hyperhive.swarm.wireguard.address to be set
|
||||
(e.g. "10.100.0.1/24").
|
||||
'';
|
||||
}
|
||||
];
|
||||
assertions =
|
||||
lib.optionals config.services.hyperhive.swarm.wireguard.enable [
|
||||
{
|
||||
assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null;
|
||||
message = ''
|
||||
services.hyperhive.swarm.wireguard.enable requires
|
||||
services.hyperhive.swarm.wireguard.privateKeyFile to be set.
|
||||
Generate a key: wg genkey > /etc/wireguard/hive.key
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = config.services.hyperhive.swarm.wireguard.address != "";
|
||||
message = ''
|
||||
services.hyperhive.swarm.wireguard.enable requires
|
||||
services.hyperhive.swarm.wireguard.address to be set
|
||||
(e.g. "10.100.0.1/24").
|
||||
'';
|
||||
}
|
||||
]
|
||||
++ lib.optionals config.services.hyperhive.otel.enable [
|
||||
{
|
||||
assertion = config.services.hyperhive.otel.endpoint != "";
|
||||
message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.hive-c0re = {
|
||||
description = "hyperhive coordinator daemon";
|
||||
|
|
@ -790,6 +866,26 @@ in
|
|||
// lib.optionalAttrs (config.services.hyperhive.swarmName != null) {
|
||||
HYPERHIVE_SWARM_NAME = config.services.hyperhive.swarmName;
|
||||
}
|
||||
// lib.optionalAttrs config.services.hyperhive.otel.enable (
|
||||
# Hive-wide OTEL config -> read by meta.rs::otel_config and
|
||||
# injected as build-time `hyperhive.otel.*` into every agent.
|
||||
# Endpoint presence is the enable signal on the meta side; the
|
||||
# optional fields are only emitted when set so absent values
|
||||
# don't render no-op env lines.
|
||||
let
|
||||
otel = config.services.hyperhive.otel;
|
||||
in
|
||||
{
|
||||
HYPERHIVE_OTEL_ENDPOINT = otel.endpoint;
|
||||
HYPERHIVE_OTEL_PROTOCOL = otel.protocol;
|
||||
}
|
||||
// lib.optionalAttrs (otel.extraResourceAttributes != "") {
|
||||
HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = otel.extraResourceAttributes;
|
||||
}
|
||||
// lib.optionalAttrs (otel.headersCredential != null) {
|
||||
HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential;
|
||||
}
|
||||
)
|
||||
// {
|
||||
# In-cluster forge URL — the gateway vhost (`forge.<domain>`), which
|
||||
# nginx proxies to forgejo. The forge is mandatory, so this is
|
||||
|
|
|
|||
|
|
@ -174,23 +174,34 @@ in
|
|||
visible = false;
|
||||
};
|
||||
|
||||
# OTEL stats export is configured ONCE at host level via
|
||||
# `services.hyperhive.otel.*` (see nix/modules/hive-c0re.nix) and
|
||||
# injected into every agent's build by the meta-flake renderer
|
||||
# (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are
|
||||
# the build-time implementation surface that injection writes into;
|
||||
# they are not meant to be set directly in an agent.nix. Marked
|
||||
# `internal` so the host option is the only documented operator knob.
|
||||
options.hyperhive.otel = {
|
||||
enable = lib.mkEnableOption ''
|
||||
exporting this agent's Claude Code stats (token usage, cost, tool
|
||||
calls) to an OTLP endpoint via Claude Code's built-in OpenTelemetry.
|
||||
Each agent's harness exports its own stats directly to the collector,
|
||||
so it keeps working even when hive-c0re is down. Meant to be enabled
|
||||
hive-wide (one switch for every agent) - there is no per-agent
|
||||
opt-in flag beyond this option
|
||||
'';
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
internal = true;
|
||||
description = ''
|
||||
Export this agent's Claude Code stats (token usage, cost, tool
|
||||
calls) to an OTLP endpoint via Claude Code's built-in
|
||||
OpenTelemetry. Each agent's harness exports directly to the
|
||||
collector, so it keeps working even when hive-c0re is down.
|
||||
Host-driven: set `services.hyperhive.otel.enable` instead.
|
||||
'';
|
||||
};
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "https://collector.example.com/otel";
|
||||
internal = true;
|
||||
description = ''
|
||||
OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`.
|
||||
Required when `enable` is true.
|
||||
Host-driven via `services.hyperhive.otel.endpoint`.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -201,8 +212,10 @@ in
|
|||
"grpc"
|
||||
];
|
||||
default = "http/protobuf";
|
||||
internal = true;
|
||||
description = ''
|
||||
OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
|
||||
Host-driven via `services.hyperhive.otel.protocol`.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -214,27 +227,27 @@ in
|
|||
# is only ever read from disk by systemd at start, never nix-stored.
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/run/secrets/otel-headers";
|
||||
internal = true;
|
||||
description = ''
|
||||
Absolute path to an operator-provided secret file whose contents
|
||||
become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
|
||||
`Authorization=Bearer <token>`). Loaded via systemd
|
||||
`LoadCredential` into the unit-private credential store at
|
||||
runtime, so the token is never copied into the nix store or
|
||||
exposed in the process argv. Must be an absolute path (systemd
|
||||
`LoadCredential` requires one). Leave null if the endpoint needs
|
||||
no auth header.
|
||||
exposed in the process argv. Host-driven via
|
||||
`services.hyperhive.otel.headersCredential`.
|
||||
'';
|
||||
};
|
||||
|
||||
extraResourceAttributes = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "deployment.environment=prod";
|
||||
internal = true;
|
||||
description = ''
|
||||
Extra comma-separated entries appended to
|
||||
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
|
||||
`service.name` / `agent` / `hive` / `swarm` labels.
|
||||
Host-driven via `services.hyperhive.otel.extraResourceAttributes`.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
|
@ -787,11 +800,6 @@ in
|
|||
'';
|
||||
|
||||
assertions = [
|
||||
# OTEL export needs an endpoint to point at.
|
||||
{
|
||||
assertion = !config.hyperhive.otel.enable || config.hyperhive.otel.endpoint != "";
|
||||
message = "hyperhive.otel.enable is true but hyperhive.otel.endpoint is empty.";
|
||||
}
|
||||
# Guard the inputs-routed-as-output pattern: the agent flake.nix is
|
||||
# expected to set `_module.args.flakeInputs = builtins.removeAttrs inputs ["self"]`.
|
||||
# If `self` leaks into flakeInputs the agent gets a spurious attrset
|
||||
|
|
|
|||
Loading…
Reference in a new issue