vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: the upstream export is a swarm-otel deploy option

Browse source 
The external export is a setting of the swarm collector, so it moves to
services.hyperhive.deploy.swarm-otel.upstream. Renamed:
- otel.endpoint -> upstream.endpoint
- otel.protocol -> upstream.protocol
- otel.headersCredential -> upstream.headersCredential
- otel.collector.upstreamHeaderName -> upstream.headerName
The unreleased otel.upstreamSignals is declared as upstream.signals.

Hosts keep working through the rename warnings. Eval test covers the old
paths reaching the new options. Docs and stale comments point at the
new paths.
This commit is contained in:
atlas 2026-10-21 10:25:26 +02:00
commit 41a7974b9e
7 changed files with 206 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

3
nix/host-modules/hive-c0re/default.nix
View file

@ -145,7 +145,8 @@ in
#
# ⚠️ `SSL_CERT_FILE` REPLACES the trust store, and unlike the other
# consumers of this helper, hive-c0re already makes a *public*-TLS call:
# the OTEL exporter reaches whatever `otel.endpoint` names, typically a
# the OTEL exporter reaches whatever
# `deploy.swarm-otel.upstream.endpoint` names, typically a
# normal internet host. The bundle is system CAs + the store's root
# precisely so that path keeps working — narrowing it to the store's root
# alone would fix a case nobody hits yet and break one that runs today.

130
nix/host-modules/otel.nix
View file

@ -8,8 +8,9 @@
# collector (./swarm-otel.nix). It holds no credential and picks no
# destination: an agent's samples cross a hive boundary exactly once, and
# what happens after that is the swarm's decision, not a hive's. The
# upstream options declared below describe that far end and are read one
# tier up — they stay here because they mean what they have always meant.
# upstream export is a swarm-collector deploy option,
# `services.hyperhive.deploy.swarm-otel.upstream` (./swarm-otel.nix); the
# old `otel.*` paths forward to it below.
{
pkgs,
lib,
@ -56,6 +57,22 @@ in
hostUnit = true;
enable = runsHive;
})
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "otel" "endpoint" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "endpoint" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "otel" "protocol" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "protocol" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "otel" "headersCredential" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headersCredential" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "otel" "collector" "upstreamHeaderName" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headerName" ]
)
(lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "enable" ] ''
Every hive runs its own telemetry collector, and every agent exports to
it by default, so there is no switch. Remove this definition.
@ -67,115 +84,6 @@ in
];
options.services.hyperhive.otel = {
endpoint = lib.mkOption {
type = lib.types.str;
default = "";
example = "https://collector.example.com/otel";
description = ''
Upstream OTLP endpoint: where telemetry ultimately goes, after it
has left the swarm.
Read by the swarm's collector
({option}`services.hyperhive.deploy.swarm-otel.enable`), which is the only
tier that holds the upstream credential. An agent is handed the
*first* hop instead — this hive's own collector — so this value is
never given to a container.
Optional. Leave it empty and the swarm's own metrics store
({option}`services.hyperhive.deploy.victoriametrics.enable`) is the
destination; that is a complete deployment, not a degraded one.
Set both and telemetry goes to both.
'';
};
upstreamSignals = lib.mkOption {
type = lib.types.listOf (
lib.types.enum [
"metrics"
"logs"
]
);
default = [
"metrics"
"logs"
];
example = [ "metrics" ];
description = ''
Which signals the swarm's collector sends to
{option}`services.hyperhive.otel.endpoint`. The swarm's own metrics
and log stores always receive everything, whatever this lists.
Use it when the upstream does not accept a signal, for example one
that answers 404 on `/v1/logs`: list only `"metrics"` and the logs
pipelines stop exporting upstream. An empty list defines no upstream
exporter at all. No pipeline exports traces, so they have no entry.
Has no effect while {option}`services.hyperhive.otel.endpoint` is empty.
'';
};
protocol = lib.mkOption {
type = lib.types.enum [
"http/protobuf"
"http/json"
"grpc"
];
default = "http/protobuf";
description = ''
OTLP wire protocol for the **upstream** link, honoured by the
swarm collector's exporter.
Not what agents speak: their first hop is this hive's collector,
whose OTLP/HTTP receiver takes protobuf whatever the upstream
wants (see `hive-c0re/environment.nix`).
'';
};
headersCredential = lib.mkOption {
# `str`, not `path`: a `path`-typed relative literal is hash-copied
# into the world-readable nix store at eval time, defeating the
# point. Keep it a string + require an absolute runtime path so the
# secret is only ever read from disk by systemd at start.
type = lib.types.nullOr lib.types.str;
default = null;
example = "/run/secrets/otel-headers";
description = ''
Absolute path to an operator-provided secret file holding the
upstream auth header as `NAME=value` (e.g.
`Authorization=Bearer <token>`).
**Only the swarm's collector reads this** — the one tier that
talks to the upstream. It arrives as an `EnvironmentFile`, so the
value is never read by nix, never copied into the store or the
generated config, and never passed in argv; and it reaches
neither an agent container nor this hive's own collector, which
is the point of the tiers existing. Must be absolute.
Leave null if the upstream needs no auth header; the collector
then sends none rather than an empty one.
'';
};
collector.upstreamHeaderName = lib.mkOption {
type = lib.types.str;
default = "Authorization";
description = ''
Name of the HTTP header the collector sends upstream, whose
*value* comes from `headersCredential`.
The name is here and the value is not, and that split is forced
rather than chosen: the collector models exporter headers as a
static map, so rendering them means nix reading the value — the
one thing `headersCredential` being a path exists to prevent.
A name is public, a value is not.
⇒ exactly one header is expressible this way. A credential
carrying several (`a=1,b=2`) would be read as a single value,
which is why the shape is a named header rather than an opaque
blob: a second header has to be *declared*, not smuggled.
'';
};
collector.enable = lib.mkOption {
type = lib.types.bool;
readOnly = true;

153
nix/host-modules/swarm-otel.nix
View file

@ -17,7 +17,7 @@
let
cfg = config.services.hyperhive.swarm.otel;
deployCfg = config.services.hyperhive.deploy;
otelCfg = config.services.hyperhive.otel;
upstream = deployCfg.swarm-otel.upstream;
vmCfg = config.services.hyperhive.swarm.victoriametrics;
vlCfg = config.services.hyperhive.swarm.victorialogs;
hyperhiveCfg = config.services.hyperhive;
@ -273,13 +273,13 @@ let
storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
# The operator-configured upstream, named once: the same exporter carries
# every signal listed in `upstreamSignals`, so metrics and logs both reach
# every signal listed in `upstream.signals`, so metrics and logs both reach
# it without a second definition.
upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp";
upstreamExporterName = if upstream.protocol == "grpc" then "otlp" else "otlphttp";
upstreamFor =
signal:
lib.optional (
otelCfg.endpoint != "" && builtins.elem signal otelCfg.upstreamSignals
upstream.endpoint != "" && builtins.elem signal upstream.signals
) upstreamExporterName;
# The collector refuses an exporter that no pipeline uses, so the exporter
# is only defined when at least one signal goes to it.
@ -310,11 +310,126 @@ let
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
in
{
# What the collector IS to the swarm — the client it is registered as, where
# it exports — is `swarm.otel` in ./swarm-otel-service.nix. The secret is a
# path on the machine that runs it, so it hangs off the deployment. `enable`
# already lives in ./deploy.nix, which also carries the rename.
# What the collector IS to the swarm — the client it is registered as — is
# `swarm.otel` in ./swarm-otel-service.nix. The secret is a path on the
# machine that runs it, and the external export (`upstream`) is the
# operator's choice for this deployment, so both hang off the deployment.
# `enable` already lives in ./deploy.nix, which also carries the rename.
options.services.hyperhive.deploy.swarm-otel = {
upstream = {
endpoint = lib.mkOption {
type = lib.types.str;
default = "";
example = "https://collector.example.com/otel";
description = ''
Upstream OTLP endpoint: where telemetry ultimately goes, after it
has left the swarm.
Read by the swarm's collector
({option}`services.hyperhive.deploy.swarm-otel.enable`), which is the only
tier that holds the upstream credential. An agent is handed the
*first* hop instead — this hive's own collector — so this value is
never given to a container.
Optional. Leave it empty and the swarm's own metrics store
({option}`services.hyperhive.deploy.victoriametrics.enable`) is the
destination; that is a complete deployment, not a degraded one.
Set both and telemetry goes to both.
'';
};
signals = lib.mkOption {
type = lib.types.listOf (
lib.types.enum [
"metrics"
"logs"
]
);
default = [
"metrics"
"logs"
];
example = [ "metrics" ];
description = ''
Which signals the swarm's collector sends to
{option}`services.hyperhive.deploy.swarm-otel.upstream.endpoint`. The
swarm's own metrics and log stores always receive everything,
whatever this lists.
Use it when the upstream does not accept a signal, for example one
that answers 404 on `/v1/logs`: list only `"metrics"` and the logs
pipelines stop exporting upstream. An empty list defines no upstream
exporter at all. No pipeline exports traces, so they have no entry.
Has no effect while
{option}`services.hyperhive.deploy.swarm-otel.upstream.endpoint` is empty.
'';
};
protocol = lib.mkOption {
type = lib.types.enum [
"http/protobuf"
"http/json"
"grpc"
];
default = "http/protobuf";
description = ''
OTLP wire protocol for the **upstream** link, honoured by the
swarm collector's exporter.
Not what agents speak: their first hop is this hive's collector,
whose OTLP/HTTP receiver takes protobuf whatever the upstream
wants (see `hive-c0re/environment.nix`).
'';
};
headersCredential = lib.mkOption {
# `str`, not `path`: a `path`-typed relative literal is hash-copied
# into the world-readable nix store at eval time, defeating the
# point. Keep it a string + require an absolute runtime path so the
# secret is only ever read from disk by systemd at start.
type = lib.types.nullOr lib.types.str;
default = null;
example = "/run/secrets/otel-headers";
description = ''
Absolute path to an operator-provided secret file holding the
upstream auth header as `NAME=value` (e.g.
`Authorization=Bearer <token>`).
**Only the swarm's collector reads this** — the one tier that
talks to the upstream. It arrives as an `EnvironmentFile`, so the
value is never read by nix, never copied into the store or the
generated config, and never passed in argv; and it reaches
neither an agent container nor this hive's own collector, which
is the point of the tiers existing. Must be absolute.
Leave null if the upstream needs no auth header; the collector
then sends none rather than an empty one.
'';
};
headerName = lib.mkOption {
type = lib.types.str;
default = "Authorization";
description = ''
Name of the HTTP header the collector sends upstream, whose
*value* comes from
{option}`services.hyperhive.deploy.swarm-otel.upstream.headersCredential`.
The name is here and the value is not, and that split is forced
rather than chosen: the collector models exporter headers as a
static map, so rendering them means nix reading the value — the
one thing `headersCredential` being a path exists to prevent.
A name is public, a value is not.
⇒ exactly one header is expressible this way. A credential
carrying several (`a=1,b=2`) would be read as a single value,
which is why the shape is a named header rather than an opaque
blob: a second header has to be *declared*, not smuggled.
'';
};
};
clientSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -816,9 +931,9 @@ in
isReadOnly = true;
};
}
// lib.optionalAttrs (otelCfg.headersCredential != null) {
${otelCfg.headersCredential} = {
hostPath = otelCfg.headersCredential;
// lib.optionalAttrs (upstream.headersCredential != null) {
${upstream.headersCredential} = {
hostPath = upstream.headersCredential;
isReadOnly = true;
};
}
@ -853,7 +968,7 @@ in
{ ... }:
{
# This tier is the one that resolves an operator-configured
# hostname: `otel.endpoint` is an external URL, and reaching it
# hostname: `upstream.endpoint` is an external URL, and reaching it
# is the entire reason this container holds a credential. The
# `/etc/resolv.conf` nixos-containers copies in is a snapshot
# taken once at boot, so without this the upstream export
@ -1076,16 +1191,16 @@ in
};
}
// lib.optionalAttrs haveUpstream {
${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = {
endpoint = otelCfg.endpoint;
${if upstream.protocol == "grpc" then "otlp" else "otlphttp"} = {
endpoint = upstream.endpoint;
}
// lib.optionalAttrs (otelCfg.headersCredential != null) {
// lib.optionalAttrs (upstream.headersCredential != null) {
# Interpolated by the collector from its environment at
# runtime, never by nix: `EnvironmentFile` below is what
# puts it there, so the value is not read into the store.
headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
headers.${upstream.headerName} = "\${env:${upstream.headerName}}";
}
// lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
// lib.optionalAttrs (upstream.protocol == "http/json") { encoding = "json"; };
}
// {
# `logs_endpoint`, NOT `endpoint`: the latter is a base the
@ -1512,8 +1627,8 @@ in
# EnvironmentFile format — so the secret reaches the process as
# an environment variable without being read by nix, written to
# the store, or passed in argv.
(lib.optionalAttrs (otelCfg.headersCredential != null) {
serviceConfig.EnvironmentFile = otelCfg.headersCredential;
(lib.optionalAttrs (upstream.headersCredential != null) {
serviceConfig.EnvironmentFile = upstream.headersCredential;
})
# The other credential, and the other direction: the one above

43
nix/module-eval/swarm-otel-core.nix
View file

@ -41,14 +41,28 @@ let
deploy.victorialogs.enable = false;
# An upstream makes `logExporterNames` longer than the store alone, which
# is what the self-journal case below needs to tell the two apart.
otel.endpoint = "https://upstream.example/otel";
otel.upstreamSignals = upstreamSignals;
deploy.swarm-otel.upstream.endpoint = "https://upstream.example/otel";
deploy.swarm-otel.upstream.signals = upstreamSignals;
};
otelNoStores = mkOtelNoStores [
"metrics"
"logs"
];
# A host config still on the `otel.*` upstream paths, every shipped one.
otelOldUpstreamPaths = hive {
deploy.swarm-otel.enable = true;
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
otel.endpoint = "https://old.example/otel";
otel.protocol = "http/json";
otel.headersCredential = "/run/secrets/old-otel-headers";
otel.collector.upstreamHeaderName = "X-Old-Token";
};
# The same collector with the upstream limited to one signal, and with none.
otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ];
otelUpstreamNone = mkOtelNoStores [ ];
@ -356,10 +370,33 @@ let
&& !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"))
&& lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/");
}
{
# The control is `otelNoStores`: it sets the new paths and warns about
# none of the old ones, so the warnings below come from the renames.
name = "the old otel.* upstream paths set deploy.swarm-otel.upstream and warn";
ok =
let
u = otelOldUpstreamPaths.services.hyperhive.deploy.swarm-otel.upstream;
warnsAbout = m: path: lib.any (lib.hasInfix "services.hyperhive.${path}") m.warnings;
in
u.endpoint == "https://old.example/otel"
&& u.protocol == "http/json"
&& u.headersCredential == "/run/secrets/old-otel-headers"
&& u.headerName == "X-Old-Token"
&& warnsAbout otelOldUpstreamPaths "otel.endpoint"
&& warnsAbout otelOldUpstreamPaths "otel.protocol"
&& warnsAbout otelOldUpstreamPaths "otel.headersCredential"
&& warnsAbout otelOldUpstreamPaths "otel.collector.upstreamHeaderName"
&& !(warnsAbout otelNoStores "otel.endpoint")
&&
otelNoStores.services.hyperhive.deploy.swarm-otel.upstream.endpoint
== "https://upstream.example/otel"
&& (otelSettings otelOldUpstreamPaths).exporters ? otlphttp;
}
{
# The collector rejects an exporter no pipeline uses, so with nothing
# going upstream the exporter must not be defined.
name = "an empty upstreamSignals defines no upstream exporter and keeps the stores";
name = "an empty upstream.signals defines no upstream exporter and keeps the stores";
ok =
let
s = otelSettings otelUpstreamNone;