vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: the operator chooses which signals go upstream

Browse source 
Adds services.hyperhive.otel.upstreamSignals, a list of "metrics" and/or
"logs" naming the signals the swarm collector sends to otel.endpoint. The
swarm's own stores always receive everything. The upstream exporter is
only defined when at least one signal uses it, since otelcol refuses an
exporter no pipeline references.

On deploy: nothing changes until the option is set; the default is
[ "metrics" "logs" ], today's behaviour. To stop the 404 flood from an
upstream that has no /v1/logs, set services.hyperhive.otel.upstreamSignals
= [ "metrics" ].
This commit is contained in:
atlas 2026-10-21 09:31:39 +02:00
commit 2977dc91d1
4 changed files with 114 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/swarm/services.md
View file

@ -313,7 +313,9 @@ read side stays open to any authenticated caller.
The swarm's collector receives from every hive's own collector and is the
only process that decides where telemetry goes: it always writes the stores
above, and exports to `otel.endpoint` as well when that's set. It holds
above, and exports to `otel.endpoint` as well when that's set. Which signals
go there is `otel.upstreamSignals` (default `[ "metrics" "logs" ]`); the stores
always get both. It holds
the upstream credential, which is why no hive and no agent needs one.
It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to

26
nix/host-modules/otel.nix
View file

@ -88,6 +88,32 @@ in
'';
};
upstreamSignals = lib.mkOption {
type = lib.types.listOf (
lib.types.enum [
"metrics"
"logs"
]
);
default = [
"metrics"
"logs"
];
example = [ "metrics" ];
description = ''
Which signals the swarm's collector sends to
{option}`services.hyperhive.otel.endpoint`. The swarm's own metrics
and log stores always receive everything, whatever this lists.
Use it when the upstream does not accept a signal, for example one
that answers 404 on `/v1/logs`: list only `"metrics"` and the logs
pipelines stop exporting upstream. An empty list defines no upstream
exporter at all. No pipeline exports traces, so they have no entry.
Has no effect while {option}`services.hyperhive.otel.endpoint` is empty.
'';
};
protocol = lib.mkOption {
type = lib.types.enum [
"http/protobuf"

19
nix/host-modules/swarm-otel.nix
View file

@ -273,10 +273,17 @@ let
storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
# The operator-configured upstream, named once: the same exporter carries
# every signal, so metrics and logs both reach it without a second
# definition.
# every signal listed in `upstreamSignals`, so metrics and logs both reach
# it without a second definition.
upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp";
upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName;
upstreamFor =
signal:
lib.optional (
otelCfg.endpoint != "" && builtins.elem signal otelCfg.upstreamSignals
) upstreamExporterName;
# The collector refuses an exporter that no pipeline uses, so the exporter
# is only defined when at least one signal goes to it.
haveUpstream = upstreamFor "metrics" ++ upstreamFor "logs" != [ ];
# One list, read by every pipeline: the per-hive pipelines fan out to
# exactly the same destinations as the single pipeline they replace.
@ -285,10 +292,10 @@ let
# ⚠️ Unconditional: `deploy.victoriametrics.enable` says "this host RUNS the
# store", and a swarm has one either way. Gating on it left a collector
# elsewhere with no exporter at all, dropping everything silently.
exporterNames = upstreamExporters ++ [ "otlphttp/victoriametrics" ];
exporterNames = upstreamFor "metrics" ++ [ "otlphttp/victoriametrics" ];
# Same fan-out for logs, unconditional for the same reason.
logExporterNames = upstreamExporters ++ [ "otlphttp/victorialogs" ];
logExporterNames = upstreamFor "logs" ++ [ "otlphttp/victorialogs" ];
# An empty exporter list is not a quiet no-op — the collector rejects it.
# Now always satisfied, as a consequence of the log store always existing.
@ -1068,7 +1075,7 @@ in
auth.authenticator = pushAuthenticator "victoriametrics";
};
}
// lib.optionalAttrs (otelCfg.endpoint != "") {
// lib.optionalAttrs haveUpstream {
${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = {
endpoint = otelCfg.endpoint;
}

85
nix/module-eval/swarm-otel-core.nix
View file

@ -28,19 +28,30 @@ let
# shape from docs/swarm/services.md, and the one the old per-host gates made
# inexpressible. It is the whole point of the cases below that this hive is
# not a degenerate configuration but a supported one.
otelNoStores = hive {
deploy.swarm-otel.enable = true;
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
deploy.victoriametrics.enable = false;
deploy.victorialogs.enable = false;
# An upstream makes `logExporterNames` longer than the store alone, which
# is what the self-journal case below needs to tell the two apart.
otel.endpoint = "https://upstream.example/otel";
};
mkOtelNoStores =
upstreamSignals:
hive {
deploy.swarm-otel.enable = true;
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
deploy.victoriametrics.enable = false;
deploy.victorialogs.enable = false;
# An upstream makes `logExporterNames` longer than the store alone, which
# is what the self-journal case below needs to tell the two apart.
otel.endpoint = "https://upstream.example/otel";
otel.upstreamSignals = upstreamSignals;
};
otelNoStores = mkOtelNoStores [
"metrics"
"logs"
];
# The same collector with the upstream limited to one signal, and with none.
otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ];
otelUpstreamNone = mkOtelNoStores [ ];
# The collector beside authelia, reading its own OIDC secret out of the
# store like every other collector — the cert pair here is not scenery, it
@ -313,6 +324,54 @@ let
&& lib.hasInfix "Exporting failed" (lib.head f.logs.log_record)
&& lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record);
}
{
# Presence control: the default sends both signals to the upstream, so
# the negative arms below cannot pass on a misspelled exporter name.
name = "the upstream receives metrics and logs by default";
ok =
let
s = otelSettings otelNoStores;
ps = s.service.pipelines;
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
in
s.exporters ? otlphttp
&& names "metrics/" != [ ]
&& names "logs/" != [ ]
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
&& lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/");
}
{
name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything";
ok =
let
s = otelSettings otelUpstreamMetricsOnly;
ps = s.service.pipelines;
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
in
s.exporters ? otlphttp
&& names "metrics/" != [ ]
&& names "logs/" != [ ]
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
&& lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/")
&& !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"))
&& lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/");
}
{
# The collector rejects an exporter no pipeline uses, so with nothing
# going upstream the exporter must not be defined.
name = "an empty upstreamSignals defines no upstream exporter and keeps the stores";
ok =
let
s = otelSettings otelUpstreamNone;
ps = lib.attrValues s.service.pipelines;
in
!(s.exporters ? otlphttp)
&& !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps)
&& s.exporters ? "otlphttp/victoriametrics"
&& s.exporters ? "otlphttp/victorialogs"
&& lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps
&& lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps;
}
];
in
runGroup "swarm-otel-core" cases