vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: the operator chooses which signals go upstream

Browse source 
Adds services.hyperhive.otel.upstreamSignals, a list of "metrics" and/or
"logs" naming the signals the swarm collector sends to otel.endpoint. The
swarm's own stores always receive everything. The upstream exporter is
only defined when at least one signal uses it, since otelcol refuses an
exporter no pipeline references.

On deploy: nothing changes until the option is set; the default is
[ "metrics" "logs" ], today's behaviour. To stop the 404 flood from an
upstream that has no /v1/logs, set services.hyperhive.otel.upstreamSignals
= [ "metrics" ].
This commit is contained in:
atlas 2026-10-21 09:31:39 +02:00
commit 2977dc91d1
4 changed files with 114 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

85
nix/module-eval/swarm-otel-core.nix
View file

@ -28,19 +28,30 @@ let
# shape from docs/swarm/services.md, and the one the old per-host gates made
# inexpressible. It is the whole point of the cases below that this hive is
# not a degenerate configuration but a supported one.
otelNoStores = hive {
deploy.swarm-otel.enable = true;
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
deploy.victoriametrics.enable = false;
deploy.victorialogs.enable = false;
# An upstream makes `logExporterNames` longer than the store alone, which
# is what the self-journal case below needs to tell the two apart.
otel.endpoint = "https://upstream.example/otel";
};
mkOtelNoStores =
upstreamSignals:
hive {
deploy.swarm-otel.enable = true;
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
deploy.victoriametrics.enable = false;
deploy.victorialogs.enable = false;
# An upstream makes `logExporterNames` longer than the store alone, which
# is what the self-journal case below needs to tell the two apart.
otel.endpoint = "https://upstream.example/otel";
otel.upstreamSignals = upstreamSignals;
};
otelNoStores = mkOtelNoStores [
"metrics"
"logs"
];
# The same collector with the upstream limited to one signal, and with none.
otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ];
otelUpstreamNone = mkOtelNoStores [ ];
# The collector beside authelia, reading its own OIDC secret out of the
# store like every other collector — the cert pair here is not scenery, it
@ -313,6 +324,54 @@ let
&& lib.hasInfix "Exporting failed" (lib.head f.logs.log_record)
&& lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record);
}
{
# Presence control: the default sends both signals to the upstream, so
# the negative arms below cannot pass on a misspelled exporter name.
name = "the upstream receives metrics and logs by default";
ok =
let
s = otelSettings otelNoStores;
ps = s.service.pipelines;
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
in
s.exporters ? otlphttp
&& names "metrics/" != [ ]
&& names "logs/" != [ ]
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
&& lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/");
}
{
name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything";
ok =
let
s = otelSettings otelUpstreamMetricsOnly;
ps = s.service.pipelines;
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
in
s.exporters ? otlphttp
&& names "metrics/" != [ ]
&& names "logs/" != [ ]
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
&& lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/")
&& !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"))
&& lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/");
}
{
# The collector rejects an exporter no pipeline uses, so with nothing
# going upstream the exporter must not be defined.
name = "an empty upstreamSignals defines no upstream exporter and keeps the stores";
ok =
let
s = otelSettings otelUpstreamNone;
ps = lib.attrValues s.service.pipelines;
in
!(s.exporters ? otlphttp)
&& !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps)
&& s.exporters ? "otlphttp/victoriametrics"
&& s.exporters ? "otlphttp/victorialogs"
&& lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps
&& lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps;
}
];
in
runGroup "swarm-otel-core" cases