mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

otel: collectors drop their own export-failure lines instead of shipping them

Browse source
This commit is contained in:
atlas 2026-11-14 22:13:39 +01:00 • committed by mara
commit 3fa926d7d6
3 changed files with 115 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

18
nix/module-eval/agent-otel.nix
View file

@ -103,6 +103,24 @@ let
name = "the agent forwarder's journald receiver carries the shared PRIORITY mapping";
ok = carriesJournaldSeverity (agentSettings agentOtel).receivers.journald;
}
{
# The forwarder reads its own journal, which holds its own exporter
# failures. Presence control: the resource processor is still listed.
name = "the agent forwarder drops its own export-failure lines before exporting";
ok =
let
s = agentSettings agentOtel;
p = s.service.pipelines.logs.processors;
in
p == [
"filter/drop-own-export-failures"
"resource"
]
&& s.processors ? "filter/drop-own-export-failures"
&& lib.hasInfix "opentelemetry-collector.service" (
lib.head s.processors."filter/drop-own-export-failures".logs.log_record
);
}
{
# The hop's two ends: what it reads, and where what it reads goes.
# The endpoint is compared against the value the fixture handed the

13
nix/module-eval/hive-otel.nix
View file

@ -117,6 +117,19 @@ let
]
&& hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
}
{
# Presence control: the resource processor is still listed after it.
name = "the hive collector drops its own export-failure lines before its logs pipeline exports";
ok =
hiveOtelPipelines.logs.processors == [
"filter/drop-own-export-failures"
"resourcedetection"
]
&& hiveOtelSettings.processors ? "filter/drop-own-export-failures"
&& lib.hasInfix "opentelemetry-collector.service" (
lib.head hiveOtelSettings.processors."filter/drop-own-export-failures".logs.log_record
);
}
{
# `deltatocumulative` is metrics-only: naming it in a logs pipeline
# kills the collector at startup rather than doing nothing. The second

28
nix/module-eval/swarm-otel-core.nix
View file

@ -341,6 +341,34 @@ let
&& lib.hasInfix "Exporting failed" (lib.head f.logs.log_record)
&& lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record);
}
{
# 🩸 The host journal holds the host collector's own failure lines, and
# shipping them is more traffic into an outage. The condition is read
# from the shared file, so every tier's wiring check below asks about
# the same text. Presence controls: the exporter is named, the pipeline
# lists the processor before the resource stamp, and the self pipeline
# keeps its own store-only filter.
name = "the host-journal pipeline drops the collector's own export-failure lines before it exports";
ok =
let
s = otelSettings otelNoStores;
host = lib.filter (p: builtins.elem "journald" p.receivers) (lib.attrValues s.service.pipelines);
f = s.processors."filter/drop-own-export-failures";
cond = lib.head f.logs.log_record;
i =
lib.lists.findFirstIndex (x: x == "filter/drop-own-export-failures") (-1)
(lib.head host).processors;
in
lib.length host == 1
&& (lib.head host).exporters != [ ]
&& i >= 0
&& i < lib.length (lib.head host).processors - 1
&& lib.length f.logs.log_record == 1
&& lib.hasInfix "opentelemetry-collector.service" cond
&& lib.hasInfix "Exporting failed" cond
&& lib.hasInfix "sending queue is full" cond
&& s.processors ? "filter/drop-store-export-failures";
}
{
# The journald receiver saves its cursor before an entry reaches the
# exporter, so an entry in an in-memory queue when the store is down or