From 8d70afd93e7af9afba5fe18b349fb34608897a3e Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 20 Oct 2026 00:13:25 +0200 Subject: [PATCH] swarm-otel's own logs go only to the store, so an upstream export failure no longer feeds itself The logs/self pipeline shipped the collector's own journal to the upstream exporter too. When the upstream failed, the collector logged the failure, journald/self read it, and logs/self sent it upstream again: ~8 lines/s. logs/self now exports only to otlphttp/victorialogs, and a filter processor drops the collector's failure lines that name that exporter, so the store exporter cannot rebuild the loop one hop later. Other collector lines, including upstream failures, still reach the store. On deploy, swarm-otel's store rows drop from ~8/s of error spam to its real output. --- docs/swarm/services.md | 7 ++++++- nix/host-modules/swarm-otel.nix | 20 ++++++++++++++++++-- nix/module-eval/swarm-otel-core.nix | 28 ++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 791f2c94..2ba212c6 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -351,7 +351,12 @@ authelia, forge and homeserver have a forwarder client of their own The swarm collector's own container, `swarm-otel`, has no forwarder: its collector reads the container's journal itself (receiver `journald/self`) and exports it to the log store like the host's journal, stamped `service.name` -`swarm-otel` and with no `hive` label. The container mounts the host's journal +`swarm-otel` and with no `hive` label. Unlike the host's journal, it goes to the +log store only, never the upstream: the collector writes an upstream export failure to this +journal, and shipping it upstream would fail again. A `filter` processor also +drops the collector's own failure lines that name the store exporter +(`otlphttp/victorialogs`), which can't reach the store anyway; failures of +every other exporter still arrive. The container mounts the host's journal read-only at `/run/host-journal`, not over its own `/var/log/journal`. diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index ce3ef972..f50eeccd 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1245,16 +1245,21 @@ in ]; exporters = logExporterNames; }; - # This container's own journal goes straight to the stores, + # This container's own journal goes straight to the store, # not through `otlp/`: a collector forwarding to # itself would need its own receiver up first. + # + # The store only, never the upstream: an upstream export + # failure is logged here, and shipping that line upstream + # fails again and logs again, forever. "logs/self" = { receivers = [ "journald/self" ]; processors = [ + "filter/drop-store-export-failures" "resource/self-journal" "resource/${swarmTierName}" ]; - exporters = logExporterNames; + exporters = [ "otlphttp/victorialogs" ]; }; }; } @@ -1458,6 +1463,17 @@ in ''IsMatch(body["_SYSTEMD_SLICE"], "^user-[0-9]+\\.slice$")'' ]; }; + + # The store exporter's own failure lines cannot reach the + # store, and re-feeding them to it repeats the failure. + # Failures of any other exporter pass. `body` is the journal + # entry as a map, the collector's message under `MESSAGE`. + "filter/drop-store-export-failures" = { + error_mode = "ignore"; + logs.log_record = [ + ''IsMatch(body["MESSAGE"], "Exporting failed") and IsMatch(body["MESSAGE"], "otlphttp/victorialogs")'' + ]; + }; }; }; }; diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 3b17f943..9d5248bf 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -37,6 +37,9 @@ let deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; deploy.victoriametrics.enable = false; deploy.victorialogs.enable = false; + # An upstream makes `logExporterNames` longer than the store alone, which + # is what the self-journal case below needs to tell the two apart. + otel.endpoint = "https://upstream.example/otel"; }; # The collector beside authelia, reading its own OIDC secret out of the @@ -285,6 +288,31 @@ let && lib.elem "resource/self-journal" (lib.head self).processors && builtins.elem "resource/self-journal" (lib.attrNames s.processors); } + { + # 🩸 The collector logs its own export failures to the journal that + # `journald/self` reads. Shipping that to the upstream feeds each + # failure back into the failing exporter. Presence controls: the + # upstream exporter exists and the host-journal pipeline does send to + # it, so an exporter-name typo cannot make the negative pass. + name = "the collector's own journal never reaches the upstream, and drops the store exporter's own failures"; + ok = + let + s = otelSettings otelNoStores; + pipelines = lib.attrValues s.service.pipelines; + self = lib.filter (p: builtins.elem "journald/self" p.receivers) pipelines; + host = lib.filter (p: builtins.elem "journald" p.receivers) pipelines; + f = s.processors."filter/drop-store-export-failures"; + in + s.exporters ? otlphttp + && lib.length host == 1 + && lib.all (p: builtins.elem "otlphttp" p.exporters) host + && lib.length self == 1 + && !(lib.any (p: builtins.elem "otlphttp" p.exporters) self) + && lib.elem "filter/drop-store-export-failures" (lib.head self).processors + && lib.length f.logs.log_record == 1 + && lib.hasInfix "Exporting failed" (lib.head f.logs.log_record) + && lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record); + } ]; in runGroup "swarm-otel-core" cases