diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 791f2c94..2ba212c6 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -351,7 +351,12 @@ authelia, forge and homeserver have a forwarder client of their own The swarm collector's own container, `swarm-otel`, has no forwarder: its collector reads the container's journal itself (receiver `journald/self`) and exports it to the log store like the host's journal, stamped `service.name` -`swarm-otel` and with no `hive` label. The container mounts the host's journal +`swarm-otel` and with no `hive` label. Unlike the host's journal, it goes to the +log store only, never the upstream: the collector writes an upstream export failure to this +journal, and shipping it upstream would fail again. A `filter` processor also +drops the collector's own failure lines that name the store exporter +(`otlphttp/victorialogs`), which can't reach the store anyway; failures of +every other exporter still arrive. The container mounts the host's journal read-only at `/run/host-journal`, not over its own `/var/log/journal`. diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index ce3ef972..f50eeccd 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1245,16 +1245,21 @@ in ]; exporters = logExporterNames; }; - # This container's own journal goes straight to the stores, + # This container's own journal goes straight to the store, # not through `otlp/`: a collector forwarding to # itself would need its own receiver up first. + # + # The store only, never the upstream: an upstream export + # failure is logged here, and shipping that line upstream + # fails again and logs again, forever. "logs/self" = { receivers = [ "journald/self" ]; processors = [ + "filter/drop-store-export-failures" "resource/self-journal" "resource/${swarmTierName}" ]; - exporters = logExporterNames; + exporters = [ "otlphttp/victorialogs" ]; }; }; } @@ -1458,6 +1463,17 @@ in ''IsMatch(body["_SYSTEMD_SLICE"], "^user-[0-9]+\\.slice$")'' ]; }; + + # The store exporter's own failure lines cannot reach the + # store, and re-feeding them to it repeats the failure. + # Failures of any other exporter pass. `body` is the journal + # entry as a map, the collector's message under `MESSAGE`. + "filter/drop-store-export-failures" = { + error_mode = "ignore"; + logs.log_record = [ + ''IsMatch(body["MESSAGE"], "Exporting failed") and IsMatch(body["MESSAGE"], "otlphttp/victorialogs")'' + ]; + }; }; }; }; diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 3b17f943..9d5248bf 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -37,6 +37,9 @@ let deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; deploy.victoriametrics.enable = false; deploy.victorialogs.enable = false; + # An upstream makes `logExporterNames` longer than the store alone, which + # is what the self-journal case below needs to tell the two apart. + otel.endpoint = "https://upstream.example/otel"; }; # The collector beside authelia, reading its own OIDC secret out of the @@ -285,6 +288,31 @@ let && lib.elem "resource/self-journal" (lib.head self).processors && builtins.elem "resource/self-journal" (lib.attrNames s.processors); } + { + # 🩸 The collector logs its own export failures to the journal that + # `journald/self` reads. Shipping that to the upstream feeds each + # failure back into the failing exporter. Presence controls: the + # upstream exporter exists and the host-journal pipeline does send to + # it, so an exporter-name typo cannot make the negative pass. + name = "the collector's own journal never reaches the upstream, and drops the store exporter's own failures"; + ok = + let + s = otelSettings otelNoStores; + pipelines = lib.attrValues s.service.pipelines; + self = lib.filter (p: builtins.elem "journald/self" p.receivers) pipelines; + host = lib.filter (p: builtins.elem "journald" p.receivers) pipelines; + f = s.processors."filter/drop-store-export-failures"; + in + s.exporters ? otlphttp + && lib.length host == 1 + && lib.all (p: builtins.elem "otlphttp" p.exporters) host + && lib.length self == 1 + && !(lib.any (p: builtins.elem "otlphttp" p.exporters) self) + && lib.elem "filter/drop-store-export-failures" (lib.head self).processors + && lib.length f.logs.log_record == 1 + && lib.hasInfix "Exporting failed" (lib.head f.logs.log_record) + && lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record); + } ]; in runGroup "swarm-otel-core" cases