From 2977dc91d16b8cdb04227a4a709392eb0ed4ea95 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 21 Oct 2026 09:31:39 +0200 Subject: [PATCH] swarm-otel: the operator chooses which signals go upstream Adds services.hyperhive.otel.upstreamSignals, a list of "metrics" and/or "logs" naming the signals the swarm collector sends to otel.endpoint. The swarm's own stores always receive everything. The upstream exporter is only defined when at least one signal uses it, since otelcol refuses an exporter no pipeline references. On deploy: nothing changes until the option is set; the default is [ "metrics" "logs" ], today's behaviour. To stop the 404 flood from an upstream that has no /v1/logs, set services.hyperhive.otel.upstreamSignals = [ "metrics" ]. --- docs/swarm/services.md | 4 +- nix/host-modules/otel.nix | 26 +++++++++ nix/host-modules/swarm-otel.nix | 19 +++++-- nix/module-eval/swarm-otel-core.nix | 85 ++++++++++++++++++++++++----- 4 files changed, 114 insertions(+), 20 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 2ba212c6..dd650ea9 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -313,7 +313,9 @@ read side stays open to any authenticated caller. The swarm's collector receives from every hive's own collector and is the only process that decides where telemetry goes: it always writes the stores -above, and exports to `otel.endpoint` as well when that's set. It holds +above, and exports to `otel.endpoint` as well when that's set. Which signals +go there is `otel.upstreamSignals` (default `[ "metrics" "logs" ]`); the stores +always get both. It holds the upstream credential, which is why no hive and no agent needs one. It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index cf284450..6104581b 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -88,6 +88,32 @@ in ''; }; + upstreamSignals = lib.mkOption { + type = lib.types.listOf ( + lib.types.enum [ + "metrics" + "logs" + ] + ); + default = [ + "metrics" + "logs" + ]; + example = [ "metrics" ]; + description = '' + Which signals the swarm's collector sends to + {option}`services.hyperhive.otel.endpoint`. The swarm's own metrics + and log stores always receive everything, whatever this lists. + + Use it when the upstream does not accept a signal, for example one + that answers 404 on `/v1/logs`: list only `"metrics"` and the logs + pipelines stop exporting upstream. An empty list defines no upstream + exporter at all. No pipeline exports traces, so they have no entry. + + Has no effect while {option}`services.hyperhive.otel.endpoint` is empty. + ''; + }; + protocol = lib.mkOption { type = lib.types.enum [ "http/protobuf" diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f50eeccd..16cc61a9 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -273,10 +273,17 @@ let storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client"; # The operator-configured upstream, named once: the same exporter carries - # every signal, so metrics and logs both reach it without a second - # definition. + # every signal listed in `upstreamSignals`, so metrics and logs both reach + # it without a second definition. upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"; - upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName; + upstreamFor = + signal: + lib.optional ( + otelCfg.endpoint != "" && builtins.elem signal otelCfg.upstreamSignals + ) upstreamExporterName; + # The collector refuses an exporter that no pipeline uses, so the exporter + # is only defined when at least one signal goes to it. + haveUpstream = upstreamFor "metrics" ++ upstreamFor "logs" != [ ]; # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. @@ -285,10 +292,10 @@ let # ⚠️ Unconditional: `deploy.victoriametrics.enable` says "this host RUNS the # store", and a swarm has one either way. Gating on it left a collector # elsewhere with no exporter at all, dropping everything silently. - exporterNames = upstreamExporters ++ [ "otlphttp/victoriametrics" ]; + exporterNames = upstreamFor "metrics" ++ [ "otlphttp/victoriametrics" ]; # Same fan-out for logs, unconditional for the same reason. - logExporterNames = upstreamExporters ++ [ "otlphttp/victorialogs" ]; + logExporterNames = upstreamFor "logs" ++ [ "otlphttp/victorialogs" ]; # An empty exporter list is not a quiet no-op — the collector rejects it. # Now always satisfied, as a consequence of the log store always existing. @@ -1068,7 +1075,7 @@ in auth.authenticator = pushAuthenticator "victoriametrics"; }; } - // lib.optionalAttrs (otelCfg.endpoint != "") { + // lib.optionalAttrs haveUpstream { ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { endpoint = otelCfg.endpoint; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 9d5248bf..214b272a 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -28,19 +28,30 @@ let # shape from docs/swarm/services.md, and the one the old per-host gates made # inexpressible. It is the whole point of the cases below that this hive is # not a degenerate configuration but a supported one. - otelNoStores = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = true; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; - deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; - deploy.victoriametrics.enable = false; - deploy.victorialogs.enable = false; - # An upstream makes `logExporterNames` longer than the store alone, which - # is what the self-journal case below needs to tell the two apart. - otel.endpoint = "https://upstream.example/otel"; - }; + mkOtelNoStores = + upstreamSignals: + hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; + deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; + deploy.victoriametrics.enable = false; + deploy.victorialogs.enable = false; + # An upstream makes `logExporterNames` longer than the store alone, which + # is what the self-journal case below needs to tell the two apart. + otel.endpoint = "https://upstream.example/otel"; + otel.upstreamSignals = upstreamSignals; + }; + otelNoStores = mkOtelNoStores [ + "metrics" + "logs" + ]; + + # The same collector with the upstream limited to one signal, and with none. + otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ]; + otelUpstreamNone = mkOtelNoStores [ ]; # The collector beside authelia, reading its own OIDC secret out of the # store like every other collector — the cert pair here is not scenery, it @@ -313,6 +324,54 @@ let && lib.hasInfix "Exporting failed" (lib.head f.logs.log_record) && lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record); } + { + # Presence control: the default sends both signals to the upstream, so + # the negative arms below cannot pass on a misspelled exporter name. + name = "the upstream receives metrics and logs by default"; + ok = + let + s = otelSettings otelNoStores; + ps = s.service.pipelines; + names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps); + in + s.exporters ? otlphttp + && names "metrics/" != [ ] + && names "logs/" != [ ] + && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/") + && lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"); + } + { + name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything"; + ok = + let + s = otelSettings otelUpstreamMetricsOnly; + ps = s.service.pipelines; + names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps); + in + s.exporters ? otlphttp + && names "metrics/" != [ ] + && names "logs/" != [ ] + && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/") + && lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/") + && !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/")) + && lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/"); + } + { + # The collector rejects an exporter no pipeline uses, so with nothing + # going upstream the exporter must not be defined. + name = "an empty upstreamSignals defines no upstream exporter and keeps the stores"; + ok = + let + s = otelSettings otelUpstreamNone; + ps = lib.attrValues s.service.pipelines; + in + !(s.exporters ? otlphttp) + && !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps) + && s.exporters ? "otlphttp/victoriametrics" + && s.exporters ? "otlphttp/victorialogs" + && lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps + && lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps; + } ]; in runGroup "swarm-otel-core" cases