diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 2ba212c6..dd650ea9 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -313,7 +313,9 @@ read side stays open to any authenticated caller. The swarm's collector receives from every hive's own collector and is the only process that decides where telemetry goes: it always writes the stores -above, and exports to `otel.endpoint` as well when that's set. It holds +above, and exports to `otel.endpoint` as well when that's set. Which signals +go there is `otel.upstreamSignals` (default `[ "metrics" "logs" ]`); the stores +always get both. It holds the upstream credential, which is why no hive and no agent needs one. It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index cf284450..6104581b 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -88,6 +88,32 @@ in ''; }; + upstreamSignals = lib.mkOption { + type = lib.types.listOf ( + lib.types.enum [ + "metrics" + "logs" + ] + ); + default = [ + "metrics" + "logs" + ]; + example = [ "metrics" ]; + description = '' + Which signals the swarm's collector sends to + {option}`services.hyperhive.otel.endpoint`. The swarm's own metrics + and log stores always receive everything, whatever this lists. + + Use it when the upstream does not accept a signal, for example one + that answers 404 on `/v1/logs`: list only `"metrics"` and the logs + pipelines stop exporting upstream. An empty list defines no upstream + exporter at all. No pipeline exports traces, so they have no entry. + + Has no effect while {option}`services.hyperhive.otel.endpoint` is empty. + ''; + }; + protocol = lib.mkOption { type = lib.types.enum [ "http/protobuf" diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f50eeccd..16cc61a9 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -273,10 +273,17 @@ let storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client"; # The operator-configured upstream, named once: the same exporter carries - # every signal, so metrics and logs both reach it without a second - # definition. + # every signal listed in `upstreamSignals`, so metrics and logs both reach + # it without a second definition. upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"; - upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName; + upstreamFor = + signal: + lib.optional ( + otelCfg.endpoint != "" && builtins.elem signal otelCfg.upstreamSignals + ) upstreamExporterName; + # The collector refuses an exporter that no pipeline uses, so the exporter + # is only defined when at least one signal goes to it. + haveUpstream = upstreamFor "metrics" ++ upstreamFor "logs" != [ ]; # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. @@ -285,10 +292,10 @@ let # ⚠️ Unconditional: `deploy.victoriametrics.enable` says "this host RUNS the # store", and a swarm has one either way. Gating on it left a collector # elsewhere with no exporter at all, dropping everything silently. - exporterNames = upstreamExporters ++ [ "otlphttp/victoriametrics" ]; + exporterNames = upstreamFor "metrics" ++ [ "otlphttp/victoriametrics" ]; # Same fan-out for logs, unconditional for the same reason. - logExporterNames = upstreamExporters ++ [ "otlphttp/victorialogs" ]; + logExporterNames = upstreamFor "logs" ++ [ "otlphttp/victorialogs" ]; # An empty exporter list is not a quiet no-op — the collector rejects it. # Now always satisfied, as a consequence of the log store always existing. @@ -1068,7 +1075,7 @@ in auth.authenticator = pushAuthenticator "victoriametrics"; }; } - // lib.optionalAttrs (otelCfg.endpoint != "") { + // lib.optionalAttrs haveUpstream { ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { endpoint = otelCfg.endpoint; } diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 9d5248bf..214b272a 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -28,19 +28,30 @@ let # shape from docs/swarm/services.md, and the one the old per-host gates made # inexpressible. It is the whole point of the cases below that this hive is # not a degenerate configuration but a supported one. - otelNoStores = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = true; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; - deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; - deploy.victoriametrics.enable = false; - deploy.victorialogs.enable = false; - # An upstream makes `logExporterNames` longer than the store alone, which - # is what the self-journal case below needs to tell the two apart. - otel.endpoint = "https://upstream.example/otel"; - }; + mkOtelNoStores = + upstreamSignals: + hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; + deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; + deploy.victoriametrics.enable = false; + deploy.victorialogs.enable = false; + # An upstream makes `logExporterNames` longer than the store alone, which + # is what the self-journal case below needs to tell the two apart. + otel.endpoint = "https://upstream.example/otel"; + otel.upstreamSignals = upstreamSignals; + }; + otelNoStores = mkOtelNoStores [ + "metrics" + "logs" + ]; + + # The same collector with the upstream limited to one signal, and with none. + otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ]; + otelUpstreamNone = mkOtelNoStores [ ]; # The collector beside authelia, reading its own OIDC secret out of the # store like every other collector — the cert pair here is not scenery, it @@ -313,6 +324,54 @@ let && lib.hasInfix "Exporting failed" (lib.head f.logs.log_record) && lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record); } + { + # Presence control: the default sends both signals to the upstream, so + # the negative arms below cannot pass on a misspelled exporter name. + name = "the upstream receives metrics and logs by default"; + ok = + let + s = otelSettings otelNoStores; + ps = s.service.pipelines; + names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps); + in + s.exporters ? otlphttp + && names "metrics/" != [ ] + && names "logs/" != [ ] + && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/") + && lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"); + } + { + name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything"; + ok = + let + s = otelSettings otelUpstreamMetricsOnly; + ps = s.service.pipelines; + names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps); + in + s.exporters ? otlphttp + && names "metrics/" != [ ] + && names "logs/" != [ ] + && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/") + && lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/") + && !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/")) + && lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/"); + } + { + # The collector rejects an exporter no pipeline uses, so with nothing + # going upstream the exporter must not be defined. + name = "an empty upstreamSignals defines no upstream exporter and keeps the stores"; + ok = + let + s = otelSettings otelUpstreamNone; + ps = lib.attrValues s.service.pipelines; + in + !(s.exporters ? otlphttp) + && !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps) + && s.exporters ? "otlphttp/victoriametrics" + && s.exporters ? "otlphttp/victorialogs" + && lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps + && lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps; + } ]; in runGroup "swarm-otel-core" cases