diff --git a/docs/networking/network.md b/docs/networking/network.md index ebd5f4cf..658f8f7a 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -183,7 +183,7 @@ host-side service on another port — for example a dev OTLP collector you want agents to reach directly — is unreachable. (hyperhive's own services need none of this: a module that means to be reachable from agents opens its own port here, which the hive collector and `nats.enable` both do. Note -that `otel.endpoint` is the _upstream_, which no agent ever dials. See +that `deploy.swarm-otel.upstream.endpoint` is the _upstream_, which no agent ever dials. See `docs/scheduler/observability.md`.) `services.hyperhive.network.exposeHostPorts = [ 4318 ];` opens each diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index ab701adf..66212053 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -23,7 +23,7 @@ the host is: An all-local host runs both; the hop between them configures itself. ```nix -services.hyperhive.otel = { +services.hyperhive.deploy.swarm-otel.upstream = { endpoint = "https://collector.example.com/otel"; # the upstream headersCredential = "/run/secrets/otel-headers"; # only the swarm tier reads it }; @@ -33,13 +33,13 @@ services.hyperhive.otel = { Every hive runs its collector and every agent exports to it, with nothing to switch on: there is no per-hive or per-agent opt-in. -`endpoint` is where telemetry ends up after it leaves the swarm — optional, +`upstream.endpoint` is where telemetry ends up after it leaves the swarm — optional, because the swarm's own metrics store (`deploy.victoriametrics`) is a destination in its own right. With both, telemetry goes to both. See [`swarm/services.md`](../swarm/services.md#metrics-victoriametrics--grafana). **Telemetry leaves a hive exactly one way: through the collector that -runs on the host.** Agents never talk to `endpoint` themselves — +runs on the host.** Agents never talk to `upstream.endpoint` themselves — they export unauthenticated to a collector address only their own containers can reach. They dial it as `http://hive-collector.hive:4318`, a name identical on @@ -62,7 +62,7 @@ the control plane, so degraded telemetry isn't degraded operation. ### Why two tiers -**Agents export only to the hive tier.** Exporting straight to `endpoint` would mean +**Agents export only to the hive tier.** Exporting straight to `upstream.endpoint` would mean every agent needs the credential — and the only place to hand it to an agent container is somewhere the agent itself can read, its own claude settings among them. `0600` protects a secret from other containers, not from the @@ -159,8 +159,8 @@ belongs to exactly one hive. Getting the secret wrong shows up as the hive's collector logging 401s from the swarm tier and no metrics appearing for that hive. -⚠️ **`endpoint` must be valid for `protocol`.** The upstream exporter follows -`otel.protocol` (`grpc` → the gRPC exporter, otherwise OTLP/HTTP), and the gRPC +⚠️ **`upstream.endpoint` must be valid for `upstream.protocol`.** The upstream exporter follows +`upstream.protocol` (`grpc` → the gRPC exporter, otherwise OTLP/HTTP), and the gRPC exporter takes an _address_: `https://host/path` is a legal `OTEL_EXPORTER_OTLP_ENDPOINT` for HTTP but fails as gRPC with _"missing port in address"_. nix validates the collector's config at build time, so a mismatch is @@ -175,7 +175,8 @@ To let agents reach some other host-local service of your own → ## Options reference -The nix module (`nix/host-modules/otel.nix`) generates every +The nix modules (`nix/host-modules/otel.nix`, and `nix/host-modules/swarm-otel.nix` +for the `services.hyperhive.deploy.swarm-otel.upstream.*` export) generate every `services.hyperhive.otel.*` option's full type, default, description and example into the [options reference](https://hyperhive.darkest.space/options/) (host options — `nix build .#docs-host` for a local render). The same module diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 2ba212c6..d93a87ee 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -313,7 +313,9 @@ read side stays open to any authenticated caller. The swarm's collector receives from every hive's own collector and is the only process that decides where telemetry goes: it always writes the stores -above, and exports to `otel.endpoint` as well when that's set. It holds +above, and exports to `deploy.swarm-otel.upstream.endpoint` as well when that's set. Which signals +go there is `deploy.swarm-otel.upstream.signals` (default `[ "metrics" "logs" ]`); the stores +always get both. It holds the upstream credential, which is why no hive and no agent needs one. It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 557b497b..c2603a70 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -145,7 +145,8 @@ in # # ⚠️ `SSL_CERT_FILE` REPLACES the trust store, and unlike the other # consumers of this helper, hive-c0re already makes a *public*-TLS call: - # the OTEL exporter reaches whatever `otel.endpoint` names, typically a + # the OTEL exporter reaches whatever + # `deploy.swarm-otel.upstream.endpoint` names, typically a # normal internet host. The bundle is system CAs + the store's root # precisely so that path keeps working — narrowing it to the store's root # alone would fix a case nobody hits yet and break one that runs today. diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index cf284450..b162f8b8 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -8,8 +8,9 @@ # collector (./swarm-otel.nix). It holds no credential and picks no # destination: an agent's samples cross a hive boundary exactly once, and # what happens after that is the swarm's decision, not a hive's. The -# upstream options declared below describe that far end and are read one -# tier up — they stay here because they mean what they have always meant. +# upstream export is a swarm-collector deploy option, +# `services.hyperhive.deploy.swarm-otel.upstream` (./swarm-otel.nix); the +# old `otel.*` paths forward to it below. { pkgs, lib, @@ -56,6 +57,22 @@ in hostUnit = true; enable = runsHive; }) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "endpoint" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "endpoint" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "protocol" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "protocol" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "headersCredential" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headersCredential" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "collector" "upstreamHeaderName" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headerName" ] + ) (lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "enable" ] '' Every hive runs its own telemetry collector, and every agent exports to it by default, so there is no switch. Remove this definition. @@ -67,89 +84,6 @@ in ]; options.services.hyperhive.otel = { - endpoint = lib.mkOption { - type = lib.types.str; - default = ""; - example = "https://collector.example.com/otel"; - description = '' - Upstream OTLP endpoint: where telemetry ultimately goes, after it - has left the swarm. - - Read by the swarm's collector - ({option}`services.hyperhive.deploy.swarm-otel.enable`), which is the only - tier that holds the upstream credential. An agent is handed the - *first* hop instead — this hive's own collector — so this value is - never given to a container. - - Optional. Leave it empty and the swarm's own metrics store - ({option}`services.hyperhive.deploy.victoriametrics.enable`) is the - destination; that is a complete deployment, not a degraded one. - Set both and telemetry goes to both. - ''; - }; - - protocol = lib.mkOption { - type = lib.types.enum [ - "http/protobuf" - "http/json" - "grpc" - ]; - default = "http/protobuf"; - description = '' - OTLP wire protocol for the **upstream** link, honoured by the - swarm collector's exporter. - - Not what agents speak: their first hop is this hive's collector, - whose OTLP/HTTP receiver takes protobuf whatever the upstream - wants (see `hive-c0re/environment.nix`). - ''; - }; - - headersCredential = lib.mkOption { - # `str`, not `path`: a `path`-typed relative literal is hash-copied - # into the world-readable nix store at eval time, defeating the - # point. Keep it a string + require an absolute runtime path so the - # secret is only ever read from disk by systemd at start. - type = lib.types.nullOr lib.types.str; - default = null; - example = "/run/secrets/otel-headers"; - description = '' - Absolute path to an operator-provided secret file holding the - upstream auth header as `NAME=value` (e.g. - `Authorization=Bearer `). - - **Only the swarm's collector reads this** — the one tier that - talks to the upstream. It arrives as an `EnvironmentFile`, so the - value is never read by nix, never copied into the store or the - generated config, and never passed in argv; and it reaches - neither an agent container nor this hive's own collector, which - is the point of the tiers existing. Must be absolute. - - Leave null if the upstream needs no auth header; the collector - then sends none rather than an empty one. - ''; - }; - - collector.upstreamHeaderName = lib.mkOption { - type = lib.types.str; - default = "Authorization"; - description = '' - Name of the HTTP header the collector sends upstream, whose - *value* comes from `headersCredential`. - - The name is here and the value is not, and that split is forced - rather than chosen: the collector models exporter headers as a - static map, so rendering them means nix reading the value — the - one thing `headersCredential` being a path exists to prevent. - A name is public, a value is not. - - ⇒ exactly one header is expressible this way. A credential - carrying several (`a=1,b=2`) would be read as a single value, - which is why the shape is a named header rather than an opaque - blob: a second header has to be *declared*, not smuggled. - ''; - }; - collector.enable = lib.mkOption { type = lib.types.bool; readOnly = true; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f50eeccd..8016c5fc 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -17,7 +17,7 @@ let cfg = config.services.hyperhive.swarm.otel; deployCfg = config.services.hyperhive.deploy; - otelCfg = config.services.hyperhive.otel; + upstream = deployCfg.swarm-otel.upstream; vmCfg = config.services.hyperhive.swarm.victoriametrics; vlCfg = config.services.hyperhive.swarm.victorialogs; hyperhiveCfg = config.services.hyperhive; @@ -273,10 +273,17 @@ let storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client"; # The operator-configured upstream, named once: the same exporter carries - # every signal, so metrics and logs both reach it without a second - # definition. - upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"; - upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName; + # every signal listed in `upstream.signals`, so metrics and logs both reach + # it without a second definition. + upstreamExporterName = if upstream.protocol == "grpc" then "otlp" else "otlphttp"; + upstreamFor = + signal: + lib.optional ( + upstream.endpoint != "" && builtins.elem signal upstream.signals + ) upstreamExporterName; + # The collector refuses an exporter that no pipeline uses, so the exporter + # is only defined when at least one signal goes to it. + haveUpstream = upstreamFor "metrics" ++ upstreamFor "logs" != [ ]; # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. @@ -285,10 +292,10 @@ let # ⚠️ Unconditional: `deploy.victoriametrics.enable` says "this host RUNS the # store", and a swarm has one either way. Gating on it left a collector # elsewhere with no exporter at all, dropping everything silently. - exporterNames = upstreamExporters ++ [ "otlphttp/victoriametrics" ]; + exporterNames = upstreamFor "metrics" ++ [ "otlphttp/victoriametrics" ]; # Same fan-out for logs, unconditional for the same reason. - logExporterNames = upstreamExporters ++ [ "otlphttp/victorialogs" ]; + logExporterNames = upstreamFor "logs" ++ [ "otlphttp/victorialogs" ]; # An empty exporter list is not a quiet no-op — the collector rejects it. # Now always satisfied, as a consequence of the log store always existing. @@ -303,11 +310,126 @@ let storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in { - # What the collector IS to the swarm — the client it is registered as, where - # it exports — is `swarm.otel` in ./swarm-otel-service.nix. The secret is a - # path on the machine that runs it, so it hangs off the deployment. `enable` - # already lives in ./deploy.nix, which also carries the rename. + # What the collector IS to the swarm — the client it is registered as — is + # `swarm.otel` in ./swarm-otel-service.nix. The secret is a path on the + # machine that runs it, and the external export (`upstream`) is the + # operator's choice for this deployment, so both hang off the deployment. + # `enable` already lives in ./deploy.nix, which also carries the rename. options.services.hyperhive.deploy.swarm-otel = { + upstream = { + endpoint = lib.mkOption { + type = lib.types.str; + default = ""; + example = "https://collector.example.com/otel"; + description = '' + Upstream OTLP endpoint: where telemetry ultimately goes, after it + has left the swarm. + + Read by the swarm's collector + ({option}`services.hyperhive.deploy.swarm-otel.enable`), which is the only + tier that holds the upstream credential. An agent is handed the + *first* hop instead — this hive's own collector — so this value is + never given to a container. + + Optional. Leave it empty and the swarm's own metrics store + ({option}`services.hyperhive.deploy.victoriametrics.enable`) is the + destination; that is a complete deployment, not a degraded one. + Set both and telemetry goes to both. + ''; + }; + + signals = lib.mkOption { + type = lib.types.listOf ( + lib.types.enum [ + "metrics" + "logs" + ] + ); + default = [ + "metrics" + "logs" + ]; + example = [ "metrics" ]; + description = '' + Which signals the swarm's collector sends to + {option}`services.hyperhive.deploy.swarm-otel.upstream.endpoint`. The + swarm's own metrics and log stores always receive everything, + whatever this lists. + + Use it when the upstream does not accept a signal, for example one + that answers 404 on `/v1/logs`: list only `"metrics"` and the logs + pipelines stop exporting upstream. An empty list defines no upstream + exporter at all. No pipeline exports traces, so they have no entry. + + Has no effect while + {option}`services.hyperhive.deploy.swarm-otel.upstream.endpoint` is empty. + ''; + }; + + protocol = lib.mkOption { + type = lib.types.enum [ + "http/protobuf" + "http/json" + "grpc" + ]; + default = "http/protobuf"; + description = '' + OTLP wire protocol for the **upstream** link, honoured by the + swarm collector's exporter. + + Not what agents speak: their first hop is this hive's collector, + whose OTLP/HTTP receiver takes protobuf whatever the upstream + wants (see `hive-c0re/environment.nix`). + ''; + }; + + headersCredential = lib.mkOption { + # `str`, not `path`: a `path`-typed relative literal is hash-copied + # into the world-readable nix store at eval time, defeating the + # point. Keep it a string + require an absolute runtime path so the + # secret is only ever read from disk by systemd at start. + type = lib.types.nullOr lib.types.str; + default = null; + example = "/run/secrets/otel-headers"; + description = '' + Absolute path to an operator-provided secret file holding the + upstream auth header as `NAME=value` (e.g. + `Authorization=Bearer `). + + **Only the swarm's collector reads this** — the one tier that + talks to the upstream. It arrives as an `EnvironmentFile`, so the + value is never read by nix, never copied into the store or the + generated config, and never passed in argv; and it reaches + neither an agent container nor this hive's own collector, which + is the point of the tiers existing. Must be absolute. + + Leave null if the upstream needs no auth header; the collector + then sends none rather than an empty one. + ''; + }; + + headerName = lib.mkOption { + type = lib.types.str; + default = "Authorization"; + description = '' + Name of the HTTP header the collector sends upstream, whose + *value* comes from + {option}`services.hyperhive.deploy.swarm-otel.upstream.headersCredential`. + + The name is here and the value is not, and that split is forced + rather than chosen: the collector models exporter headers as a + static map, so rendering them means nix reading the value — the + one thing `headersCredential` being a path exists to prevent. + A name is public, a value is not. + + ⇒ exactly one header is expressible this way. A credential + carrying several (`a=1,b=2`) would be read as a single value, + which is why the shape is a named header rather than an opaque + blob: a second header has to be *declared*, not smuggled. + ''; + }; + }; + clientSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -809,9 +931,9 @@ in isReadOnly = true; }; } - // lib.optionalAttrs (otelCfg.headersCredential != null) { - ${otelCfg.headersCredential} = { - hostPath = otelCfg.headersCredential; + // lib.optionalAttrs (upstream.headersCredential != null) { + ${upstream.headersCredential} = { + hostPath = upstream.headersCredential; isReadOnly = true; }; } @@ -846,7 +968,7 @@ in { ... }: { # This tier is the one that resolves an operator-configured - # hostname: `otel.endpoint` is an external URL, and reaching it + # hostname: `upstream.endpoint` is an external URL, and reaching it # is the entire reason this container holds a credential. The # `/etc/resolv.conf` nixos-containers copies in is a snapshot # taken once at boot, so without this the upstream export @@ -1068,17 +1190,17 @@ in auth.authenticator = pushAuthenticator "victoriametrics"; }; } - // lib.optionalAttrs (otelCfg.endpoint != "") { - ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { - endpoint = otelCfg.endpoint; + // lib.optionalAttrs haveUpstream { + ${if upstream.protocol == "grpc" then "otlp" else "otlphttp"} = { + endpoint = upstream.endpoint; } - // lib.optionalAttrs (otelCfg.headersCredential != null) { + // lib.optionalAttrs (upstream.headersCredential != null) { # Interpolated by the collector from its environment at # runtime, never by nix: `EnvironmentFile` below is what # puts it there, so the value is not read into the store. - headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; + headers.${upstream.headerName} = "\${env:${upstream.headerName}}"; } - // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; + // lib.optionalAttrs (upstream.protocol == "http/json") { encoding = "json"; }; } // { # `logs_endpoint`, NOT `endpoint`: the latter is a base the @@ -1505,8 +1627,8 @@ in # EnvironmentFile format — so the secret reaches the process as # an environment variable without being read by nix, written to # the store, or passed in argv. - (lib.optionalAttrs (otelCfg.headersCredential != null) { - serviceConfig.EnvironmentFile = otelCfg.headersCredential; + (lib.optionalAttrs (upstream.headersCredential != null) { + serviceConfig.EnvironmentFile = upstream.headersCredential; }) # The other credential, and the other direction: the one above diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 9d5248bf..697950bb 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -28,20 +28,45 @@ let # shape from docs/swarm/services.md, and the one the old per-host gates made # inexpressible. It is the whole point of the cases below that this hive is # not a degenerate configuration but a supported one. - otelNoStores = hive { + mkOtelNoStores = + upstreamSignals: + hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; + deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; + deploy.victoriametrics.enable = false; + deploy.victorialogs.enable = false; + # An upstream makes `logExporterNames` longer than the store alone, which + # is what the self-journal case below needs to tell the two apart. + deploy.swarm-otel.upstream.endpoint = "https://upstream.example/otel"; + deploy.swarm-otel.upstream.signals = upstreamSignals; + }; + otelNoStores = mkOtelNoStores [ + "metrics" + "logs" + ]; + + # A host config still on the `otel.*` upstream paths, every shipped one. + otelOldUpstreamPaths = hive { deploy.swarm-otel.enable = true; deploy.authelia.enable = true; deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; - deploy.victoriametrics.enable = false; - deploy.victorialogs.enable = false; - # An upstream makes `logExporterNames` longer than the store alone, which - # is what the self-journal case below needs to tell the two apart. - otel.endpoint = "https://upstream.example/otel"; + otel.endpoint = "https://old.example/otel"; + otel.protocol = "http/json"; + otel.headersCredential = "/run/secrets/old-otel-headers"; + otel.collector.upstreamHeaderName = "X-Old-Token"; }; + # The same collector with the upstream limited to one signal, and with none. + otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ]; + otelUpstreamNone = mkOtelNoStores [ ]; + # The collector beside authelia, reading its own OIDC secret out of the # store like every other collector — the cert pair here is not scenery, it # is the arm that would catch the deleted co-located copy unit coming back. @@ -313,6 +338,77 @@ let && lib.hasInfix "Exporting failed" (lib.head f.logs.log_record) && lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record); } + { + # Presence control: the default sends both signals to the upstream, so + # the negative arms below cannot pass on a misspelled exporter name. + name = "the upstream receives metrics and logs by default"; + ok = + let + s = otelSettings otelNoStores; + ps = s.service.pipelines; + names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps); + in + s.exporters ? otlphttp + && names "metrics/" != [ ] + && names "logs/" != [ ] + && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/") + && lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"); + } + { + name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything"; + ok = + let + s = otelSettings otelUpstreamMetricsOnly; + ps = s.service.pipelines; + names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps); + in + s.exporters ? otlphttp + && names "metrics/" != [ ] + && names "logs/" != [ ] + && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/") + && lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/") + && !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/")) + && lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/"); + } + { + # The control is `otelNoStores`: it sets the new paths and warns about + # none of the old ones, so the warnings below come from the renames. + name = "the old otel.* upstream paths set deploy.swarm-otel.upstream and warn"; + ok = + let + u = otelOldUpstreamPaths.services.hyperhive.deploy.swarm-otel.upstream; + warnsAbout = m: path: lib.any (lib.hasInfix "services.hyperhive.${path}") m.warnings; + in + u.endpoint == "https://old.example/otel" + && u.protocol == "http/json" + && u.headersCredential == "/run/secrets/old-otel-headers" + && u.headerName == "X-Old-Token" + && warnsAbout otelOldUpstreamPaths "otel.endpoint" + && warnsAbout otelOldUpstreamPaths "otel.protocol" + && warnsAbout otelOldUpstreamPaths "otel.headersCredential" + && warnsAbout otelOldUpstreamPaths "otel.collector.upstreamHeaderName" + && !(warnsAbout otelNoStores "otel.endpoint") + && + otelNoStores.services.hyperhive.deploy.swarm-otel.upstream.endpoint + == "https://upstream.example/otel" + && (otelSettings otelOldUpstreamPaths).exporters ? otlphttp; + } + { + # The collector rejects an exporter no pipeline uses, so with nothing + # going upstream the exporter must not be defined. + name = "an empty upstream.signals defines no upstream exporter and keeps the stores"; + ok = + let + s = otelSettings otelUpstreamNone; + ps = lib.attrValues s.service.pipelines; + in + !(s.exporters ? otlphttp) + && !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps) + && s.exporters ? "otlphttp/victoriametrics" + && s.exporters ? "otlphttp/victorialogs" + && lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps + && lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps; + } ]; in runGroup "swarm-otel-core" cases