swarm-otel: the operator chooses which signals go upstream

Adds services.hyperhive.otel.upstreamSignals, a list of "metrics" and/or
"logs" naming the signals the swarm collector sends to otel.endpoint. The
swarm's own stores always receive everything. The upstream exporter is
only defined when at least one signal uses it, since otelcol refuses an
exporter no pipeline references.

On deploy: nothing changes until the option is set; the default is
[ "metrics" "logs" ], today's behaviour. To stop the 404 flood from an
upstream that has no /v1/logs, set services.hyperhive.otel.upstreamSignals
= [ "metrics" ].

docs/swarm/services.md

@@ -313,7 +313,9 @@ read side stays open to any authenticated caller.
 
 The swarm's collector receives from every hive's own collector and is the
 only process that decides where telemetry goes: it always writes the stores
-above, and exports to `otel.endpoint` as well when that's set. It holds
+above, and exports to `otel.endpoint` as well when that's set. Which signals
+go there is `otel.upstreamSignals` (default `[ "metrics" "logs" ]`); the stores
+always get both. It holds
 the upstream credential, which is why no hive and no agent needs one.
 
 It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to

nix/host-modules/otel.nix

@@ -88,6 +88,32 @@ in
       '';
     };
 
+    upstreamSignals = lib.mkOption {
+      type = lib.types.listOf (
+        lib.types.enum [
+          "metrics"
+          "logs"
+        ]
+      );
+      default = [
+        "metrics"
+        "logs"
+      ];
+      example = [ "metrics" ];
+      description = ''
+        Which signals the swarm's collector sends to
+        {option}`services.hyperhive.otel.endpoint`. The swarm's own metrics
+        and log stores always receive everything, whatever this lists.
+
+        Use it when the upstream does not accept a signal, for example one
+        that answers 404 on `/v1/logs`: list only `"metrics"` and the logs
+        pipelines stop exporting upstream. An empty list defines no upstream
+        exporter at all. No pipeline exports traces, so they have no entry.
+
+        Has no effect while {option}`services.hyperhive.otel.endpoint` is empty.
+      '';
+    };
+
     protocol = lib.mkOption {
       type = lib.types.enum [
         "http/protobuf"

nix/host-modules/swarm-otel.nix

@@ -273,10 +273,17 @@ let
   storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
 
   # The operator-configured upstream, named once: the same exporter carries
-  # every signal, so metrics and logs both reach it without a second
+  # every signal listed in `upstreamSignals`, so metrics and logs both reach
-  # definition.
+  # it without a second definition.
   upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp";
-  upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName;
+  upstreamFor =
+    signal:
+    lib.optional (
+      otelCfg.endpoint != "" && builtins.elem signal otelCfg.upstreamSignals
+    ) upstreamExporterName;
+  # The collector refuses an exporter that no pipeline uses, so the exporter
+  # is only defined when at least one signal goes to it.
+  haveUpstream = upstreamFor "metrics" ++ upstreamFor "logs" != [ ];
 
   # One list, read by every pipeline: the per-hive pipelines fan out to
   # exactly the same destinations as the single pipeline they replace.
@@ -285,10 +292,10 @@ let
   # ⚠️ Unconditional: `deploy.victoriametrics.enable` says "this host RUNS the
   # store", and a swarm has one either way. Gating on it left a collector
   # elsewhere with no exporter at all, dropping everything silently.
-  exporterNames = upstreamExporters ++ [ "otlphttp/victoriametrics" ];
+  exporterNames = upstreamFor "metrics" ++ [ "otlphttp/victoriametrics" ];
 
   # Same fan-out for logs, unconditional for the same reason.
-  logExporterNames = upstreamExporters ++ [ "otlphttp/victorialogs" ];
+  logExporterNames = upstreamFor "logs" ++ [ "otlphttp/victorialogs" ];
 
   # An empty exporter list is not a quiet no-op — the collector rejects it.
   # Now always satisfied, as a consequence of the log store always existing.
@@ -1068,7 +1075,7 @@ in
                   auth.authenticator = pushAuthenticator "victoriametrics";
                 };
               }
-              // lib.optionalAttrs (otelCfg.endpoint != "") {
+              // lib.optionalAttrs haveUpstream {
                 ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = {
                   endpoint = otelCfg.endpoint;
                 }

nix/module-eval/swarm-otel-core.nix

@@ -28,19 +28,30 @@ let
   # shape from docs/swarm/services.md, and the one the old per-host gates made
   # inexpressible. It is the whole point of the cases below that this hive is
   # not a degenerate configuration but a supported one.
-  otelNoStores = hive {
+  mkOtelNoStores =
-    deploy.swarm-otel.enable = true;
+    upstreamSignals:
-    deploy.authelia.enable = true;
+    hive {
-    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
+      deploy.swarm-otel.enable = true;
-    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
+      deploy.authelia.enable = true;
-    deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
+      deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
-    deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
+      deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
-    deploy.victoriametrics.enable = false;
+      deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
-    deploy.victorialogs.enable = false;
+      deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
-    # An upstream makes `logExporterNames` longer than the store alone, which
+      deploy.victoriametrics.enable = false;
-    # is what the self-journal case below needs to tell the two apart.
+      deploy.victorialogs.enable = false;
-    otel.endpoint = "https://upstream.example/otel";
+      # An upstream makes `logExporterNames` longer than the store alone, which
-  };
+      # is what the self-journal case below needs to tell the two apart.
+      otel.endpoint = "https://upstream.example/otel";
+      otel.upstreamSignals = upstreamSignals;
+    };
+  otelNoStores = mkOtelNoStores [
+    "metrics"
+    "logs"
+  ];
+
+  # The same collector with the upstream limited to one signal, and with none.
+  otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ];
+  otelUpstreamNone = mkOtelNoStores [ ];
 
   # The collector beside authelia, reading its own OIDC secret out of the
   # store like every other collector — the cert pair here is not scenery, it
@@ -313,6 +324,54 @@ let
         && lib.hasInfix "Exporting failed" (lib.head f.logs.log_record)
         && lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record);
     }
+    {
+      # Presence control: the default sends both signals to the upstream, so
+      # the negative arms below cannot pass on a misspelled exporter name.
+      name = "the upstream receives metrics and logs by default";
+      ok =
+        let
+          s = otelSettings otelNoStores;
+          ps = s.service.pipelines;
+          names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
+        in
+        s.exporters ? otlphttp
+        && names "metrics/" != [ ]
+        && names "logs/" != [ ]
+        && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
+        && lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/");
+    }
+    {
+      name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything";
+      ok =
+        let
+          s = otelSettings otelUpstreamMetricsOnly;
+          ps = s.service.pipelines;
+          names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
+        in
+        s.exporters ? otlphttp
+        && names "metrics/" != [ ]
+        && names "logs/" != [ ]
+        && lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
+        && lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/")
+        && !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"))
+        && lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/");
+    }
+    {
+      # The collector rejects an exporter no pipeline uses, so with nothing
+      # going upstream the exporter must not be defined.
+      name = "an empty upstreamSignals defines no upstream exporter and keeps the stores";
+      ok =
+        let
+          s = otelSettings otelUpstreamNone;
+          ps = lib.attrValues s.service.pipelines;
+        in
+        !(s.exporters ? otlphttp)
+        && !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps)
+        && s.exporters ? "otlphttp/victoriametrics"
+        && s.exporters ? "otlphttp/victorialogs"
+        && lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps
+        && lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps;
+    }
   ];
 in
 runGroup "swarm-otel-core" cases