From e2ff4f5281120a6378493583ae4da42402392f9c Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 23 Sep 2026 18:04:33 +0200 Subject: [PATCH] swarm-bao: give the store's collector an explicit self-telemetry port 8890, so it stops claiming the hive collector's 8888 in the shared netns. Extends the module-eval port case to all three tiers. --- nix/host-modules/swarm-bao.nix | 42 ++++++++++++++++++++++++++++++++ nix/module-eval/hive-otel.nix | 44 +++++++++++++++++++++++++++------- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 8fd7474c..aebab744 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1295,6 +1295,29 @@ in Two spellings present as a healthy-looking 401. ''; }; + + otel.telemetryPort = lib.mkOption { + type = lib.types.port; + default = 8890; + description = '' + Port the collector inside the store's container serves its **own** + metrics on — queue depth, refused and dropped samples, exporter + failures. How you find out that telemetry is being lost, so it is + worth keeping rather than switching off. + + ⚠️ **Deliberately neither 8888 nor 8889.** 8888 is the collector + binary's built-in default, which the hive tier + ({option}`services.hyperhive.otel.telemetryPort`) already binds, and + 8889 is the swarm tier's + ({option}`services.hyperhive.swarm.otel.telemetryPort`). This + container runs with `privateNetwork = false`, so all three share the + host's network namespace whenever they are co-located — and unlike + the OTLP ports this one appears nowhere in either config when it is + left undeclared, so nothing that compares configured ports can see + the clash. The second collector to start simply dies with + `bind: address already in use`. + ''; + }; }; # ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than @@ -2373,6 +2396,25 @@ in exporters = [ "otlphttp" ]; }; + # Moves this collector's self-metrics off the built-in + # default of `localhost:8888`, which the hive tier holds — + # this container shares the host's netns, so leaving the + # default in place is the hive collector failing to bind. + # + # ⚠️ `metrics.address` is the spelling that looks right and + # is REJECTED by this collector version: + # `'migration.MetricsConfigV030' has invalid keys: + # address`. `readers` is the schema it accepts, and the + # difference is a startup failure rather than a warning. + service.telemetry.metrics.readers = [ + { + pull.exporter.prometheus = { + host = "127.0.0.1"; + port = cfg.otel.telemetryPort; + }; + } + ]; + # An extension configured but not listed here is INERT — the # receiver's `storage: file_storage` above would name a # component the collector never starts, and the exporter's diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 656483ec..8fb51d65 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -54,6 +54,23 @@ let # one carries none. swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; + + # Also duplicated from bao-otel-collector.nix, where it is the fixture for + # the store standing beside a HIVE collector. That co-location is exactly + # the topology the port case below needs: two collectors, one netns, and + # nothing but their self-telemetry ports keeping them apart. + # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion + # demands of any hive with the tier on. + baoWithHiveOtel = hive { + deploy.bao.enable = true; + otel.enable = true; + otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; + }; + + # The collector INSIDE the store's container — neither the host's nor the + # swarm tier's. + baoForwarderSettings = + machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings; cases = [ { # The tier in the middle. Its OTLP receiver takes both signals on one @@ -133,17 +150,28 @@ let && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst; } { - # Both collectors share a network namespace whenever they are - # co-located, and this port appears in no config the port-collision - # assertion can read — so equal defaults mean the second to start dies - # at `bind()`. Pinned as a case rather than an assertion: enforcing it - # belongs with the other port checks, not here. - name = "the two collector tiers do not claim the same self-telemetry port"; + # Collectors share a network namespace whenever they are co-located, + # and this port appears in no config the port-collision assertion can + # read — so equal defaults mean the second to start dies at `bind()`. + # Pinned as a case rather than an assertion: enforcing it belongs with + # the other port checks, not here. + # + # THREE tiers, because the store's container grew a collector of its + # own and this case could not see it: an UNDECLARED port is not an + # absent one, it is the binary's 8888, which is the hive tier's. That + # is why `portOf` falls back to 8888 rather than throwing — a tier that + # declares nothing has to fail this case loudly, not evaluate away. + name = "no two collector tiers claim the same self-telemetry port"; ok = let - portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port; + portOf = + s: + (lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888; + hivePort = portOf hiveOtelSettings; + swarmPort = portOf (otelSettings baoWithCollector); + baoPort = portOf (baoForwarderSettings baoWithHiveOtel); in - portOf hiveOtelSettings != portOf (otelSettings baoWithCollector); + hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort; } ]; in