swarm-bao: give the store's collector an explicit self-telemetry port

8890, so it stops claiming the hive collector's 8888 in the shared netns.
Extends the module-eval port case to all three tiers.
This commit is contained in:
atlas 2026-09-23 18:04:33 +02:00
commit e2ff4f5281
2 changed files with 78 additions and 8 deletions

View file

@ -1295,6 +1295,29 @@ in
Two spellings present as a healthy-looking 401.
'';
};
otel.telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8890;
description = ''
Port the collector inside the store's container serves its **own**
metrics on queue depth, refused and dropped samples, exporter
failures. How you find out that telemetry is being lost, so it is
worth keeping rather than switching off.
**Deliberately neither 8888 nor 8889.** 8888 is the collector
binary's built-in default, which the hive tier
({option}`services.hyperhive.otel.telemetryPort`) already binds, and
8889 is the swarm tier's
({option}`services.hyperhive.swarm.otel.telemetryPort`). This
container runs with `privateNetwork = false`, so all three share the
host's network namespace whenever they are co-located and unlike
the OTLP ports this one appears nowhere in either config when it is
left undeclared, so nothing that compares configured ports can see
the clash. The second collector to start simply dies with
`bind: address already in use`.
'';
};
};
# ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
@ -2373,6 +2396,25 @@ in
exporters = [ "otlphttp" ];
};
# Moves this collector's self-metrics off the built-in
# default of `localhost:8888`, which the hive tier holds —
# this container shares the host's netns, so leaving the
# default in place is the hive collector failing to bind.
#
# ⚠️ `metrics.address` is the spelling that looks right and
# is REJECTED by this collector version:
# `'migration.MetricsConfigV030' has invalid keys:
# address`. `readers` is the schema it accepts, and the
# difference is a startup failure rather than a warning.
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.otel.telemetryPort;
};
}
];
# An extension configured but not listed here is INERT — the
# receiver's `storage: file_storage` above would name a
# component the collector never starts, and the exporter's

View file

@ -54,6 +54,23 @@ let
# one carries none.
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
};
# Also duplicated from bao-otel-collector.nix, where it is the fixture for
# the store standing beside a HIVE collector. That co-location is exactly
# the topology the port case below needs: two collectors, one netns, and
# nothing but their self-telemetry ports keeping them apart.
# `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
# demands of any hive with the tier on.
baoWithHiveOtel = hive {
deploy.bao.enable = true;
otel.enable = true;
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
};
# The collector INSIDE the store's container — neither the host's nor the
# swarm tier's.
baoForwarderSettings =
machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
cases = [
{
# The tier in the middle. Its OTLP receiver takes both signals on one
@ -133,17 +150,28 @@ let
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
}
{
# Both collectors share a network namespace whenever they are
# co-located, and this port appears in no config the port-collision
# assertion can read — so equal defaults mean the second to start dies
# at `bind()`. Pinned as a case rather than an assertion: enforcing it
# belongs with the other port checks, not here.
name = "the two collector tiers do not claim the same self-telemetry port";
# Collectors share a network namespace whenever they are co-located,
# and this port appears in no config the port-collision assertion can
# read — so equal defaults mean the second to start dies at `bind()`.
# Pinned as a case rather than an assertion: enforcing it belongs with
# the other port checks, not here.
#
# THREE tiers, because the store's container grew a collector of its
# own and this case could not see it: an UNDECLARED port is not an
# absent one, it is the binary's 8888, which is the hive tier's. That
# is why `portOf` falls back to 8888 rather than throwing — a tier that
# declares nothing has to fail this case loudly, not evaluate away.
name = "no two collector tiers claim the same self-telemetry port";
ok =
let
portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port;
portOf =
s:
(lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
hivePort = portOf hiveOtelSettings;
swarmPort = portOf (otelSettings baoWithCollector);
baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
in
portOf hiveOtelSettings != portOf (otelSettings baoWithCollector);
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
}
];
in