swarm-bao: give the store's collector an explicit self-telemetry port
8890, so it stops claiming the hive collector's 8888 in the shared netns. Extends the module-eval port case to all three tiers.
This commit is contained in:
parent
5cec69bd21
commit
e2ff4f5281
2 changed files with 78 additions and 8 deletions
|
|
@ -1295,6 +1295,29 @@ in
|
|||
Two spellings present as a healthy-looking 401.
|
||||
'';
|
||||
};
|
||||
|
||||
otel.telemetryPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8890;
|
||||
description = ''
|
||||
Port the collector inside the store's container serves its **own**
|
||||
metrics on — queue depth, refused and dropped samples, exporter
|
||||
failures. How you find out that telemetry is being lost, so it is
|
||||
worth keeping rather than switching off.
|
||||
|
||||
⚠️ **Deliberately neither 8888 nor 8889.** 8888 is the collector
|
||||
binary's built-in default, which the hive tier
|
||||
({option}`services.hyperhive.otel.telemetryPort`) already binds, and
|
||||
8889 is the swarm tier's
|
||||
({option}`services.hyperhive.swarm.otel.telemetryPort`). This
|
||||
container runs with `privateNetwork = false`, so all three share the
|
||||
host's network namespace whenever they are co-located — and unlike
|
||||
the OTLP ports this one appears nowhere in either config when it is
|
||||
left undeclared, so nothing that compares configured ports can see
|
||||
the clash. The second collector to start simply dies with
|
||||
`bind: address already in use`.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
|
||||
|
|
@ -2373,6 +2396,25 @@ in
|
|||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# Moves this collector's self-metrics off the built-in
|
||||
# default of `localhost:8888`, which the hive tier holds —
|
||||
# this container shares the host's netns, so leaving the
|
||||
# default in place is the hive collector failing to bind.
|
||||
#
|
||||
# ⚠️ `metrics.address` is the spelling that looks right and
|
||||
# is REJECTED by this collector version:
|
||||
# `'migration.MetricsConfigV030' has invalid keys:
|
||||
# address`. `readers` is the schema it accepts, and the
|
||||
# difference is a startup failure rather than a warning.
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
port = cfg.otel.telemetryPort;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts, and the exporter's
|
||||
|
|
|
|||
|
|
@ -54,6 +54,23 @@ let
|
|||
# one carries none.
|
||||
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
|
||||
};
|
||||
|
||||
# Also duplicated from bao-otel-collector.nix, where it is the fixture for
|
||||
# the store standing beside a HIVE collector. That co-location is exactly
|
||||
# the topology the port case below needs: two collectors, one netns, and
|
||||
# nothing but their self-telemetry ports keeping them apart.
|
||||
# `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
|
||||
# demands of any hive with the tier on.
|
||||
baoWithHiveOtel = hive {
|
||||
deploy.bao.enable = true;
|
||||
otel.enable = true;
|
||||
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
||||
};
|
||||
|
||||
# The collector INSIDE the store's container — neither the host's nor the
|
||||
# swarm tier's.
|
||||
baoForwarderSettings =
|
||||
machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
|
||||
cases = [
|
||||
{
|
||||
# The tier in the middle. Its OTLP receiver takes both signals on one
|
||||
|
|
@ -133,17 +150,28 @@ let
|
|||
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
||||
}
|
||||
{
|
||||
# Both collectors share a network namespace whenever they are
|
||||
# co-located, and this port appears in no config the port-collision
|
||||
# assertion can read — so equal defaults mean the second to start dies
|
||||
# at `bind()`. Pinned as a case rather than an assertion: enforcing it
|
||||
# belongs with the other port checks, not here.
|
||||
name = "the two collector tiers do not claim the same self-telemetry port";
|
||||
# Collectors share a network namespace whenever they are co-located,
|
||||
# and this port appears in no config the port-collision assertion can
|
||||
# read — so equal defaults mean the second to start dies at `bind()`.
|
||||
# Pinned as a case rather than an assertion: enforcing it belongs with
|
||||
# the other port checks, not here.
|
||||
#
|
||||
# THREE tiers, because the store's container grew a collector of its
|
||||
# own and this case could not see it: an UNDECLARED port is not an
|
||||
# absent one, it is the binary's 8888, which is the hive tier's. That
|
||||
# is why `portOf` falls back to 8888 rather than throwing — a tier that
|
||||
# declares nothing has to fail this case loudly, not evaluate away.
|
||||
name = "no two collector tiers claim the same self-telemetry port";
|
||||
ok =
|
||||
let
|
||||
portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port;
|
||||
portOf =
|
||||
s:
|
||||
(lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
|
||||
hivePort = portOf hiveOtelSettings;
|
||||
swarmPort = portOf (otelSettings baoWithCollector);
|
||||
baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
|
||||
in
|
||||
portOf hiveOtelSettings != portOf (otelSettings baoWithCollector);
|
||||
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
|
||||
}
|
||||
];
|
||||
in
|
||||
|
|
|
|||
Loading…
Reference in a new issue