swarm-bao: give the store's collector an explicit self-telemetry port
8890, so it stops claiming the hive collector's 8888 in the shared netns. Extends the module-eval port case to all three tiers.
This commit is contained in:
parent
5cec69bd21
commit
e2ff4f5281
2 changed files with 78 additions and 8 deletions
|
|
@ -1295,6 +1295,29 @@ in
|
||||||
Two spellings present as a healthy-looking 401.
|
Two spellings present as a healthy-looking 401.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
otel.telemetryPort = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 8890;
|
||||||
|
description = ''
|
||||||
|
Port the collector inside the store's container serves its **own**
|
||||||
|
metrics on — queue depth, refused and dropped samples, exporter
|
||||||
|
failures. How you find out that telemetry is being lost, so it is
|
||||||
|
worth keeping rather than switching off.
|
||||||
|
|
||||||
|
⚠️ **Deliberately neither 8888 nor 8889.** 8888 is the collector
|
||||||
|
binary's built-in default, which the hive tier
|
||||||
|
({option}`services.hyperhive.otel.telemetryPort`) already binds, and
|
||||||
|
8889 is the swarm tier's
|
||||||
|
({option}`services.hyperhive.swarm.otel.telemetryPort`). This
|
||||||
|
container runs with `privateNetwork = false`, so all three share the
|
||||||
|
host's network namespace whenever they are co-located — and unlike
|
||||||
|
the OTLP ports this one appears nowhere in either config when it is
|
||||||
|
left undeclared, so nothing that compares configured ports can see
|
||||||
|
the clash. The second collector to start simply dies with
|
||||||
|
`bind: address already in use`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
# ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
|
# ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
|
||||||
|
|
@ -2373,6 +2396,25 @@ in
|
||||||
exporters = [ "otlphttp" ];
|
exporters = [ "otlphttp" ];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Moves this collector's self-metrics off the built-in
|
||||||
|
# default of `localhost:8888`, which the hive tier holds —
|
||||||
|
# this container shares the host's netns, so leaving the
|
||||||
|
# default in place is the hive collector failing to bind.
|
||||||
|
#
|
||||||
|
# ⚠️ `metrics.address` is the spelling that looks right and
|
||||||
|
# is REJECTED by this collector version:
|
||||||
|
# `'migration.MetricsConfigV030' has invalid keys:
|
||||||
|
# address`. `readers` is the schema it accepts, and the
|
||||||
|
# difference is a startup failure rather than a warning.
|
||||||
|
service.telemetry.metrics.readers = [
|
||||||
|
{
|
||||||
|
pull.exporter.prometheus = {
|
||||||
|
host = "127.0.0.1";
|
||||||
|
port = cfg.otel.telemetryPort;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
# An extension configured but not listed here is INERT — the
|
# An extension configured but not listed here is INERT — the
|
||||||
# receiver's `storage: file_storage` above would name a
|
# receiver's `storage: file_storage` above would name a
|
||||||
# component the collector never starts, and the exporter's
|
# component the collector never starts, and the exporter's
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,23 @@ let
|
||||||
# one carries none.
|
# one carries none.
|
||||||
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
|
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Also duplicated from bao-otel-collector.nix, where it is the fixture for
|
||||||
|
# the store standing beside a HIVE collector. That co-location is exactly
|
||||||
|
# the topology the port case below needs: two collectors, one netns, and
|
||||||
|
# nothing but their self-telemetry ports keeping them apart.
|
||||||
|
# `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
|
||||||
|
# demands of any hive with the tier on.
|
||||||
|
baoWithHiveOtel = hive {
|
||||||
|
deploy.bao.enable = true;
|
||||||
|
otel.enable = true;
|
||||||
|
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The collector INSIDE the store's container — neither the host's nor the
|
||||||
|
# swarm tier's.
|
||||||
|
baoForwarderSettings =
|
||||||
|
machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
|
||||||
cases = [
|
cases = [
|
||||||
{
|
{
|
||||||
# The tier in the middle. Its OTLP receiver takes both signals on one
|
# The tier in the middle. Its OTLP receiver takes both signals on one
|
||||||
|
|
@ -133,17 +150,28 @@ let
|
||||||
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# Both collectors share a network namespace whenever they are
|
# Collectors share a network namespace whenever they are co-located,
|
||||||
# co-located, and this port appears in no config the port-collision
|
# and this port appears in no config the port-collision assertion can
|
||||||
# assertion can read — so equal defaults mean the second to start dies
|
# read — so equal defaults mean the second to start dies at `bind()`.
|
||||||
# at `bind()`. Pinned as a case rather than an assertion: enforcing it
|
# Pinned as a case rather than an assertion: enforcing it belongs with
|
||||||
# belongs with the other port checks, not here.
|
# the other port checks, not here.
|
||||||
name = "the two collector tiers do not claim the same self-telemetry port";
|
#
|
||||||
|
# THREE tiers, because the store's container grew a collector of its
|
||||||
|
# own and this case could not see it: an UNDECLARED port is not an
|
||||||
|
# absent one, it is the binary's 8888, which is the hive tier's. That
|
||||||
|
# is why `portOf` falls back to 8888 rather than throwing — a tier that
|
||||||
|
# declares nothing has to fail this case loudly, not evaluate away.
|
||||||
|
name = "no two collector tiers claim the same self-telemetry port";
|
||||||
ok =
|
ok =
|
||||||
let
|
let
|
||||||
portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port;
|
portOf =
|
||||||
|
s:
|
||||||
|
(lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
|
||||||
|
hivePort = portOf hiveOtelSettings;
|
||||||
|
swarmPort = portOf (otelSettings baoWithCollector);
|
||||||
|
baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
|
||||||
in
|
in
|
||||||
portOf hiveOtelSettings != portOf (otelSettings baoWithCollector);
|
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
in
|
in
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue