container collectors scrape their own telemetry and push it up their hop
This commit is contained in:
atlas
mara
parent
2095bb2d0f
commit
73c966a7eb
11 changed files with 163 additions and 49 deletions
|
|
@ -219,10 +219,12 @@ and failure series on `collector` appear once something fails, and the
|
|||
collector and of the swarm's collector, which both carry the collector
|
||||
binary's default `service.name`, `otelcol-contrib`. The timeseries panels
|
||||
split by `server.port`: 8888 is the hive collector, 8889 the swarm
|
||||
collector. The per-container collectors (the secret store, NATS, Grafana,
|
||||
Authelia, the Victoria stores and the agents' forwarders) don't report
|
||||
their own metrics yet, so they have no lines here; a separate issue on
|
||||
the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI
|
||||
collector. Every per-container collector (the secret store, the other service
|
||||
containers and the agents' forwarders) also scrapes its own endpoint on
|
||||
loopback and pushes the result up its normal hop, so each has a line of its
|
||||
own, on its own port: agents all report on 8904, service containers on
|
||||
8890 to 8903. The series carry `container` (services) or `agent` (agents)
|
||||
to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI
|
||||
and in `scripts/pre-push`. It fails when a dashboard query selects a
|
||||
`service.name` that nothing emits.
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,11 @@ let
|
|||
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
||||
};
|
||||
|
||||
# Where the collector serves its own metrics, inside the container's own
|
||||
# netns. Not the binary's default 8888, which the collector dashboard
|
||||
# reads as the hive collector's.
|
||||
selfTelemetryPort = 8904;
|
||||
|
||||
# The exporter's NAME picks the wire protocol, so `protocol` has to be
|
||||
# read here as well as handed to the SDKs above — same expression as the
|
||||
# swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
|
||||
|
|
@ -296,5 +301,14 @@ in
|
|||
service.extensions = [ "file_storage" ];
|
||||
};
|
||||
};
|
||||
|
||||
# Every agent container has a network namespace of its own, so they can all
|
||||
# use the same port.
|
||||
services.opentelemetry-collector.settings = import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = selfTelemetryPort;
|
||||
exporter = logExporter;
|
||||
attributes.agent = userName;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
52
nix/collector-self-scrape.nix
Normal file
52
nix/collector-self-scrape.nix
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Collector settings that make a container's OpenTelemetry collector report its
|
||||
# own health (refused and failed records, exporter queue, memory), imported by
|
||||
# every container collector: the agent forwarder, the service journal
|
||||
# forwarder and the secret store's collector. The two host-side tiers scrape
|
||||
# themselves through their `scrapeTargets` instead.
|
||||
#
|
||||
# One function renders both halves, so the endpoint and the job that reads it
|
||||
# cannot name different ports. The endpoint binds loopback: the collector
|
||||
# scrapes itself, and nothing outside the container has a reason to.
|
||||
#
|
||||
# The job runs in its own pipeline. A container's other metrics pipeline may
|
||||
# carry a `resource` processor that upserts `service.name`, which would
|
||||
# relabel these series away from the `otelcol-contrib` the collector dashboard
|
||||
# selects on. The attributes stamped here tell the container's collectors
|
||||
# apart.
|
||||
#
|
||||
# `readers`, not `metrics.address`: this collector version rejects `address`
|
||||
# at startup.
|
||||
{
|
||||
lib,
|
||||
port,
|
||||
exporter,
|
||||
attributes,
|
||||
}:
|
||||
{
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
inherit port;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
receivers."prometheus/self".config.scrape_configs = [
|
||||
{
|
||||
job_name = "collector";
|
||||
static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ];
|
||||
}
|
||||
];
|
||||
|
||||
processors."resource/self".attributes = lib.mapAttrsToList (key: value: {
|
||||
inherit key value;
|
||||
action = "upsert";
|
||||
}) attributes;
|
||||
|
||||
service.pipelines."metrics/self" = {
|
||||
receivers = [ "prometheus/self" ];
|
||||
processors = [ "resource/self" ];
|
||||
exporters = [ exporter ];
|
||||
};
|
||||
}
|
||||
|
|
@ -96,8 +96,9 @@ in
|
|||
type = lib.types.port;
|
||||
description = ''
|
||||
Loopback port for the collector's own metrics (refused and dropped
|
||||
records, exporter failures). Required because the container may share
|
||||
the host netns, where the binary's default 8888 is the hive tier's.
|
||||
records, exporter failures), which it scrapes and pushes to the swarm
|
||||
collector. Required because the container may share the host netns,
|
||||
where the binary's default 8888 is the hive tier's.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -242,17 +243,6 @@ in
|
|||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
|
||||
# `address` at startup.
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
port = cfg.telemetryPort;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# An extension not listed here is inert: the receiver's storage would
|
||||
# name a component that never starts, and the exporter would push
|
||||
# unauthenticated.
|
||||
|
|
@ -261,6 +251,12 @@ in
|
|||
authName
|
||||
];
|
||||
}
|
||||
(import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = cfg.telemetryPort;
|
||||
exporter = "otlphttp";
|
||||
attributes.container = cfg.serviceName;
|
||||
})
|
||||
# A separate set the module system merges in: joining it to the first with
|
||||
# `//` would replace the first set's whole `service` attrset.
|
||||
(lib.mkIf (cfg.otlpMetricsPort != null) {
|
||||
|
|
|
|||
|
|
@ -4575,25 +4575,6 @@ in
|
|||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# Moves this collector's self-metrics off the built-in
|
||||
# default of `localhost:8888`, which the hive tier holds —
|
||||
# this container shares the host's netns, so leaving the
|
||||
# default in place is the hive collector failing to bind.
|
||||
#
|
||||
# ⚠️ `metrics.address` is the spelling that looks right and
|
||||
# is REJECTED by this collector version:
|
||||
# `'migration.MetricsConfigV030' has invalid keys:
|
||||
# address`. `readers` is the schema it accepts, and the
|
||||
# difference is a startup failure rather than a warning.
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
port = cfg.otel.telemetryPort;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts, and the exporter's
|
||||
|
|
@ -4606,6 +4587,15 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# Off the built-in default of `localhost:8888`, which the hive tier
|
||||
# holds: this container shares the host's netns.
|
||||
services.opentelemetry-collector.settings = import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = cfg.otel.telemetryPort;
|
||||
exporter = "otlphttp";
|
||||
attributes.container = collectorServiceName;
|
||||
};
|
||||
|
||||
# How the secret crosses the sandbox. The host unit
|
||||
# `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this
|
||||
# path on the host's /run, which the container bind-mounts
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"title": "hyperhive \u00b7 collector (self-telemetry)",
|
||||
"uid": "hyperhive-swarm-collector",
|
||||
"description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
|
||||
"description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.",
|
||||
"editable": true,
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 39,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ let
|
|||
agent
|
||||
carriesJournaldSeverity
|
||||
runGroup
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
# The log path's three hops, one fixture each. Nothing carries a journal
|
||||
|
|
@ -55,6 +56,20 @@ let
|
|||
|
||||
agentSettings = machine: machine.services.opentelemetry-collector.settings;
|
||||
cases = [
|
||||
{
|
||||
# The collector's own counters are the only signal that it is dropping
|
||||
# telemetry. 8904 keeps agents off 8888, which the collector dashboard
|
||||
# reads as the hive collector; the exporter follows the wire protocol.
|
||||
name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
|
||||
ok =
|
||||
selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
|
||||
&& selfScrapes (agentSettings agentOtelGrpc) 8904 {
|
||||
agent = agentOtelGrpc.services.hyperhive.agent.user.name;
|
||||
}
|
||||
# Control: with the switch off the lookup finds no collector config.
|
||||
&& !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
|
||||
&& (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
|
||||
}
|
||||
{
|
||||
# The journald receiver's own default directory is the RUNTIME
|
||||
# journal, and a container that stores persistently leaves that
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ let
|
|||
runGroup
|
||||
baoSettings
|
||||
forceServiceConfigs
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
baoTwoAddresses = hive {
|
||||
|
|
@ -185,6 +186,20 @@ let
|
|||
&& m.exporters == [ "otlphttp" ]
|
||||
&& m.exporters == s.service.pipelines.logs.exporters;
|
||||
}
|
||||
{
|
||||
# The store's collector reports its own health through a pipeline of its
|
||||
# own: the store pipeline above upserts `service.name`, and sharing it
|
||||
# would relabel these series away from the collector dashboard.
|
||||
name = "the store's collector scrapes its own telemetry endpoint on loopback";
|
||||
ok =
|
||||
let
|
||||
s = (baoForwarder baoWithCollector).settings;
|
||||
port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
|
||||
in
|
||||
selfScrapes s port { container = "bao"; }
|
||||
&& s.service.pipelines.metrics.receivers == [ "prometheus" ]
|
||||
&& s.service.pipelines.metrics.processors == [ "resource" ];
|
||||
}
|
||||
{
|
||||
# Presence control for the case above: both fields are omitted rather
|
||||
# than defaulted, so a target declared as bare `host:port` renders what
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ let
|
|||
})
|
||||
hive
|
||||
runGroup
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
inherit (shared) bare;
|
||||
|
|
@ -558,11 +559,10 @@ let
|
|||
ok =
|
||||
let
|
||||
c = ciAlone.containers.swarm-ci-runner.config;
|
||||
reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
|
||||
s = c.services.opentelemetry-collector.settings;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == 8895;
|
||||
&& selfScrapes s 8895 { container = "ci-runner"; };
|
||||
}
|
||||
];
|
||||
in
|
||||
|
|
|
|||
|
|
@ -208,6 +208,37 @@ let
|
|||
# the two tier cases differ only in which receiver they hand it.
|
||||
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
|
||||
|
||||
# Whether a container collector's settings carry the self-scrape of
|
||||
# ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
|
||||
# a job scraping exactly that address, and a metrics pipeline that reads the
|
||||
# job and leaves through an exporter the settings define. `attrs` are the
|
||||
# resource attributes the pipeline stamps.
|
||||
selfScrapes =
|
||||
s: port: attrs:
|
||||
let
|
||||
job = lib.findFirst (j: j.job_name == "collector") null (
|
||||
s.receivers."prometheus/self".config.scrape_configs or [ ]
|
||||
);
|
||||
pipeline = s.service.pipelines."metrics/self" or null;
|
||||
reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
|
||||
stamped = lib.mapAttrsToList (key: value: {
|
||||
inherit key value;
|
||||
action = "upsert";
|
||||
}) attrs;
|
||||
in
|
||||
job != null
|
||||
&& job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
|
||||
&&
|
||||
(reader.pull.exporter.prometheus or null) == {
|
||||
host = "127.0.0.1";
|
||||
inherit port;
|
||||
}
|
||||
&& pipeline != null
|
||||
&& pipeline.receivers == [ "prometheus/self" ]
|
||||
&& (s.processors."resource/self".attributes or null) == stamped
|
||||
&& pipeline.exporters != [ ]
|
||||
&& lib.all (e: s.exporters ? ${e}) pipeline.exporters;
|
||||
|
||||
# The enables that switch owns. The hive tier's collector is not among them: it
|
||||
# follows `deploy.hive-controller.enable`, and a services host without a hive
|
||||
# runs none.
|
||||
|
|
@ -283,6 +314,7 @@ in
|
|||
inherit agentHarness;
|
||||
inherit journaldSeverityParser;
|
||||
inherit carriesJournaldSeverity;
|
||||
inherit selfScrapes;
|
||||
inherit swarmServiceEnables;
|
||||
inherit runGroup;
|
||||
inherit forceServiceConfigs;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ let
|
|||
})
|
||||
hive
|
||||
runGroup
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
# No store identity, which the first case below relies on.
|
||||
|
|
@ -157,12 +158,10 @@ let
|
|||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||
]
|
||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||
&& !(lib.elem svc.port taken)
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
||||
&& selfScrapes s svc.port { container = svc.name; };
|
||||
}
|
||||
]
|
||||
) forwarderServices;
|
||||
|
|
@ -269,12 +268,10 @@ let
|
|||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||
]
|
||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||
&& !(lib.elem svc.port taken)
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
||||
&& selfScrapes s svc.port { container = svc.name; };
|
||||
}
|
||||
]
|
||||
) hostNetnsForwarders;
|
||||
|
|
@ -740,6 +737,7 @@ let
|
|||
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
||||
&& lib.elem "oauth2client/swarm" s.service.extensions
|
||||
&& s.service.pipelines.logs.receivers == [ "journald" ]
|
||||
&& selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
|
||||
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
||||
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
||||
|
|
@ -997,7 +995,7 @@ let
|
|||
registered = lib.findFirst (
|
||||
x: x.id == clientId
|
||||
) null h.services.hyperhive.swarm.authelia.oidc.clients;
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
port = c.services.hyperhive.swarmJournalForwarder.telemetryPort;
|
||||
taken = [
|
||||
8888
|
||||
8890
|
||||
|
|
@ -1022,8 +1020,8 @@ let
|
|||
&& registered.audience == [ clientId ]
|
||||
&& !(lib.elem "--link-journal=host" container.extraFlags)
|
||||
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& !(lib.elem reader.pull.exporter.prometheus.port taken)
|
||||
&& selfScrapes s port { container = "authelia"; }
|
||||
&& !(lib.elem port taken)
|
||||
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
|
||||
&& !(
|
||||
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc
|
||||
|
|
|
|||
Loading…
Reference in a new issue