Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a00bcb8c05 | ||
|
|
73c966a7eb |
11 changed files with 388 additions and 272 deletions
|
|
@ -219,10 +219,12 @@ and failure series on `collector` appear once something fails, and the
|
||||||
collector and of the swarm's collector, which both carry the collector
|
collector and of the swarm's collector, which both carry the collector
|
||||||
binary's default `service.name`, `otelcol-contrib`. The timeseries panels
|
binary's default `service.name`, `otelcol-contrib`. The timeseries panels
|
||||||
split by `server.port`: 8888 is the hive collector, 8889 the swarm
|
split by `server.port`: 8888 is the hive collector, 8889 the swarm
|
||||||
collector. The per-container collectors (the secret store, NATS, Grafana,
|
collector. Every per-container collector (the secret store, the other service
|
||||||
Authelia, the Victoria stores and the agents' forwarders) don't report
|
containers and the agents' forwarders) also scrapes its own endpoint on
|
||||||
their own metrics yet, so they have no lines here; a separate issue on
|
loopback and pushes the result up its normal hop, so each has a line of its
|
||||||
the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI
|
own, on its own port: agents all report on 8904, service containers on
|
||||||
|
8890 to 8903. The series carry `container` (services) or `agent` (agents)
|
||||||
|
to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI
|
||||||
and in `scripts/pre-push`. It fails when a dashboard query selects a
|
and in `scripts/pre-push`. It fails when a dashboard query selects a
|
||||||
`service.name` that nothing emits.
|
`service.name` that nothing emits.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,11 @@ let
|
||||||
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Where the collector serves its own metrics, inside the container's own
|
||||||
|
# netns. Not the binary's default 8888, which the collector dashboard
|
||||||
|
# reads as the hive collector's.
|
||||||
|
selfTelemetryPort = 8904;
|
||||||
|
|
||||||
# The exporter's NAME picks the wire protocol, so `protocol` has to be
|
# The exporter's NAME picks the wire protocol, so `protocol` has to be
|
||||||
# read here as well as handed to the SDKs above — same expression as the
|
# read here as well as handed to the SDKs above — same expression as the
|
||||||
# swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
|
# swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
|
||||||
|
|
@ -210,91 +215,101 @@ in
|
||||||
# and the collector then dies at startup. Same caveat as both other
|
# and the collector then dies at startup. Same caveat as both other
|
||||||
# tiers; see nix/host-modules/otel.nix for the measurement.
|
# tiers; see nix/host-modules/otel.nix for the measurement.
|
||||||
validateConfigFile = true;
|
validateConfigFile = true;
|
||||||
settings = {
|
settings = lib.mkMerge [
|
||||||
# `journalctl --follow --lines=0` (what this receiver runs under
|
{
|
||||||
# the hood) ships only what's written *after* it starts — every
|
# `journalctl --follow --lines=0` (what this receiver runs under
|
||||||
# collector start has a silent gap at the front. `storage:` below
|
# the hood) ships only what's written *after* it starts — every
|
||||||
# persists the read cursor, so a restart resumes from where it
|
# collector start has a silent gap at the front. `storage:` below
|
||||||
# left off instead of re-opening that gap; `start_at` stays `end`
|
# persists the read cursor, so a restart resumes from where it
|
||||||
# (its default) because a cursor already covers every run after
|
# left off instead of re-opening that gap; `start_at` stays `end`
|
||||||
# the first, and `beginning` without one would re-ship the whole
|
# (its default) because a cursor already covers every run after
|
||||||
# journal on every restart. The cursor lives in the unit's own
|
# the first, and `beginning` without one would re-ship the whole
|
||||||
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below
|
# journal on every restart. The cursor lives in the unit's own
|
||||||
# as `WorkingDirectory` too) — container-lifetime, not the
|
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below
|
||||||
# host-persisted state bind mount: it indexes /var/log/journal,
|
# as `WorkingDirectory` too) — container-lifetime, not the
|
||||||
# itself fresh on every container start, so the two must share
|
# host-persisted state bind mount: it indexes /var/log/journal,
|
||||||
# a lifetime or the cursor outlives the journal it points into.
|
# itself fresh on every container start, so the two must share
|
||||||
# `create_directory` matters here for a build-time-vs-runtime
|
# a lifetime or the cursor outlives the journal it points into.
|
||||||
# reason, not a convenience one: `validateConfigFile` above runs
|
# `create_directory` matters here for a build-time-vs-runtime
|
||||||
# `otelcol validate` in the nix build sandbox, before systemd's
|
# reason, not a convenience one: `validateConfigFile` above runs
|
||||||
# `StateDirectory=` has ever created this path. Without it the
|
# `otelcol validate` in the nix build sandbox, before systemd's
|
||||||
# validator refuses with "directory must exist" and the build fails.
|
# `StateDirectory=` has ever created this path. Without it the
|
||||||
extensions.file_storage = {
|
# validator refuses with "directory must exist" and the build fails.
|
||||||
directory = "/var/lib/opentelemetry-collector";
|
extensions.file_storage = {
|
||||||
create_directory = true;
|
directory = "/var/lib/opentelemetry-collector";
|
||||||
};
|
create_directory = true;
|
||||||
|
};
|
||||||
|
|
||||||
receivers.journald = {
|
receivers.journald = {
|
||||||
# ⚠️ STATED, and it must stay stated: the receiver's own default
|
# ⚠️ STATED, and it must stay stated: the receiver's own default
|
||||||
# is the RUNTIME journal (`/run/log/journal`), which in an agent
|
# is the RUNTIME journal (`/run/log/journal`), which in an agent
|
||||||
# container is empty — journald stores persistently here, so
|
# container is empty — journald stores persistently here, so
|
||||||
# every entry is under /var/log/journal. Dropping this line
|
# every entry is under /var/log/journal. Dropping this line
|
||||||
# leaves a collector that validates, starts, reports healthy and
|
# leaves a collector that validates, starts, reports healthy and
|
||||||
# forwards nothing.
|
# forwards nothing.
|
||||||
directory = "/var/log/journal";
|
directory = "/var/log/journal";
|
||||||
storage = "file_storage";
|
storage = "file_storage";
|
||||||
# No `units` allowlist, and no user-session filter like the swarm
|
# No `units` allowlist, and no user-session filter like the swarm
|
||||||
# tier's: a container's journal is the harness and what the
|
# tier's: a container's journal is the harness and what the
|
||||||
# harness spawns, so there is no foreign traffic to filter out and
|
# harness spawns, so there is no foreign traffic to filter out and
|
||||||
# an allowlist would only be a list to forget to update.
|
# an allowlist would only be a list to forget to update.
|
||||||
|
|
||||||
# journald's PRIORITY carries the level every line already has;
|
# journald's PRIORITY carries the level every line already has;
|
||||||
# without this the record reaches VictoriaLogs with
|
# without this the record reaches VictoriaLogs with
|
||||||
# `severity_text: Unspecified` and the store cannot tell an error
|
# `severity_text: Unspecified` and the store cannot tell an error
|
||||||
# from a debug line. The table — and the inverted direction that
|
# from a debug line. The table — and the inverted direction that
|
||||||
# makes it worth a file of its own — lives in
|
# makes it worth a file of its own — lives in
|
||||||
# ../journald-severity.nix, shared with the swarm tier's receiver.
|
# ../journald-severity.nix, shared with the swarm tier's receiver.
|
||||||
operators = import ../journald-severity.nix;
|
operators = import ../journald-severity.nix;
|
||||||
};
|
};
|
||||||
|
|
||||||
# Identity the hop above cannot supply. A host-side reader can say
|
# Identity the hop above cannot supply. A host-side reader can say
|
||||||
# which machine a line came from; only a collector running as the
|
# which machine a line came from; only a collector running as the
|
||||||
# agent can say which agent. `hive` and `swarm` are deliberately
|
# agent can say which agent. `hive` and `swarm` are deliberately
|
||||||
# NOT stamped here: the swarm tier upserts both from the receiver
|
# NOT stamped here: the swarm tier upserts both from the receiver
|
||||||
# that accepted the sample, precisely so the labels come from
|
# that accepted the sample, precisely so the labels come from
|
||||||
# something the sender cannot write.
|
# something the sender cannot write.
|
||||||
processors.resource.attributes = [
|
processors.resource.attributes = [
|
||||||
{
|
{
|
||||||
key = "service.name";
|
key = "service.name";
|
||||||
value = "hyperhive-agent";
|
value = "hyperhive-agent";
|
||||||
action = "upsert";
|
action = "upsert";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
key = "agent";
|
||||||
|
value = userName;
|
||||||
|
action = "upsert";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
# The hive's collector, at the same base address every in-process
|
||||||
|
# producer already exports to. `endpoint` is a BASE the exporter
|
||||||
|
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
|
||||||
|
exporters.${logExporter} = {
|
||||||
|
endpoint = cfg.endpoint;
|
||||||
}
|
}
|
||||||
{
|
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
|
||||||
key = "agent";
|
|
||||||
value = userName;
|
|
||||||
action = "upsert";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
# The hive's collector, at the same base address every in-process
|
service.pipelines.logs = {
|
||||||
# producer already exports to. `endpoint` is a BASE the exporter
|
receivers = [ "journald" ];
|
||||||
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
|
processors = [ "resource" ];
|
||||||
exporters.${logExporter} = {
|
exporters = [ logExporter ];
|
||||||
endpoint = cfg.endpoint;
|
};
|
||||||
|
|
||||||
|
# An extension configured but not listed here is INERT — the
|
||||||
|
# journald receiver's `storage: file_storage` above would name a
|
||||||
|
# component the collector never starts.
|
||||||
|
service.extensions = [ "file_storage" ];
|
||||||
}
|
}
|
||||||
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
|
# Every agent container has a network namespace of its own, so they can all
|
||||||
|
# use the same port.
|
||||||
service.pipelines.logs = {
|
(import ../collector-self-scrape.nix {
|
||||||
receivers = [ "journald" ];
|
inherit lib;
|
||||||
processors = [ "resource" ];
|
port = selfTelemetryPort;
|
||||||
exporters = [ logExporter ];
|
exporter = logExporter;
|
||||||
};
|
attributes.agent = userName;
|
||||||
|
})
|
||||||
# An extension configured but not listed here is INERT — the
|
];
|
||||||
# journald receiver's `storage: file_storage` above would name a
|
|
||||||
# component the collector never starts.
|
|
||||||
service.extensions = [ "file_storage" ];
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
52
nix/collector-self-scrape.nix
Normal file
52
nix/collector-self-scrape.nix
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
# Collector settings that make a container's OpenTelemetry collector report its
|
||||||
|
# own health (refused and failed records, exporter queue, memory), imported by
|
||||||
|
# every container collector: the agent forwarder, the service journal
|
||||||
|
# forwarder and the secret store's collector. The two host-side tiers scrape
|
||||||
|
# themselves through their `scrapeTargets` instead.
|
||||||
|
#
|
||||||
|
# One function renders both halves, so the endpoint and the job that reads it
|
||||||
|
# cannot name different ports. The endpoint binds loopback: the collector
|
||||||
|
# scrapes itself, and nothing outside the container has a reason to.
|
||||||
|
#
|
||||||
|
# The job runs in its own pipeline. A container's other metrics pipeline may
|
||||||
|
# carry a `resource` processor that upserts `service.name`, which would
|
||||||
|
# relabel these series away from the `otelcol-contrib` the collector dashboard
|
||||||
|
# selects on. The attributes stamped here tell the container's collectors
|
||||||
|
# apart.
|
||||||
|
#
|
||||||
|
# `readers`, not `metrics.address`: this collector version rejects `address`
|
||||||
|
# at startup.
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
port,
|
||||||
|
exporter,
|
||||||
|
attributes,
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
service.telemetry.metrics.readers = [
|
||||||
|
{
|
||||||
|
pull.exporter.prometheus = {
|
||||||
|
host = "127.0.0.1";
|
||||||
|
inherit port;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
receivers."prometheus/self".config.scrape_configs = [
|
||||||
|
{
|
||||||
|
job_name = "collector";
|
||||||
|
static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
processors."resource/self".attributes = lib.mapAttrsToList (key: value: {
|
||||||
|
inherit key value;
|
||||||
|
action = "upsert";
|
||||||
|
}) attributes;
|
||||||
|
|
||||||
|
service.pipelines."metrics/self" = {
|
||||||
|
receivers = [ "prometheus/self" ];
|
||||||
|
processors = [ "resource/self" ];
|
||||||
|
exporters = [ exporter ];
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -96,8 +96,9 @@ in
|
||||||
type = lib.types.port;
|
type = lib.types.port;
|
||||||
description = ''
|
description = ''
|
||||||
Loopback port for the collector's own metrics (refused and dropped
|
Loopback port for the collector's own metrics (refused and dropped
|
||||||
records, exporter failures). Required because the container may share
|
records, exporter failures), which it scrapes and pushes to the swarm
|
||||||
the host netns, where the binary's default 8888 is the hive tier's.
|
collector. Required because the container may share the host netns,
|
||||||
|
where the binary's default 8888 is the hive tier's.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -242,17 +243,6 @@ in
|
||||||
exporters = [ "otlphttp" ];
|
exporters = [ "otlphttp" ];
|
||||||
};
|
};
|
||||||
|
|
||||||
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
|
|
||||||
# `address` at startup.
|
|
||||||
service.telemetry.metrics.readers = [
|
|
||||||
{
|
|
||||||
pull.exporter.prometheus = {
|
|
||||||
host = "127.0.0.1";
|
|
||||||
port = cfg.telemetryPort;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
# An extension not listed here is inert: the receiver's storage would
|
# An extension not listed here is inert: the receiver's storage would
|
||||||
# name a component that never starts, and the exporter would push
|
# name a component that never starts, and the exporter would push
|
||||||
# unauthenticated.
|
# unauthenticated.
|
||||||
|
|
@ -261,6 +251,12 @@ in
|
||||||
authName
|
authName
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
(import ../collector-self-scrape.nix {
|
||||||
|
inherit lib;
|
||||||
|
port = cfg.telemetryPort;
|
||||||
|
exporter = "otlphttp";
|
||||||
|
attributes.container = cfg.serviceName;
|
||||||
|
})
|
||||||
# A separate set the module system merges in: joining it to the first with
|
# A separate set the module system merges in: joining it to the first with
|
||||||
# `//` would replace the first set's whole `service` attrset.
|
# `//` would replace the first set's whole `service` attrset.
|
||||||
(lib.mkIf (cfg.otlpMetricsPort != null) {
|
(lib.mkIf (cfg.otlpMetricsPort != null) {
|
||||||
|
|
|
||||||
|
|
@ -4438,172 +4438,163 @@ in
|
||||||
# caveat as every other tier; see nix/host-modules/otel.nix for
|
# caveat as every other tier; see nix/host-modules/otel.nix for
|
||||||
# the measurement.
|
# the measurement.
|
||||||
validateConfigFile = true;
|
validateConfigFile = true;
|
||||||
settings = {
|
settings = lib.mkMerge [
|
||||||
# `journalctl --follow --lines=0` (what the receiver runs under
|
{
|
||||||
# the hood) ships only what is written *after* it starts, so
|
# `journalctl --follow --lines=0` (what the receiver runs under
|
||||||
# every collector start has a silent gap at the front. This
|
# the hood) ships only what is written *after* it starts, so
|
||||||
# persists the read cursor, so a restart resumes instead of
|
# every collector start has a silent gap at the front. This
|
||||||
# re-opening that gap; `start_at` stays `end` (its default),
|
# persists the read cursor, so a restart resumes instead of
|
||||||
# since a cursor covers every run after the first and
|
# re-opening that gap; `start_at` stays `end` (its default),
|
||||||
# `beginning` without one would re-ship the whole journal on
|
# since a cursor covers every run after the first and
|
||||||
# every restart. `create_directory` is required for a
|
# `beginning` without one would re-ship the whole journal on
|
||||||
# build-time reason rather than a convenience one:
|
# every restart. `create_directory` is required for a
|
||||||
# `validateConfigFile` above runs `otelcol validate` in the nix
|
# build-time reason rather than a convenience one:
|
||||||
# sandbox, before systemd's `StateDirectory=` has created the
|
# `validateConfigFile` above runs `otelcol validate` in the nix
|
||||||
# path, and the validator refuses with "directory must exist".
|
# sandbox, before systemd's `StateDirectory=` has created the
|
||||||
extensions.file_storage = {
|
# path, and the validator refuses with "directory must exist".
|
||||||
directory = "/var/lib/opentelemetry-collector";
|
extensions.file_storage = {
|
||||||
create_directory = true;
|
directory = "/var/lib/opentelemetry-collector";
|
||||||
};
|
create_directory = true;
|
||||||
|
};
|
||||||
|
|
||||||
receivers.journald = {
|
receivers.journald = {
|
||||||
# ⚠️ STATED, and it must stay stated: the receiver's own
|
# ⚠️ STATED, and it must stay stated: the receiver's own
|
||||||
# default is the RUNTIME journal (`/run/log/journal`), which
|
# default is the RUNTIME journal (`/run/log/journal`), which
|
||||||
# in this container is empty — journald stores persistently
|
# in this container is empty — journald stores persistently
|
||||||
# here, so every entry is under /var/log/journal. Dropping
|
# here, so every entry is under /var/log/journal. Dropping
|
||||||
# this line leaves a collector that validates, starts,
|
# this line leaves a collector that validates, starts,
|
||||||
# reports healthy and forwards nothing.
|
# reports healthy and forwards nothing.
|
||||||
directory = "/var/log/journal";
|
directory = "/var/log/journal";
|
||||||
# ⚠️ `--merge` is a guard: an unqualified
|
# ⚠️ `--merge` is a guard: an unqualified
|
||||||
# `journalctl --follow` over a journal with no entry for the
|
# `journalctl --follow` over a journal with no entry for the
|
||||||
# current boot (a host-linked one) crash-loops the receiver
|
# current boot (a host-linked one) crash-loops the receiver
|
||||||
# every ~2s with "No journal boot entry found for the
|
# every ~2s with "No journal boot entry found for the
|
||||||
# specified boot (+0)". This journal is container-local, so
|
# specified boot (+0)". This journal is container-local, so
|
||||||
# the current boot has an entry and the flag changes nothing.
|
# the current boot has an entry and the flag changes nothing.
|
||||||
merge = true;
|
merge = true;
|
||||||
storage = "file_storage";
|
storage = "file_storage";
|
||||||
};
|
};
|
||||||
|
|
||||||
# The store's scrape, run by this container's own collector.
|
# The store's scrape, run by this container's own collector.
|
||||||
#
|
#
|
||||||
# ⚠️ `metrics_path` and `params` are STATED, and both have to
|
# ⚠️ `metrics_path` and `params` are STATED, and both have to
|
||||||
# be: openbao serves no `/metrics` at all, and
|
# be: openbao serves no `/metrics` at all, and
|
||||||
# `/v1/sys/metrics` answers JSON unless the format is asked
|
# `/v1/sys/metrics` answers JSON unless the format is asked
|
||||||
# for. A scrape of the default path 404s and a scrape without
|
# for. A scrape of the default path 404s and a scrape without
|
||||||
# the query parses as nothing — each reads as a dead exporter
|
# the query parses as nothing — each reads as a dead exporter
|
||||||
# rather than as a wrong request. The spelling matches
|
# rather than as a wrong request. The spelling matches
|
||||||
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
|
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
|
||||||
# every other service's target.
|
# every other service's target.
|
||||||
#
|
#
|
||||||
# No credential, unlike the swarm collector's
|
# No credential, unlike the swarm collector's
|
||||||
# `publishedScrapeTargets` jobs: the target is the loopback
|
# `publishedScrapeTargets` jobs: the target is the loopback
|
||||||
# listener above, inside this container's own netns, which is
|
# listener above, inside this container's own netns, which is
|
||||||
# what lets the scrape stay unauthenticated without the API
|
# what lets the scrape stay unauthenticated without the API
|
||||||
# listener's client-cert requirement being touched.
|
# listener's client-cert requirement being touched.
|
||||||
receivers.prometheus.config.scrape_configs = [
|
receivers.prometheus.config.scrape_configs = [
|
||||||
{
|
{
|
||||||
job_name = "bao";
|
job_name = "bao";
|
||||||
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
|
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
|
||||||
metrics_path = "/v1/sys/metrics";
|
metrics_path = "/v1/sys/metrics";
|
||||||
params.format = [ "prometheus" ];
|
params.format = [ "prometheus" ];
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
# Identity the hop above cannot supply: a host-side reader can
|
# Identity the hop above cannot supply: a host-side reader can
|
||||||
# say which machine a line came from, and only a collector
|
# say which machine a line came from, and only a collector
|
||||||
# inside this container can say it was the secret store's. No
|
# inside this container can say it was the secret store's. No
|
||||||
# `hive` or `swarm` label — the swarm tier upserts `hive` from
|
# `hive` or `swarm` label — the swarm tier upserts `hive` from
|
||||||
# the receiver that accepted the record, precisely so it comes
|
# the receiver that accepted the record, precisely so it comes
|
||||||
# from something the sender cannot write.
|
# from something the sender cannot write.
|
||||||
#
|
#
|
||||||
# `collectorServiceName`, not `cfg.machine`: what the
|
# `collectorServiceName`, not `cfg.machine`: what the
|
||||||
# dashboard selects on is a display identity, not the
|
# dashboard selects on is a display identity, not the
|
||||||
# container/receiver name — see that binding's own comment.
|
# container/receiver name — see that binding's own comment.
|
||||||
processors.resource.attributes = [
|
processors.resource.attributes = [
|
||||||
{
|
{
|
||||||
key = "service.name";
|
key = "service.name";
|
||||||
value = collectorServiceName;
|
value = collectorServiceName;
|
||||||
action = "upsert";
|
action = "upsert";
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
# `otlphttp` unconditionally, unlike the agent forwarder's
|
# `otlphttp` unconditionally, unlike the agent forwarder's
|
||||||
# protocol-derived exporter name: the swarm receiver speaks
|
# protocol-derived exporter name: the swarm receiver speaks
|
||||||
# OTLP/HTTP protobuf whatever this tier's *upstream*
|
# OTLP/HTTP protobuf whatever this tier's *upstream*
|
||||||
# protocol is, which is the same reason
|
# protocol is, which is the same reason
|
||||||
# ./hive-c0re/environment.nix pins the protocol it hands out.
|
# ./hive-c0re/environment.nix pins the protocol it hands out.
|
||||||
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
|
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
|
||||||
exporters.otlphttp = {
|
exporters.otlphttp = {
|
||||||
endpoint = otelFirstHop;
|
endpoint = otelFirstHop;
|
||||||
# The receiver at that address accepts an OIDC token and
|
# The receiver at that address accepts an OIDC token and
|
||||||
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
|
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
|
||||||
# so this is what makes the hop deliver rather than collect
|
# so this is what makes the hop deliver rather than collect
|
||||||
# 401s — unconditional, because a forwarder that exports
|
# 401s — unconditional, because a forwarder that exports
|
||||||
# unauthenticated has no working mode to fall back to.
|
# unauthenticated has no working mode to fall back to.
|
||||||
auth.authenticator = forwarderAuthName;
|
auth.authenticator = forwarderAuthName;
|
||||||
};
|
};
|
||||||
|
|
||||||
# The push side, opposite in direction to the receiver's
|
# The push side, opposite in direction to the receiver's
|
||||||
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
|
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
|
||||||
# to send. Hence `oauth2client`, and hence a client id and
|
# to send. Hence `oauth2client`, and hence a client id and
|
||||||
# secret rather than an issuer and an audience to check.
|
# secret rather than an issuer and an audience to check.
|
||||||
extensions.${forwarderAuthName} = {
|
extensions.${forwarderAuthName} = {
|
||||||
client_id = cfg.otel.clientId;
|
client_id = cfg.otel.clientId;
|
||||||
# A path, never a value — and one systemd resolves at
|
# A path, never a value — and one systemd resolves at
|
||||||
# runtime, so neither the secret nor the credentials
|
# runtime, so neither the secret nor the credentials
|
||||||
# directory is rendered into the nix store.
|
# directory is rendered into the nix store.
|
||||||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
|
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
|
||||||
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
||||||
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
||||||
# Registering it on the client only makes it permissible; a
|
# Registering it on the client only makes it permissible; a
|
||||||
# token minted without asking carries `aud: []` and the
|
# token minted without asking carries `aud: []` and the
|
||||||
# receiver refuses it — with a config that reads perfectly
|
# receiver refuses it — with a config that reads perfectly
|
||||||
# at both ends.
|
# at both ends.
|
||||||
endpoint_params.audience = cfg.otel.clientId;
|
endpoint_params.audience = cfg.otel.clientId;
|
||||||
};
|
};
|
||||||
|
|
||||||
service.pipelines.logs = {
|
service.pipelines.logs = {
|
||||||
receivers = [ "journald" ];
|
receivers = [ "journald" ];
|
||||||
processors = [ "resource" ];
|
processors = [ "resource" ];
|
||||||
exporters = [ "otlphttp" ];
|
exporters = [ "otlphttp" ];
|
||||||
};
|
};
|
||||||
|
|
||||||
# A SECOND pipeline, not a second receiver on the logs one: a
|
# A SECOND pipeline, not a second receiver on the logs one: a
|
||||||
# pipeline is typed, and a prometheus receiver in `logs` is a
|
# pipeline is typed, and a prometheus receiver in `logs` is a
|
||||||
# config the collector rejects at startup rather than one that
|
# config the collector rejects at startup rather than one that
|
||||||
# quietly ships metrics as log records.
|
# quietly ships metrics as log records.
|
||||||
#
|
#
|
||||||
# Same `resource` processor and same `otlphttp` exporter, so
|
# Same `resource` processor and same `otlphttp` exporter, so
|
||||||
# the store's metrics carry the same `service.name` its logs
|
# the store's metrics carry the same `service.name` its logs
|
||||||
# do and leave by the same hop — no otel bypass, one exit from
|
# do and leave by the same hop — no otel bypass, one exit from
|
||||||
# this container. The exporter treats `endpoint` as a base and
|
# this container. The exporter treats `endpoint` as a base and
|
||||||
# appends the signal's own path, so `/v1/metrics` here and
|
# appends the signal's own path, so `/v1/metrics` here and
|
||||||
# `/v1/logs` above come from the one address.
|
# `/v1/logs` above come from the one address.
|
||||||
service.pipelines.metrics = {
|
service.pipelines.metrics = {
|
||||||
receivers = [ "prometheus" ];
|
receivers = [ "prometheus" ];
|
||||||
processors = [ "resource" ];
|
processors = [ "resource" ];
|
||||||
exporters = [ "otlphttp" ];
|
exporters = [ "otlphttp" ];
|
||||||
};
|
};
|
||||||
|
|
||||||
# Moves this collector's self-metrics off the built-in
|
# An extension configured but not listed here is INERT — the
|
||||||
# default of `localhost:8888`, which the hive tier holds —
|
# receiver's `storage: file_storage` above would name a
|
||||||
# this container shares the host's netns, so leaving the
|
# component the collector never starts, and the exporter's
|
||||||
# default in place is the hive collector failing to bind.
|
# authenticator would push unauthenticated while the
|
||||||
#
|
# collector reports healthy.
|
||||||
# ⚠️ `metrics.address` is the spelling that looks right and
|
service.extensions = [
|
||||||
# is REJECTED by this collector version:
|
"file_storage"
|
||||||
# `'migration.MetricsConfigV030' has invalid keys:
|
forwarderAuthName
|
||||||
# address`. `readers` is the schema it accepts, and the
|
];
|
||||||
# difference is a startup failure rather than a warning.
|
}
|
||||||
service.telemetry.metrics.readers = [
|
# Off the built-in default of `localhost:8888`, which the hive tier
|
||||||
{
|
# holds: this container shares the host's netns.
|
||||||
pull.exporter.prometheus = {
|
(import ../collector-self-scrape.nix {
|
||||||
host = "127.0.0.1";
|
inherit lib;
|
||||||
port = cfg.otel.telemetryPort;
|
port = cfg.otel.telemetryPort;
|
||||||
};
|
exporter = "otlphttp";
|
||||||
}
|
attributes.container = collectorServiceName;
|
||||||
];
|
})
|
||||||
|
];
|
||||||
# An extension configured but not listed here is INERT — the
|
|
||||||
# receiver's `storage: file_storage` above would name a
|
|
||||||
# component the collector never starts, and the exporter's
|
|
||||||
# authenticator would push unauthenticated while the
|
|
||||||
# collector reports healthy.
|
|
||||||
service.extensions = [
|
|
||||||
"file_storage"
|
|
||||||
forwarderAuthName
|
|
||||||
];
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# How the secret crosses the sandbox. The host unit
|
# How the secret crosses the sandbox. The host unit
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"title": "hyperhive \u00b7 collector (self-telemetry)",
|
"title": "hyperhive \u00b7 collector (self-telemetry)",
|
||||||
"uid": "hyperhive-swarm-collector",
|
"uid": "hyperhive-swarm-collector",
|
||||||
"description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
|
"description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.",
|
||||||
"editable": true,
|
"editable": true,
|
||||||
"refresh": "1m",
|
"refresh": "1m",
|
||||||
"schemaVersion": 39,
|
"schemaVersion": 39,
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ let
|
||||||
agent
|
agent
|
||||||
carriesJournaldSeverity
|
carriesJournaldSeverity
|
||||||
runGroup
|
runGroup
|
||||||
|
selfScrapes
|
||||||
;
|
;
|
||||||
|
|
||||||
# The log path's three hops, one fixture each. Nothing carries a journal
|
# The log path's three hops, one fixture each. Nothing carries a journal
|
||||||
|
|
@ -55,6 +56,20 @@ let
|
||||||
|
|
||||||
agentSettings = machine: machine.services.opentelemetry-collector.settings;
|
agentSettings = machine: machine.services.opentelemetry-collector.settings;
|
||||||
cases = [
|
cases = [
|
||||||
|
{
|
||||||
|
# The collector's own counters are the only signal that it is dropping
|
||||||
|
# telemetry. 8904 keeps agents off 8888, which the collector dashboard
|
||||||
|
# reads as the hive collector; the exporter follows the wire protocol.
|
||||||
|
name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
|
||||||
|
ok =
|
||||||
|
selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
|
||||||
|
&& selfScrapes (agentSettings agentOtelGrpc) 8904 {
|
||||||
|
agent = agentOtelGrpc.services.hyperhive.agent.user.name;
|
||||||
|
}
|
||||||
|
# Control: with the switch off the lookup finds no collector config.
|
||||||
|
&& !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
|
||||||
|
&& (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# The journald receiver's own default directory is the RUNTIME
|
# The journald receiver's own default directory is the RUNTIME
|
||||||
# journal, and a container that stores persistently leaves that
|
# journal, and a container that stores persistently leaves that
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ let
|
||||||
runGroup
|
runGroup
|
||||||
baoSettings
|
baoSettings
|
||||||
forceServiceConfigs
|
forceServiceConfigs
|
||||||
|
selfScrapes
|
||||||
;
|
;
|
||||||
|
|
||||||
baoTwoAddresses = hive {
|
baoTwoAddresses = hive {
|
||||||
|
|
@ -185,6 +186,20 @@ let
|
||||||
&& m.exporters == [ "otlphttp" ]
|
&& m.exporters == [ "otlphttp" ]
|
||||||
&& m.exporters == s.service.pipelines.logs.exporters;
|
&& m.exporters == s.service.pipelines.logs.exporters;
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# The store's collector reports its own health through a pipeline of its
|
||||||
|
# own: the store pipeline above upserts `service.name`, and sharing it
|
||||||
|
# would relabel these series away from the collector dashboard.
|
||||||
|
name = "the store's collector scrapes its own telemetry endpoint on loopback";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
s = (baoForwarder baoWithCollector).settings;
|
||||||
|
port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
|
||||||
|
in
|
||||||
|
selfScrapes s port { container = "bao"; }
|
||||||
|
&& s.service.pipelines.metrics.receivers == [ "prometheus" ]
|
||||||
|
&& s.service.pipelines.metrics.processors == [ "resource" ];
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# Presence control for the case above: both fields are omitted rather
|
# Presence control for the case above: both fields are omitted rather
|
||||||
# than defaulted, so a target declared as bare `host:port` renders what
|
# than defaulted, so a target declared as bare `host:port` renders what
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ let
|
||||||
})
|
})
|
||||||
hive
|
hive
|
||||||
runGroup
|
runGroup
|
||||||
|
selfScrapes
|
||||||
;
|
;
|
||||||
|
|
||||||
inherit (shared) bare;
|
inherit (shared) bare;
|
||||||
|
|
@ -558,11 +559,10 @@ let
|
||||||
ok =
|
ok =
|
||||||
let
|
let
|
||||||
c = ciAlone.containers.swarm-ci-runner.config;
|
c = ciAlone.containers.swarm-ci-runner.config;
|
||||||
reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
|
s = c.services.opentelemetry-collector.settings;
|
||||||
in
|
in
|
||||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
|
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
|
||||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
&& selfScrapes s 8895 { container = "ci-runner"; };
|
||||||
&& reader.pull.exporter.prometheus.port == 8895;
|
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
in
|
in
|
||||||
|
|
|
||||||
|
|
@ -208,6 +208,37 @@ let
|
||||||
# the two tier cases differ only in which receiver they hand it.
|
# the two tier cases differ only in which receiver they hand it.
|
||||||
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
|
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
|
||||||
|
|
||||||
|
# Whether a container collector's settings carry the self-scrape of
|
||||||
|
# ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
|
||||||
|
# a job scraping exactly that address, and a metrics pipeline that reads the
|
||||||
|
# job and leaves through an exporter the settings define. `attrs` are the
|
||||||
|
# resource attributes the pipeline stamps.
|
||||||
|
selfScrapes =
|
||||||
|
s: port: attrs:
|
||||||
|
let
|
||||||
|
job = lib.findFirst (j: j.job_name == "collector") null (
|
||||||
|
s.receivers."prometheus/self".config.scrape_configs or [ ]
|
||||||
|
);
|
||||||
|
pipeline = s.service.pipelines."metrics/self" or null;
|
||||||
|
reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
|
||||||
|
stamped = lib.mapAttrsToList (key: value: {
|
||||||
|
inherit key value;
|
||||||
|
action = "upsert";
|
||||||
|
}) attrs;
|
||||||
|
in
|
||||||
|
job != null
|
||||||
|
&& job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
|
||||||
|
&&
|
||||||
|
(reader.pull.exporter.prometheus or null) == {
|
||||||
|
host = "127.0.0.1";
|
||||||
|
inherit port;
|
||||||
|
}
|
||||||
|
&& pipeline != null
|
||||||
|
&& pipeline.receivers == [ "prometheus/self" ]
|
||||||
|
&& (s.processors."resource/self".attributes or null) == stamped
|
||||||
|
&& pipeline.exporters != [ ]
|
||||||
|
&& lib.all (e: s.exporters ? ${e}) pipeline.exporters;
|
||||||
|
|
||||||
# The enables that switch owns. The hive tier's collector is not among them: it
|
# The enables that switch owns. The hive tier's collector is not among them: it
|
||||||
# follows `deploy.hive-controller.enable`, and a services host without a hive
|
# follows `deploy.hive-controller.enable`, and a services host without a hive
|
||||||
# runs none.
|
# runs none.
|
||||||
|
|
@ -283,6 +314,7 @@ in
|
||||||
inherit agentHarness;
|
inherit agentHarness;
|
||||||
inherit journaldSeverityParser;
|
inherit journaldSeverityParser;
|
||||||
inherit carriesJournaldSeverity;
|
inherit carriesJournaldSeverity;
|
||||||
|
inherit selfScrapes;
|
||||||
inherit swarmServiceEnables;
|
inherit swarmServiceEnables;
|
||||||
inherit runGroup;
|
inherit runGroup;
|
||||||
inherit forceServiceConfigs;
|
inherit forceServiceConfigs;
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ let
|
||||||
})
|
})
|
||||||
hive
|
hive
|
||||||
runGroup
|
runGroup
|
||||||
|
selfScrapes
|
||||||
;
|
;
|
||||||
|
|
||||||
# No store identity, which the first case below relies on.
|
# No store identity, which the first case below relies on.
|
||||||
|
|
@ -157,12 +158,10 @@ let
|
||||||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||||
]
|
]
|
||||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
|
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
|
||||||
reader = lib.head s.service.telemetry.metrics.readers;
|
|
||||||
in
|
in
|
||||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||||
&& !(lib.elem svc.port taken)
|
&& !(lib.elem svc.port taken)
|
||||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
&& selfScrapes s svc.port { container = svc.name; };
|
||||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
) forwarderServices;
|
) forwarderServices;
|
||||||
|
|
@ -269,12 +268,10 @@ let
|
||||||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||||
]
|
]
|
||||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
|
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
|
||||||
reader = lib.head s.service.telemetry.metrics.readers;
|
|
||||||
in
|
in
|
||||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||||
&& !(lib.elem svc.port taken)
|
&& !(lib.elem svc.port taken)
|
||||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
&& selfScrapes s svc.port { container = svc.name; };
|
||||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
) hostNetnsForwarders;
|
) hostNetnsForwarders;
|
||||||
|
|
@ -740,6 +737,7 @@ let
|
||||||
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
||||||
&& lib.elem "oauth2client/swarm" s.service.extensions
|
&& lib.elem "oauth2client/swarm" s.service.extensions
|
||||||
&& s.service.pipelines.logs.receivers == [ "journald" ]
|
&& s.service.pipelines.logs.receivers == [ "journald" ]
|
||||||
|
&& selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
|
||||||
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
||||||
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
|
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
|
||||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
||||||
|
|
@ -997,7 +995,7 @@ let
|
||||||
registered = lib.findFirst (
|
registered = lib.findFirst (
|
||||||
x: x.id == clientId
|
x: x.id == clientId
|
||||||
) null h.services.hyperhive.swarm.authelia.oidc.clients;
|
) null h.services.hyperhive.swarm.authelia.oidc.clients;
|
||||||
reader = lib.head s.service.telemetry.metrics.readers;
|
port = c.services.hyperhive.swarmJournalForwarder.telemetryPort;
|
||||||
taken = [
|
taken = [
|
||||||
8888
|
8888
|
||||||
8890
|
8890
|
||||||
|
|
@ -1022,8 +1020,8 @@ let
|
||||||
&& registered.audience == [ clientId ]
|
&& registered.audience == [ clientId ]
|
||||||
&& !(lib.elem "--link-journal=host" container.extraFlags)
|
&& !(lib.elem "--link-journal=host" container.extraFlags)
|
||||||
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
|
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
|
||||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
&& selfScrapes s port { container = "authelia"; }
|
||||||
&& !(lib.elem reader.pull.exporter.prometheus.port taken)
|
&& !(lib.elem port taken)
|
||||||
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
|
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
|
||||||
&& !(
|
&& !(
|
||||||
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc
|
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue