Compare commits
11 changed files with 272 additions and 388 deletions
|
|
@ -219,12 +219,10 @@ and failure series on `collector` appear once something fails, and the
|
|||
collector and of the swarm's collector, which both carry the collector
|
||||
binary's default `service.name`, `otelcol-contrib`. The timeseries panels
|
||||
split by `server.port`: 8888 is the hive collector, 8889 the swarm
|
||||
collector. Every per-container collector (the secret store, the other service
|
||||
containers and the agents' forwarders) also scrapes its own endpoint on
|
||||
loopback and pushes the result up its normal hop, so each has a line of its
|
||||
own, on its own port: agents all report on 8904, service containers on
|
||||
8890 to 8903. The series carry `container` (services) or `agent` (agents)
|
||||
to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI
|
||||
collector. The per-container collectors (the secret store, NATS, Grafana,
|
||||
Authelia, the Victoria stores and the agents' forwarders) don't report
|
||||
their own metrics yet, so they have no lines here; a separate issue on
|
||||
the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI
|
||||
and in `scripts/pre-push`. It fails when a dashboard query selects a
|
||||
`service.name` that nothing emits.
|
||||
|
||||
|
|
|
|||
|
|
@ -72,11 +72,6 @@ let
|
|||
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
||||
};
|
||||
|
||||
# Where the collector serves its own metrics, inside the container's own
|
||||
# netns. Not the binary's default 8888, which the collector dashboard
|
||||
# reads as the hive collector's.
|
||||
selfTelemetryPort = 8904;
|
||||
|
||||
# The exporter's NAME picks the wire protocol, so `protocol` has to be
|
||||
# read here as well as handed to the SDKs above — same expression as the
|
||||
# swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
|
||||
|
|
@ -215,101 +210,91 @@ in
|
|||
# and the collector then dies at startup. Same caveat as both other
|
||||
# tiers; see nix/host-modules/otel.nix for the measurement.
|
||||
validateConfigFile = true;
|
||||
settings = lib.mkMerge [
|
||||
{
|
||||
# `journalctl --follow --lines=0` (what this receiver runs under
|
||||
# the hood) ships only what's written *after* it starts — every
|
||||
# collector start has a silent gap at the front. `storage:` below
|
||||
# persists the read cursor, so a restart resumes from where it
|
||||
# left off instead of re-opening that gap; `start_at` stays `end`
|
||||
# (its default) because a cursor already covers every run after
|
||||
# the first, and `beginning` without one would re-ship the whole
|
||||
# journal on every restart. The cursor lives in the unit's own
|
||||
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below
|
||||
# as `WorkingDirectory` too) — container-lifetime, not the
|
||||
# host-persisted state bind mount: it indexes /var/log/journal,
|
||||
# itself fresh on every container start, so the two must share
|
||||
# a lifetime or the cursor outlives the journal it points into.
|
||||
# `create_directory` matters here for a build-time-vs-runtime
|
||||
# reason, not a convenience one: `validateConfigFile` above runs
|
||||
# `otelcol validate` in the nix build sandbox, before systemd's
|
||||
# `StateDirectory=` has ever created this path. Without it the
|
||||
# validator refuses with "directory must exist" and the build fails.
|
||||
extensions.file_storage = {
|
||||
directory = "/var/lib/opentelemetry-collector";
|
||||
create_directory = true;
|
||||
};
|
||||
settings = {
|
||||
# `journalctl --follow --lines=0` (what this receiver runs under
|
||||
# the hood) ships only what's written *after* it starts — every
|
||||
# collector start has a silent gap at the front. `storage:` below
|
||||
# persists the read cursor, so a restart resumes from where it
|
||||
# left off instead of re-opening that gap; `start_at` stays `end`
|
||||
# (its default) because a cursor already covers every run after
|
||||
# the first, and `beginning` without one would re-ship the whole
|
||||
# journal on every restart. The cursor lives in the unit's own
|
||||
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below
|
||||
# as `WorkingDirectory` too) — container-lifetime, not the
|
||||
# host-persisted state bind mount: it indexes /var/log/journal,
|
||||
# itself fresh on every container start, so the two must share
|
||||
# a lifetime or the cursor outlives the journal it points into.
|
||||
# `create_directory` matters here for a build-time-vs-runtime
|
||||
# reason, not a convenience one: `validateConfigFile` above runs
|
||||
# `otelcol validate` in the nix build sandbox, before systemd's
|
||||
# `StateDirectory=` has ever created this path. Without it the
|
||||
# validator refuses with "directory must exist" and the build fails.
|
||||
extensions.file_storage = {
|
||||
directory = "/var/lib/opentelemetry-collector";
|
||||
create_directory = true;
|
||||
};
|
||||
|
||||
receivers.journald = {
|
||||
# ⚠️ STATED, and it must stay stated: the receiver's own default
|
||||
# is the RUNTIME journal (`/run/log/journal`), which in an agent
|
||||
# container is empty — journald stores persistently here, so
|
||||
# every entry is under /var/log/journal. Dropping this line
|
||||
# leaves a collector that validates, starts, reports healthy and
|
||||
# forwards nothing.
|
||||
directory = "/var/log/journal";
|
||||
storage = "file_storage";
|
||||
# No `units` allowlist, and no user-session filter like the swarm
|
||||
# tier's: a container's journal is the harness and what the
|
||||
# harness spawns, so there is no foreign traffic to filter out and
|
||||
# an allowlist would only be a list to forget to update.
|
||||
receivers.journald = {
|
||||
# ⚠️ STATED, and it must stay stated: the receiver's own default
|
||||
# is the RUNTIME journal (`/run/log/journal`), which in an agent
|
||||
# container is empty — journald stores persistently here, so
|
||||
# every entry is under /var/log/journal. Dropping this line
|
||||
# leaves a collector that validates, starts, reports healthy and
|
||||
# forwards nothing.
|
||||
directory = "/var/log/journal";
|
||||
storage = "file_storage";
|
||||
# No `units` allowlist, and no user-session filter like the swarm
|
||||
# tier's: a container's journal is the harness and what the
|
||||
# harness spawns, so there is no foreign traffic to filter out and
|
||||
# an allowlist would only be a list to forget to update.
|
||||
|
||||
# journald's PRIORITY carries the level every line already has;
|
||||
# without this the record reaches VictoriaLogs with
|
||||
# `severity_text: Unspecified` and the store cannot tell an error
|
||||
# from a debug line. The table — and the inverted direction that
|
||||
# makes it worth a file of its own — lives in
|
||||
# ../journald-severity.nix, shared with the swarm tier's receiver.
|
||||
operators = import ../journald-severity.nix;
|
||||
};
|
||||
# journald's PRIORITY carries the level every line already has;
|
||||
# without this the record reaches VictoriaLogs with
|
||||
# `severity_text: Unspecified` and the store cannot tell an error
|
||||
# from a debug line. The table — and the inverted direction that
|
||||
# makes it worth a file of its own — lives in
|
||||
# ../journald-severity.nix, shared with the swarm tier's receiver.
|
||||
operators = import ../journald-severity.nix;
|
||||
};
|
||||
|
||||
# Identity the hop above cannot supply. A host-side reader can say
|
||||
# which machine a line came from; only a collector running as the
|
||||
# agent can say which agent. `hive` and `swarm` are deliberately
|
||||
# NOT stamped here: the swarm tier upserts both from the receiver
|
||||
# that accepted the sample, precisely so the labels come from
|
||||
# something the sender cannot write.
|
||||
processors.resource.attributes = [
|
||||
{
|
||||
key = "service.name";
|
||||
value = "hyperhive-agent";
|
||||
action = "upsert";
|
||||
}
|
||||
{
|
||||
key = "agent";
|
||||
value = userName;
|
||||
action = "upsert";
|
||||
}
|
||||
];
|
||||
|
||||
# The hive's collector, at the same base address every in-process
|
||||
# producer already exports to. `endpoint` is a BASE the exporter
|
||||
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
|
||||
exporters.${logExporter} = {
|
||||
endpoint = cfg.endpoint;
|
||||
# Identity the hop above cannot supply. A host-side reader can say
|
||||
# which machine a line came from; only a collector running as the
|
||||
# agent can say which agent. `hive` and `swarm` are deliberately
|
||||
# NOT stamped here: the swarm tier upserts both from the receiver
|
||||
# that accepted the sample, precisely so the labels come from
|
||||
# something the sender cannot write.
|
||||
processors.resource.attributes = [
|
||||
{
|
||||
key = "service.name";
|
||||
value = "hyperhive-agent";
|
||||
action = "upsert";
|
||||
}
|
||||
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
|
||||
{
|
||||
key = "agent";
|
||||
value = userName;
|
||||
action = "upsert";
|
||||
}
|
||||
];
|
||||
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ logExporter ];
|
||||
};
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# journald receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts.
|
||||
service.extensions = [ "file_storage" ];
|
||||
# The hive's collector, at the same base address every in-process
|
||||
# producer already exports to. `endpoint` is a BASE the exporter
|
||||
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
|
||||
exporters.${logExporter} = {
|
||||
endpoint = cfg.endpoint;
|
||||
}
|
||||
# Every agent container has a network namespace of its own, so they can all
|
||||
# use the same port.
|
||||
(import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = selfTelemetryPort;
|
||||
exporter = logExporter;
|
||||
attributes.agent = userName;
|
||||
})
|
||||
];
|
||||
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
|
||||
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ logExporter ];
|
||||
};
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# journald receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts.
|
||||
service.extensions = [ "file_storage" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,52 +0,0 @@
|
|||
# Collector settings that make a container's OpenTelemetry collector report its
|
||||
# own health (refused and failed records, exporter queue, memory), imported by
|
||||
# every container collector: the agent forwarder, the service journal
|
||||
# forwarder and the secret store's collector. The two host-side tiers scrape
|
||||
# themselves through their `scrapeTargets` instead.
|
||||
#
|
||||
# One function renders both halves, so the endpoint and the job that reads it
|
||||
# cannot name different ports. The endpoint binds loopback: the collector
|
||||
# scrapes itself, and nothing outside the container has a reason to.
|
||||
#
|
||||
# The job runs in its own pipeline. A container's other metrics pipeline may
|
||||
# carry a `resource` processor that upserts `service.name`, which would
|
||||
# relabel these series away from the `otelcol-contrib` the collector dashboard
|
||||
# selects on. The attributes stamped here tell the container's collectors
|
||||
# apart.
|
||||
#
|
||||
# `readers`, not `metrics.address`: this collector version rejects `address`
|
||||
# at startup.
|
||||
{
|
||||
lib,
|
||||
port,
|
||||
exporter,
|
||||
attributes,
|
||||
}:
|
||||
{
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
inherit port;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
receivers."prometheus/self".config.scrape_configs = [
|
||||
{
|
||||
job_name = "collector";
|
||||
static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ];
|
||||
}
|
||||
];
|
||||
|
||||
processors."resource/self".attributes = lib.mapAttrsToList (key: value: {
|
||||
inherit key value;
|
||||
action = "upsert";
|
||||
}) attributes;
|
||||
|
||||
service.pipelines."metrics/self" = {
|
||||
receivers = [ "prometheus/self" ];
|
||||
processors = [ "resource/self" ];
|
||||
exporters = [ exporter ];
|
||||
};
|
||||
}
|
||||
|
|
@ -96,9 +96,8 @@ in
|
|||
type = lib.types.port;
|
||||
description = ''
|
||||
Loopback port for the collector's own metrics (refused and dropped
|
||||
records, exporter failures), which it scrapes and pushes to the swarm
|
||||
collector. Required because the container may share the host netns,
|
||||
where the binary's default 8888 is the hive tier's.
|
||||
records, exporter failures). Required because the container may share
|
||||
the host netns, where the binary's default 8888 is the hive tier's.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -243,6 +242,17 @@ in
|
|||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
|
||||
# `address` at startup.
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
port = cfg.telemetryPort;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# An extension not listed here is inert: the receiver's storage would
|
||||
# name a component that never starts, and the exporter would push
|
||||
# unauthenticated.
|
||||
|
|
@ -251,12 +261,6 @@ in
|
|||
authName
|
||||
];
|
||||
}
|
||||
(import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = cfg.telemetryPort;
|
||||
exporter = "otlphttp";
|
||||
attributes.container = cfg.serviceName;
|
||||
})
|
||||
# A separate set the module system merges in: joining it to the first with
|
||||
# `//` would replace the first set's whole `service` attrset.
|
||||
(lib.mkIf (cfg.otlpMetricsPort != null) {
|
||||
|
|
|
|||
|
|
@ -4438,163 +4438,172 @@ in
|
|||
# caveat as every other tier; see nix/host-modules/otel.nix for
|
||||
# the measurement.
|
||||
validateConfigFile = true;
|
||||
settings = lib.mkMerge [
|
||||
{
|
||||
# `journalctl --follow --lines=0` (what the receiver runs under
|
||||
# the hood) ships only what is written *after* it starts, so
|
||||
# every collector start has a silent gap at the front. This
|
||||
# persists the read cursor, so a restart resumes instead of
|
||||
# re-opening that gap; `start_at` stays `end` (its default),
|
||||
# since a cursor covers every run after the first and
|
||||
# `beginning` without one would re-ship the whole journal on
|
||||
# every restart. `create_directory` is required for a
|
||||
# build-time reason rather than a convenience one:
|
||||
# `validateConfigFile` above runs `otelcol validate` in the nix
|
||||
# sandbox, before systemd's `StateDirectory=` has created the
|
||||
# path, and the validator refuses with "directory must exist".
|
||||
extensions.file_storage = {
|
||||
directory = "/var/lib/opentelemetry-collector";
|
||||
create_directory = true;
|
||||
};
|
||||
settings = {
|
||||
# `journalctl --follow --lines=0` (what the receiver runs under
|
||||
# the hood) ships only what is written *after* it starts, so
|
||||
# every collector start has a silent gap at the front. This
|
||||
# persists the read cursor, so a restart resumes instead of
|
||||
# re-opening that gap; `start_at` stays `end` (its default),
|
||||
# since a cursor covers every run after the first and
|
||||
# `beginning` without one would re-ship the whole journal on
|
||||
# every restart. `create_directory` is required for a
|
||||
# build-time reason rather than a convenience one:
|
||||
# `validateConfigFile` above runs `otelcol validate` in the nix
|
||||
# sandbox, before systemd's `StateDirectory=` has created the
|
||||
# path, and the validator refuses with "directory must exist".
|
||||
extensions.file_storage = {
|
||||
directory = "/var/lib/opentelemetry-collector";
|
||||
create_directory = true;
|
||||
};
|
||||
|
||||
receivers.journald = {
|
||||
# ⚠️ STATED, and it must stay stated: the receiver's own
|
||||
# default is the RUNTIME journal (`/run/log/journal`), which
|
||||
# in this container is empty — journald stores persistently
|
||||
# here, so every entry is under /var/log/journal. Dropping
|
||||
# this line leaves a collector that validates, starts,
|
||||
# reports healthy and forwards nothing.
|
||||
directory = "/var/log/journal";
|
||||
# ⚠️ `--merge` is a guard: an unqualified
|
||||
# `journalctl --follow` over a journal with no entry for the
|
||||
# current boot (a host-linked one) crash-loops the receiver
|
||||
# every ~2s with "No journal boot entry found for the
|
||||
# specified boot (+0)". This journal is container-local, so
|
||||
# the current boot has an entry and the flag changes nothing.
|
||||
merge = true;
|
||||
storage = "file_storage";
|
||||
};
|
||||
receivers.journald = {
|
||||
# ⚠️ STATED, and it must stay stated: the receiver's own
|
||||
# default is the RUNTIME journal (`/run/log/journal`), which
|
||||
# in this container is empty — journald stores persistently
|
||||
# here, so every entry is under /var/log/journal. Dropping
|
||||
# this line leaves a collector that validates, starts,
|
||||
# reports healthy and forwards nothing.
|
||||
directory = "/var/log/journal";
|
||||
# ⚠️ `--merge` is a guard: an unqualified
|
||||
# `journalctl --follow` over a journal with no entry for the
|
||||
# current boot (a host-linked one) crash-loops the receiver
|
||||
# every ~2s with "No journal boot entry found for the
|
||||
# specified boot (+0)". This journal is container-local, so
|
||||
# the current boot has an entry and the flag changes nothing.
|
||||
merge = true;
|
||||
storage = "file_storage";
|
||||
};
|
||||
|
||||
# The store's scrape, run by this container's own collector.
|
||||
#
|
||||
# ⚠️ `metrics_path` and `params` are STATED, and both have to
|
||||
# be: openbao serves no `/metrics` at all, and
|
||||
# `/v1/sys/metrics` answers JSON unless the format is asked
|
||||
# for. A scrape of the default path 404s and a scrape without
|
||||
# the query parses as nothing — each reads as a dead exporter
|
||||
# rather than as a wrong request. The spelling matches
|
||||
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
|
||||
# every other service's target.
|
||||
#
|
||||
# No credential, unlike the swarm collector's
|
||||
# `publishedScrapeTargets` jobs: the target is the loopback
|
||||
# listener above, inside this container's own netns, which is
|
||||
# what lets the scrape stay unauthenticated without the API
|
||||
# listener's client-cert requirement being touched.
|
||||
receivers.prometheus.config.scrape_configs = [
|
||||
{
|
||||
job_name = "bao";
|
||||
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
|
||||
metrics_path = "/v1/sys/metrics";
|
||||
params.format = [ "prometheus" ];
|
||||
}
|
||||
];
|
||||
# The store's scrape, run by this container's own collector.
|
||||
#
|
||||
# ⚠️ `metrics_path` and `params` are STATED, and both have to
|
||||
# be: openbao serves no `/metrics` at all, and
|
||||
# `/v1/sys/metrics` answers JSON unless the format is asked
|
||||
# for. A scrape of the default path 404s and a scrape without
|
||||
# the query parses as nothing — each reads as a dead exporter
|
||||
# rather than as a wrong request. The spelling matches
|
||||
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
|
||||
# every other service's target.
|
||||
#
|
||||
# No credential, unlike the swarm collector's
|
||||
# `publishedScrapeTargets` jobs: the target is the loopback
|
||||
# listener above, inside this container's own netns, which is
|
||||
# what lets the scrape stay unauthenticated without the API
|
||||
# listener's client-cert requirement being touched.
|
||||
receivers.prometheus.config.scrape_configs = [
|
||||
{
|
||||
job_name = "bao";
|
||||
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
|
||||
metrics_path = "/v1/sys/metrics";
|
||||
params.format = [ "prometheus" ];
|
||||
}
|
||||
];
|
||||
|
||||
# Identity the hop above cannot supply: a host-side reader can
|
||||
# say which machine a line came from, and only a collector
|
||||
# inside this container can say it was the secret store's. No
|
||||
# `hive` or `swarm` label — the swarm tier upserts `hive` from
|
||||
# the receiver that accepted the record, precisely so it comes
|
||||
# from something the sender cannot write.
|
||||
#
|
||||
# `collectorServiceName`, not `cfg.machine`: what the
|
||||
# dashboard selects on is a display identity, not the
|
||||
# container/receiver name — see that binding's own comment.
|
||||
processors.resource.attributes = [
|
||||
{
|
||||
key = "service.name";
|
||||
value = collectorServiceName;
|
||||
action = "upsert";
|
||||
}
|
||||
];
|
||||
# Identity the hop above cannot supply: a host-side reader can
|
||||
# say which machine a line came from, and only a collector
|
||||
# inside this container can say it was the secret store's. No
|
||||
# `hive` or `swarm` label — the swarm tier upserts `hive` from
|
||||
# the receiver that accepted the record, precisely so it comes
|
||||
# from something the sender cannot write.
|
||||
#
|
||||
# `collectorServiceName`, not `cfg.machine`: what the
|
||||
# dashboard selects on is a display identity, not the
|
||||
# container/receiver name — see that binding's own comment.
|
||||
processors.resource.attributes = [
|
||||
{
|
||||
key = "service.name";
|
||||
value = collectorServiceName;
|
||||
action = "upsert";
|
||||
}
|
||||
];
|
||||
|
||||
# `otlphttp` unconditionally, unlike the agent forwarder's
|
||||
# protocol-derived exporter name: the swarm receiver speaks
|
||||
# OTLP/HTTP protobuf whatever this tier's *upstream*
|
||||
# protocol is, which is the same reason
|
||||
# ./hive-c0re/environment.nix pins the protocol it hands out.
|
||||
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
|
||||
exporters.otlphttp = {
|
||||
endpoint = otelFirstHop;
|
||||
# The receiver at that address accepts an OIDC token and
|
||||
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
|
||||
# so this is what makes the hop deliver rather than collect
|
||||
# 401s — unconditional, because a forwarder that exports
|
||||
# unauthenticated has no working mode to fall back to.
|
||||
auth.authenticator = forwarderAuthName;
|
||||
};
|
||||
# `otlphttp` unconditionally, unlike the agent forwarder's
|
||||
# protocol-derived exporter name: the swarm receiver speaks
|
||||
# OTLP/HTTP protobuf whatever this tier's *upstream*
|
||||
# protocol is, which is the same reason
|
||||
# ./hive-c0re/environment.nix pins the protocol it hands out.
|
||||
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
|
||||
exporters.otlphttp = {
|
||||
endpoint = otelFirstHop;
|
||||
# The receiver at that address accepts an OIDC token and
|
||||
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
|
||||
# so this is what makes the hop deliver rather than collect
|
||||
# 401s — unconditional, because a forwarder that exports
|
||||
# unauthenticated has no working mode to fall back to.
|
||||
auth.authenticator = forwarderAuthName;
|
||||
};
|
||||
|
||||
# The push side, opposite in direction to the receiver's
|
||||
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
|
||||
# to send. Hence `oauth2client`, and hence a client id and
|
||||
# secret rather than an issuer and an audience to check.
|
||||
extensions.${forwarderAuthName} = {
|
||||
client_id = cfg.otel.clientId;
|
||||
# A path, never a value — and one systemd resolves at
|
||||
# runtime, so neither the secret nor the credentials
|
||||
# directory is rendered into the nix store.
|
||||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
|
||||
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
||||
# Registering it on the client only makes it permissible; a
|
||||
# token minted without asking carries `aud: []` and the
|
||||
# receiver refuses it — with a config that reads perfectly
|
||||
# at both ends.
|
||||
endpoint_params.audience = cfg.otel.clientId;
|
||||
};
|
||||
# The push side, opposite in direction to the receiver's
|
||||
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
|
||||
# to send. Hence `oauth2client`, and hence a client id and
|
||||
# secret rather than an issuer and an audience to check.
|
||||
extensions.${forwarderAuthName} = {
|
||||
client_id = cfg.otel.clientId;
|
||||
# A path, never a value — and one systemd resolves at
|
||||
# runtime, so neither the secret nor the credentials
|
||||
# directory is rendered into the nix store.
|
||||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
|
||||
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
||||
# Registering it on the client only makes it permissible; a
|
||||
# token minted without asking carries `aud: []` and the
|
||||
# receiver refuses it — with a config that reads perfectly
|
||||
# at both ends.
|
||||
endpoint_params.audience = cfg.otel.clientId;
|
||||
};
|
||||
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# A SECOND pipeline, not a second receiver on the logs one: a
|
||||
# pipeline is typed, and a prometheus receiver in `logs` is a
|
||||
# config the collector rejects at startup rather than one that
|
||||
# quietly ships metrics as log records.
|
||||
#
|
||||
# Same `resource` processor and same `otlphttp` exporter, so
|
||||
# the store's metrics carry the same `service.name` its logs
|
||||
# do and leave by the same hop — no otel bypass, one exit from
|
||||
# this container. The exporter treats `endpoint` as a base and
|
||||
# appends the signal's own path, so `/v1/metrics` here and
|
||||
# `/v1/logs` above come from the one address.
|
||||
service.pipelines.metrics = {
|
||||
receivers = [ "prometheus" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
# A SECOND pipeline, not a second receiver on the logs one: a
|
||||
# pipeline is typed, and a prometheus receiver in `logs` is a
|
||||
# config the collector rejects at startup rather than one that
|
||||
# quietly ships metrics as log records.
|
||||
#
|
||||
# Same `resource` processor and same `otlphttp` exporter, so
|
||||
# the store's metrics carry the same `service.name` its logs
|
||||
# do and leave by the same hop — no otel bypass, one exit from
|
||||
# this container. The exporter treats `endpoint` as a base and
|
||||
# appends the signal's own path, so `/v1/metrics` here and
|
||||
# `/v1/logs` above come from the one address.
|
||||
service.pipelines.metrics = {
|
||||
receivers = [ "prometheus" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts, and the exporter's
|
||||
# authenticator would push unauthenticated while the
|
||||
# collector reports healthy.
|
||||
service.extensions = [
|
||||
"file_storage"
|
||||
forwarderAuthName
|
||||
];
|
||||
}
|
||||
# Off the built-in default of `localhost:8888`, which the hive tier
|
||||
# holds: this container shares the host's netns.
|
||||
(import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = cfg.otel.telemetryPort;
|
||||
exporter = "otlphttp";
|
||||
attributes.container = collectorServiceName;
|
||||
})
|
||||
];
|
||||
# Moves this collector's self-metrics off the built-in
|
||||
# default of `localhost:8888`, which the hive tier holds —
|
||||
# this container shares the host's netns, so leaving the
|
||||
# default in place is the hive collector failing to bind.
|
||||
#
|
||||
# ⚠️ `metrics.address` is the spelling that looks right and
|
||||
# is REJECTED by this collector version:
|
||||
# `'migration.MetricsConfigV030' has invalid keys:
|
||||
# address`. `readers` is the schema it accepts, and the
|
||||
# difference is a startup failure rather than a warning.
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
port = cfg.otel.telemetryPort;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts, and the exporter's
|
||||
# authenticator would push unauthenticated while the
|
||||
# collector reports healthy.
|
||||
service.extensions = [
|
||||
"file_storage"
|
||||
forwarderAuthName
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# How the secret crosses the sandbox. The host unit
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"title": "hyperhive \u00b7 collector (self-telemetry)",
|
||||
"uid": "hyperhive-swarm-collector",
|
||||
"description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.",
|
||||
"description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
|
||||
"editable": true,
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 39,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ let
|
|||
agent
|
||||
carriesJournaldSeverity
|
||||
runGroup
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
# The log path's three hops, one fixture each. Nothing carries a journal
|
||||
|
|
@ -56,20 +55,6 @@ let
|
|||
|
||||
agentSettings = machine: machine.services.opentelemetry-collector.settings;
|
||||
cases = [
|
||||
{
|
||||
# The collector's own counters are the only signal that it is dropping
|
||||
# telemetry. 8904 keeps agents off 8888, which the collector dashboard
|
||||
# reads as the hive collector; the exporter follows the wire protocol.
|
||||
name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
|
||||
ok =
|
||||
selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
|
||||
&& selfScrapes (agentSettings agentOtelGrpc) 8904 {
|
||||
agent = agentOtelGrpc.services.hyperhive.agent.user.name;
|
||||
}
|
||||
# Control: with the switch off the lookup finds no collector config.
|
||||
&& !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
|
||||
&& (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
|
||||
}
|
||||
{
|
||||
# The journald receiver's own default directory is the RUNTIME
|
||||
# journal, and a container that stores persistently leaves that
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ let
|
|||
runGroup
|
||||
baoSettings
|
||||
forceServiceConfigs
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
baoTwoAddresses = hive {
|
||||
|
|
@ -186,20 +185,6 @@ let
|
|||
&& m.exporters == [ "otlphttp" ]
|
||||
&& m.exporters == s.service.pipelines.logs.exporters;
|
||||
}
|
||||
{
|
||||
# The store's collector reports its own health through a pipeline of its
|
||||
# own: the store pipeline above upserts `service.name`, and sharing it
|
||||
# would relabel these series away from the collector dashboard.
|
||||
name = "the store's collector scrapes its own telemetry endpoint on loopback";
|
||||
ok =
|
||||
let
|
||||
s = (baoForwarder baoWithCollector).settings;
|
||||
port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
|
||||
in
|
||||
selfScrapes s port { container = "bao"; }
|
||||
&& s.service.pipelines.metrics.receivers == [ "prometheus" ]
|
||||
&& s.service.pipelines.metrics.processors == [ "resource" ];
|
||||
}
|
||||
{
|
||||
# Presence control for the case above: both fields are omitted rather
|
||||
# than defaulted, so a target declared as bare `host:port` renders what
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ let
|
|||
})
|
||||
hive
|
||||
runGroup
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
inherit (shared) bare;
|
||||
|
|
@ -559,10 +558,11 @@ let
|
|||
ok =
|
||||
let
|
||||
c = ciAlone.containers.swarm-ci-runner.config;
|
||||
s = c.services.opentelemetry-collector.settings;
|
||||
reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
|
||||
&& selfScrapes s 8895 { container = "ci-runner"; };
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == 8895;
|
||||
}
|
||||
];
|
||||
in
|
||||
|
|
|
|||
|
|
@ -208,37 +208,6 @@ let
|
|||
# the two tier cases differ only in which receiver they hand it.
|
||||
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
|
||||
|
||||
# Whether a container collector's settings carry the self-scrape of
|
||||
# ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
|
||||
# a job scraping exactly that address, and a metrics pipeline that reads the
|
||||
# job and leaves through an exporter the settings define. `attrs` are the
|
||||
# resource attributes the pipeline stamps.
|
||||
selfScrapes =
|
||||
s: port: attrs:
|
||||
let
|
||||
job = lib.findFirst (j: j.job_name == "collector") null (
|
||||
s.receivers."prometheus/self".config.scrape_configs or [ ]
|
||||
);
|
||||
pipeline = s.service.pipelines."metrics/self" or null;
|
||||
reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
|
||||
stamped = lib.mapAttrsToList (key: value: {
|
||||
inherit key value;
|
||||
action = "upsert";
|
||||
}) attrs;
|
||||
in
|
||||
job != null
|
||||
&& job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
|
||||
&&
|
||||
(reader.pull.exporter.prometheus or null) == {
|
||||
host = "127.0.0.1";
|
||||
inherit port;
|
||||
}
|
||||
&& pipeline != null
|
||||
&& pipeline.receivers == [ "prometheus/self" ]
|
||||
&& (s.processors."resource/self".attributes or null) == stamped
|
||||
&& pipeline.exporters != [ ]
|
||||
&& lib.all (e: s.exporters ? ${e}) pipeline.exporters;
|
||||
|
||||
# The enables that switch owns. The hive tier's collector is not among them: it
|
||||
# follows `deploy.hive-controller.enable`, and a services host without a hive
|
||||
# runs none.
|
||||
|
|
@ -314,7 +283,6 @@ in
|
|||
inherit agentHarness;
|
||||
inherit journaldSeverityParser;
|
||||
inherit carriesJournaldSeverity;
|
||||
inherit selfScrapes;
|
||||
inherit swarmServiceEnables;
|
||||
inherit runGroup;
|
||||
inherit forceServiceConfigs;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ let
|
|||
})
|
||||
hive
|
||||
runGroup
|
||||
selfScrapes
|
||||
;
|
||||
|
||||
# No store identity, which the first case below relies on.
|
||||
|
|
@ -158,10 +157,12 @@ let
|
|||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||
]
|
||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||
&& !(lib.elem svc.port taken)
|
||||
&& selfScrapes s svc.port { container = svc.name; };
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
||||
}
|
||||
]
|
||||
) forwarderServices;
|
||||
|
|
@ -268,10 +269,12 @@ let
|
|||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||
]
|
||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||
&& !(lib.elem svc.port taken)
|
||||
&& selfScrapes s svc.port { container = svc.name; };
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
||||
}
|
||||
]
|
||||
) hostNetnsForwarders;
|
||||
|
|
@ -737,7 +740,6 @@ let
|
|||
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
||||
&& lib.elem "oauth2client/swarm" s.service.extensions
|
||||
&& s.service.pipelines.logs.receivers == [ "journald" ]
|
||||
&& selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
|
||||
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
||||
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
||||
|
|
@ -995,7 +997,7 @@ let
|
|||
registered = lib.findFirst (
|
||||
x: x.id == clientId
|
||||
) null h.services.hyperhive.swarm.authelia.oidc.clients;
|
||||
port = c.services.hyperhive.swarmJournalForwarder.telemetryPort;
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
taken = [
|
||||
8888
|
||||
8890
|
||||
|
|
@ -1020,8 +1022,8 @@ let
|
|||
&& registered.audience == [ clientId ]
|
||||
&& !(lib.elem "--link-journal=host" container.extraFlags)
|
||||
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
|
||||
&& selfScrapes s port { container = "authelia"; }
|
||||
&& !(lib.elem port taken)
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& !(lib.elem reader.pull.exporter.prometheus.port taken)
|
||||
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
|
||||
&& !(
|
||||
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc
|
||||
|
|
|
|||
Loading…
Reference in a new issue