merge the self-scrape into each collector's settings instead of defining them twice
This commit is contained in:
atlas
mara
parent
73c966a7eb
commit
a00bcb8c05
1 changed files with 243 additions and 241 deletions
|
|
@ -4438,162 +4438,163 @@ in
|
|||
# caveat as every other tier; see nix/host-modules/otel.nix for
|
||||
# the measurement.
|
||||
validateConfigFile = true;
|
||||
settings = {
|
||||
# `journalctl --follow --lines=0` (what the receiver runs under
|
||||
# the hood) ships only what is written *after* it starts, so
|
||||
# every collector start has a silent gap at the front. This
|
||||
# persists the read cursor, so a restart resumes instead of
|
||||
# re-opening that gap; `start_at` stays `end` (its default),
|
||||
# since a cursor covers every run after the first and
|
||||
# `beginning` without one would re-ship the whole journal on
|
||||
# every restart. `create_directory` is required for a
|
||||
# build-time reason rather than a convenience one:
|
||||
# `validateConfigFile` above runs `otelcol validate` in the nix
|
||||
# sandbox, before systemd's `StateDirectory=` has created the
|
||||
# path, and the validator refuses with "directory must exist".
|
||||
extensions.file_storage = {
|
||||
directory = "/var/lib/opentelemetry-collector";
|
||||
create_directory = true;
|
||||
};
|
||||
settings = lib.mkMerge [
|
||||
{
|
||||
# `journalctl --follow --lines=0` (what the receiver runs under
|
||||
# the hood) ships only what is written *after* it starts, so
|
||||
# every collector start has a silent gap at the front. This
|
||||
# persists the read cursor, so a restart resumes instead of
|
||||
# re-opening that gap; `start_at` stays `end` (its default),
|
||||
# since a cursor covers every run after the first and
|
||||
# `beginning` without one would re-ship the whole journal on
|
||||
# every restart. `create_directory` is required for a
|
||||
# build-time reason rather than a convenience one:
|
||||
# `validateConfigFile` above runs `otelcol validate` in the nix
|
||||
# sandbox, before systemd's `StateDirectory=` has created the
|
||||
# path, and the validator refuses with "directory must exist".
|
||||
extensions.file_storage = {
|
||||
directory = "/var/lib/opentelemetry-collector";
|
||||
create_directory = true;
|
||||
};
|
||||
|
||||
receivers.journald = {
|
||||
# ⚠️ STATED, and it must stay stated: the receiver's own
|
||||
# default is the RUNTIME journal (`/run/log/journal`), which
|
||||
# in this container is empty — journald stores persistently
|
||||
# here, so every entry is under /var/log/journal. Dropping
|
||||
# this line leaves a collector that validates, starts,
|
||||
# reports healthy and forwards nothing.
|
||||
directory = "/var/log/journal";
|
||||
# ⚠️ `--merge` is a guard: an unqualified
|
||||
# `journalctl --follow` over a journal with no entry for the
|
||||
# current boot (a host-linked one) crash-loops the receiver
|
||||
# every ~2s with "No journal boot entry found for the
|
||||
# specified boot (+0)". This journal is container-local, so
|
||||
# the current boot has an entry and the flag changes nothing.
|
||||
merge = true;
|
||||
storage = "file_storage";
|
||||
};
|
||||
receivers.journald = {
|
||||
# ⚠️ STATED, and it must stay stated: the receiver's own
|
||||
# default is the RUNTIME journal (`/run/log/journal`), which
|
||||
# in this container is empty — journald stores persistently
|
||||
# here, so every entry is under /var/log/journal. Dropping
|
||||
# this line leaves a collector that validates, starts,
|
||||
# reports healthy and forwards nothing.
|
||||
directory = "/var/log/journal";
|
||||
# ⚠️ `--merge` is a guard: an unqualified
|
||||
# `journalctl --follow` over a journal with no entry for the
|
||||
# current boot (a host-linked one) crash-loops the receiver
|
||||
# every ~2s with "No journal boot entry found for the
|
||||
# specified boot (+0)". This journal is container-local, so
|
||||
# the current boot has an entry and the flag changes nothing.
|
||||
merge = true;
|
||||
storage = "file_storage";
|
||||
};
|
||||
|
||||
# The store's scrape, run by this container's own collector.
|
||||
#
|
||||
# ⚠️ `metrics_path` and `params` are STATED, and both have to
|
||||
# be: openbao serves no `/metrics` at all, and
|
||||
# `/v1/sys/metrics` answers JSON unless the format is asked
|
||||
# for. A scrape of the default path 404s and a scrape without
|
||||
# the query parses as nothing — each reads as a dead exporter
|
||||
# rather than as a wrong request. The spelling matches
|
||||
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
|
||||
# every other service's target.
|
||||
#
|
||||
# No credential, unlike the swarm collector's
|
||||
# `publishedScrapeTargets` jobs: the target is the loopback
|
||||
# listener above, inside this container's own netns, which is
|
||||
# what lets the scrape stay unauthenticated without the API
|
||||
# listener's client-cert requirement being touched.
|
||||
receivers.prometheus.config.scrape_configs = [
|
||||
{
|
||||
job_name = "bao";
|
||||
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
|
||||
metrics_path = "/v1/sys/metrics";
|
||||
params.format = [ "prometheus" ];
|
||||
}
|
||||
];
|
||||
# The store's scrape, run by this container's own collector.
|
||||
#
|
||||
# ⚠️ `metrics_path` and `params` are STATED, and both have to
|
||||
# be: openbao serves no `/metrics` at all, and
|
||||
# `/v1/sys/metrics` answers JSON unless the format is asked
|
||||
# for. A scrape of the default path 404s and a scrape without
|
||||
# the query parses as nothing — each reads as a dead exporter
|
||||
# rather than as a wrong request. The spelling matches
|
||||
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
|
||||
# every other service's target.
|
||||
#
|
||||
# No credential, unlike the swarm collector's
|
||||
# `publishedScrapeTargets` jobs: the target is the loopback
|
||||
# listener above, inside this container's own netns, which is
|
||||
# what lets the scrape stay unauthenticated without the API
|
||||
# listener's client-cert requirement being touched.
|
||||
receivers.prometheus.config.scrape_configs = [
|
||||
{
|
||||
job_name = "bao";
|
||||
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
|
||||
metrics_path = "/v1/sys/metrics";
|
||||
params.format = [ "prometheus" ];
|
||||
}
|
||||
];
|
||||
|
||||
# Identity the hop above cannot supply: a host-side reader can
|
||||
# say which machine a line came from, and only a collector
|
||||
# inside this container can say it was the secret store's. No
|
||||
# `hive` or `swarm` label — the swarm tier upserts `hive` from
|
||||
# the receiver that accepted the record, precisely so it comes
|
||||
# from something the sender cannot write.
|
||||
#
|
||||
# `collectorServiceName`, not `cfg.machine`: what the
|
||||
# dashboard selects on is a display identity, not the
|
||||
# container/receiver name — see that binding's own comment.
|
||||
processors.resource.attributes = [
|
||||
{
|
||||
key = "service.name";
|
||||
value = collectorServiceName;
|
||||
action = "upsert";
|
||||
}
|
||||
];
|
||||
# Identity the hop above cannot supply: a host-side reader can
|
||||
# say which machine a line came from, and only a collector
|
||||
# inside this container can say it was the secret store's. No
|
||||
# `hive` or `swarm` label — the swarm tier upserts `hive` from
|
||||
# the receiver that accepted the record, precisely so it comes
|
||||
# from something the sender cannot write.
|
||||
#
|
||||
# `collectorServiceName`, not `cfg.machine`: what the
|
||||
# dashboard selects on is a display identity, not the
|
||||
# container/receiver name — see that binding's own comment.
|
||||
processors.resource.attributes = [
|
||||
{
|
||||
key = "service.name";
|
||||
value = collectorServiceName;
|
||||
action = "upsert";
|
||||
}
|
||||
];
|
||||
|
||||
# `otlphttp` unconditionally, unlike the agent forwarder's
|
||||
# protocol-derived exporter name: the swarm receiver speaks
|
||||
# OTLP/HTTP protobuf whatever this tier's *upstream*
|
||||
# protocol is, which is the same reason
|
||||
# ./hive-c0re/environment.nix pins the protocol it hands out.
|
||||
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
|
||||
exporters.otlphttp = {
|
||||
endpoint = otelFirstHop;
|
||||
# The receiver at that address accepts an OIDC token and
|
||||
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
|
||||
# so this is what makes the hop deliver rather than collect
|
||||
# 401s — unconditional, because a forwarder that exports
|
||||
# unauthenticated has no working mode to fall back to.
|
||||
auth.authenticator = forwarderAuthName;
|
||||
};
|
||||
# `otlphttp` unconditionally, unlike the agent forwarder's
|
||||
# protocol-derived exporter name: the swarm receiver speaks
|
||||
# OTLP/HTTP protobuf whatever this tier's *upstream*
|
||||
# protocol is, which is the same reason
|
||||
# ./hive-c0re/environment.nix pins the protocol it hands out.
|
||||
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
|
||||
exporters.otlphttp = {
|
||||
endpoint = otelFirstHop;
|
||||
# The receiver at that address accepts an OIDC token and
|
||||
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
|
||||
# so this is what makes the hop deliver rather than collect
|
||||
# 401s — unconditional, because a forwarder that exports
|
||||
# unauthenticated has no working mode to fall back to.
|
||||
auth.authenticator = forwarderAuthName;
|
||||
};
|
||||
|
||||
# The push side, opposite in direction to the receiver's
|
||||
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
|
||||
# to send. Hence `oauth2client`, and hence a client id and
|
||||
# secret rather than an issuer and an audience to check.
|
||||
extensions.${forwarderAuthName} = {
|
||||
client_id = cfg.otel.clientId;
|
||||
# A path, never a value — and one systemd resolves at
|
||||
# runtime, so neither the secret nor the credentials
|
||||
# directory is rendered into the nix store.
|
||||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
|
||||
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
||||
# Registering it on the client only makes it permissible; a
|
||||
# token minted without asking carries `aud: []` and the
|
||||
# receiver refuses it — with a config that reads perfectly
|
||||
# at both ends.
|
||||
endpoint_params.audience = cfg.otel.clientId;
|
||||
};
|
||||
# The push side, opposite in direction to the receiver's
|
||||
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
|
||||
# to send. Hence `oauth2client`, and hence a client id and
|
||||
# secret rather than an issuer and an audience to check.
|
||||
extensions.${forwarderAuthName} = {
|
||||
client_id = cfg.otel.clientId;
|
||||
# A path, never a value — and one systemd resolves at
|
||||
# runtime, so neither the secret nor the credentials
|
||||
# directory is rendered into the nix store.
|
||||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
|
||||
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
||||
# Registering it on the client only makes it permissible; a
|
||||
# token minted without asking carries `aud: []` and the
|
||||
# receiver refuses it — with a config that reads perfectly
|
||||
# at both ends.
|
||||
endpoint_params.audience = cfg.otel.clientId;
|
||||
};
|
||||
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# A SECOND pipeline, not a second receiver on the logs one: a
|
||||
# pipeline is typed, and a prometheus receiver in `logs` is a
|
||||
# config the collector rejects at startup rather than one that
|
||||
# quietly ships metrics as log records.
|
||||
#
|
||||
# Same `resource` processor and same `otlphttp` exporter, so
|
||||
# the store's metrics carry the same `service.name` its logs
|
||||
# do and leave by the same hop — no otel bypass, one exit from
|
||||
# this container. The exporter treats `endpoint` as a base and
|
||||
# appends the signal's own path, so `/v1/metrics` here and
|
||||
# `/v1/logs` above come from the one address.
|
||||
service.pipelines.metrics = {
|
||||
receivers = [ "prometheus" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
# A SECOND pipeline, not a second receiver on the logs one: a
|
||||
# pipeline is typed, and a prometheus receiver in `logs` is a
|
||||
# config the collector rejects at startup rather than one that
|
||||
# quietly ships metrics as log records.
|
||||
#
|
||||
# Same `resource` processor and same `otlphttp` exporter, so
|
||||
# the store's metrics carry the same `service.name` its logs
|
||||
# do and leave by the same hop — no otel bypass, one exit from
|
||||
# this container. The exporter treats `endpoint` as a base and
|
||||
# appends the signal's own path, so `/v1/metrics` here and
|
||||
# `/v1/logs` above come from the one address.
|
||||
service.pipelines.metrics = {
|
||||
receivers = [ "prometheus" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts, and the exporter's
|
||||
# authenticator would push unauthenticated while the
|
||||
# collector reports healthy.
|
||||
service.extensions = [
|
||||
"file_storage"
|
||||
forwarderAuthName
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# Off the built-in default of `localhost:8888`, which the hive tier
|
||||
# holds: this container shares the host's netns.
|
||||
services.opentelemetry-collector.settings = import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = cfg.otel.telemetryPort;
|
||||
exporter = "otlphttp";
|
||||
attributes.container = collectorServiceName;
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts, and the exporter's
|
||||
# authenticator would push unauthenticated while the
|
||||
# collector reports healthy.
|
||||
service.extensions = [
|
||||
"file_storage"
|
||||
forwarderAuthName
|
||||
];
|
||||
}
|
||||
# Off the built-in default of `localhost:8888`, which the hive tier
|
||||
# holds: this container shares the host's netns.
|
||||
(import ../collector-self-scrape.nix {
|
||||
inherit lib;
|
||||
port = cfg.otel.telemetryPort;
|
||||
exporter = "otlphttp";
|
||||
attributes.container = collectorServiceName;
|
||||
})
|
||||
];
|
||||
};
|
||||
|
||||
# How the secret crosses the sandbox. The host unit
|
||||
|
|
|
|||
Loading…
Reference in a new issue