Watch
0
0
Fork
You've already forked hyperhive
0

Compare commits

..
11 changed files with 388 additions and 272 deletions

View file

@ -219,10 +219,12 @@ and failure series on `collector` appear once something fails, and the
collector and of the swarm's collector, which both carry the collector
binary's default `service.name`, `otelcol-contrib`. The timeseries panels
split by `server.port`: 8888 is the hive collector, 8889 the swarm
collector. The per-container collectors (the secret store, NATS, Grafana,
Authelia, the Victoria stores and the agents' forwarders) don't report
their own metrics yet, so they have no lines here; a separate issue on
the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI
collector. Every per-container collector (the secret store, the other service
containers and the agents' forwarders) also scrapes its own endpoint on
loopback and pushes the result up its normal hop, so each has a line of its
own, on its own port: agents all report on 8904, service containers on
8890 to 8903. The series carry `container` (services) or `agent` (agents)
to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI
and in `scripts/pre-push`. It fails when a dashboard query selects a
`service.name` that nothing emits.

View file

@ -72,6 +72,11 @@ let
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
};
# Where the collector serves its own metrics, inside the container's own
# netns. Not the binary's default 8888, which the collector dashboard
# reads as the hive collector's.
selfTelemetryPort = 8904;
# The exporter's NAME picks the wire protocol, so `protocol` has to be
# read here as well as handed to the SDKs above — same expression as the
# swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
@ -210,91 +215,101 @@ in
# and the collector then dies at startup. Same caveat as both other
# tiers; see nix/host-modules/otel.nix for the measurement.
validateConfigFile = true;
settings = {
# `journalctl --follow --lines=0` (what this receiver runs under
# the hood) ships only what's written *after* it starts — every
# collector start has a silent gap at the front. `storage:` below
# persists the read cursor, so a restart resumes from where it
# left off instead of re-opening that gap; `start_at` stays `end`
# (its default) because a cursor already covers every run after
# the first, and `beginning` without one would re-ship the whole
# journal on every restart. The cursor lives in the unit's own
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below
# as `WorkingDirectory` too) — container-lifetime, not the
# host-persisted state bind mount: it indexes /var/log/journal,
# itself fresh on every container start, so the two must share
# a lifetime or the cursor outlives the journal it points into.
# `create_directory` matters here for a build-time-vs-runtime
# reason, not a convenience one: `validateConfigFile` above runs
# `otelcol validate` in the nix build sandbox, before systemd's
# `StateDirectory=` has ever created this path. Without it the
# validator refuses with "directory must exist" and the build fails.
extensions.file_storage = {
directory = "/var/lib/opentelemetry-collector";
create_directory = true;
};
settings = lib.mkMerge [
{
# `journalctl --follow --lines=0` (what this receiver runs under
# the hood) ships only what's written *after* it starts — every
# collector start has a silent gap at the front. `storage:` below
# persists the read cursor, so a restart resumes from where it
# left off instead of re-opening that gap; `start_at` stays `end`
# (its default) because a cursor already covers every run after
# the first, and `beginning` without one would re-ship the whole
# journal on every restart. The cursor lives in the unit's own
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below
# as `WorkingDirectory` too) — container-lifetime, not the
# host-persisted state bind mount: it indexes /var/log/journal,
# itself fresh on every container start, so the two must share
# a lifetime or the cursor outlives the journal it points into.
# `create_directory` matters here for a build-time-vs-runtime
# reason, not a convenience one: `validateConfigFile` above runs
# `otelcol validate` in the nix build sandbox, before systemd's
# `StateDirectory=` has ever created this path. Without it the
# validator refuses with "directory must exist" and the build fails.
extensions.file_storage = {
directory = "/var/lib/opentelemetry-collector";
create_directory = true;
};
receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own default
# is the RUNTIME journal (`/run/log/journal`), which in an agent
# container is empty — journald stores persistently here, so
# every entry is under /var/log/journal. Dropping this line
# leaves a collector that validates, starts, reports healthy and
# forwards nothing.
directory = "/var/log/journal";
storage = "file_storage";
# No `units` allowlist, and no user-session filter like the swarm
# tier's: a container's journal is the harness and what the
# harness spawns, so there is no foreign traffic to filter out and
# an allowlist would only be a list to forget to update.
receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own default
# is the RUNTIME journal (`/run/log/journal`), which in an agent
# container is empty — journald stores persistently here, so
# every entry is under /var/log/journal. Dropping this line
# leaves a collector that validates, starts, reports healthy and
# forwards nothing.
directory = "/var/log/journal";
storage = "file_storage";
# No `units` allowlist, and no user-session filter like the swarm
# tier's: a container's journal is the harness and what the
# harness spawns, so there is no foreign traffic to filter out and
# an allowlist would only be a list to forget to update.
# journald's PRIORITY carries the level every line already has;
# without this the record reaches VictoriaLogs with
# `severity_text: Unspecified` and the store cannot tell an error
# from a debug line. The table — and the inverted direction that
# makes it worth a file of its own — lives in
# ../journald-severity.nix, shared with the swarm tier's receiver.
operators = import ../journald-severity.nix;
};
# journald's PRIORITY carries the level every line already has;
# without this the record reaches VictoriaLogs with
# `severity_text: Unspecified` and the store cannot tell an error
# from a debug line. The table — and the inverted direction that
# makes it worth a file of its own — lives in
# ../journald-severity.nix, shared with the swarm tier's receiver.
operators = import ../journald-severity.nix;
};
# Identity the hop above cannot supply. A host-side reader can say
# which machine a line came from; only a collector running as the
# agent can say which agent. `hive` and `swarm` are deliberately
# NOT stamped here: the swarm tier upserts both from the receiver
# that accepted the sample, precisely so the labels come from
# something the sender cannot write.
processors.resource.attributes = [
{
key = "service.name";
value = "hyperhive-agent";
action = "upsert";
# Identity the hop above cannot supply. A host-side reader can say
# which machine a line came from; only a collector running as the
# agent can say which agent. `hive` and `swarm` are deliberately
# NOT stamped here: the swarm tier upserts both from the receiver
# that accepted the sample, precisely so the labels come from
# something the sender cannot write.
processors.resource.attributes = [
{
key = "service.name";
value = "hyperhive-agent";
action = "upsert";
}
{
key = "agent";
value = userName;
action = "upsert";
}
];
# The hive's collector, at the same base address every in-process
# producer already exports to. `endpoint` is a BASE the exporter
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
exporters.${logExporter} = {
endpoint = cfg.endpoint;
}
{
key = "agent";
value = userName;
action = "upsert";
}
];
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
# The hive's collector, at the same base address every in-process
# producer already exports to. `endpoint` is a BASE the exporter
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
exporters.${logExporter} = {
endpoint = cfg.endpoint;
service.pipelines.logs = {
receivers = [ "journald" ];
processors = [ "resource" ];
exporters = [ logExporter ];
};
# An extension configured but not listed here is INERT — the
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
}
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
service.pipelines.logs = {
receivers = [ "journald" ];
processors = [ "resource" ];
exporters = [ logExporter ];
};
# An extension configured but not listed here is INERT — the
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
};
# Every agent container has a network namespace of its own, so they can all
# use the same port.
(import ../collector-self-scrape.nix {
inherit lib;
port = selfTelemetryPort;
exporter = logExporter;
attributes.agent = userName;
})
];
};
};
}

View file

@ -0,0 +1,52 @@
# Collector settings that make a container's OpenTelemetry collector report its
# own health (refused and failed records, exporter queue, memory), imported by
# every container collector: the agent forwarder, the service journal
# forwarder and the secret store's collector. The two host-side tiers scrape
# themselves through their `scrapeTargets` instead.
#
# One function renders both halves, so the endpoint and the job that reads it
# cannot name different ports. The endpoint binds loopback: the collector
# scrapes itself, and nothing outside the container has a reason to.
#
# The job runs in its own pipeline. A container's other metrics pipeline may
# carry a `resource` processor that upserts `service.name`, which would
# relabel these series away from the `otelcol-contrib` the collector dashboard
# selects on. The attributes stamped here tell the container's collectors
# apart.
#
# `readers`, not `metrics.address`: this collector version rejects `address`
# at startup.
{
lib,
port,
exporter,
attributes,
}:
{
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
inherit port;
};
}
];
receivers."prometheus/self".config.scrape_configs = [
{
job_name = "collector";
static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ];
}
];
processors."resource/self".attributes = lib.mapAttrsToList (key: value: {
inherit key value;
action = "upsert";
}) attributes;
service.pipelines."metrics/self" = {
receivers = [ "prometheus/self" ];
processors = [ "resource/self" ];
exporters = [ exporter ];
};
}

View file

@ -96,8 +96,9 @@ in
type = lib.types.port;
description = ''
Loopback port for the collector's own metrics (refused and dropped
records, exporter failures). Required because the container may share
the host netns, where the binary's default 8888 is the hive tier's.
records, exporter failures), which it scrapes and pushes to the swarm
collector. Required because the container may share the host netns,
where the binary's default 8888 is the hive tier's.
'';
};
@ -242,17 +243,6 @@ in
exporters = [ "otlphttp" ];
};
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
# `address` at startup.
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.telemetryPort;
};
}
];
# An extension not listed here is inert: the receiver's storage would
# name a component that never starts, and the exporter would push
# unauthenticated.
@ -261,6 +251,12 @@ in
authName
];
}
(import ../collector-self-scrape.nix {
inherit lib;
port = cfg.telemetryPort;
exporter = "otlphttp";
attributes.container = cfg.serviceName;
})
# A separate set the module system merges in: joining it to the first with
# `//` would replace the first set's whole `service` attrset.
(lib.mkIf (cfg.otlpMetricsPort != null) {

View file

@ -4438,172 +4438,163 @@ in
# caveat as every other tier; see nix/host-modules/otel.nix for
# the measurement.
validateConfigFile = true;
settings = {
# `journalctl --follow --lines=0` (what the receiver runs under
# the hood) ships only what is written *after* it starts, so
# every collector start has a silent gap at the front. This
# persists the read cursor, so a restart resumes instead of
# re-opening that gap; `start_at` stays `end` (its default),
# since a cursor covers every run after the first and
# `beginning` without one would re-ship the whole journal on
# every restart. `create_directory` is required for a
# build-time reason rather than a convenience one:
# `validateConfigFile` above runs `otelcol validate` in the nix
# sandbox, before systemd's `StateDirectory=` has created the
# path, and the validator refuses with "directory must exist".
extensions.file_storage = {
directory = "/var/lib/opentelemetry-collector";
create_directory = true;
};
settings = lib.mkMerge [
{
# `journalctl --follow --lines=0` (what the receiver runs under
# the hood) ships only what is written *after* it starts, so
# every collector start has a silent gap at the front. This
# persists the read cursor, so a restart resumes instead of
# re-opening that gap; `start_at` stays `end` (its default),
# since a cursor covers every run after the first and
# `beginning` without one would re-ship the whole journal on
# every restart. `create_directory` is required for a
# build-time reason rather than a convenience one:
# `validateConfigFile` above runs `otelcol validate` in the nix
# sandbox, before systemd's `StateDirectory=` has created the
# path, and the validator refuses with "directory must exist".
extensions.file_storage = {
directory = "/var/lib/opentelemetry-collector";
create_directory = true;
};
receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own
# default is the RUNTIME journal (`/run/log/journal`), which
# in this container is empty — journald stores persistently
# here, so every entry is under /var/log/journal. Dropping
# this line leaves a collector that validates, starts,
# reports healthy and forwards nothing.
directory = "/var/log/journal";
# ⚠️ `--merge` is a guard: an unqualified
# `journalctl --follow` over a journal with no entry for the
# current boot (a host-linked one) crash-loops the receiver
# every ~2s with "No journal boot entry found for the
# specified boot (+0)". This journal is container-local, so
# the current boot has an entry and the flag changes nothing.
merge = true;
storage = "file_storage";
};
receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own
# default is the RUNTIME journal (`/run/log/journal`), which
# in this container is empty — journald stores persistently
# here, so every entry is under /var/log/journal. Dropping
# this line leaves a collector that validates, starts,
# reports healthy and forwards nothing.
directory = "/var/log/journal";
# ⚠️ `--merge` is a guard: an unqualified
# `journalctl --follow` over a journal with no entry for the
# current boot (a host-linked one) crash-loops the receiver
# every ~2s with "No journal boot entry found for the
# specified boot (+0)". This journal is container-local, so
# the current boot has an entry and the flag changes nothing.
merge = true;
storage = "file_storage";
};
# The store's scrape, run by this container's own collector.
#
# ⚠️ `metrics_path` and `params` are STATED, and both have to
# be: openbao serves no `/metrics` at all, and
# `/v1/sys/metrics` answers JSON unless the format is asked
# for. A scrape of the default path 404s and a scrape without
# the query parses as nothing — each reads as a dead exporter
# rather than as a wrong request. The spelling matches
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
# every other service's target.
#
# No credential, unlike the swarm collector's
# `publishedScrapeTargets` jobs: the target is the loopback
# listener above, inside this container's own netns, which is
# what lets the scrape stay unauthenticated without the API
# listener's client-cert requirement being touched.
receivers.prometheus.config.scrape_configs = [
{
job_name = "bao";
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
metrics_path = "/v1/sys/metrics";
params.format = [ "prometheus" ];
}
];
# The store's scrape, run by this container's own collector.
#
# ⚠️ `metrics_path` and `params` are STATED, and both have to
# be: openbao serves no `/metrics` at all, and
# `/v1/sys/metrics` answers JSON unless the format is asked
# for. A scrape of the default path 404s and a scrape without
# the query parses as nothing — each reads as a dead exporter
# rather than as a wrong request. The spelling matches
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
# every other service's target.
#
# No credential, unlike the swarm collector's
# `publishedScrapeTargets` jobs: the target is the loopback
# listener above, inside this container's own netns, which is
# what lets the scrape stay unauthenticated without the API
# listener's client-cert requirement being touched.
receivers.prometheus.config.scrape_configs = [
{
job_name = "bao";
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
metrics_path = "/v1/sys/metrics";
params.format = [ "prometheus" ];
}
];
# Identity the hop above cannot supply: a host-side reader can
# say which machine a line came from, and only a collector
# inside this container can say it was the secret store's. No
# `hive` or `swarm` label — the swarm tier upserts `hive` from
# the receiver that accepted the record, precisely so it comes
# from something the sender cannot write.
#
# `collectorServiceName`, not `cfg.machine`: what the
# dashboard selects on is a display identity, not the
# container/receiver name — see that binding's own comment.
processors.resource.attributes = [
{
key = "service.name";
value = collectorServiceName;
action = "upsert";
}
];
# Identity the hop above cannot supply: a host-side reader can
# say which machine a line came from, and only a collector
# inside this container can say it was the secret store's. No
# `hive` or `swarm` label — the swarm tier upserts `hive` from
# the receiver that accepted the record, precisely so it comes
# from something the sender cannot write.
#
# `collectorServiceName`, not `cfg.machine`: what the
# dashboard selects on is a display identity, not the
# container/receiver name — see that binding's own comment.
processors.resource.attributes = [
{
key = "service.name";
value = collectorServiceName;
action = "upsert";
}
];
# `otlphttp` unconditionally, unlike the agent forwarder's
# protocol-derived exporter name: the swarm receiver speaks
# OTLP/HTTP protobuf whatever this tier's *upstream*
# protocol is, which is the same reason
# ./hive-c0re/environment.nix pins the protocol it hands out.
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
exporters.otlphttp = {
endpoint = otelFirstHop;
# The receiver at that address accepts an OIDC token and
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
# so this is what makes the hop deliver rather than collect
# 401s — unconditional, because a forwarder that exports
# unauthenticated has no working mode to fall back to.
auth.authenticator = forwarderAuthName;
};
# `otlphttp` unconditionally, unlike the agent forwarder's
# protocol-derived exporter name: the swarm receiver speaks
# OTLP/HTTP protobuf whatever this tier's *upstream*
# protocol is, which is the same reason
# ./hive-c0re/environment.nix pins the protocol it hands out.
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
exporters.otlphttp = {
endpoint = otelFirstHop;
# The receiver at that address accepts an OIDC token and
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
# so this is what makes the hop deliver rather than collect
# 401s — unconditional, because a forwarder that exports
# unauthenticated has no working mode to fall back to.
auth.authenticator = forwarderAuthName;
};
# The push side, opposite in direction to the receiver's
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
# to send. Hence `oauth2client`, and hence a client id and
# secret rather than an issuer and an audience to check.
extensions.${forwarderAuthName} = {
client_id = cfg.otel.clientId;
# A path, never a value — and one systemd resolves at
# runtime, so neither the secret nor the credentials
# directory is rendered into the nix store.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
token_url = "${toString autheliaCfg.url}/api/oidc/token";
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
# Registering it on the client only makes it permissible; a
# token minted without asking carries `aud: []` and the
# receiver refuses it — with a config that reads perfectly
# at both ends.
endpoint_params.audience = cfg.otel.clientId;
};
# The push side, opposite in direction to the receiver's
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
# to send. Hence `oauth2client`, and hence a client id and
# secret rather than an issuer and an audience to check.
extensions.${forwarderAuthName} = {
client_id = cfg.otel.clientId;
# A path, never a value — and one systemd resolves at
# runtime, so neither the secret nor the credentials
# directory is rendered into the nix store.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
token_url = "${toString autheliaCfg.url}/api/oidc/token";
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
# Registering it on the client only makes it permissible; a
# token minted without asking carries `aud: []` and the
# receiver refuses it — with a config that reads perfectly
# at both ends.
endpoint_params.audience = cfg.otel.clientId;
};
service.pipelines.logs = {
receivers = [ "journald" ];
processors = [ "resource" ];
exporters = [ "otlphttp" ];
};
service.pipelines.logs = {
receivers = [ "journald" ];
processors = [ "resource" ];
exporters = [ "otlphttp" ];
};
# A SECOND pipeline, not a second receiver on the logs one: a
# pipeline is typed, and a prometheus receiver in `logs` is a
# config the collector rejects at startup rather than one that
# quietly ships metrics as log records.
#
# Same `resource` processor and same `otlphttp` exporter, so
# the store's metrics carry the same `service.name` its logs
# do and leave by the same hop — no otel bypass, one exit from
# this container. The exporter treats `endpoint` as a base and
# appends the signal's own path, so `/v1/metrics` here and
# `/v1/logs` above come from the one address.
service.pipelines.metrics = {
receivers = [ "prometheus" ];
processors = [ "resource" ];
exporters = [ "otlphttp" ];
};
# A SECOND pipeline, not a second receiver on the logs one: a
# pipeline is typed, and a prometheus receiver in `logs` is a
# config the collector rejects at startup rather than one that
# quietly ships metrics as log records.
#
# Same `resource` processor and same `otlphttp` exporter, so
# the store's metrics carry the same `service.name` its logs
# do and leave by the same hop — no otel bypass, one exit from
# this container. The exporter treats `endpoint` as a base and
# appends the signal's own path, so `/v1/metrics` here and
# `/v1/logs` above come from the one address.
service.pipelines.metrics = {
receivers = [ "prometheus" ];
processors = [ "resource" ];
exporters = [ "otlphttp" ];
};
# Moves this collector's self-metrics off the built-in
# default of `localhost:8888`, which the hive tier holds —
# this container shares the host's netns, so leaving the
# default in place is the hive collector failing to bind.
#
# ⚠️ `metrics.address` is the spelling that looks right and
# is REJECTED by this collector version:
# `'migration.MetricsConfigV030' has invalid keys:
# address`. `readers` is the schema it accepts, and the
# difference is a startup failure rather than a warning.
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.otel.telemetryPort;
};
}
];
# An extension configured but not listed here is INERT — the
# receiver's `storage: file_storage` above would name a
# component the collector never starts, and the exporter's
# authenticator would push unauthenticated while the
# collector reports healthy.
service.extensions = [
"file_storage"
forwarderAuthName
];
};
# An extension configured but not listed here is INERT — the
# receiver's `storage: file_storage` above would name a
# component the collector never starts, and the exporter's
# authenticator would push unauthenticated while the
# collector reports healthy.
service.extensions = [
"file_storage"
forwarderAuthName
];
}
# Off the built-in default of `localhost:8888`, which the hive tier
# holds: this container shares the host's netns.
(import ../collector-self-scrape.nix {
inherit lib;
port = cfg.otel.telemetryPort;
exporter = "otlphttp";
attributes.container = collectorServiceName;
})
];
};
# How the secret crosses the sandbox. The host unit

View file

@ -1,7 +1,7 @@
{
"title": "hyperhive \u00b7 collector (self-telemetry)",
"uid": "hyperhive-swarm-collector",
"description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
"description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.",
"editable": true,
"refresh": "1m",
"schemaVersion": 39,

View file

@ -21,6 +21,7 @@ let
agent
carriesJournaldSeverity
runGroup
selfScrapes
;
# The log path's three hops, one fixture each. Nothing carries a journal
@ -55,6 +56,20 @@ let
agentSettings = machine: machine.services.opentelemetry-collector.settings;
cases = [
{
# The collector's own counters are the only signal that it is dropping
# telemetry. 8904 keeps agents off 8888, which the collector dashboard
# reads as the hive collector; the exporter follows the wire protocol.
name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
ok =
selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
&& selfScrapes (agentSettings agentOtelGrpc) 8904 {
agent = agentOtelGrpc.services.hyperhive.agent.user.name;
}
# Control: with the switch off the lookup finds no collector config.
&& !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
&& (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
}
{
# The journald receiver's own default directory is the RUNTIME
# journal, and a container that stores persistently leaves that

View file

@ -22,6 +22,7 @@ let
runGroup
baoSettings
forceServiceConfigs
selfScrapes
;
baoTwoAddresses = hive {
@ -185,6 +186,20 @@ let
&& m.exporters == [ "otlphttp" ]
&& m.exporters == s.service.pipelines.logs.exporters;
}
{
# The store's collector reports its own health through a pipeline of its
# own: the store pipeline above upserts `service.name`, and sharing it
# would relabel these series away from the collector dashboard.
name = "the store's collector scrapes its own telemetry endpoint on loopback";
ok =
let
s = (baoForwarder baoWithCollector).settings;
port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
in
selfScrapes s port { container = "bao"; }
&& s.service.pipelines.metrics.receivers == [ "prometheus" ]
&& s.service.pipelines.metrics.processors == [ "resource" ];
}
{
# Presence control for the case above: both fields are omitted rather
# than defaulted, so a target declared as bare `host:port` renders what

View file

@ -25,6 +25,7 @@ let
})
hive
runGroup
selfScrapes
;
inherit (shared) bare;
@ -558,11 +559,10 @@ let
ok =
let
c = ciAlone.containers.swarm-ci-runner.config;
reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
s = c.services.opentelemetry-collector.settings;
in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == 8895;
&& selfScrapes s 8895 { container = "ci-runner"; };
}
];
in

View file

@ -208,6 +208,37 @@ let
# the two tier cases differ only in which receiver they hand it.
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
# Whether a container collector's settings carry the self-scrape of
# ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
# a job scraping exactly that address, and a metrics pipeline that reads the
# job and leaves through an exporter the settings define. `attrs` are the
# resource attributes the pipeline stamps.
selfScrapes =
s: port: attrs:
let
job = lib.findFirst (j: j.job_name == "collector") null (
s.receivers."prometheus/self".config.scrape_configs or [ ]
);
pipeline = s.service.pipelines."metrics/self" or null;
reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
stamped = lib.mapAttrsToList (key: value: {
inherit key value;
action = "upsert";
}) attrs;
in
job != null
&& job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
&&
(reader.pull.exporter.prometheus or null) == {
host = "127.0.0.1";
inherit port;
}
&& pipeline != null
&& pipeline.receivers == [ "prometheus/self" ]
&& (s.processors."resource/self".attributes or null) == stamped
&& pipeline.exporters != [ ]
&& lib.all (e: s.exporters ? ${e}) pipeline.exporters;
# The enables that switch owns. The hive tier's collector is not among them: it
# follows `deploy.hive-controller.enable`, and a services host without a hive
# runs none.
@ -283,6 +314,7 @@ in
inherit agentHarness;
inherit journaldSeverityParser;
inherit carriesJournaldSeverity;
inherit selfScrapes;
inherit swarmServiceEnables;
inherit runGroup;
inherit forceServiceConfigs;

View file

@ -20,6 +20,7 @@ let
})
hive
runGroup
selfScrapes
;
# No store identity, which the first case below relies on.
@ -157,12 +158,10 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort
]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
reader = lib.head s.service.telemetry.metrics.readers;
in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken)
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == svc.port;
&& selfScrapes s svc.port { container = svc.name; };
}
]
) forwarderServices;
@ -269,12 +268,10 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort
]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
reader = lib.head s.service.telemetry.metrics.readers;
in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken)
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == svc.port;
&& selfScrapes s svc.port { container = svc.name; };
}
]
) hostNetnsForwarders;
@ -740,6 +737,7 @@ let
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& lib.elem "oauth2client/swarm" s.service.extensions
&& s.service.pipelines.logs.receivers == [ "journald" ]
&& selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
@ -997,7 +995,7 @@ let
registered = lib.findFirst (
x: x.id == clientId
) null h.services.hyperhive.swarm.authelia.oidc.clients;
reader = lib.head s.service.telemetry.metrics.readers;
port = c.services.hyperhive.swarmJournalForwarder.telemetryPort;
taken = [
8888
8890
@ -1022,8 +1020,8 @@ let
&& registered.audience == [ clientId ]
&& !(lib.elem "--link-journal=host" container.extraFlags)
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& !(lib.elem reader.pull.exporter.prometheus.port taken)
&& selfScrapes s port { container = "authelia"; }
&& !(lib.elem port taken)
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
&& !(
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc