Watch
0
0
Fork
You've already forked hyperhive
0

Compare commits

...
11 changed files with 388 additions and 272 deletions

View file

@ -219,10 +219,12 @@ and failure series on `collector` appear once something fails, and the
collector and of the swarm's collector, which both carry the collector collector and of the swarm's collector, which both carry the collector
binary's default `service.name`, `otelcol-contrib`. The timeseries panels binary's default `service.name`, `otelcol-contrib`. The timeseries panels
split by `server.port`: 8888 is the hive collector, 8889 the swarm split by `server.port`: 8888 is the hive collector, 8889 the swarm
collector. The per-container collectors (the secret store, NATS, Grafana, collector. Every per-container collector (the secret store, the other service
Authelia, the Victoria stores and the agents' forwarders) don't report containers and the agents' forwarders) also scrapes its own endpoint on
their own metrics yet, so they have no lines here; a separate issue on loopback and pushes the result up its normal hop, so each has a line of its
the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI own, on its own port: agents all report on 8904, service containers on
8890 to 8903. The series carry `container` (services) or `agent` (agents)
to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI
and in `scripts/pre-push`. It fails when a dashboard query selects a and in `scripts/pre-push`. It fails when a dashboard query selects a
`service.name` that nothing emits. `service.name` that nothing emits.

View file

@ -72,6 +72,11 @@ let
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
}; };
# Where the collector serves its own metrics, inside the container's own
# netns. Not the binary's default 8888, which the collector dashboard
# reads as the hive collector's.
selfTelemetryPort = 8904;
# The exporter's NAME picks the wire protocol, so `protocol` has to be # The exporter's NAME picks the wire protocol, so `protocol` has to be
# read here as well as handed to the SDKs above — same expression as the # read here as well as handed to the SDKs above — same expression as the
# swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix. # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
@ -210,91 +215,101 @@ in
# and the collector then dies at startup. Same caveat as both other # and the collector then dies at startup. Same caveat as both other
# tiers; see nix/host-modules/otel.nix for the measurement. # tiers; see nix/host-modules/otel.nix for the measurement.
validateConfigFile = true; validateConfigFile = true;
settings = { settings = lib.mkMerge [
# `journalctl --follow --lines=0` (what this receiver runs under {
# the hood) ships only what's written *after* it starts — every # `journalctl --follow --lines=0` (what this receiver runs under
# collector start has a silent gap at the front. `storage:` below # the hood) ships only what's written *after* it starts — every
# persists the read cursor, so a restart resumes from where it # collector start has a silent gap at the front. `storage:` below
# left off instead of re-opening that gap; `start_at` stays `end` # persists the read cursor, so a restart resumes from where it
# (its default) because a cursor already covers every run after # left off instead of re-opening that gap; `start_at` stays `end`
# the first, and `beginning` without one would re-ship the whole # (its default) because a cursor already covers every run after
# journal on every restart. The cursor lives in the unit's own # the first, and `beginning` without one would re-ship the whole
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below # journal on every restart. The cursor lives in the unit's own
# as `WorkingDirectory` too) — container-lifetime, not the # `StateDirectory` (nixpkgs' module already sets one, `%S`, below
# host-persisted state bind mount: it indexes /var/log/journal, # as `WorkingDirectory` too) — container-lifetime, not the
# itself fresh on every container start, so the two must share # host-persisted state bind mount: it indexes /var/log/journal,
# a lifetime or the cursor outlives the journal it points into. # itself fresh on every container start, so the two must share
# `create_directory` matters here for a build-time-vs-runtime # a lifetime or the cursor outlives the journal it points into.
# reason, not a convenience one: `validateConfigFile` above runs # `create_directory` matters here for a build-time-vs-runtime
# `otelcol validate` in the nix build sandbox, before systemd's # reason, not a convenience one: `validateConfigFile` above runs
# `StateDirectory=` has ever created this path. Without it the # `otelcol validate` in the nix build sandbox, before systemd's
# validator refuses with "directory must exist" and the build fails. # `StateDirectory=` has ever created this path. Without it the
extensions.file_storage = { # validator refuses with "directory must exist" and the build fails.
directory = "/var/lib/opentelemetry-collector"; extensions.file_storage = {
create_directory = true; directory = "/var/lib/opentelemetry-collector";
}; create_directory = true;
};
receivers.journald = { receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own default # ⚠️ STATED, and it must stay stated: the receiver's own default
# is the RUNTIME journal (`/run/log/journal`), which in an agent # is the RUNTIME journal (`/run/log/journal`), which in an agent
# container is empty — journald stores persistently here, so # container is empty — journald stores persistently here, so
# every entry is under /var/log/journal. Dropping this line # every entry is under /var/log/journal. Dropping this line
# leaves a collector that validates, starts, reports healthy and # leaves a collector that validates, starts, reports healthy and
# forwards nothing. # forwards nothing.
directory = "/var/log/journal"; directory = "/var/log/journal";
storage = "file_storage"; storage = "file_storage";
# No `units` allowlist, and no user-session filter like the swarm # No `units` allowlist, and no user-session filter like the swarm
# tier's: a container's journal is the harness and what the # tier's: a container's journal is the harness and what the
# harness spawns, so there is no foreign traffic to filter out and # harness spawns, so there is no foreign traffic to filter out and
# an allowlist would only be a list to forget to update. # an allowlist would only be a list to forget to update.
# journald's PRIORITY carries the level every line already has; # journald's PRIORITY carries the level every line already has;
# without this the record reaches VictoriaLogs with # without this the record reaches VictoriaLogs with
# `severity_text: Unspecified` and the store cannot tell an error # `severity_text: Unspecified` and the store cannot tell an error
# from a debug line. The table — and the inverted direction that # from a debug line. The table — and the inverted direction that
# makes it worth a file of its own — lives in # makes it worth a file of its own — lives in
# ../journald-severity.nix, shared with the swarm tier's receiver. # ../journald-severity.nix, shared with the swarm tier's receiver.
operators = import ../journald-severity.nix; operators = import ../journald-severity.nix;
}; };
# Identity the hop above cannot supply. A host-side reader can say # Identity the hop above cannot supply. A host-side reader can say
# which machine a line came from; only a collector running as the # which machine a line came from; only a collector running as the
# agent can say which agent. `hive` and `swarm` are deliberately # agent can say which agent. `hive` and `swarm` are deliberately
# NOT stamped here: the swarm tier upserts both from the receiver # NOT stamped here: the swarm tier upserts both from the receiver
# that accepted the sample, precisely so the labels come from # that accepted the sample, precisely so the labels come from
# something the sender cannot write. # something the sender cannot write.
processors.resource.attributes = [ processors.resource.attributes = [
{ {
key = "service.name"; key = "service.name";
value = "hyperhive-agent"; value = "hyperhive-agent";
action = "upsert"; action = "upsert";
}
{
key = "agent";
value = userName;
action = "upsert";
}
];
# The hive's collector, at the same base address every in-process
# producer already exports to. `endpoint` is a BASE the exporter
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
exporters.${logExporter} = {
endpoint = cfg.endpoint;
} }
{ // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
key = "agent";
value = userName;
action = "upsert";
}
];
# The hive's collector, at the same base address every in-process service.pipelines.logs = {
# producer already exports to. `endpoint` is a BASE the exporter receivers = [ "journald" ];
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. processors = [ "resource" ];
exporters.${logExporter} = { exporters = [ logExporter ];
endpoint = cfg.endpoint; };
# An extension configured but not listed here is INERT — the
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
} }
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; # Every agent container has a network namespace of its own, so they can all
# use the same port.
service.pipelines.logs = { (import ../collector-self-scrape.nix {
receivers = [ "journald" ]; inherit lib;
processors = [ "resource" ]; port = selfTelemetryPort;
exporters = [ logExporter ]; exporter = logExporter;
}; attributes.agent = userName;
})
# An extension configured but not listed here is INERT — the ];
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
};
}; };
}; };
} }

View file

@ -0,0 +1,52 @@
# Collector settings that make a container's OpenTelemetry collector report its
# own health (refused and failed records, exporter queue, memory), imported by
# every container collector: the agent forwarder, the service journal
# forwarder and the secret store's collector. The two host-side tiers scrape
# themselves through their `scrapeTargets` instead.
#
# One function renders both halves, so the endpoint and the job that reads it
# cannot name different ports. The endpoint binds loopback: the collector
# scrapes itself, and nothing outside the container has a reason to.
#
# The job runs in its own pipeline. A container's other metrics pipeline may
# carry a `resource` processor that upserts `service.name`, which would
# relabel these series away from the `otelcol-contrib` the collector dashboard
# selects on. The attributes stamped here tell the container's collectors
# apart.
#
# `readers`, not `metrics.address`: this collector version rejects `address`
# at startup.
{
lib,
port,
exporter,
attributes,
}:
{
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
inherit port;
};
}
];
receivers."prometheus/self".config.scrape_configs = [
{
job_name = "collector";
static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ];
}
];
processors."resource/self".attributes = lib.mapAttrsToList (key: value: {
inherit key value;
action = "upsert";
}) attributes;
service.pipelines."metrics/self" = {
receivers = [ "prometheus/self" ];
processors = [ "resource/self" ];
exporters = [ exporter ];
};
}

View file

@ -96,8 +96,9 @@ in
type = lib.types.port; type = lib.types.port;
description = '' description = ''
Loopback port for the collector's own metrics (refused and dropped Loopback port for the collector's own metrics (refused and dropped
records, exporter failures). Required because the container may share records, exporter failures), which it scrapes and pushes to the swarm
the host netns, where the binary's default 8888 is the hive tier's. collector. Required because the container may share the host netns,
where the binary's default 8888 is the hive tier's.
''; '';
}; };
@ -242,17 +243,6 @@ in
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
# `address` at startup.
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.telemetryPort;
};
}
];
# An extension not listed here is inert: the receiver's storage would # An extension not listed here is inert: the receiver's storage would
# name a component that never starts, and the exporter would push # name a component that never starts, and the exporter would push
# unauthenticated. # unauthenticated.
@ -261,6 +251,12 @@ in
authName authName
]; ];
} }
(import ../collector-self-scrape.nix {
inherit lib;
port = cfg.telemetryPort;
exporter = "otlphttp";
attributes.container = cfg.serviceName;
})
# A separate set the module system merges in: joining it to the first with # A separate set the module system merges in: joining it to the first with
# `//` would replace the first set's whole `service` attrset. # `//` would replace the first set's whole `service` attrset.
(lib.mkIf (cfg.otlpMetricsPort != null) { (lib.mkIf (cfg.otlpMetricsPort != null) {

View file

@ -4438,172 +4438,163 @@ in
# caveat as every other tier; see nix/host-modules/otel.nix for # caveat as every other tier; see nix/host-modules/otel.nix for
# the measurement. # the measurement.
validateConfigFile = true; validateConfigFile = true;
settings = { settings = lib.mkMerge [
# `journalctl --follow --lines=0` (what the receiver runs under {
# the hood) ships only what is written *after* it starts, so # `journalctl --follow --lines=0` (what the receiver runs under
# every collector start has a silent gap at the front. This # the hood) ships only what is written *after* it starts, so
# persists the read cursor, so a restart resumes instead of # every collector start has a silent gap at the front. This
# re-opening that gap; `start_at` stays `end` (its default), # persists the read cursor, so a restart resumes instead of
# since a cursor covers every run after the first and # re-opening that gap; `start_at` stays `end` (its default),
# `beginning` without one would re-ship the whole journal on # since a cursor covers every run after the first and
# every restart. `create_directory` is required for a # `beginning` without one would re-ship the whole journal on
# build-time reason rather than a convenience one: # every restart. `create_directory` is required for a
# `validateConfigFile` above runs `otelcol validate` in the nix # build-time reason rather than a convenience one:
# sandbox, before systemd's `StateDirectory=` has created the # `validateConfigFile` above runs `otelcol validate` in the nix
# path, and the validator refuses with "directory must exist". # sandbox, before systemd's `StateDirectory=` has created the
extensions.file_storage = { # path, and the validator refuses with "directory must exist".
directory = "/var/lib/opentelemetry-collector"; extensions.file_storage = {
create_directory = true; directory = "/var/lib/opentelemetry-collector";
}; create_directory = true;
};
receivers.journald = { receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own # ⚠️ STATED, and it must stay stated: the receiver's own
# default is the RUNTIME journal (`/run/log/journal`), which # default is the RUNTIME journal (`/run/log/journal`), which
# in this container is empty — journald stores persistently # in this container is empty — journald stores persistently
# here, so every entry is under /var/log/journal. Dropping # here, so every entry is under /var/log/journal. Dropping
# this line leaves a collector that validates, starts, # this line leaves a collector that validates, starts,
# reports healthy and forwards nothing. # reports healthy and forwards nothing.
directory = "/var/log/journal"; directory = "/var/log/journal";
# ⚠️ `--merge` is a guard: an unqualified # ⚠️ `--merge` is a guard: an unqualified
# `journalctl --follow` over a journal with no entry for the # `journalctl --follow` over a journal with no entry for the
# current boot (a host-linked one) crash-loops the receiver # current boot (a host-linked one) crash-loops the receiver
# every ~2s with "No journal boot entry found for the # every ~2s with "No journal boot entry found for the
# specified boot (+0)". This journal is container-local, so # specified boot (+0)". This journal is container-local, so
# the current boot has an entry and the flag changes nothing. # the current boot has an entry and the flag changes nothing.
merge = true; merge = true;
storage = "file_storage"; storage = "file_storage";
}; };
# The store's scrape, run by this container's own collector. # The store's scrape, run by this container's own collector.
# #
# ⚠️ `metrics_path` and `params` are STATED, and both have to # ⚠️ `metrics_path` and `params` are STATED, and both have to
# be: openbao serves no `/metrics` at all, and # be: openbao serves no `/metrics` at all, and
# `/v1/sys/metrics` answers JSON unless the format is asked # `/v1/sys/metrics` answers JSON unless the format is asked
# for. A scrape of the default path 404s and a scrape without # for. A scrape of the default path 404s and a scrape without
# the query parses as nothing — each reads as a dead exporter # the query parses as nothing — each reads as a dead exporter
# rather than as a wrong request. The spelling matches # rather than as a wrong request. The spelling matches
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
# every other service's target. # every other service's target.
# #
# No credential, unlike the swarm collector's # No credential, unlike the swarm collector's
# `publishedScrapeTargets` jobs: the target is the loopback # `publishedScrapeTargets` jobs: the target is the loopback
# listener above, inside this container's own netns, which is # listener above, inside this container's own netns, which is
# what lets the scrape stay unauthenticated without the API # what lets the scrape stay unauthenticated without the API
# listener's client-cert requirement being touched. # listener's client-cert requirement being touched.
receivers.prometheus.config.scrape_configs = [ receivers.prometheus.config.scrape_configs = [
{ {
job_name = "bao"; job_name = "bao";
static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
metrics_path = "/v1/sys/metrics"; metrics_path = "/v1/sys/metrics";
params.format = [ "prometheus" ]; params.format = [ "prometheus" ];
} }
]; ];
# Identity the hop above cannot supply: a host-side reader can # Identity the hop above cannot supply: a host-side reader can
# say which machine a line came from, and only a collector # say which machine a line came from, and only a collector
# inside this container can say it was the secret store's. No # inside this container can say it was the secret store's. No
# `hive` or `swarm` label — the swarm tier upserts `hive` from # `hive` or `swarm` label — the swarm tier upserts `hive` from
# the receiver that accepted the record, precisely so it comes # the receiver that accepted the record, precisely so it comes
# from something the sender cannot write. # from something the sender cannot write.
# #
# `collectorServiceName`, not `cfg.machine`: what the # `collectorServiceName`, not `cfg.machine`: what the
# dashboard selects on is a display identity, not the # dashboard selects on is a display identity, not the
# container/receiver name — see that binding's own comment. # container/receiver name — see that binding's own comment.
processors.resource.attributes = [ processors.resource.attributes = [
{ {
key = "service.name"; key = "service.name";
value = collectorServiceName; value = collectorServiceName;
action = "upsert"; action = "upsert";
} }
]; ];
# `otlphttp` unconditionally, unlike the agent forwarder's # `otlphttp` unconditionally, unlike the agent forwarder's
# protocol-derived exporter name: the swarm receiver speaks # protocol-derived exporter name: the swarm receiver speaks
# OTLP/HTTP protobuf whatever this tier's *upstream* # OTLP/HTTP protobuf whatever this tier's *upstream*
# protocol is, which is the same reason # protocol is, which is the same reason
# ./hive-c0re/environment.nix pins the protocol it hands out. # ./hive-c0re/environment.nix pins the protocol it hands out.
# `endpoint` is a BASE the exporter appends `/v1/logs` to. # `endpoint` is a BASE the exporter appends `/v1/logs` to.
exporters.otlphttp = { exporters.otlphttp = {
endpoint = otelFirstHop; endpoint = otelFirstHop;
# The receiver at that address accepts an OIDC token and # The receiver at that address accepts an OIDC token and
# nothing else (./swarm-otel.nix's `oidc/*` authenticators), # nothing else (./swarm-otel.nix's `oidc/*` authenticators),
# so this is what makes the hop deliver rather than collect # so this is what makes the hop deliver rather than collect
# 401s — unconditional, because a forwarder that exports # 401s — unconditional, because a forwarder that exports
# unauthenticated has no working mode to fall back to. # unauthenticated has no working mode to fall back to.
auth.authenticator = forwarderAuthName; auth.authenticator = forwarderAuthName;
}; };
# The push side, opposite in direction to the receiver's # The push side, opposite in direction to the receiver's
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
# to send. Hence `oauth2client`, and hence a client id and # to send. Hence `oauth2client`, and hence a client id and
# secret rather than an issuer and an audience to check. # secret rather than an issuer and an audience to check.
extensions.${forwarderAuthName} = { extensions.${forwarderAuthName} = {
client_id = cfg.otel.clientId; client_id = cfg.otel.clientId;
# A path, never a value — and one systemd resolves at # A path, never a value — and one systemd resolves at
# runtime, so neither the secret nor the credentials # runtime, so neither the secret nor the credentials
# directory is rendered into the nix store. # directory is rendered into the nix store.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
token_url = "${toString autheliaCfg.url}/api/oidc/token"; token_url = "${toString autheliaCfg.url}/api/oidc/token";
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
# Registering it on the client only makes it permissible; a # Registering it on the client only makes it permissible; a
# token minted without asking carries `aud: []` and the # token minted without asking carries `aud: []` and the
# receiver refuses it — with a config that reads perfectly # receiver refuses it — with a config that reads perfectly
# at both ends. # at both ends.
endpoint_params.audience = cfg.otel.clientId; endpoint_params.audience = cfg.otel.clientId;
}; };
service.pipelines.logs = { service.pipelines.logs = {
receivers = [ "journald" ]; receivers = [ "journald" ];
processors = [ "resource" ]; processors = [ "resource" ];
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# A SECOND pipeline, not a second receiver on the logs one: a # A SECOND pipeline, not a second receiver on the logs one: a
# pipeline is typed, and a prometheus receiver in `logs` is a # pipeline is typed, and a prometheus receiver in `logs` is a
# config the collector rejects at startup rather than one that # config the collector rejects at startup rather than one that
# quietly ships metrics as log records. # quietly ships metrics as log records.
# #
# Same `resource` processor and same `otlphttp` exporter, so # Same `resource` processor and same `otlphttp` exporter, so
# the store's metrics carry the same `service.name` its logs # the store's metrics carry the same `service.name` its logs
# do and leave by the same hop — no otel bypass, one exit from # do and leave by the same hop — no otel bypass, one exit from
# this container. The exporter treats `endpoint` as a base and # this container. The exporter treats `endpoint` as a base and
# appends the signal's own path, so `/v1/metrics` here and # appends the signal's own path, so `/v1/metrics` here and
# `/v1/logs` above come from the one address. # `/v1/logs` above come from the one address.
service.pipelines.metrics = { service.pipelines.metrics = {
receivers = [ "prometheus" ]; receivers = [ "prometheus" ];
processors = [ "resource" ]; processors = [ "resource" ];
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# Moves this collector's self-metrics off the built-in # An extension configured but not listed here is INERT — the
# default of `localhost:8888`, which the hive tier holds — # receiver's `storage: file_storage` above would name a
# this container shares the host's netns, so leaving the # component the collector never starts, and the exporter's
# default in place is the hive collector failing to bind. # authenticator would push unauthenticated while the
# # collector reports healthy.
# ⚠️ `metrics.address` is the spelling that looks right and service.extensions = [
# is REJECTED by this collector version: "file_storage"
# `'migration.MetricsConfigV030' has invalid keys: forwarderAuthName
# address`. `readers` is the schema it accepts, and the ];
# difference is a startup failure rather than a warning. }
service.telemetry.metrics.readers = [ # Off the built-in default of `localhost:8888`, which the hive tier
{ # holds: this container shares the host's netns.
pull.exporter.prometheus = { (import ../collector-self-scrape.nix {
host = "127.0.0.1"; inherit lib;
port = cfg.otel.telemetryPort; port = cfg.otel.telemetryPort;
}; exporter = "otlphttp";
} attributes.container = collectorServiceName;
]; })
];
# An extension configured but not listed here is INERT — the
# receiver's `storage: file_storage` above would name a
# component the collector never starts, and the exporter's
# authenticator would push unauthenticated while the
# collector reports healthy.
service.extensions = [
"file_storage"
forwarderAuthName
];
};
}; };
# How the secret crosses the sandbox. The host unit # How the secret crosses the sandbox. The host unit

View file

@ -1,7 +1,7 @@
{ {
"title": "hyperhive \u00b7 collector (self-telemetry)", "title": "hyperhive \u00b7 collector (self-telemetry)",
"uid": "hyperhive-swarm-collector", "uid": "hyperhive-swarm-collector",
"description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.", "description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.",
"editable": true, "editable": true,
"refresh": "1m", "refresh": "1m",
"schemaVersion": 39, "schemaVersion": 39,

View file

@ -21,6 +21,7 @@ let
agent agent
carriesJournaldSeverity carriesJournaldSeverity
runGroup runGroup
selfScrapes
; ;
# The log path's three hops, one fixture each. Nothing carries a journal # The log path's three hops, one fixture each. Nothing carries a journal
@ -55,6 +56,20 @@ let
agentSettings = machine: machine.services.opentelemetry-collector.settings; agentSettings = machine: machine.services.opentelemetry-collector.settings;
cases = [ cases = [
{
# The collector's own counters are the only signal that it is dropping
# telemetry. 8904 keeps agents off 8888, which the collector dashboard
# reads as the hive collector; the exporter follows the wire protocol.
name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
ok =
selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
&& selfScrapes (agentSettings agentOtelGrpc) 8904 {
agent = agentOtelGrpc.services.hyperhive.agent.user.name;
}
# Control: with the switch off the lookup finds no collector config.
&& !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
&& (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
}
{ {
# The journald receiver's own default directory is the RUNTIME # The journald receiver's own default directory is the RUNTIME
# journal, and a container that stores persistently leaves that # journal, and a container that stores persistently leaves that

View file

@ -22,6 +22,7 @@ let
runGroup runGroup
baoSettings baoSettings
forceServiceConfigs forceServiceConfigs
selfScrapes
; ;
baoTwoAddresses = hive { baoTwoAddresses = hive {
@ -185,6 +186,20 @@ let
&& m.exporters == [ "otlphttp" ] && m.exporters == [ "otlphttp" ]
&& m.exporters == s.service.pipelines.logs.exporters; && m.exporters == s.service.pipelines.logs.exporters;
} }
{
# The store's collector reports its own health through a pipeline of its
# own: the store pipeline above upserts `service.name`, and sharing it
# would relabel these series away from the collector dashboard.
name = "the store's collector scrapes its own telemetry endpoint on loopback";
ok =
let
s = (baoForwarder baoWithCollector).settings;
port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
in
selfScrapes s port { container = "bao"; }
&& s.service.pipelines.metrics.receivers == [ "prometheus" ]
&& s.service.pipelines.metrics.processors == [ "resource" ];
}
{ {
# Presence control for the case above: both fields are omitted rather # Presence control for the case above: both fields are omitted rather
# than defaulted, so a target declared as bare `host:port` renders what # than defaulted, so a target declared as bare `host:port` renders what

View file

@ -25,6 +25,7 @@ let
}) })
hive hive
runGroup runGroup
selfScrapes
; ;
inherit (shared) bare; inherit (shared) bare;
@ -558,11 +559,10 @@ let
ok = ok =
let let
c = ciAlone.containers.swarm-ci-runner.config; c = ciAlone.containers.swarm-ci-runner.config;
reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers; s = c.services.opentelemetry-collector.settings;
in in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895 c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
&& reader.pull.exporter.prometheus.host == "127.0.0.1" && selfScrapes s 8895 { container = "ci-runner"; };
&& reader.pull.exporter.prometheus.port == 8895;
} }
]; ];
in in

View file

@ -208,6 +208,37 @@ let
# the two tier cases differ only in which receiver they hand it. # the two tier cases differ only in which receiver they hand it.
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
# Whether a container collector's settings carry the self-scrape of
# ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
# a job scraping exactly that address, and a metrics pipeline that reads the
# job and leaves through an exporter the settings define. `attrs` are the
# resource attributes the pipeline stamps.
selfScrapes =
s: port: attrs:
let
job = lib.findFirst (j: j.job_name == "collector") null (
s.receivers."prometheus/self".config.scrape_configs or [ ]
);
pipeline = s.service.pipelines."metrics/self" or null;
reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
stamped = lib.mapAttrsToList (key: value: {
inherit key value;
action = "upsert";
}) attrs;
in
job != null
&& job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
&&
(reader.pull.exporter.prometheus or null) == {
host = "127.0.0.1";
inherit port;
}
&& pipeline != null
&& pipeline.receivers == [ "prometheus/self" ]
&& (s.processors."resource/self".attributes or null) == stamped
&& pipeline.exporters != [ ]
&& lib.all (e: s.exporters ? ${e}) pipeline.exporters;
# The enables that switch owns. The hive tier's collector is not among them: it # The enables that switch owns. The hive tier's collector is not among them: it
# follows `deploy.hive-controller.enable`, and a services host without a hive # follows `deploy.hive-controller.enable`, and a services host without a hive
# runs none. # runs none.
@ -283,6 +314,7 @@ in
inherit agentHarness; inherit agentHarness;
inherit journaldSeverityParser; inherit journaldSeverityParser;
inherit carriesJournaldSeverity; inherit carriesJournaldSeverity;
inherit selfScrapes;
inherit swarmServiceEnables; inherit swarmServiceEnables;
inherit runGroup; inherit runGroup;
inherit forceServiceConfigs; inherit forceServiceConfigs;

View file

@ -20,6 +20,7 @@ let
}) })
hive hive
runGroup runGroup
selfScrapes
; ;
# No store identity, which the first case below relies on. # No store identity, which the first case below relies on.
@ -157,12 +158,10 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort h.services.hyperhive.swarm.nats.otel.telemetryPort
] ]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices); ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
reader = lib.head s.service.telemetry.metrics.readers;
in in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken) && !(lib.elem svc.port taken)
&& reader.pull.exporter.prometheus.host == "127.0.0.1" && selfScrapes s svc.port { container = svc.name; };
&& reader.pull.exporter.prometheus.port == svc.port;
} }
] ]
) forwarderServices; ) forwarderServices;
@ -269,12 +268,10 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort h.services.hyperhive.swarm.nats.otel.telemetryPort
] ]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders); ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
reader = lib.head s.service.telemetry.metrics.readers;
in in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken) && !(lib.elem svc.port taken)
&& reader.pull.exporter.prometheus.host == "127.0.0.1" && selfScrapes s svc.port { container = svc.name; };
&& reader.pull.exporter.prometheus.port == svc.port;
} }
] ]
) hostNetnsForwarders; ) hostNetnsForwarders;
@ -740,6 +737,7 @@ let
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& lib.elem "oauth2client/swarm" s.service.extensions && lib.elem "oauth2client/swarm" s.service.extensions
&& s.service.pipelines.logs.receivers == [ "journald" ] && s.service.pipelines.logs.receivers == [ "journald" ]
&& selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential && lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem" && fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
@ -997,7 +995,7 @@ let
registered = lib.findFirst ( registered = lib.findFirst (
x: x.id == clientId x: x.id == clientId
) null h.services.hyperhive.swarm.authelia.oidc.clients; ) null h.services.hyperhive.swarm.authelia.oidc.clients;
reader = lib.head s.service.telemetry.metrics.readers; port = c.services.hyperhive.swarmJournalForwarder.telemetryPort;
taken = [ taken = [
8888 8888
8890 8890
@ -1022,8 +1020,8 @@ let
&& registered.audience == [ clientId ] && registered.audience == [ clientId ]
&& !(lib.elem "--link-journal=host" container.extraFlags) && !(lib.elem "--link-journal=host" container.extraFlags)
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags && lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
&& reader.pull.exporter.prometheus.host == "127.0.0.1" && selfScrapes s port { container = "authelia"; }
&& !(lib.elem reader.pull.exporter.prometheus.port taken) && !(lib.elem port taken)
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable && !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
&& !( && !(
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc